From 392bfa873777ebe6fae464fa07be9311afc62983 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 13:28:21 +0000 Subject: [PATCH 1/5] direct: use the postgres branch when its create request times out Creating a postgres branch sometimes fails with 504 DEADLINE_EXCEEDED, "The create request timed out and may still be completing. Check whether the resource was created before retrying." The deploy failed even when the branch had been created. On a transient create error, read the branch: if it exists, wait until it is READY and use it; if it does not, repeat the create (retrySafe). With replace_existing the request is repeated directly, since a branch found then may predate it. cloud: TestAccept/bundle/resources/postgres_branches TestAccept/bundle/resources/postgres_roles/replace_existing Co-authored-by: Isaac --- .../create_504/databricks.yml | 18 +++++++ .../create_504/out.test.toml | 3 ++ .../postgres_branches/create_504/output.txt | 30 +++++++++++ .../postgres_branches/create_504/script | 12 +++++ .../postgres_branches/create_504/test.toml | 6 +++ bundle/direct/dresources/postgres_branch.go | 51 ++++++++++++++++++- 6 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/resources/postgres_branches/create_504/databricks.yml create mode 100644 acceptance/bundle/resources/postgres_branches/create_504/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_branches/create_504/output.txt create mode 100644 acceptance/bundle/resources/postgres_branches/create_504/script create mode 100644 acceptance/bundle/resources/postgres_branches/create_504/test.toml diff --git a/acceptance/bundle/resources/postgres_branches/create_504/databricks.yml b/acceptance/bundle/resources/postgres_branches/create_504/databricks.yml new file mode 100644 index 00000000000..3a5d296fbfe --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test-bundle + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj + display_name: "Test Postgres Project" + pg_version: 16 + + postgres_branches: + dev: + parent: ${resources.postgres_projects.my_project.id} + branch_id: dev + no_expiry: true diff --git a/acceptance/bundle/resources/postgres_branches/create_504/out.test.toml b/acceptance/bundle/resources/postgres_branches/create_504/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/postgres_branches/create_504/output.txt b/acceptance/bundle/resources/postgres_branches/create_504/output.txt new file mode 100644 index 00000000000..9efc60f88f6 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/output.txt @@ -0,0 +1,30 @@ + +=== The create request times out but the branch was created: deploy uses the branch + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Warn: deploying resources.postgres_branches.dev: the branch was created despite the failed create request: Fault injected by test. +Created postgres_branches.dev +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres/projects/test-pg-proj/branches +POST /api/2.0/postgres/projects/test-pg-proj/branches +GET /api/2.0/postgres/projects/test-pg-proj/branches/dev + +=== The create request times out and the branch was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Warn: deploying resources.postgres_branches.dev: retrying after 504 Gateway Timeout from POST /api/2.0/postgres/projects/test-pg-proj/branches +Created postgres_branches.dev +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres/projects/test-pg-proj/branches +POST /api/2.0/postgres/projects/test-pg-proj/branches +GET /api/2.0/postgres/projects/test-pg-proj/branches/dev +POST /api/2.0/postgres/projects/test-pg-proj/branches +GET /api/2.0/postgres/projects/test-pg-proj/branches/dev/operations/[UUID] diff --git a/acceptance/bundle/resources/postgres_branches/create_504/script b/acceptance/bundle/resources/postgres_branches/create_504/script new file mode 100644 index 00000000000..1ea194b4dc4 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/script @@ -0,0 +1,12 @@ +title "The create request times out but the branch was created: deploy uses the branch" +fault.py --after-handler "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The create request times out and the branch was not created: deploy repeats the create" +fault.py "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 diff --git a/acceptance/bundle/resources/postgres_branches/create_504/test.toml b/acceptance/bundle/resources/postgres_branches/create_504/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_branches/create_504/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/bundle/direct/dresources/postgres_branch.go b/bundle/direct/dresources/postgres_branch.go index e134d5d8876..b678fafd088 100644 --- a/bundle/direct/dresources/postgres_branch.go +++ b/bundle/direct/dresources/postgres_branch.go @@ -2,10 +2,15 @@ package dresources import ( "context" + "fmt" "slices" + "time" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/common/types/fieldmask" sdktime "github.com/databricks/databricks-sdk-go/common/types/time" "github.com/databricks/databricks-sdk-go/marshal" @@ -125,7 +130,7 @@ func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresB ForceSendFields: nil, }) if err != nil { - return "", nil, err + return r.recoverFailedCreate(ctx, config, err) } // Wait for the branch to be ready (long-running operation) @@ -138,6 +143,50 @@ func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresB return remote.Name, remote, nil } +// branchReadyTimeout bounds the wait for a branch that recoverFailedCreate found still initializing. +const branchReadyTimeout = 10 * time.Minute + +// recoverFailedCreate handles a create request that failed with a transient error. The backend +// answers a slow create with 504 DEADLINE_EXCEEDED ("may still be completing"), so the branch may +// exist anyway: use it once it is ready instead of failing, or, if it does not exist, let the +// caller repeat the create. +func (r *ResourcePostgresBranch) recoverFailedCreate(ctx context.Context, config *PostgresBranchState, createErr error) (string, *PostgresBranchRemote, error) { + if !retry.IsTransient(ctx, createErr) { + return "", nil, createErr + } + // With replace_existing, a branch found now may predate the request, and repeating the + // request is safe. + if config.ReplaceExisting != nil && *config.ReplaceExisting { + return "", nil, retrySafe(createErr) + } + + name := config.Parent + "/branches/" + config.BranchId + deadline := time.Now().Add(branchReadyTimeout) + for { + branch, err := r.client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name}) + if apierr.IsMissing(err) { + return "", nil, retrySafe(createErr) + } + if err != nil { + log.Warnf(ctx, "checking whether %s was created: %s", name, err) + return "", nil, createErr + } + if branch.Status != nil && branch.Status.CurrentState == postgres.BranchStatusStateReady { + log.Warnf(ctx, "the branch was created despite the failed create request: %s", createErr) + remote := makePostgresBranchRemote(branch) + return remote.Name, remote, nil + } + if time.Now().After(deadline) { + return "", nil, fmt.Errorf("%w; branch %s exists but did not become ready within %s", createErr, name, branchReadyTimeout) + } + select { + case <-ctx.Done(): + return "", nil, ctx.Err() + case <-time.After(5 * time.Second): + } + } +} + // expire_time, no_expiry and ttl are three sides of one oneof, and the API accepts them // in update_mask only under the group name: masking the field itself is answered with // "Unknown field path in update_mask". Probed against a real workspace on 2026-08-31. From e52f186ada4875b20ae8c55d5cd9e8630a3197e8 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 13:28:59 +0000 Subject: [PATCH 2/5] Add changelog entry cloud: TestAccept/bundle/resources/postgres_branches TestAccept/bundle/resources/postgres_roles/replace_existing Co-authored-by: Isaac --- .nextchanges/bundles/7602.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/7602.md diff --git a/.nextchanges/bundles/7602.md b/.nextchanges/bundles/7602.md new file mode 100644 index 00000000000..9fe798f6fe3 --- /dev/null +++ b/.nextchanges/bundles/7602.md @@ -0,0 +1 @@ +* Use the Lakebase branch when its create request times out but the branch was created, instead of failing the deploy, in the direct deployment engine. ([#7602](https://github.com/databricks/cli/pull/7602)) From 03c8c111e089f97609b1974a901ac093fea438a2 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Fri, 13 Nov 2026 13:32:48 +0000 Subject: [PATCH 3/5] Stop waiting on a branch that is not initializing; retry its reads Addresses review: a found branch in DELETED/ARCHIVED never becomes READY, so fail at once instead of polling for 10 minutes; retry transient errors on the GET used to check the branch. cloud: TestAccept/bundle/resources/postgres_branches TestAccept/bundle/resources/postgres_roles/replace_existing Co-authored-by: Isaac --- bundle/direct/dresources/postgres_branch.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/postgres_branch.go b/bundle/direct/dresources/postgres_branch.go index b678fafd088..782b92aa59d 100644 --- a/bundle/direct/dresources/postgres_branch.go +++ b/bundle/direct/dresources/postgres_branch.go @@ -143,7 +143,7 @@ func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresB return remote.Name, remote, nil } -// branchReadyTimeout bounds the wait for a branch that recoverFailedCreate found still initializing. +// branchReadyTimeout bounds the wait for a branch that recoverFailedCreate found in INIT. const branchReadyTimeout = 10 * time.Minute // recoverFailedCreate handles a create request that failed with a transient error. The backend @@ -163,7 +163,9 @@ func (r *ResourcePostgresBranch) recoverFailedCreate(ctx context.Context, config name := config.Parent + "/branches/" + config.BranchId deadline := time.Now().Add(branchReadyTimeout) for { - branch, err := r.client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name}) + branch, err := retry.With(ctx, retry.IsTransientRead, func() (*postgres.Branch, error) { + return r.client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name}) + }) if apierr.IsMissing(err) { return "", nil, retrySafe(createErr) } @@ -171,11 +173,20 @@ func (r *ResourcePostgresBranch) recoverFailedCreate(ctx context.Context, config log.Warnf(ctx, "checking whether %s was created: %s", name, err) return "", nil, createErr } - if branch.Status != nil && branch.Status.CurrentState == postgres.BranchStatusStateReady { + var state postgres.BranchStatusState + if branch.Status != nil { + state = branch.Status.CurrentState + } + if state == postgres.BranchStatusStateReady { log.Warnf(ctx, "the branch was created despite the failed create request: %s", createErr) remote := makePostgresBranchRemote(branch) return remote.Name, remote, nil } + // Only a new branch still initializing is worth waiting for; any other state (e.g. DELETED, + // ARCHIVED) will not become READY. + if state != postgres.BranchStatusStateInit { + return "", nil, fmt.Errorf("%w; branch %s exists in state %q", createErr, name, state) + } if time.Now().After(deadline) { return "", nil, fmt.Errorf("%w; branch %s exists but did not become ready within %s", createErr, name, branchReadyTimeout) } From 6cb360a2e84197fd166d3ddca624081cfff7ace8 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 13:23:48 +0000 Subject: [PATCH 4/5] direct: address review of postgres branch create recovery - Repeat the create request inside DoCreate while the branch is missing, and treat ALREADY_EXISTS on a repeat as the earlier request having landed, so the next read finds the branch. The repeat used to go back to the caller's retry, where ALREADY_EXISTS ended the deploy. - Use a found branch only if it was created at or after the first request (with a minute of clock skew), so a branch that predates it is not taken over. - Once the branch is ours, return it even if the wait for READY ends early, so state records it and the next deploy does not fail with ALREADY_EXISTS. - Pass a cancelled context through instead of reporting a failed existence check. - Log using the recovered branch at Info, so acceptance goldens are not affected when the cloud deploy recovers. - Make the wait timeout and poll interval variables, and cover every recovery path in unit tests; add acceptance scenarios for replace_existing and a failed read. - Add retry.Pause, the log-and-wait step of retry.With, for loops like this one. cloud: TestAccept/bundle/resources/postgres_branches TestAccept/bundle/resources/postgres_roles/replace_existing TestAccept/bundle/resources/postgres_databases/update Co-authored-by: Isaac --- .../postgres_branches/create_504/output.txt | 41 ++- .../postgres_branches/create_504/script | 17 ++ bundle/direct/dresources/postgres_branch.go | 141 +++++++--- .../direct/dresources/postgres_branch_test.go | 266 ++++++++++++++++++ libs/retry/retry.go | 20 +- 5 files changed, 438 insertions(+), 47 deletions(-) create mode 100644 bundle/direct/dresources/postgres_branch_test.go diff --git a/acceptance/bundle/resources/postgres_branches/create_504/output.txt b/acceptance/bundle/resources/postgres_branches/create_504/output.txt index 9efc60f88f6..1caeac8cd64 100644 --- a/acceptance/bundle/resources/postgres_branches/create_504/output.txt +++ b/acceptance/bundle/resources/postgres_branches/create_504/output.txt @@ -3,9 +3,8 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... -Created postgres_projects.my_project -Warn: deploying resources.postgres_branches.dev: the branch was created despite the failed create request: Fault injected by test. Created postgres_branches.dev +Created postgres_projects.my_project Files: 0 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged @@ -27,4 +26,40 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged POST /api/2.0/postgres/projects/test-pg-proj/branches GET /api/2.0/postgres/projects/test-pg-proj/branches/dev POST /api/2.0/postgres/projects/test-pg-proj/branches -GET /api/2.0/postgres/projects/test-pg-proj/branches/dev/operations/[UUID] +GET /api/2.0/postgres/projects/test-pg-proj/branches/dev/operations/[UUID][0] + +=== The branch cannot be read after the create request times out: deploy fails with the create error + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Warn: deploying resources.postgres_branches.dev: checking whether projects/test-pg-proj/branches/dev was created: Fault injected by test. +Error: cannot create resources.postgres_branches.dev: Fault injected by test. (504 DEADLINE_EXCEEDED) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-pg-proj/branches?branch_id=dev +HTTP Status: 504 Gateway Timeout +API error_code: DEADLINE_EXCEEDED +API message: Fault injected by test. + +Files: 0 uploaded, 0 deleted + +Exit code: 1 + +>>> print_requests.py --get //postgres/projects/test-pg-proj/branches +POST /api/2.0/postgres/projects/test-pg-proj/branches +GET /api/2.0/postgres/projects/test-pg-proj/branches/dev + +=== With replace_existing, a timed-out create request is repeated without reading the branch + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Warn: deploying resources.postgres_branches.dev: retrying after 504 Gateway Timeout from POST /api/2.0/postgres/projects/test-pg-proj/branches +Created postgres_branches.dev +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres/projects/test-pg-proj/branches +POST /api/2.0/postgres/projects/test-pg-proj/branches +POST /api/2.0/postgres/projects/test-pg-proj/branches +GET /api/2.0/postgres/projects/test-pg-proj/branches/dev/operations/[UUID][1] diff --git a/acceptance/bundle/resources/postgres_branches/create_504/script b/acceptance/bundle/resources/postgres_branches/create_504/script index 1ea194b4dc4..48ff7fbe56e 100644 --- a/acceptance/bundle/resources/postgres_branches/create_504/script +++ b/acceptance/bundle/resources/postgres_branches/create_504/script @@ -10,3 +10,20 @@ fault.py "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED trace $CLI bundle deploy trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path' $CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The branch cannot be read after the create request times out: deploy fails with the create error" +fault.py "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED +fault.py "GET /api/2.0/postgres/projects/*/branches/dev" 403 0 1 PERMISSION_DENIED +errcode trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "With replace_existing, a timed-out create request is repeated without reading the branch" +update_file.py databricks.yml "no_expiry: true" "no_expiry: true + replace_existing: true" +fault.py --after-handler "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 diff --git a/bundle/direct/dresources/postgres_branch.go b/bundle/direct/dresources/postgres_branch.go index 782b92aa59d..162aa77dbbd 100644 --- a/bundle/direct/dresources/postgres_branch.go +++ b/bundle/direct/dresources/postgres_branch.go @@ -2,6 +2,7 @@ package dresources import ( "context" + "errors" "fmt" "slices" "time" @@ -110,7 +111,7 @@ func (r *ResourcePostgresBranch) DoRead(ctx context.Context, id string) (*Postgr } func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresBranchState) (string, *PostgresBranchRemote, error) { - waiter, err := r.client.Postgres.CreateBranch(ctx, postgres.CreateBranchRequest{ + req := postgres.CreateBranchRequest{ BranchId: config.BranchId, Parent: config.Parent, Branch: postgres.Branch{ @@ -128,74 +129,138 @@ func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresB }, ReplaceExisting: config.ReplaceExisting != nil && *config.ReplaceExisting, ForceSendFields: nil, - }) + } + start := time.Now() + waiter, err := r.client.Postgres.CreateBranch(ctx, req) if err != nil { - return r.recoverFailedCreate(ctx, config, err) + return r.recoverFailedCreate(ctx, req, start, err) } + return r.waitCreated(ctx, waiter) +} - // Wait for the branch to be ready (long-running operation) +// waitCreated waits for the create operation of a branch that a create request returned. +func (r *ResourcePostgresBranch) waitCreated(ctx context.Context, waiter postgres.CreateBranchOperationInterface) (string, *PostgresBranchRemote, error) { result, err := waitCreateOperation(ctx, waiter.Wait) if err != nil { return "", nil, err } - remote := makePostgresBranchRemote(result) return remote.Name, remote, nil } -// branchReadyTimeout bounds the wait for a branch that recoverFailedCreate found in INIT. -const branchReadyTimeout = 10 * time.Minute +// Tuning for recoverFailedCreate, variables so that tests can shorten them. +var ( + // branchReadyTimeout bounds the wait for a recovered branch that is still in INIT. + branchReadyTimeout = 10 * time.Minute + // branchReadyPollInterval is the pause between reads of a recovered branch in INIT. + branchReadyPollInterval = 5 * time.Second + // branchCreateTimeSkew is how much earlier than our first create request a found branch's + // create_time may be, to allow for clock skew, and still count as created by that request. + branchCreateTimeSkew = time.Minute +) -// recoverFailedCreate handles a create request that failed with a transient error. The backend -// answers a slow create with 504 DEADLINE_EXCEEDED ("may still be completing"), so the branch may -// exist anyway: use it once it is ready instead of failing, or, if it does not exist, let the -// caller repeat the create. -func (r *ResourcePostgresBranch) recoverFailedCreate(ctx context.Context, config *PostgresBranchState, createErr error) (string, *PostgresBranchRemote, error) { - if !retry.IsTransient(ctx, createErr) { +// recoverFailedCreate handles a create request that failed with a transient error. A slow create +// is answered with 504 DEADLINE_EXCEEDED ("The create request timed out and may still be +// completing"), so the request may create the branch anyway. Look for the branch: use it if our +// request created it, and repeat the request while it is missing. A repeated request that fails +// with ALREADY_EXISTS means an earlier one has landed since, so the next look finds that branch. +func (r *ResourcePostgresBranch) recoverFailedCreate(ctx context.Context, req postgres.CreateBranchRequest, start time.Time, createErr error) (string, *PostgresBranchRemote, error) { + if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) { return "", nil, createErr } - // With replace_existing, a branch found now may predate the request, and repeating the - // request is safe. - if config.ReplaceExisting != nil && *config.ReplaceExisting { + // A branch found after a replace may predate it, and repeating a replace is safe. + if req.ReplaceExisting { return "", nil, retrySafe(createErr) } - name := config.Parent + "/branches/" + config.BranchId - deadline := time.Now().Add(branchReadyTimeout) - for { - branch, err := retry.With(ctx, retry.IsTransientRead, func() (*postgres.Branch, error) { - return r.client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name}) - }) - if apierr.IsMissing(err) { - return "", nil, retrySafe(createErr) + name := req.Parent + "/branches/" + req.BranchId + lastErr := createErr + for attempt := 1; ; attempt++ { + branch, err := r.readBranch(ctx, name) + if ctx.Err() != nil { + return "", nil, ctx.Err() } - if err != nil { + if err == nil { + return r.useRecoveredBranch(ctx, branch, start, createErr) + } + if !apierr.IsMissing(err) { log.Warnf(ctx, "checking whether %s was created: %s", name, err) return "", nil, createErr } - var state postgres.BranchStatusState - if branch.Status != nil { - state = branch.Status.CurrentState + if attempt > retry.MaxRetries { + return "", nil, lastErr } - if state == postgres.BranchStatusStateReady { - log.Warnf(ctx, "the branch was created despite the failed create request: %s", createErr) - remote := makePostgresBranchRemote(branch) - return remote.Name, remote, nil + if err := retry.Pause(ctx, lastErr); err != nil { + return "", nil, err } - // Only a new branch still initializing is worth waiting for; any other state (e.g. DELETED, - // ARCHIVED) will not become READY. - if state != postgres.BranchStatusStateInit { - return "", nil, fmt.Errorf("%w; branch %s exists in state %q", createErr, name, state) + waiter, err := r.client.Postgres.CreateBranch(ctx, req) + if err == nil { + return r.waitCreated(ctx, waiter) } + if !retry.IsTransient(ctx, err) && !errors.Is(err, apierr.ErrAlreadyExists) { + return "", nil, err + } + lastErr = err + } +} + +func (r *ResourcePostgresBranch) readBranch(ctx context.Context, name string) (*postgres.Branch, error) { + return retry.With(ctx, retry.IsTransientRead, func() (*postgres.Branch, error) { + return r.client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name}) + }) +} + +// useRecoveredBranch returns the branch recoverFailedCreate found, once it is READY. +// +// The wait is inline rather than in WaitAfterCreate, like the wait for the create operation on the +// success path: a WaitAfterCreate would read every created branch once more. Once the branch is +// known to be ours, it is returned even when the wait ends early, so that state records it and the +// next deploy does not fail with ALREADY_EXISTS. +func (r *ResourcePostgresBranch) useRecoveredBranch(ctx context.Context, branch *postgres.Branch, start time.Time, createErr error) (string, *PostgresBranchRemote, error) { + // Without replace_existing, a branch that existed before our request would have made it fail + // with ALREADY_EXISTS. That holds only if the request reached the backend's check, so look at + // when the branch was created rather than adopt a branch the bundle did not create. + if branch.CreateTime == nil || branch.CreateTime.AsTime().Before(start.Add(-branchCreateTimeSkew)) { + return "", nil, fmt.Errorf("%w; branch %s exists but was not created by this deployment", createErr, branch.Name) + } + + deadline := time.Now().Add(branchReadyTimeout) + for branchState(branch) == postgres.BranchStatusStateInit { if time.Now().After(deadline) { - return "", nil, fmt.Errorf("%w; branch %s exists but did not become ready within %s", createErr, name, branchReadyTimeout) + log.Warnf(ctx, "branch %s is still initializing after %s", branch.Name, branchReadyTimeout) + break } select { case <-ctx.Done(): return "", nil, ctx.Err() - case <-time.After(5 * time.Second): + case <-time.After(branchReadyPollInterval): + } + next, err := r.readBranch(ctx, branch.Name) + if err != nil { + if ctx.Err() != nil { + return "", nil, ctx.Err() + } + log.Warnf(ctx, "waiting for branch %s to become ready: %s", branch.Name, err) + break } + branch = next + } + + // Other states, such as DELETED or ARCHIVED, will not become READY. + if state := branchState(branch); state != postgres.BranchStatusStateReady && state != postgres.BranchStatusStateInit { + return "", nil, fmt.Errorf("%w; branch %s exists in state %q", createErr, branch.Name, state) + } + // Info, not Warn: the deploy got the branch it asked for, so there is nothing to act on. + log.Infof(ctx, "using branch %s, created despite the failed create request: %s", branch.Name, createErr) + remote := makePostgresBranchRemote(branch) + return remote.Name, remote, nil +} + +func branchState(branch *postgres.Branch) postgres.BranchStatusState { + if branch.Status == nil { + return "" } + return branch.Status.CurrentState } // expire_time, no_expiry and ttl are three sides of one oneof, and the API accepts them diff --git a/bundle/direct/dresources/postgres_branch_test.go b/bundle/direct/dresources/postgres_branch_test.go new file mode 100644 index 00000000000..89eb30e4ef8 --- /dev/null +++ b/bundle/direct/dresources/postgres_branch_test.go @@ -0,0 +1,266 @@ +package dresources + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/service/postgres" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testBranchParent = "projects/p" + testBranchName = "projects/p/branches/dev" + // The message the backend sends with a slow create; the SDK does not retry it. + testBranchTimeoutMessage = "The create request timed out and may still be completing. Check whether the resource was created before retrying." +) + +// branchCreateScript plays the given responses, in order, to the branch create and get requests. +// A response is a postgres.Branch (for a create: the finished operation that returns it) or a +// testserver.Response for an error. +type branchCreateScript struct { + t *testing.T + creates []any + gets []any + // onGet runs when a get request arrives, before its response is sent. + onGet func() + + createCount int + getCount int + operation *postgres.Operation +} + +func (s *branchCreateScript) client() *databricks.WorkspaceClient { + server := testserver.New(s.t) + server.Handle("POST", "/api/2.0/postgres/projects/{project_id}/branches", func(req testserver.Request) any { + s.createCount++ + require.LessOrEqual(s.t, s.createCount, len(s.creates), "unexpected create request") + resp := s.creates[s.createCount-1] + if branch, ok := resp.(postgres.Branch); ok { + body, err := json.Marshal(branch) + require.NoError(s.t, err) + s.operation = &postgres.Operation{Name: testBranchName + "/operations/op", Done: true, Response: body} + return s.operation + } + return resp + }) + // The SDK polls the operation even when the create response says it is done. + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/operations/{operation_id}", func(req testserver.Request) any { + require.NotNil(s.t, s.operation, "unexpected operation request") + return s.operation + }) + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}", func(req testserver.Request) any { + s.getCount++ + require.LessOrEqual(s.t, s.getCount, len(s.gets), "unexpected get request") + if s.onGet != nil { + s.onGet() + } + return s.gets[s.getCount-1] + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(s.t, err) + return client +} + +func branchError(status int, code, message string) testserver.Response { + return testserver.Response{StatusCode: status, Body: map[string]string{"error_code": code, "message": message}} +} + +var ( + branchTimeout = branchError(504, "DEADLINE_EXCEEDED", testBranchTimeoutMessage) + branchNotFound = branchError(404, "NOT_FOUND", "branch not found") + branchAlreadyExists = branchError(409, "ALREADY_EXISTS", "branch with such id already exists") +) + +func testBranch(state postgres.BranchStatusState, created time.Time) postgres.Branch { + return postgres.Branch{ + Name: testBranchName, + BranchId: "dev", + Parent: testBranchParent, + CreateTime: sdktime.New(created), + Status: &postgres.BranchStatus{CurrentState: state}, + } +} + +// createTestBranch runs DoCreate with fast retries and polling. +func createTestBranch(t *testing.T, s *branchCreateScript, ctx context.Context, replaceExisting bool) (string, *PostgresBranchRemote, error) { + t.Helper() + ctx = env.Set(ctx, retry.IntervalMsVariable, "1") + setForTest(t, &branchReadyPollInterval, time.Millisecond) + r := (&ResourcePostgresBranch{}).New(s.client()) + return r.DoCreate(ctx, &PostgresBranchState{BranchId: "dev", Parent: testBranchParent, ReplaceExisting: &replaceExisting}) +} + +func setForTest[T any](t *testing.T, v *T, value T) { + old := *v + *v = value + t.Cleanup(func() { *v = old }) +} + +func TestPostgresBranchCreateWaitsForBranchCreatedDespiteTimeout(t *testing.T) { + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout}, + gets: []any{testBranch(postgres.BranchStatusStateInit, time.Now()), testBranch(postgres.BranchStatusStateReady, time.Now())}, + } + + id, remote, err := createTestBranch(t, s, t.Context(), false) + + require.NoError(t, err) + assert.Equal(t, testBranchName, id) + assert.Equal(t, postgres.BranchStatusStateReady, remote.Status.CurrentState) + assert.Equal(t, 1, s.createCount) + assert.Equal(t, 2, s.getCount) +} + +func TestPostgresBranchCreateReturnsInitBranchAfterReadyTimeout(t *testing.T) { + setForTest(t, &branchReadyTimeout, 0) + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout}, + gets: []any{testBranch(postgres.BranchStatusStateInit, time.Now())}, + } + + id, remote, err := createTestBranch(t, s, t.Context(), false) + + // Returned so that state records it: the branch exists and is ours. + require.NoError(t, err) + assert.Equal(t, testBranchName, id) + assert.Equal(t, postgres.BranchStatusStateInit, remote.Status.CurrentState) + assert.Equal(t, 1, s.getCount) +} + +func TestPostgresBranchCreateRepeatsCreateWhileBranchIsMissing(t *testing.T) { + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout, testBranch(postgres.BranchStatusStateReady, time.Now())}, + gets: []any{branchNotFound}, + } + + id, _, err := createTestBranch(t, s, t.Context(), false) + + require.NoError(t, err) + assert.Equal(t, testBranchName, id) + assert.Equal(t, 2, s.createCount) + assert.Equal(t, 1, s.getCount) +} + +func TestPostgresBranchCreateUsesBranchWhenRepeatedCreateFindsIt(t *testing.T) { + // The first request was still completing when the branch was read, and has landed since. + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout, branchAlreadyExists}, + gets: []any{branchNotFound, testBranch(postgres.BranchStatusStateReady, time.Now())}, + } + + id, _, err := createTestBranch(t, s, t.Context(), false) + + require.NoError(t, err) + assert.Equal(t, testBranchName, id) + assert.Equal(t, 2, s.createCount) + assert.Equal(t, 2, s.getCount) +} + +func TestPostgresBranchCreateGivesUpWhenBranchStaysMissing(t *testing.T) { + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout, branchTimeout, branchTimeout}, + gets: []any{branchNotFound, branchNotFound, branchNotFound}, + } + + _, _, err := createTestBranch(t, s, t.Context(), false) + + require.ErrorContains(t, err, testBranchTimeoutMessage) + assert.Equal(t, 1+retry.MaxRetries, s.createCount) + assert.Equal(t, 1+retry.MaxRetries, s.getCount) +} + +func TestPostgresBranchCreateDoesNotUseBranchThatPredatesRequest(t *testing.T) { + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout}, + gets: []any{testBranch(postgres.BranchStatusStateReady, time.Now().Add(-time.Hour))}, + } + + _, _, err := createTestBranch(t, s, t.Context(), false) + + require.ErrorContains(t, err, testBranchTimeoutMessage) + require.ErrorContains(t, err, "branch "+testBranchName+" exists but was not created by this deployment") + assert.Equal(t, 1, s.createCount) +} + +func TestPostgresBranchCreateFailsOnBranchThatWillNotBecomeReady(t *testing.T) { + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout}, + gets: []any{testBranch(postgres.BranchStatusStateDeleted, time.Now())}, + } + + _, _, err := createTestBranch(t, s, t.Context(), false) + + require.ErrorContains(t, err, testBranchTimeoutMessage) + require.ErrorContains(t, err, `exists in state "DELETED"`) + assert.Equal(t, 1, s.getCount) +} + +func TestPostgresBranchCreateReturnsCreateErrorWhenReadFails(t *testing.T) { + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout}, + gets: []any{branchError(403, "PERMISSION_DENIED", "no access")}, + } + + _, _, err := createTestBranch(t, s, t.Context(), false) + + require.ErrorContains(t, err, testBranchTimeoutMessage) + assert.NotContains(t, err.Error(), "no access") + assert.Equal(t, 1, s.createCount) +} + +func TestPostgresBranchCreateReturnsCancellationDuringRead(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + s := &branchCreateScript{ + t: t, + creates: []any{branchTimeout}, + gets: []any{branchNotFound}, + onGet: cancel, + } + + _, _, err := createTestBranch(t, s, ctx, false) + + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, s.createCount) +} + +func TestPostgresBranchCreateLeavesReplaceToCallerRetry(t *testing.T) { + s := &branchCreateScript{t: t, creates: []any{branchTimeout}} + + _, _, err := createTestBranch(t, s, t.Context(), true) + + // Marked safe to retry for DeploymentUnit.Create, without reading the branch. + _, ok := errors.AsType[*RetrySafeError](err) + assert.True(t, ok, "want a RetrySafeError, got %v", err) + assert.Equal(t, 0, s.getCount) +} + +func TestPostgresBranchCreateDoesNotRecoverFromNonTransientError(t *testing.T) { + s := &branchCreateScript{t: t, creates: []any{branchError(400, "INVALID_PARAMETER_VALUE", "bad spec")}} + + _, _, err := createTestBranch(t, s, t.Context(), false) + + require.ErrorContains(t, err, "bad spec") + _, ok := errors.AsType[*RetrySafeError](err) + assert.False(t, ok) + assert.Equal(t, 0, s.getCount) +} diff --git a/libs/retry/retry.go b/libs/retry/retry.go index 95bf5a7250e..db2de0d4524 100644 --- a/libs/retry/retry.go +++ b/libs/retry/retry.go @@ -102,22 +102,30 @@ func IsTransientRead(ctx context.Context, err error) bool { // With retries fn while check returns true for the error, up to MaxRetries times. func With[T any](ctx context.Context, check func(context.Context, error) bool, fn func() (T, error)) (T, error) { - wait := interval(ctx) for attempt := 0; ; attempt++ { result, err := fn() if err == nil || attempt >= MaxRetries || !check(ctx, err) { return result, err } - log.Warnf(ctx, "%s", retryMessage(err)) - select { - case <-ctx.Done(): + if err := Pause(ctx, err); err != nil { var zero T - return zero, ctx.Err() - case <-time.After(wait): + return zero, err } } } +// Pause reports that the call that failed with err is retried and waits out the pause between +// attempts, for callers that run their own retry loop. It returns ctx.Err() if ctx is done first. +func Pause(ctx context.Context, err error) error { + log.Warnf(ctx, "%s", retryMessage(err)) + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval(ctx)): + return nil + } +} + // retryMessage describes the error that triggered a retry. func retryMessage(err error) string { // The "retrying after " prefix is what acceptance tests suppress (SuppressLines.warn_retry). From 1b7ce29378caec357ed094c92b58583f063155a4 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Tue, 17 Nov 2026 13:56:35 +0000 Subject: [PATCH 5/5] Stop the READY wait once the deadline is reached, not after it On Windows the clock can return the same instant twice, so with a zero timeout the wait read the branch once more. cloud: TestAccept/bundle/resources/postgres_branches TestAccept/bundle/resources/postgres_roles/replace_existing TestAccept/bundle/resources/postgres_databases/update Co-authored-by: Isaac --- bundle/direct/dresources/postgres_branch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/direct/dresources/postgres_branch.go b/bundle/direct/dresources/postgres_branch.go index 162aa77dbbd..da755e3ae14 100644 --- a/bundle/direct/dresources/postgres_branch.go +++ b/bundle/direct/dresources/postgres_branch.go @@ -226,7 +226,7 @@ func (r *ResourcePostgresBranch) useRecoveredBranch(ctx context.Context, branch deadline := time.Now().Add(branchReadyTimeout) for branchState(branch) == postgres.BranchStatusStateInit { - if time.Now().After(deadline) { + if !time.Now().Before(deadline) { log.Warnf(ctx, "branch %s is still initializing after %s", branch.Name, branchReadyTimeout) break }