Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7602.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Use the Lakebase branch when its create request times out but the branch was created, instead of failing the deploy, in the direct deployment engine. ([#7602](https://github.com/databricks/cli/pull/7602))
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
bundle:
name: test-bundle

sync:
paths: []

resources:
postgres_projects:
my_project:
project_id: test-pg-proj
display_name: "Test Postgres Project"
pg_version: 16

postgres_branches:
dev:
parent: ${resources.postgres_projects.my_project.id}
branch_id: dev
no_expiry: true
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
EnvMatrix.READPLAN = []
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@

=== The create request times out but the branch was created: deploy uses the branch

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_branches.dev
Created postgres_projects.my_project
Files: 0 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //postgres/projects/test-pg-proj/branches
POST /api/2.0/postgres/projects/test-pg-proj/branches
GET /api/2.0/postgres/projects/test-pg-proj/branches/dev

=== The create request times out and the branch was not created: deploy repeats the create

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_projects.my_project
Warn: deploying resources.postgres_branches.dev: retrying after 504 Gateway Timeout from POST /api/2.0/postgres/projects/test-pg-proj/branches
Created postgres_branches.dev
Files: 0 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //postgres/projects/test-pg-proj/branches
POST /api/2.0/postgres/projects/test-pg-proj/branches
GET /api/2.0/postgres/projects/test-pg-proj/branches/dev
POST /api/2.0/postgres/projects/test-pg-proj/branches
GET /api/2.0/postgres/projects/test-pg-proj/branches/dev/operations/[UUID][0]

=== The branch cannot be read after the create request times out: deploy fails with the create error

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_projects.my_project
Warn: deploying resources.postgres_branches.dev: checking whether projects/test-pg-proj/branches/dev was created: Fault injected by test.
Error: cannot create resources.postgres_branches.dev: Fault injected by test. (504 DEADLINE_EXCEEDED)

Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-pg-proj/branches?branch_id=dev
HTTP Status: 504 Gateway Timeout
API error_code: DEADLINE_EXCEEDED
API message: Fault injected by test.

Files: 0 uploaded, 0 deleted

Exit code: 1

>>> print_requests.py --get //postgres/projects/test-pg-proj/branches
POST /api/2.0/postgres/projects/test-pg-proj/branches
GET /api/2.0/postgres/projects/test-pg-proj/branches/dev

=== With replace_existing, a timed-out create request is repeated without reading the branch

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_projects.my_project
Warn: deploying resources.postgres_branches.dev: retrying after 504 Gateway Timeout from POST /api/2.0/postgres/projects/test-pg-proj/branches
Created postgres_branches.dev
Files: 0 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //postgres/projects/test-pg-proj/branches
POST /api/2.0/postgres/projects/test-pg-proj/branches
POST /api/2.0/postgres/projects/test-pg-proj/branches
GET /api/2.0/postgres/projects/test-pg-proj/branches/dev/operations/[UUID][1]
29 changes: 29 additions & 0 deletions acceptance/bundle/resources/postgres_branches/create_504/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
title "The create request times out but the branch was created: deploy uses the branch"
fault.py --after-handler "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "The create request times out and the branch was not created: deploy repeats the create"
fault.py "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "The branch cannot be read after the create request times out: deploy fails with the create error"
fault.py "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED
fault.py "GET /api/2.0/postgres/projects/*/branches/dev" 403 0 1 PERMISSION_DENIED
errcode trace $CLI bundle deploy
trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "With replace_existing, a timed-out create request is repeated without reading the branch"
update_file.py databricks.yml "no_expiry: true" "no_expiry: true
replace_existing: true"
fault.py --after-handler "POST /api/2.0/postgres/projects/*/branches" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //postgres/projects/test-pg-proj/branches | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Cloud = false
RecordRequests = true
EnvMatrix.READPLAN = []
Env.DATABRICKS_BUNDLE_ENGINE = "direct"
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
135 changes: 130 additions & 5 deletions bundle/direct/dresources/postgres_branch.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,16 @@ package dresources

import (
"context"
"errors"
"fmt"
"slices"
"time"

"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/common/types/fieldmask"
sdktime "github.com/databricks/databricks-sdk-go/common/types/time"
"github.com/databricks/databricks-sdk-go/marshal"
Expand Down Expand Up @@ -105,7 +111,7 @@ func (r *ResourcePostgresBranch) DoRead(ctx context.Context, id string) (*Postgr
}

func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresBranchState) (string, *PostgresBranchRemote, error) {
waiter, err := r.client.Postgres.CreateBranch(ctx, postgres.CreateBranchRequest{
req := postgres.CreateBranchRequest{
BranchId: config.BranchId,
Parent: config.Parent,
Branch: postgres.Branch{
Expand All @@ -123,21 +129,140 @@ func (r *ResourcePostgresBranch) DoCreate(ctx context.Context, config *PostgresB
},
ReplaceExisting: config.ReplaceExisting != nil && *config.ReplaceExisting,
ForceSendFields: nil,
})
}
start := time.Now()
waiter, err := r.client.Postgres.CreateBranch(ctx, req)
if err != nil {
return "", nil, err
return r.recoverFailedCreate(ctx, req, start, err)
}
return r.waitCreated(ctx, waiter)
}

// Wait for the branch to be ready (long-running operation)
// waitCreated waits for the create operation of a branch that a create request returned.
func (r *ResourcePostgresBranch) waitCreated(ctx context.Context, waiter postgres.CreateBranchOperationInterface) (string, *PostgresBranchRemote, error) {
result, err := waitCreateOperation(ctx, waiter.Wait)
if err != nil {
return "", nil, err
}

remote := makePostgresBranchRemote(result)
return remote.Name, remote, nil
}

// Tuning for recoverFailedCreate, variables so that tests can shorten them.
var (
// branchReadyTimeout bounds the wait for a recovered branch that is still in INIT.
branchReadyTimeout = 10 * time.Minute
// branchReadyPollInterval is the pause between reads of a recovered branch in INIT.
branchReadyPollInterval = 5 * time.Second
// branchCreateTimeSkew is how much earlier than our first create request a found branch's
// create_time may be, to allow for clock skew, and still count as created by that request.
branchCreateTimeSkew = time.Minute
)

// recoverFailedCreate handles a create request that failed with a transient error. A slow create
// is answered with 504 DEADLINE_EXCEEDED ("The create request timed out and may still be
// completing"), so the request may create the branch anyway. Look for the branch: use it if our
// request created it, and repeat the request while it is missing. A repeated request that fails
// with ALREADY_EXISTS means an earlier one has landed since, so the next look finds that branch.
func (r *ResourcePostgresBranch) recoverFailedCreate(ctx context.Context, req postgres.CreateBranchRequest, start time.Time, createErr error) (string, *PostgresBranchRemote, error) {
if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) {
return "", nil, createErr
}
// A branch found after a replace may predate it, and repeating a replace is safe.
if req.ReplaceExisting {
return "", nil, retrySafe(createErr)
}

name := req.Parent + "/branches/" + req.BranchId
lastErr := createErr
for attempt := 1; ; attempt++ {
branch, err := r.readBranch(ctx, name)
if ctx.Err() != nil {
return "", nil, ctx.Err()
}
if err == nil {
return r.useRecoveredBranch(ctx, branch, start, createErr)
}
if !apierr.IsMissing(err) {
log.Warnf(ctx, "checking whether %s was created: %s", name, err)
return "", nil, createErr
}
if attempt > retry.MaxRetries {
return "", nil, lastErr
}
if err := retry.Pause(ctx, lastErr); err != nil {
return "", nil, err
}
waiter, err := r.client.Postgres.CreateBranch(ctx, req)
if err == nil {
return r.waitCreated(ctx, waiter)
}
if !retry.IsTransient(ctx, err) && !errors.Is(err, apierr.ErrAlreadyExists) {
return "", nil, err
}
lastErr = err
}
}

func (r *ResourcePostgresBranch) readBranch(ctx context.Context, name string) (*postgres.Branch, error) {
return retry.With(ctx, retry.IsTransientRead, func() (*postgres.Branch, error) {
return r.client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name})
})
}

// useRecoveredBranch returns the branch recoverFailedCreate found, once it is READY.
//
// The wait is inline rather than in WaitAfterCreate, like the wait for the create operation on the
// success path: a WaitAfterCreate would read every created branch once more. Once the branch is
// known to be ours, it is returned even when the wait ends early, so that state records it and the
// next deploy does not fail with ALREADY_EXISTS.
func (r *ResourcePostgresBranch) useRecoveredBranch(ctx context.Context, branch *postgres.Branch, start time.Time, createErr error) (string, *PostgresBranchRemote, error) {
// Without replace_existing, a branch that existed before our request would have made it fail
// with ALREADY_EXISTS. That holds only if the request reached the backend's check, so look at
// when the branch was created rather than adopt a branch the bundle did not create.
if branch.CreateTime == nil || branch.CreateTime.AsTime().Before(start.Add(-branchCreateTimeSkew)) {
return "", nil, fmt.Errorf("%w; branch %s exists but was not created by this deployment", createErr, branch.Name)
}

deadline := time.Now().Add(branchReadyTimeout)
for branchState(branch) == postgres.BranchStatusStateInit {
if !time.Now().Before(deadline) {
log.Warnf(ctx, "branch %s is still initializing after %s", branch.Name, branchReadyTimeout)
break
}
select {
case <-ctx.Done():
return "", nil, ctx.Err()
case <-time.After(branchReadyPollInterval):
}
next, err := r.readBranch(ctx, branch.Name)
if err != nil {
if ctx.Err() != nil {
return "", nil, ctx.Err()
}
log.Warnf(ctx, "waiting for branch %s to become ready: %s", branch.Name, err)
break
}
branch = next
}

// Other states, such as DELETED or ARCHIVED, will not become READY.
if state := branchState(branch); state != postgres.BranchStatusStateReady && state != postgres.BranchStatusStateInit {
return "", nil, fmt.Errorf("%w; branch %s exists in state %q", createErr, branch.Name, state)
}
// Info, not Warn: the deploy got the branch it asked for, so there is nothing to act on.
log.Infof(ctx, "using branch %s, created despite the failed create request: %s", branch.Name, createErr)
remote := makePostgresBranchRemote(branch)
return remote.Name, remote, nil
}

func branchState(branch *postgres.Branch) postgres.BranchStatusState {
if branch.Status == nil {
return ""
}
return branch.Status.CurrentState
}

// expire_time, no_expiry and ttl are three sides of one oneof, and the API accepts them
// in update_mask only under the group name: masking the field itself is answered with
// "Unknown field path in update_mask". Probed against a real workspace on 2026-08-31.
Expand Down
Loading
Loading