Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7747.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Use the Lakebase project when its create request times out but the project was created, instead of failing the deploy with "already exists", in the direct deployment engine. ([#7747](https://github.com/databricks/cli/pull/7747))
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
bundle:
name: test-bundle

sync:
paths: []

resources:
postgres_projects:
my_project:
project_id: test-pg-proj
display_name: "Test Postgres Project"
pg_version: 16
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
EnvMatrix.READPLAN = []
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@

=== The create request times out but the project was created: deploy uses the project

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_projects.my_project
Files: 0 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //postgres/projects
POST /api/2.0/postgres/projects
GET /api/2.0/postgres/projects/test-pg-proj
GET /api/2.0/postgres/projects/test-pg-proj/branches/production

=== The project cannot be read after the create request times out: deploy fails with the create error

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Warn: deploying resources.postgres_projects.my_project: checking whether projects/test-pg-proj was created: Fault injected by test.
Error: cannot create resources.postgres_projects.my_project: Fault injected by test. (504 DEADLINE_EXCEEDED)

Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects?project_id=test-pg-proj
HTTP Status: 504 Gateway Timeout
API error_code: DEADLINE_EXCEEDED
API message: Fault injected by test.

Files: 0 uploaded, 0 deleted

Exit code: 1

>>> print_requests.py --get //postgres/projects
POST /api/2.0/postgres/projects
GET /api/2.0/postgres/projects/test-pg-proj
12 changes: 12 additions & 0 deletions acceptance/bundle/resources/postgres_projects/create_504/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
title "The create request times out but the project was created: deploy uses the project"
fault.py --after-handler "POST /api/2.0/postgres/projects" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //postgres/projects | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve > /dev/null 2>&1
rm -f "$OUT_REQUESTS"

title "The project cannot be read after the create request times out: deploy fails with the create error"
fault.py "POST /api/2.0/postgres/projects" 504 0 1 DEADLINE_EXCEEDED
fault.py "GET /api/2.0/postgres/projects/test-pg-proj" 403 0 1 PERMISSION_DENIED
errcode trace $CLI bundle deploy
trace print_requests.py --get //postgres/projects | jq -r '.method + " " + .path'
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Cloud = false
RecordRequests = true
EnvMatrix.READPLAN = []
Env.DATABRICKS_BUNDLE_ENGINE = "direct"
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //postgres
POST /api/2.0/postgres/projects
GET /api/2.0/postgres/projects/test-pg-proj
POST /api/2.0/postgres/projects
GET /api/2.0/postgres/projects/test-pg-proj/operations/[UUID]
GET /api/2.0/postgres/projects/test-pg-proj/operations/[UUID]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ fault.py "GET /api/2.0/postgres/projects/*/operations/*" 500 0 1 INTERNAL_ERROR

trace $CLI bundle deploy

# Two create requests and two operation polls: the 500 and the retry of each.
# Two create requests and two operation polls: the 500 and the retry of each. Before repeating
# the create, the deploy looks for the project in case the failed request created it.
trace print_requests.py --get //postgres | jq -r '.method + " " + .path'

trace $CLI bundle destroy --auto-approve
99 changes: 43 additions & 56 deletions bundle/direct/dresources/postgres_branch.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package dresources

import (
"context"
"errors"
"fmt"
"slices"
"time"
Expand All @@ -11,7 +10,6 @@ import (
"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/common/types/fieldmask"
sdktime "github.com/databricks/databricks-sdk-go/common/types/time"
"github.com/databricks/databricks-sdk-go/marshal"
Expand Down Expand Up @@ -148,22 +146,15 @@ func (r *ResourcePostgresBranch) waitCreated(ctx context.Context, waiter postgre
return remote.Name, remote, nil
}

// Tuning for recoverFailedCreate, variables so that tests can shorten them.
// Tuning for waitBranchReady, variables so that tests can shorten them.
var (
// branchReadyTimeout bounds the wait for a recovered branch that is still in INIT.
branchReadyTimeout = 10 * time.Minute
// branchReadyPollInterval is the pause between reads of a recovered branch in INIT.
branchReadyPollInterval = 5 * time.Second
// branchCreateTimeSkew is how much earlier than our first create request a found branch's
// create_time may be, to allow for clock skew, and still count as created by that request.
branchCreateTimeSkew = time.Minute
)

// recoverFailedCreate handles a create request that failed with a transient error. A slow create
// is answered with 504 DEADLINE_EXCEEDED ("The create request timed out and may still be
// completing"), so the request may create the branch anyway. Look for the branch: use it if our
// request created it, and repeat the request while it is missing. A repeated request that fails
// with ALREADY_EXISTS means an earlier one has landed since, so the next look finds that branch.
// recoverFailedCreate handles a create request that failed; see recoverPostgresCreate.
func (r *ResourcePostgresBranch) recoverFailedCreate(ctx context.Context, req postgres.CreateBranchRequest, start time.Time, createErr error) (string, *PostgresBranchRemote, error) {
if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) {
return "", nil, createErr
Expand All @@ -174,39 +165,27 @@ func (r *ResourcePostgresBranch) recoverFailedCreate(ctx context.Context, req po
}

name := req.Parent + "/branches/" + req.BranchId
lastErr := createErr
for attempt := 1; ; attempt++ {
branch, err := r.readBranch(ctx, name)
if ctx.Err() != nil {
return "", nil, ctx.Err()
}
if err == nil {
return recoverPostgresCreate(ctx, postgresCreateRecovery[postgres.Branch, *PostgresBranchRemote]{
name: name,
get: func(ctx context.Context) (*postgres.Branch, error) {
return r.client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name})
},
create: func(ctx context.Context) (func() (string, *PostgresBranchRemote, error), error) {
waiter, err := r.client.Postgres.CreateBranch(ctx, req)
if err != nil {
return nil, err
}
return func() (string, *PostgresBranchRemote, error) { return r.waitCreated(ctx, waiter) }, nil
},
use: func(ctx context.Context, branch *postgres.Branch) (string, *PostgresBranchRemote, error) {
return r.useRecoveredBranch(ctx, branch, start, createErr)
}
if !apierr.IsMissing(err) {
log.Warnf(ctx, "checking whether %s was created: %s", name, err)
return "", nil, createErr
}
if attempt > retry.MaxRetries {
return "", nil, lastErr
}
if err := retry.Pause(ctx, lastErr); err != nil {
return "", nil, err
}
waiter, err := r.client.Postgres.CreateBranch(ctx, req)
if err == nil {
return r.waitCreated(ctx, waiter)
}
if !retry.IsTransient(ctx, err) && !errors.Is(err, apierr.ErrAlreadyExists) {
return "", nil, err
}
lastErr = err
}
},
}, createErr)
}

func (r *ResourcePostgresBranch) readBranch(ctx context.Context, name string) (*postgres.Branch, error) {
func readBranch(ctx context.Context, client *databricks.WorkspaceClient, name string) (*postgres.Branch, error) {
return retry.With(ctx, retry.IsTransientRead, func() (*postgres.Branch, error) {
return r.client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name})
return client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name})
})
}

Expand All @@ -217,13 +196,29 @@ func (r *ResourcePostgresBranch) readBranch(ctx context.Context, name string) (*
// known to be ours, it is returned even when the wait ends early, so that state records it and the
// next deploy does not fail with ALREADY_EXISTS.
func (r *ResourcePostgresBranch) useRecoveredBranch(ctx context.Context, branch *postgres.Branch, start time.Time, createErr error) (string, *PostgresBranchRemote, error) {
// Without replace_existing, a branch that existed before our request would have made it fail
// with ALREADY_EXISTS. That holds only if the request reached the backend's check, so look at
// when the branch was created rather than adopt a branch the bundle did not create.
if branch.CreateTime == nil || branch.CreateTime.AsTime().Before(start.Add(-branchCreateTimeSkew)) {
if !createdByRequestAt(branch.CreateTime, start) {
return "", nil, fmt.Errorf("%w; branch %s exists but was not created by this deployment", createErr, branch.Name)
}

branch, err := waitBranchReady(ctx, r.client, branch)
if err != nil {
return "", nil, err
}

// Other states, such as DELETED or ARCHIVED, will not become READY.
if state := branchState(branch); state != postgres.BranchStatusStateReady && state != postgres.BranchStatusStateInit {
return "", nil, fmt.Errorf("%w; branch %s exists in state %q", createErr, branch.Name, state)
}
// Info, not Warn: the deploy got the branch it asked for, so there is nothing to act on.
log.Infof(ctx, "using branch %s, created despite the failed create request: %s", branch.Name, createErr)
remote := makePostgresBranchRemote(branch)
return remote.Name, remote, nil
}

// waitBranchReady polls branch while it is in INIT, up to branchReadyTimeout, and returns the last
// branch it read. A timeout or a failed read ends the wait early with a warning; only a done
// context is returned as an error.
func waitBranchReady(ctx context.Context, client *databricks.WorkspaceClient, branch *postgres.Branch) (*postgres.Branch, error) {
deadline := time.Now().Add(branchReadyTimeout)
for branchState(branch) == postgres.BranchStatusStateInit {
if !time.Now().Before(deadline) {
Expand All @@ -232,28 +227,20 @@ func (r *ResourcePostgresBranch) useRecoveredBranch(ctx context.Context, branch
}
select {
case <-ctx.Done():
return "", nil, ctx.Err()
return nil, ctx.Err()
case <-time.After(branchReadyPollInterval):
}
next, err := r.readBranch(ctx, branch.Name)
next, err := readBranch(ctx, client, branch.Name)
if err != nil {
if ctx.Err() != nil {
return "", nil, ctx.Err()
return nil, ctx.Err()
}
log.Warnf(ctx, "waiting for branch %s to become ready: %s", branch.Name, err)
break
}
branch = next
}

// Other states, such as DELETED or ARCHIVED, will not become READY.
if state := branchState(branch); state != postgres.BranchStatusStateReady && state != postgres.BranchStatusStateInit {
return "", nil, fmt.Errorf("%w; branch %s exists in state %q", createErr, branch.Name, state)
}
// Info, not Warn: the deploy got the branch it asked for, so there is nothing to act on.
log.Infof(ctx, "using branch %s, created despite the failed create request: %s", branch.Name, createErr)
remote := makePostgresBranchRemote(branch)
return remote.Name, remote, nil
return branch, nil
}

func branchState(branch *postgres.Branch) postgres.BranchStatusState {
Expand Down
90 changes: 90 additions & 0 deletions bundle/direct/dresources/postgres_create_recovery.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package dresources

import (
"context"
"errors"
"net/http"
"strings"
"time"

"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/databricks-sdk-go/apierr"
sdktime "github.com/databricks/databricks-sdk-go/common/types/time"
)

// postgresCreateTimeSkew is how much earlier than our first create request a found resource's
// create_time may be, to allow for clock skew, and still count as created by that request.
const postgresCreateTimeSkew = time.Minute

// postgresCreateRecovery describes a Lakebase resource whose create request failed with a
// transient error. T is the SDK type of the resource and R the remote type DoCreate returns.
type postgresCreateRecovery[T, R any] struct {
// name is the full resource name, e.g. projects/p/branches/b.
name string
// get reads the resource.
get func(ctx context.Context) (*T, error)
// create repeats the create request. On success it returns a function that waits for the
// operation the request started.
create func(ctx context.Context) (func() (string, R, error), error)
// use returns the resource the lookup found.
use func(ctx context.Context, found *T) (string, R, error)
}

// recoverPostgresCreate handles a Lakebase create request that failed with a transient error. A
// slow create is answered with 504 DEADLINE_EXCEEDED ("The create request timed out and may still
// be completing"), so the request may create the resource anyway. Look for the resource: use it if
// it is found, and repeat the request while it is missing. A repeated request that fails because
// the resource already exists means an earlier one has landed since, so the next look finds it.
func recoverPostgresCreate[T, R any](ctx context.Context, rc postgresCreateRecovery[T, R], createErr error) (string, R, error) {
var zero R
lastErr := createErr
for attempt := 1; ; attempt++ {
found, err := retry.With(ctx, retry.IsTransientRead, func() (*T, error) {
return rc.get(ctx)
})
if ctx.Err() != nil {
return "", zero, ctx.Err()
}
if err == nil {
return rc.use(ctx, found)
}
if !apierr.IsMissing(err) {
log.Warnf(ctx, "checking whether %s was created: %s", rc.name, err)
return "", zero, createErr
}
if attempt > retry.MaxRetries {
return "", zero, lastErr
}
if err := retry.Pause(ctx, lastErr); err != nil {
return "", zero, err
}
wait, err := rc.create(ctx)
if err == nil {
return wait()
}
if !retry.IsTransient(ctx, err) && !isPostgresAlreadyExists(err) {
return "", zero, err
}
lastErr = err
}
}

// isPostgresAlreadyExists reports whether a Lakebase create request failed because the resource
// exists. Branches answer with 409 ALREADY_EXISTS; projects answer with 400 BAD_REQUEST and
// "A project with this resource name already exists in the workspace."
func isPostgresAlreadyExists(err error) bool {
if errors.Is(err, apierr.ErrAlreadyExists) {
return true
}
apiErr, ok := errors.AsType[*apierr.APIError](err)
return ok && apiErr.StatusCode == http.StatusBadRequest && strings.Contains(apiErr.Message, "already exists")
}

// createdByRequestAt reports whether a resource with createTime can have been created by a request
// first sent at start. Without replace_existing, a resource that existed before the request would
// have made it fail with "already exists". That holds only if the request reached the backend's
// check, so look at when the resource was created rather than adopt one the bundle did not create.
func createdByRequestAt(createTime *sdktime.Time, start time.Time) bool {
return createTime != nil && !createTime.AsTime().Before(start.Add(-postgresCreateTimeSkew))
}
Loading
Loading