diff --git a/.nextchanges/bundles/7747.md b/.nextchanges/bundles/7747.md new file mode 100644 index 00000000000..580c648c893 --- /dev/null +++ b/.nextchanges/bundles/7747.md @@ -0,0 +1 @@ +* Use the Lakebase project when its create request times out but the project was created, instead of failing the deploy with "already exists", in the direct deployment engine. ([#7747](https://github.com/databricks/cli/pull/7747)) diff --git a/acceptance/bundle/resources/postgres_projects/create_504/databricks.yml b/acceptance/bundle/resources/postgres_projects/create_504/databricks.yml new file mode 100644 index 00000000000..b95ebe35d2c --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/create_504/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: test-bundle + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj + display_name: "Test Postgres Project" + pg_version: 16 diff --git a/acceptance/bundle/resources/postgres_projects/create_504/out.test.toml b/acceptance/bundle/resources/postgres_projects/create_504/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/create_504/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/postgres_projects/create_504/output.txt b/acceptance/bundle/resources/postgres_projects/create_504/output.txt new file mode 100644 index 00000000000..7487e064b29 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/create_504/output.txt @@ -0,0 +1,33 @@ + +=== The create request times out but the project was created: deploy uses the project + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Files: 0 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres/projects +POST /api/2.0/postgres/projects +GET /api/2.0/postgres/projects/test-pg-proj +GET /api/2.0/postgres/projects/test-pg-proj/branches/production + +=== The project cannot be read after the create request times out: deploy fails with the create error + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Warn: deploying resources.postgres_projects.my_project: checking whether projects/test-pg-proj was created: Fault injected by test. +Error: cannot create resources.postgres_projects.my_project: Fault injected by test. (504 DEADLINE_EXCEEDED) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects?project_id=test-pg-proj +HTTP Status: 504 Gateway Timeout +API error_code: DEADLINE_EXCEEDED +API message: Fault injected by test. + +Files: 0 uploaded, 0 deleted + +Exit code: 1 + +>>> print_requests.py --get //postgres/projects +POST /api/2.0/postgres/projects +GET /api/2.0/postgres/projects/test-pg-proj diff --git a/acceptance/bundle/resources/postgres_projects/create_504/script b/acceptance/bundle/resources/postgres_projects/create_504/script new file mode 100644 index 00000000000..cd36cc94c9d --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/create_504/script @@ -0,0 +1,12 @@ +title "The create request times out but the project was created: deploy uses the project" +fault.py --after-handler "POST /api/2.0/postgres/projects" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve > /dev/null 2>&1 +rm -f "$OUT_REQUESTS" + +title "The project cannot be read after the create request times out: deploy fails with the create error" +fault.py "POST /api/2.0/postgres/projects" 504 0 1 DEADLINE_EXCEEDED +fault.py "GET /api/2.0/postgres/projects/test-pg-proj" 403 0 1 PERMISSION_DENIED +errcode trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects | jq -r '.method + " " + .path' diff --git a/acceptance/bundle/resources/postgres_projects/create_504/test.toml b/acceptance/bundle/resources/postgres_projects/create_504/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/create_504/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/acceptance/bundle/resources/postgres_projects/create_transient_500/output.txt b/acceptance/bundle/resources/postgres_projects/create_transient_500/output.txt index cc28f656da5..355953512a1 100644 --- a/acceptance/bundle/resources/postgres_projects/create_transient_500/output.txt +++ b/acceptance/bundle/resources/postgres_projects/create_transient_500/output.txt @@ -9,6 +9,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --get //postgres POST /api/2.0/postgres/projects +GET /api/2.0/postgres/projects/test-pg-proj POST /api/2.0/postgres/projects GET /api/2.0/postgres/projects/test-pg-proj/operations/[UUID] GET /api/2.0/postgres/projects/test-pg-proj/operations/[UUID] diff --git a/acceptance/bundle/resources/postgres_projects/create_transient_500/script b/acceptance/bundle/resources/postgres_projects/create_transient_500/script index 0f07dbbcf59..65c782c7a1f 100644 --- a/acceptance/bundle/resources/postgres_projects/create_transient_500/script +++ b/acceptance/bundle/resources/postgres_projects/create_transient_500/script @@ -5,7 +5,8 @@ fault.py "GET /api/2.0/postgres/projects/*/operations/*" 500 0 1 INTERNAL_ERROR trace $CLI bundle deploy -# Two create requests and two operation polls: the 500 and the retry of each. +# Two create requests and two operation polls: the 500 and the retry of each. Before repeating +# the create, the deploy looks for the project in case the failed request created it. trace print_requests.py --get //postgres | jq -r '.method + " " + .path' trace $CLI bundle destroy --auto-approve diff --git a/bundle/direct/dresources/postgres_branch.go b/bundle/direct/dresources/postgres_branch.go index da755e3ae14..babdf14a0b3 100644 --- a/bundle/direct/dresources/postgres_branch.go +++ b/bundle/direct/dresources/postgres_branch.go @@ -2,7 +2,6 @@ package dresources import ( "context" - "errors" "fmt" "slices" "time" @@ -11,7 +10,6 @@ import ( "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/common/types/fieldmask" sdktime "github.com/databricks/databricks-sdk-go/common/types/time" "github.com/databricks/databricks-sdk-go/marshal" @@ -148,22 +146,15 @@ func (r *ResourcePostgresBranch) waitCreated(ctx context.Context, waiter postgre return remote.Name, remote, nil } -// Tuning for recoverFailedCreate, variables so that tests can shorten them. +// Tuning for waitBranchReady, variables so that tests can shorten them. var ( // branchReadyTimeout bounds the wait for a recovered branch that is still in INIT. branchReadyTimeout = 10 * time.Minute // branchReadyPollInterval is the pause between reads of a recovered branch in INIT. branchReadyPollInterval = 5 * time.Second - // branchCreateTimeSkew is how much earlier than our first create request a found branch's - // create_time may be, to allow for clock skew, and still count as created by that request. - branchCreateTimeSkew = time.Minute ) -// recoverFailedCreate handles a create request that failed with a transient error. A slow create -// is answered with 504 DEADLINE_EXCEEDED ("The create request timed out and may still be -// completing"), so the request may create the branch anyway. Look for the branch: use it if our -// request created it, and repeat the request while it is missing. A repeated request that fails -// with ALREADY_EXISTS means an earlier one has landed since, so the next look finds that branch. +// recoverFailedCreate handles a create request that failed; see recoverPostgresCreate. func (r *ResourcePostgresBranch) recoverFailedCreate(ctx context.Context, req postgres.CreateBranchRequest, start time.Time, createErr error) (string, *PostgresBranchRemote, error) { if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) { return "", nil, createErr @@ -174,39 +165,27 @@ func (r *ResourcePostgresBranch) recoverFailedCreate(ctx context.Context, req po } name := req.Parent + "/branches/" + req.BranchId - lastErr := createErr - for attempt := 1; ; attempt++ { - branch, err := r.readBranch(ctx, name) - if ctx.Err() != nil { - return "", nil, ctx.Err() - } - if err == nil { + return recoverPostgresCreate(ctx, postgresCreateRecovery[postgres.Branch, *PostgresBranchRemote]{ + name: name, + get: func(ctx context.Context) (*postgres.Branch, error) { + return r.client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name}) + }, + create: func(ctx context.Context) (func() (string, *PostgresBranchRemote, error), error) { + waiter, err := r.client.Postgres.CreateBranch(ctx, req) + if err != nil { + return nil, err + } + return func() (string, *PostgresBranchRemote, error) { return r.waitCreated(ctx, waiter) }, nil + }, + use: func(ctx context.Context, branch *postgres.Branch) (string, *PostgresBranchRemote, error) { return r.useRecoveredBranch(ctx, branch, start, createErr) - } - if !apierr.IsMissing(err) { - log.Warnf(ctx, "checking whether %s was created: %s", name, err) - return "", nil, createErr - } - if attempt > retry.MaxRetries { - return "", nil, lastErr - } - if err := retry.Pause(ctx, lastErr); err != nil { - return "", nil, err - } - waiter, err := r.client.Postgres.CreateBranch(ctx, req) - if err == nil { - return r.waitCreated(ctx, waiter) - } - if !retry.IsTransient(ctx, err) && !errors.Is(err, apierr.ErrAlreadyExists) { - return "", nil, err - } - lastErr = err - } + }, + }, createErr) } -func (r *ResourcePostgresBranch) readBranch(ctx context.Context, name string) (*postgres.Branch, error) { +func readBranch(ctx context.Context, client *databricks.WorkspaceClient, name string) (*postgres.Branch, error) { return retry.With(ctx, retry.IsTransientRead, func() (*postgres.Branch, error) { - return r.client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name}) + return client.Postgres.GetBranch(ctx, postgres.GetBranchRequest{Name: name}) }) } @@ -217,13 +196,29 @@ func (r *ResourcePostgresBranch) readBranch(ctx context.Context, name string) (* // known to be ours, it is returned even when the wait ends early, so that state records it and the // next deploy does not fail with ALREADY_EXISTS. func (r *ResourcePostgresBranch) useRecoveredBranch(ctx context.Context, branch *postgres.Branch, start time.Time, createErr error) (string, *PostgresBranchRemote, error) { - // Without replace_existing, a branch that existed before our request would have made it fail - // with ALREADY_EXISTS. That holds only if the request reached the backend's check, so look at - // when the branch was created rather than adopt a branch the bundle did not create. - if branch.CreateTime == nil || branch.CreateTime.AsTime().Before(start.Add(-branchCreateTimeSkew)) { + if !createdByRequestAt(branch.CreateTime, start) { return "", nil, fmt.Errorf("%w; branch %s exists but was not created by this deployment", createErr, branch.Name) } + branch, err := waitBranchReady(ctx, r.client, branch) + if err != nil { + return "", nil, err + } + + // Other states, such as DELETED or ARCHIVED, will not become READY. + if state := branchState(branch); state != postgres.BranchStatusStateReady && state != postgres.BranchStatusStateInit { + return "", nil, fmt.Errorf("%w; branch %s exists in state %q", createErr, branch.Name, state) + } + // Info, not Warn: the deploy got the branch it asked for, so there is nothing to act on. + log.Infof(ctx, "using branch %s, created despite the failed create request: %s", branch.Name, createErr) + remote := makePostgresBranchRemote(branch) + return remote.Name, remote, nil +} + +// waitBranchReady polls branch while it is in INIT, up to branchReadyTimeout, and returns the last +// branch it read. A timeout or a failed read ends the wait early with a warning; only a done +// context is returned as an error. +func waitBranchReady(ctx context.Context, client *databricks.WorkspaceClient, branch *postgres.Branch) (*postgres.Branch, error) { deadline := time.Now().Add(branchReadyTimeout) for branchState(branch) == postgres.BranchStatusStateInit { if !time.Now().Before(deadline) { @@ -232,28 +227,20 @@ func (r *ResourcePostgresBranch) useRecoveredBranch(ctx context.Context, branch } select { case <-ctx.Done(): - return "", nil, ctx.Err() + return nil, ctx.Err() case <-time.After(branchReadyPollInterval): } - next, err := r.readBranch(ctx, branch.Name) + next, err := readBranch(ctx, client, branch.Name) if err != nil { if ctx.Err() != nil { - return "", nil, ctx.Err() + return nil, ctx.Err() } log.Warnf(ctx, "waiting for branch %s to become ready: %s", branch.Name, err) break } branch = next } - - // Other states, such as DELETED or ARCHIVED, will not become READY. - if state := branchState(branch); state != postgres.BranchStatusStateReady && state != postgres.BranchStatusStateInit { - return "", nil, fmt.Errorf("%w; branch %s exists in state %q", createErr, branch.Name, state) - } - // Info, not Warn: the deploy got the branch it asked for, so there is nothing to act on. - log.Infof(ctx, "using branch %s, created despite the failed create request: %s", branch.Name, createErr) - remote := makePostgresBranchRemote(branch) - return remote.Name, remote, nil + return branch, nil } func branchState(branch *postgres.Branch) postgres.BranchStatusState { diff --git a/bundle/direct/dresources/postgres_create_recovery.go b/bundle/direct/dresources/postgres_create_recovery.go new file mode 100644 index 00000000000..dc904de1719 --- /dev/null +++ b/bundle/direct/dresources/postgres_create_recovery.go @@ -0,0 +1,90 @@ +package dresources + +import ( + "context" + "errors" + "net/http" + "strings" + "time" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/databricks-sdk-go/apierr" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" +) + +// postgresCreateTimeSkew is how much earlier than our first create request a found resource's +// create_time may be, to allow for clock skew, and still count as created by that request. +const postgresCreateTimeSkew = time.Minute + +// postgresCreateRecovery describes a Lakebase resource whose create request failed with a +// transient error. T is the SDK type of the resource and R the remote type DoCreate returns. +type postgresCreateRecovery[T, R any] struct { + // name is the full resource name, e.g. projects/p/branches/b. + name string + // get reads the resource. + get func(ctx context.Context) (*T, error) + // create repeats the create request. On success it returns a function that waits for the + // operation the request started. + create func(ctx context.Context) (func() (string, R, error), error) + // use returns the resource the lookup found. + use func(ctx context.Context, found *T) (string, R, error) +} + +// recoverPostgresCreate handles a Lakebase create request that failed with a transient error. A +// slow create is answered with 504 DEADLINE_EXCEEDED ("The create request timed out and may still +// be completing"), so the request may create the resource anyway. Look for the resource: use it if +// it is found, and repeat the request while it is missing. A repeated request that fails because +// the resource already exists means an earlier one has landed since, so the next look finds it. +func recoverPostgresCreate[T, R any](ctx context.Context, rc postgresCreateRecovery[T, R], createErr error) (string, R, error) { + var zero R + lastErr := createErr + for attempt := 1; ; attempt++ { + found, err := retry.With(ctx, retry.IsTransientRead, func() (*T, error) { + return rc.get(ctx) + }) + if ctx.Err() != nil { + return "", zero, ctx.Err() + } + if err == nil { + return rc.use(ctx, found) + } + if !apierr.IsMissing(err) { + log.Warnf(ctx, "checking whether %s was created: %s", rc.name, err) + return "", zero, createErr + } + if attempt > retry.MaxRetries { + return "", zero, lastErr + } + if err := retry.Pause(ctx, lastErr); err != nil { + return "", zero, err + } + wait, err := rc.create(ctx) + if err == nil { + return wait() + } + if !retry.IsTransient(ctx, err) && !isPostgresAlreadyExists(err) { + return "", zero, err + } + lastErr = err + } +} + +// isPostgresAlreadyExists reports whether a Lakebase create request failed because the resource +// exists. Branches answer with 409 ALREADY_EXISTS; projects answer with 400 BAD_REQUEST and +// "A project with this resource name already exists in the workspace." +func isPostgresAlreadyExists(err error) bool { + if errors.Is(err, apierr.ErrAlreadyExists) { + return true + } + apiErr, ok := errors.AsType[*apierr.APIError](err) + return ok && apiErr.StatusCode == http.StatusBadRequest && strings.Contains(apiErr.Message, "already exists") +} + +// createdByRequestAt reports whether a resource with createTime can have been created by a request +// first sent at start. Without replace_existing, a resource that existed before the request would +// have made it fail with "already exists". That holds only if the request reached the backend's +// check, so look at when the resource was created rather than adopt one the bundle did not create. +func createdByRequestAt(createTime *sdktime.Time, start time.Time) bool { + return createTime != nil && !createTime.AsTime().Before(start.Add(-postgresCreateTimeSkew)) +} diff --git a/bundle/direct/dresources/postgres_project.go b/bundle/direct/dresources/postgres_project.go index 3a1f3541268..8caf845b0e1 100644 --- a/bundle/direct/dresources/postgres_project.go +++ b/bundle/direct/dresources/postgres_project.go @@ -2,9 +2,13 @@ package dresources import ( "context" + "fmt" "slices" + "time" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/common/types/fieldmask" sdktime "github.com/databricks/databricks-sdk-go/common/types/time" @@ -101,7 +105,7 @@ func (r *ResourcePostgresProject) DoRead(ctx context.Context, id string) (*Postg } func (r *ResourcePostgresProject) DoCreate(ctx context.Context, config *PostgresProjectState) (string, *PostgresProjectRemote, error) { - waiter, err := r.client.Postgres.CreateProject(ctx, postgres.CreateProjectRequest{ + req := postgres.CreateProjectRequest{ ProjectId: config.ProjectId, Project: postgres.Project{ Spec: &config.ProjectSpec, @@ -119,24 +123,84 @@ func (r *ResourcePostgresProject) DoCreate(ctx context.Context, config *Postgres UpdateTime: nil, ForceSendFields: nil, }, - }) + } + start := time.Now() + waiter, err := r.client.Postgres.CreateProject(ctx, req) if err != nil { - // The project is keyed by the caller-chosen project_id, so repeating the request cannot - // create a duplicate: if the failed request did create the project, the retry fails with - // ALREADY_EXISTS, just as the next deploy would have. - return "", nil, retrySafe(err) + return r.recoverFailedCreate(ctx, req, start, err) } + return r.waitCreated(ctx, waiter) +} - // Wait for the project to be ready (long-running operation) +// waitCreated waits for the create operation of a project that a create request returned. +func (r *ResourcePostgresProject) waitCreated(ctx context.Context, waiter postgres.CreateProjectOperationInterface) (string, *PostgresProjectRemote, error) { result, err := waitCreateOperation(ctx, waiter.Wait) if err != nil { return "", nil, err } - remote := makePostgresProjectRemote(result) return remote.Name, remote, nil } +// recoverFailedCreate handles a create request that failed; see recoverPostgresCreate. Repeating +// the request without looking first fails with "already exists" when the failed request did create +// the project, so look for it before repeating. Without a project_id there is no name to look up; +// the API rejects such a request with 400 anyway. +func (r *ResourcePostgresProject) recoverFailedCreate(ctx context.Context, req postgres.CreateProjectRequest, start time.Time, createErr error) (string, *PostgresProjectRemote, error) { + if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) || req.ProjectId == "" { + return "", nil, createErr + } + + name := "projects/" + req.ProjectId + return recoverPostgresCreate(ctx, postgresCreateRecovery[postgres.Project, *PostgresProjectRemote]{ + name: name, + get: func(ctx context.Context) (*postgres.Project, error) { + return r.client.Postgres.GetProject(ctx, postgres.GetProjectRequest{Name: name}) + }, + create: func(ctx context.Context) (func() (string, *PostgresProjectRemote, error), error) { + waiter, err := r.client.Postgres.CreateProject(ctx, req) + if err != nil { + return nil, err + } + return func() (string, *PostgresProjectRemote, error) { return r.waitCreated(ctx, waiter) }, nil + }, + use: func(ctx context.Context, project *postgres.Project) (string, *PostgresProjectRemote, error) { + if !createdByRequestAt(project.CreateTime, start) { + return "", nil, fmt.Errorf("%w; project %s exists but was not created by this deployment", createErr, project.Name) + } + if err := r.waitDefaultBranchReady(ctx, project); err != nil { + return "", nil, err + } + // Info, not Warn: the deploy got the project it asked for, so there is nothing to act on. + log.Infof(ctx, "using project %s, created despite the failed create request: %s", project.Name, createErr) + remote := makePostgresProjectRemote(project) + return remote.Name, remote, nil + }, + }, createErr) +} + +// waitDefaultBranchReady stands in for the create operation that a recovered project's failed +// request would have waited for: dependent resources are created on the default branch, so wait +// for it to leave INIT. The project is ours either way, so a wait that ends early, or a default +// branch that cannot be read, only warns: state still records the project, and the next deploy +// does not fail with "already exists". +func (r *ResourcePostgresProject) waitDefaultBranchReady(ctx context.Context, project *postgres.Project) error { + if project.Status == nil || project.Status.DefaultBranch == "" { + return nil + } + name := project.Status.DefaultBranch + branch, err := readBranch(ctx, r.client, name) + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + log.Warnf(ctx, "waiting for branch %s to become ready: %s", name, err) + return nil + } + _, err = waitBranchReady(ctx, r.client, branch) + return err +} + // The default endpoint settings carry the same suspension oneof as an endpoint does; // see branchOneofGroups. var projectOneofGroups = map[string]string{ diff --git a/bundle/direct/dresources/postgres_project_test.go b/bundle/direct/dresources/postgres_project_test.go new file mode 100644 index 00000000000..41caa1a62c4 --- /dev/null +++ b/bundle/direct/dresources/postgres_project_test.go @@ -0,0 +1,230 @@ +package dresources + +import ( + "encoding/json" + "errors" + "testing" + "time" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/service/postgres" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testProjectName = "projects/p" + +// projectCreateScript plays the given responses, in order, to the project create and get requests. +// A response is a postgres.Project (for a create: the finished operation that returns it) or a +// testserver.Response for an error. +type projectCreateScript struct { + t *testing.T + creates []any + gets []any + // branchGets are the responses to reads of the default branch. + branchGets []any + + createCount int + getCount int + branchGetCount int + operation *postgres.Operation +} + +func (s *projectCreateScript) client() *databricks.WorkspaceClient { + server := testserver.New(s.t) + server.Handle("POST", "/api/2.0/postgres/projects", func(req testserver.Request) any { + s.createCount++ + require.LessOrEqual(s.t, s.createCount, len(s.creates), "unexpected create request") + resp := s.creates[s.createCount-1] + if project, ok := resp.(postgres.Project); ok { + body, err := json.Marshal(project) + require.NoError(s.t, err) + s.operation = &postgres.Operation{Name: testProjectName + "/operations/op", Done: true, Response: body} + return s.operation + } + return resp + }) + // The SDK polls the operation even when the create response says it is done. + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/operations/{operation_id}", func(req testserver.Request) any { + require.NotNil(s.t, s.operation, "unexpected operation request") + return s.operation + }) + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}", func(req testserver.Request) any { + s.getCount++ + require.LessOrEqual(s.t, s.getCount, len(s.gets), "unexpected get request") + return s.gets[s.getCount-1] + }) + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}", func(req testserver.Request) any { + s.branchGetCount++ + require.LessOrEqual(s.t, s.branchGetCount, len(s.branchGets), "unexpected branch get request") + return s.branchGets[s.branchGetCount-1] + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(s.t, err) + return client +} + +const testDefaultBranchName = testProjectName + "/branches/production" + +// testProjectWithDefaultBranch is a project whose status names its default branch. +func testProjectWithDefaultBranch(created time.Time) postgres.Project { + project := testProject(created) + project.Status = &postgres.ProjectStatus{DefaultBranch: testDefaultBranchName} + return project +} + +func testDefaultBranch(state postgres.BranchStatusState) postgres.Branch { + return postgres.Branch{Name: testDefaultBranchName, Status: &postgres.BranchStatus{CurrentState: state}} +} + +var ( + projectNotFound = branchError(404, "NOT_FOUND", "project not found") + // The real API answers a duplicate project with 400, not 409 ALREADY_EXISTS. + projectAlreadyExists = branchError(400, "BAD_REQUEST", "A project with this resource name already exists in the workspace.") +) + +func testProject(created time.Time) postgres.Project { + return postgres.Project{Name: testProjectName, ProjectId: "p", CreateTime: sdktime.New(created)} +} + +// createTestProject runs DoCreate with fast retries. +func createTestProject(t *testing.T, s *projectCreateScript, projectID string) (string, *PostgresProjectRemote, error) { + t.Helper() + ctx := env.Set(t.Context(), retry.IntervalMsVariable, "1") + setForTest(t, &branchReadyPollInterval, time.Millisecond) + r := (&ResourcePostgresProject{}).New(s.client()) + return r.DoCreate(ctx, &PostgresProjectState{ProjectId: projectID}) +} + +func TestPostgresProjectCreateUsesProjectCreatedDespiteTimeout(t *testing.T) { + s := &projectCreateScript{t: t, creates: []any{branchTimeout}, gets: []any{testProject(time.Now())}} + + id, remote, err := createTestProject(t, s, "p") + + require.NoError(t, err) + assert.Equal(t, testProjectName, id) + assert.Equal(t, "p", remote.ProjectId) + assert.Equal(t, 1, s.createCount) + assert.Equal(t, 1, s.getCount) +} + +func TestPostgresProjectCreateRepeatsCreateWhileProjectIsMissing(t *testing.T) { + s := &projectCreateScript{t: t, creates: []any{branchTimeout, testProject(time.Now())}, gets: []any{projectNotFound}} + + id, _, err := createTestProject(t, s, "p") + + require.NoError(t, err) + assert.Equal(t, testProjectName, id) + assert.Equal(t, 2, s.createCount) + assert.Equal(t, 1, s.getCount) +} + +func TestPostgresProjectCreateUsesProjectWhenRepeatedCreateFindsIt(t *testing.T) { + // The first request was still completing when the project was read, and has landed since. + s := &projectCreateScript{ + t: t, + creates: []any{branchTimeout, projectAlreadyExists}, + gets: []any{projectNotFound, testProject(time.Now())}, + } + + id, _, err := createTestProject(t, s, "p") + + require.NoError(t, err) + assert.Equal(t, testProjectName, id) + assert.Equal(t, 2, s.createCount) + assert.Equal(t, 2, s.getCount) +} + +func TestPostgresProjectCreateGivesUpWhenProjectStaysMissing(t *testing.T) { + s := &projectCreateScript{ + t: t, + creates: []any{branchTimeout, branchTimeout, branchTimeout}, + gets: []any{projectNotFound, projectNotFound, projectNotFound}, + } + + _, _, err := createTestProject(t, s, "p") + + require.ErrorContains(t, err, testBranchTimeoutMessage) + assert.Equal(t, 1+retry.MaxRetries, s.createCount) + assert.Equal(t, 1+retry.MaxRetries, s.getCount) +} + +func TestPostgresProjectCreateDoesNotUseProjectThatPredatesRequest(t *testing.T) { + s := &projectCreateScript{t: t, creates: []any{branchTimeout}, gets: []any{testProject(time.Now().Add(-time.Hour))}} + + _, _, err := createTestProject(t, s, "p") + + require.ErrorContains(t, err, testBranchTimeoutMessage) + require.ErrorContains(t, err, "project "+testProjectName+" exists but was not created by this deployment") + assert.Equal(t, 1, s.createCount) +} + +func TestPostgresProjectCreateReturnsCreateErrorWhenReadFails(t *testing.T) { + s := &projectCreateScript{t: t, creates: []any{branchTimeout}, gets: []any{branchError(403, "PERMISSION_DENIED", "no access")}} + + _, _, err := createTestProject(t, s, "p") + + require.ErrorContains(t, err, testBranchTimeoutMessage) + assert.NotContains(t, err.Error(), "no access") + assert.Equal(t, 1, s.createCount) +} + +func TestPostgresProjectCreateDoesNotRecoverFromNonTransientError(t *testing.T) { + s := &projectCreateScript{t: t, creates: []any{projectAlreadyExists}} + + _, _, err := createTestProject(t, s, "p") + + require.ErrorContains(t, err, "already exists") + _, ok := errors.AsType[*RetrySafeError](err) + assert.False(t, ok) + assert.Equal(t, 0, s.getCount) +} + +func TestPostgresProjectCreateWithoutProjectIdIsNotRepeated(t *testing.T) { + // Without a project_id there is no name to look up. + s := &projectCreateScript{t: t, creates: []any{branchTimeout}} + + _, _, err := createTestProject(t, s, "") + + require.ErrorContains(t, err, testBranchTimeoutMessage) + assert.Equal(t, 1, s.createCount) + assert.Equal(t, 0, s.getCount) +} + +func TestPostgresProjectCreateWaitsForDefaultBranchOfRecoveredProject(t *testing.T) { + s := &projectCreateScript{ + t: t, + creates: []any{branchTimeout}, + gets: []any{testProjectWithDefaultBranch(time.Now())}, + branchGets: []any{testDefaultBranch(postgres.BranchStatusStateInit), testDefaultBranch(postgres.BranchStatusStateReady)}, + } + + id, _, err := createTestProject(t, s, "p") + + require.NoError(t, err) + assert.Equal(t, testProjectName, id) + assert.Equal(t, 1, s.createCount) + assert.Equal(t, 2, s.branchGetCount) +} + +func TestPostgresProjectCreateReturnsRecoveredProjectWhenDefaultBranchReadFails(t *testing.T) { + // The project is ours: return it so that state records it, even though the wait ended early. + s := &projectCreateScript{ + t: t, + creates: []any{branchTimeout}, + gets: []any{testProjectWithDefaultBranch(time.Now())}, + branchGets: []any{branchError(403, "PERMISSION_DENIED", "no access")}, + } + + id, _, err := createTestProject(t, s, "p") + + require.NoError(t, err) + assert.Equal(t, testProjectName, id) + assert.Equal(t, 1, s.branchGetCount) +} diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index 05169390ac5..105522a67b9 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -104,7 +104,7 @@ func (s *FakeWorkspace) PostgresProjectCreate(req Request, projectID string) Res // Check for duplicate if _, exists := s.PostgresProjects[name]; exists { - return postgresErrorResponse(409, "ALREADY_EXISTS", "project with such id already exists in the workspace") + return postgresErrorResponse(400, "BAD_REQUEST", "A project with this resource name already exists in the workspace.") } now := nowTime() diff --git a/libs/testserver/postgres_test.go b/libs/testserver/postgres_test.go index 1a414cf6762..7f2f4c0e908 100644 --- a/libs/testserver/postgres_test.go +++ b/libs/testserver/postgres_test.go @@ -108,7 +108,7 @@ func TestPostgresProjectDuplicate(t *testing.T) { createReq2.Header.Set("Authorization", "Bearer test-token") createResp2, err := client.Do(createReq2) require.NoError(t, err) - assert.Equal(t, 409, createResp2.StatusCode) + assert.Equal(t, 400, createResp2.StatusCode) createResp2.Body.Close() }