diff --git a/.nextchanges/bundles/7801.md b/.nextchanges/bundles/7801.md new file mode 100644 index 00000000000..9bde1cf325c --- /dev/null +++ b/.nextchanges/bundles/7801.md @@ -0,0 +1 @@ +* Use the Lakebase database catalog or postgres catalog that a create request created despite failing with a transient error, instead of failing the deploy, in the direct deployment engine. ([#7801](https://github.com/databricks/cli/pull/7801)) diff --git a/acceptance/bundle/resources/database_catalogs/create_504/databricks.yml b/acceptance/bundle/resources/database_catalogs/create_504/databricks.yml new file mode 100644 index 00000000000..040099ab87e --- /dev/null +++ b/acceptance/bundle/resources/database_catalogs/create_504/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +sync: + paths: [] + +resources: + database_instances: + my_instance: + name: test-db-instance + capacity: CU_1 + database_catalogs: + my_catalog: + database_instance_name: ${resources.database_instances.my_instance.name} + database_name: my_database + name: my_catalog + create_database_if_not_exists: true diff --git a/acceptance/bundle/resources/database_catalogs/create_504/out.test.toml b/acceptance/bundle/resources/database_catalogs/create_504/out.test.toml new file mode 100644 index 00000000000..0894b57ab43 --- /dev/null +++ b/acceptance/bundle/resources/database_catalogs/create_504/out.test.toml @@ -0,0 +1,4 @@ +Cloud = false +CloudEnvs.gcp = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/database_catalogs/create_504/output.txt b/acceptance/bundle/resources/database_catalogs/create_504/output.txt new file mode 100644 index 00000000000..e26faa25b0d --- /dev/null +++ b/acceptance/bundle/resources/database_catalogs/create_504/output.txt @@ -0,0 +1,65 @@ + +=== The create request times out but the catalog was created: deploy uses the catalog + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created database_catalogs.my_catalog +Created database_instances.my_instance +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //database/catalogs //unity-catalog/catalogs +POST /api/2.0/database/catalogs +GET /api/2.0/database/catalogs/my_catalog +GET /api/2.1/unity-catalog/catalogs/my_catalog + +=== The create request times out and the catalog was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created database_instances.my_instance +Warn: deploying resources.database_catalogs.my_catalog: retrying after 504 Gateway Timeout from POST /api/2.0/database/catalogs +Created database_catalogs.my_catalog +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //database/catalogs //unity-catalog/catalogs +POST /api/2.0/database/catalogs +GET /api/2.0/database/catalogs/my_catalog +POST /api/2.0/database/catalogs + +=== The create request times out, the catalog shows up only after the create is repeated: deploy uses the catalog + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created database_instances.my_instance +Warn: deploying resources.database_catalogs.my_catalog: retrying after 504 Gateway Timeout from POST /api/2.0/database/catalogs +Created database_catalogs.my_catalog +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //database/catalogs //unity-catalog/catalogs +POST /api/2.0/database/catalogs +GET /api/2.0/database/catalogs/my_catalog +POST /api/2.0/database/catalogs +GET /api/2.0/database/catalogs/my_catalog +GET /api/2.1/unity-catalog/catalogs/my_catalog + +=== The catalog cannot be read after the create request times out: deploy fails with the create error + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created database_instances.my_instance +Warn: deploying resources.database_catalogs.my_catalog: checking whether database catalog my_catalog was created: Fault injected by test. +Error: cannot create resources.database_catalogs.my_catalog: Fault injected by test. (504 DEADLINE_EXCEEDED) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/database/catalogs +HTTP Status: 504 Gateway Timeout +API error_code: DEADLINE_EXCEEDED +API message: Fault injected by test. + +Files: 0 uploaded, 0 deleted + +>>> print_requests.py --get //database/catalogs //unity-catalog/catalogs +POST /api/2.0/database/catalogs +GET /api/2.0/database/catalogs/my_catalog diff --git a/acceptance/bundle/resources/database_catalogs/create_504/script b/acceptance/bundle/resources/database_catalogs/create_504/script new file mode 100644 index 00000000000..c97e27588ed --- /dev/null +++ b/acceptance/bundle/resources/database_catalogs/create_504/script @@ -0,0 +1,28 @@ +title "The create request times out but the catalog was created: deploy uses the catalog" +fault.py --after-handler "POST /api/2.0/database/catalogs" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //database/catalogs //unity-catalog/catalogs | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve &> LOG.destroy +rm -f "$OUT_REQUESTS" + +title "The create request times out and the catalog was not created: deploy repeats the create" +fault.py "POST /api/2.0/database/catalogs" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //database/catalogs //unity-catalog/catalogs | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve &> LOG.destroy +rm -f "$OUT_REQUESTS" + +title "The create request times out, the catalog shows up only after the create is repeated: deploy uses the catalog" +fault.py --after-handler "POST /api/2.0/database/catalogs" 504 0 1 DEADLINE_EXCEEDED +fault.py "GET /api/2.0/database/catalogs/my_catalog" 404 0 1 CATALOG_DOES_NOT_EXIST +trace $CLI bundle deploy +trace print_requests.py --get //database/catalogs //unity-catalog/catalogs | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve &> LOG.destroy +rm -f "$OUT_REQUESTS" + +title "The catalog cannot be read after the create request times out: deploy fails with the create error" +fault.py "POST /api/2.0/database/catalogs" 504 0 1 DEADLINE_EXCEEDED +fault.py "GET /api/2.0/database/catalogs/my_catalog" 403 0 1 PERMISSION_DENIED +trace musterr $CLI bundle deploy +trace print_requests.py --get //database/catalogs //unity-catalog/catalogs | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve &> LOG.destroy diff --git a/acceptance/bundle/resources/database_catalogs/create_504/test.toml b/acceptance/bundle/resources/database_catalogs/create_504/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/database_catalogs/create_504/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/acceptance/bundle/resources/postgres_catalogs/create_transient_500/databricks.yml b/acceptance/bundle/resources/postgres_catalogs/create_transient_500/databricks.yml new file mode 100644 index 00000000000..021fe525fd3 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/create_transient_500/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bundle + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj + display_name: "Test Project for Catalog" + pg_version: 16 + + postgres_catalogs: + my_catalog: + catalog_id: lakebase_test + branch: ${resources.postgres_projects.my_project.id}/branches/production + postgres_database: appdb + create_database_if_missing: true diff --git a/acceptance/bundle/resources/postgres_catalogs/create_transient_500/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/create_transient_500/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/create_transient_500/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/postgres_catalogs/create_transient_500/output.txt b/acceptance/bundle/resources/postgres_catalogs/create_transient_500/output.txt new file mode 100644 index 00000000000..fcc5260db83 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/create_transient_500/output.txt @@ -0,0 +1,48 @@ + +=== The create request fails with 500 but the catalog was created: deploy uses the catalog + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_catalogs.my_catalog +Created postgres_projects.my_project +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres/catalogs +POST /api/2.0/postgres/catalogs +GET /api/2.0/postgres/catalogs/lakebase_test + +=== The create request times out and the catalog was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Warn: deploying resources.postgres_catalogs.my_catalog: retrying after 504 Gateway Timeout from POST /api/2.0/postgres/catalogs +Created postgres_catalogs.my_catalog +Files: 0 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres/catalogs +POST /api/2.0/postgres/catalogs +GET /api/2.0/postgres/catalogs/lakebase_test +POST /api/2.0/postgres/catalogs +GET /api/2.0/postgres/catalogs/lakebase_test/operations/[UUID] + +=== The catalog cannot be read after the create request fails: deploy fails with the create error + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_projects.my_project +Warn: deploying resources.postgres_catalogs.my_catalog: checking whether catalogs/lakebase_test was created: Fault injected by test. +Error: cannot create resources.postgres_catalogs.my_catalog: Fault injected by test. (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/catalogs?catalog_id=lakebase_test +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Fault injected by test. + +Files: 0 uploaded, 0 deleted + +>>> print_requests.py --get //postgres/catalogs +POST /api/2.0/postgres/catalogs +GET /api/2.0/postgres/catalogs/lakebase_test diff --git a/acceptance/bundle/resources/postgres_catalogs/create_transient_500/script b/acceptance/bundle/resources/postgres_catalogs/create_transient_500/script new file mode 100644 index 00000000000..d7511628b7d --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/create_transient_500/script @@ -0,0 +1,20 @@ +title "The create request fails with 500 but the catalog was created: deploy uses the catalog" +fault.py --after-handler "POST /api/2.0/postgres/catalogs" 500 0 1 INTERNAL_ERROR +trace $CLI bundle deploy +trace print_requests.py --get //postgres/catalogs | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve &> LOG.destroy +rm -f "$OUT_REQUESTS" + +title "The create request times out and the catalog was not created: deploy repeats the create" +fault.py "POST /api/2.0/postgres/catalogs" 504 0 1 DEADLINE_EXCEEDED +trace $CLI bundle deploy +trace print_requests.py --get //postgres/catalogs | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve &> LOG.destroy +rm -f "$OUT_REQUESTS" + +title "The catalog cannot be read after the create request fails: deploy fails with the create error" +fault.py "POST /api/2.0/postgres/catalogs" 500 0 1 INTERNAL_ERROR +fault.py "GET /api/2.0/postgres/catalogs/lakebase_test" 403 0 1 PERMISSION_DENIED +trace musterr $CLI bundle deploy +trace print_requests.py --get //postgres/catalogs | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve &> LOG.destroy diff --git a/acceptance/bundle/resources/postgres_catalogs/create_transient_500/test.toml b/acceptance/bundle/resources/postgres_catalogs/create_transient_500/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_catalogs/create_transient_500/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/bundle/direct/dresources/create_recovery.go b/bundle/direct/dresources/create_recovery.go index e854dc51fbf..a7b4697a52a 100644 --- a/bundle/direct/dresources/create_recovery.go +++ b/bundle/direct/dresources/create_recovery.go @@ -145,8 +145,12 @@ func (e *recoveryError) Unwrap() error { return e.err } // isAlreadyExists reports whether a create request failed because the resource exists. Features, // dashboards, vector search endpoints and AI Gateway services (checked for model provider services) -// answer with 409 ALREADY_EXISTS, serving endpoints with -// 400 RESOURCE_ALREADY_EXISTS (observed on AWS, 2026-11). +// answer with 409 ALREADY_EXISTS, serving endpoints with 400 RESOURCE_ALREADY_EXISTS and database +// catalogs with 400 CATALOG_ALREADY_EXISTS (observed on AWS, 2026-11). func isAlreadyExists(err error) bool { - return errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists) + if errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists) { + return true + } + apiErr, ok := errors.AsType[*apierr.APIError](err) + return ok && apiErr.ErrorCode == "CATALOG_ALREADY_EXISTS" } diff --git a/bundle/direct/dresources/database_catalog.go b/bundle/direct/dresources/database_catalog.go index a1a7ebdef44..c4da2454ba7 100644 --- a/bundle/direct/dresources/database_catalog.go +++ b/bundle/direct/dresources/database_catalog.go @@ -2,9 +2,11 @@ package dresources import ( "context" + "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/database" ) @@ -25,13 +27,66 @@ func (r *ResourceDatabaseCatalog) DoRead(ctx context.Context, id string) (*datab } func (r *ResourceDatabaseCatalog) DoCreate(ctx context.Context, config *database.DatabaseCatalog) (string, *database.DatabaseCatalog, error) { - result, err := r.client.Database.CreateDatabaseCatalog(ctx, database.CreateDatabaseCatalogRequest{ - Catalog: *config, - }) + create := func(ctx context.Context) (*foundDatabaseCatalog, error) { + result, err := r.client.Database.CreateDatabaseCatalog(ctx, database.CreateDatabaseCatalogRequest{ + Catalog: *config, + }) + if err != nil { + return nil, err + } + return &foundDatabaseCatalog{catalog: result, createdAt: 0}, nil + } + start := time.Now() + found, err := create(ctx) if err != nil { - return "", nil, err + found, err = r.recoverFailedCreate(ctx, config.Name, start, create, err) + if err != nil { + return "", nil, err + } } - return result.Name, nil, nil + return found.catalog.Name, nil, nil +} + +// foundDatabaseCatalog is a database catalog with the create time of the Unity Catalog catalog it +// registers, which the Database API does not return. createdAt is only set on a catalog that the +// lookup after a failed create found. +type foundDatabaseCatalog struct { + catalog *database.DatabaseCatalog + createdAt int64 +} + +// recoverFailedCreate handles a create request that failed; see recoverCreate. Catalog creates +// are answered with 504 DEADLINE_EXCEEDED now and then (observed on Azure, 2026-11), and the +// request may have created the catalog anyway. The catalog name is unique in the metastore, so it +// names the catalog to look up. +func (r *ResourceDatabaseCatalog) recoverFailedCreate(ctx context.Context, name string, start time.Time, create func(context.Context) (*foundDatabaseCatalog, error), createErr error) (*foundDatabaseCatalog, error) { + return recoverCreate(ctx, createRecovery[foundDatabaseCatalog]{ + name: "database catalog " + name, + start: start, + get: func(ctx context.Context) (*foundDatabaseCatalog, error) { + dbCatalog, err := r.client.Database.GetDatabaseCatalogByName(ctx, name) + if err != nil { + return nil, err + } + ucCatalog, err := r.client.Catalogs.GetByName(ctx, name) + if apierr.IsMissing(err) { + // The database catalog exists, so repeating the create would only fail. Without + // a create time it is not known to be ours, which recoverCreate reports. + return &foundDatabaseCatalog{catalog: dbCatalog, createdAt: 0}, nil + } + if err != nil { + return nil, err + } + return &foundDatabaseCatalog{catalog: dbCatalog, createdAt: ucCatalog.CreatedAt}, nil + }, + createTime: func(found *foundDatabaseCatalog) (time.Time, bool) { + if found.createdAt == 0 { + return time.Time{}, false + } + return time.UnixMilli(found.createdAt), true + }, + create: create, + }, createErr) } func (r *ResourceDatabaseCatalog) DoDelete(ctx context.Context, id string, _ *database.DatabaseCatalog) error { diff --git a/bundle/direct/dresources/database_catalog_test.go b/bundle/direct/dresources/database_catalog_test.go new file mode 100644 index 00000000000..ddeea75dca3 --- /dev/null +++ b/bundle/direct/dresources/database_catalog_test.go @@ -0,0 +1,72 @@ +package dresources + +import ( + "testing" + "time" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/database" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testDatabaseCatalogName = "my_catalog" + +// createTestDatabaseCatalog runs DoCreate against a server whose create request times out after +// the catalog exists, with the UC catalog reporting ucCreatedAt (milliseconds) as its create time. +// A negative ucCreatedAt makes the UC lookup fail with 404. +func createTestDatabaseCatalog(t *testing.T, ucCreatedAt int64) (string, int, error) { + t.Helper() + dbCatalog := database.DatabaseCatalog{Name: testDatabaseCatalogName, DatabaseInstanceName: "inst", DatabaseName: "db"} + creates := 0 + server := testserver.New(t) + server.Handle("POST", "/api/2.0/database/catalogs", func(req testserver.Request) any { + creates++ + return featureError(504, "DEADLINE_EXCEEDED", "") + }) + server.Handle("GET", "/api/2.0/database/catalogs/{name}", func(req testserver.Request) any { + return dbCatalog + }) + server.Handle("GET", "/api/2.1/unity-catalog/catalogs/{name}", func(req testserver.Request) any { + if ucCreatedAt < 0 { + return featureError(404, "CATALOG_DOES_NOT_EXIST", "Catalog 'my_catalog' does not exist.") + } + return catalog.CatalogInfo{Name: testDatabaseCatalogName, CreatedAt: ucCreatedAt} + }) + testserver.AddDefaultHandlers(server) + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + + ctx := env.Set(t.Context(), retry.IntervalMsVariable, "1") + id, _, err := (&ResourceDatabaseCatalog{}).New(client).DoCreate(ctx, &dbCatalog) + return id, creates, err +} + +func TestDatabaseCatalogCreateUsesCatalogCreatedDespiteTimeout(t *testing.T) { + id, creates, err := createTestDatabaseCatalog(t, time.Now().UnixMilli()) + require.NoError(t, err) + assert.Equal(t, testDatabaseCatalogName, id) + assert.Equal(t, 1, creates) +} + +func TestDatabaseCatalogCreateRejectsCatalogThatPredatesRequest(t *testing.T) { + _, creates, err := createTestDatabaseCatalog(t, time.Now().Add(-time.Hour).UnixMilli()) + require.EqualError(t, err, "database catalog my_catalog exists but was not created by this deployment") + requireStatus(t, err, 504) + assert.Equal(t, 1, creates) +} + +func TestDatabaseCatalogCreateRejectsCatalogWithoutCreateTime(t *testing.T) { + _, _, err := createTestDatabaseCatalog(t, 0) + require.EqualError(t, err, "database catalog my_catalog exists but has no create time, so it is not known to be created by this deployment") +} + +func TestDatabaseCatalogCreateDoesNotRepeatCreateWhenOnlyUCLookupMisses(t *testing.T) { + _, creates, err := createTestDatabaseCatalog(t, -1) + require.EqualError(t, err, "database catalog my_catalog exists but has no create time, so it is not known to be created by this deployment") + assert.Equal(t, 1, creates) +} diff --git a/bundle/direct/dresources/postgres_catalog.go b/bundle/direct/dresources/postgres_catalog.go index e2b32665d85..39fd6bde042 100644 --- a/bundle/direct/dresources/postgres_catalog.go +++ b/bundle/direct/dresources/postgres_catalog.go @@ -2,9 +2,13 @@ package dresources import ( "context" + "fmt" "strings" + "time" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" sdktime "github.com/databricks/databricks-sdk-go/common/types/time" "github.com/databricks/databricks-sdk-go/marshal" @@ -92,7 +96,7 @@ func (r *ResourcePostgresCatalog) DoRead(ctx context.Context, id string) (*Postg } func (r *ResourcePostgresCatalog) DoCreate(ctx context.Context, config *PostgresCatalogState) (string, *PostgresCatalogRemote, error) { - waiter, err := r.client.Postgres.CreateCatalog(ctx, postgres.CreateCatalogRequest{ + req := postgres.CreateCatalogRequest{ CatalogId: config.CatalogId, Catalog: postgres.Catalog{ Spec: &config.CatalogCatalogSpec, @@ -106,11 +110,17 @@ func (r *ResourcePostgresCatalog) DoCreate(ctx context.Context, config *Postgres UpdateTime: nil, ForceSendFields: nil, }, - }) + } + start := time.Now() + waiter, err := r.client.Postgres.CreateCatalog(ctx, req) if err != nil { - return "", nil, err + return r.recoverFailedCreate(ctx, req, start, err) } + return r.waitCreated(ctx, waiter) +} +// waitCreated waits for the create operation of a catalog that a create request returned. +func (r *ResourcePostgresCatalog) waitCreated(ctx context.Context, waiter postgres.CreateCatalogOperationInterface) (string, *PostgresCatalogRemote, error) { result, err := waitCreateOperation(ctx, waiter.Wait) if err != nil { return "", nil, err @@ -119,6 +129,39 @@ func (r *ResourcePostgresCatalog) DoCreate(ctx context.Context, config *Postgres return remote.Name, remote, nil } +// recoverFailedCreate handles a create request that failed; see recoverPostgresCreate. Catalog +// creates are answered with 500 INTERNAL_ERROR now and then (observed on AWS and GCP, 2026-11), +// and the request may have created the catalog anyway, so look for it before repeating the request. +func (r *ResourcePostgresCatalog) recoverFailedCreate(ctx context.Context, req postgres.CreateCatalogRequest, start time.Time, createErr error) (string, *PostgresCatalogRemote, error) { + if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) || req.CatalogId == "" { + return "", nil, createErr + } + + name := "catalogs/" + req.CatalogId + return recoverPostgresCreate(ctx, postgresCreateRecovery[postgres.Catalog, *PostgresCatalogRemote]{ + name: name, + get: func(ctx context.Context) (*postgres.Catalog, error) { + return r.client.Postgres.GetCatalog(ctx, postgres.GetCatalogRequest{Name: name}) + }, + create: func(ctx context.Context) (func() (string, *PostgresCatalogRemote, error), error) { + waiter, err := r.client.Postgres.CreateCatalog(ctx, req) + if err != nil { + return nil, err + } + return func() (string, *PostgresCatalogRemote, error) { return r.waitCreated(ctx, waiter) }, nil + }, + use: func(ctx context.Context, catalog *postgres.Catalog) (string, *PostgresCatalogRemote, error) { + if !createdByRequestAt(catalog.CreateTime, start) { + return "", nil, fmt.Errorf("%w; catalog %s exists but was not created by this deployment", createErr, catalog.Name) + } + // Info, not Warn: the deploy got the catalog it asked for, so there is nothing to act on. + log.Infof(ctx, "using catalog %s, created despite the failed create request: %s", catalog.Name, createErr) + remote := makePostgresCatalogRemote(catalog) + return remote.Name, remote, nil + }, + }, createErr) +} + func (r *ResourcePostgresCatalog) DoDelete(ctx context.Context, id string, _ *PostgresCatalogState) error { waiter, err := r.client.Postgres.DeleteCatalog(ctx, postgres.DeleteCatalogRequest{ Name: id, diff --git a/bundle/direct/dresources/postgres_catalog_test.go b/bundle/direct/dresources/postgres_catalog_test.go new file mode 100644 index 00000000000..3683bf338a3 --- /dev/null +++ b/bundle/direct/dresources/postgres_catalog_test.go @@ -0,0 +1,51 @@ +package dresources + +import ( + "testing" + "time" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/service/postgres" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// createTestPostgresCatalog runs DoCreate against a server whose create request fails with 500 +// while a catalog created at created exists. +func createTestPostgresCatalog(t *testing.T, created time.Time) (string, int, error) { + t.Helper() + creates := 0 + server := testserver.New(t) + server.Handle("POST", "/api/2.0/postgres/catalogs", func(req testserver.Request) any { + creates++ + return branchError(500, "INTERNAL_ERROR", "The request could not be processed.") + }) + server.Handle("GET", "/api/2.0/postgres/catalogs/{id}", func(req testserver.Request) any { + return postgres.Catalog{Name: "catalogs/cat", CatalogId: "cat", CreateTime: sdktime.New(created)} + }) + testserver.AddDefaultHandlers(server) + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + + ctx := env.Set(t.Context(), retry.IntervalMsVariable, "1") + id, _, err := (&ResourcePostgresCatalog{}).New(client).DoCreate(ctx, &PostgresCatalogState{CatalogId: "cat"}) + return id, creates, err +} + +func TestPostgresCatalogCreateUsesCatalogCreatedDespiteError(t *testing.T) { + id, creates, err := createTestPostgresCatalog(t, time.Now()) + require.NoError(t, err) + assert.Equal(t, "catalogs/cat", id) + assert.Equal(t, 1, creates) +} + +func TestPostgresCatalogCreateRejectsCatalogThatPredatesRequest(t *testing.T) { + _, creates, err := createTestPostgresCatalog(t, time.Now().Add(-time.Hour)) + require.ErrorContains(t, err, "catalog catalogs/cat exists but was not created by this deployment") + requireStatus(t, err, 500) + assert.Equal(t, 1, creates) +} diff --git a/libs/testserver/database_catalogs.go b/libs/testserver/database_catalogs.go index d6dbdc49687..474b4c7caa0 100644 --- a/libs/testserver/database_catalogs.go +++ b/libs/testserver/database_catalogs.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" + "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/databricks/databricks-sdk-go/service/database" ) @@ -34,11 +35,38 @@ func (s *FakeWorkspace) DatabaseCatalogCreate(req Request) Response { } } + // The catalog name is unique in the metastore, whichever API created the catalog. + _, dbExists := s.DatabaseCatalogs[databaseCatalog.Name] + _, ucExists := s.Catalogs[databaseCatalog.Name] + if dbExists || ucExists { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "CATALOG_ALREADY_EXISTS", + "message": fmt.Sprintf("Catalog '%s' already exists", databaseCatalog.Name), + }, + } + } + // Assign uid like the real backend does databaseCatalog.Uid = nextUUID() s.DatabaseCatalogs[databaseCatalog.Name] = databaseCatalog + // The catalog is registered in Unity Catalog, which is the only place its create time shows. + now := nowMilli() + s.Catalogs[databaseCatalog.Name] = catalog.CatalogInfo{ + Name: databaseCatalog.Name, + FullName: databaseCatalog.Name, + CreatedAt: now, + UpdatedAt: now, + CreatedBy: s.CurrentUser().UserName, + UpdatedBy: s.CurrentUser().UserName, + Owner: s.CurrentUser().UserName, + MetastoreId: nextUUID(), + CatalogType: catalog.CatalogTypeManagedOnlineCatalog, + } + return Response{ Body: databaseCatalog, } @@ -50,8 +78,11 @@ func (s *FakeWorkspace) DatabaseCatalogGet(name string) Response { catalog, ok := s.DatabaseCatalogs[name] if !ok { return Response{ - Body: fmt.Sprintf("database catalog with name '%s' not found", name), StatusCode: 404, + Body: map[string]string{ + "error_code": "CATALOG_DOES_NOT_EXIST", + "message": fmt.Sprintf("Catalog '%s' does not exist.", name), + }, } } @@ -75,3 +106,14 @@ func (s *FakeWorkspace) DatabaseCatalogUpdate(req Request, name string) Response }, } } + +func (s *FakeWorkspace) DatabaseCatalogDelete(name string) Response { + defer s.LockUnlock()() + + if _, ok := s.DatabaseCatalogs[name]; !ok { + return Response{StatusCode: 404} + } + delete(s.DatabaseCatalogs, name) + delete(s.Catalogs, name) + return Response{} +} diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 7a9584792c4..f5128622ee9 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -1402,7 +1402,7 @@ func AddDefaultHandlers(server *Server) { }) server.Handle("DELETE", "/api/2.0/database/catalogs/{name}", func(req Request) any { - return MapDelete(req.Workspace, req.Workspace.DatabaseCatalogs, req.Vars["name"]) + return req.Workspace.DatabaseCatalogDelete(req.Vars["name"]) }) // Synced Database Tables: diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index 105522a67b9..47bd4d02cb1 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -1604,7 +1604,7 @@ func (s *FakeWorkspace) PostgresCatalogCreate(req Request, catalogID string) Res name := "catalogs/" + catalogID if _, exists := s.PostgresCatalogs[name]; exists { - return postgresErrorResponse(409, "ALREADY_EXISTS", "catalog with such id already exists") + return postgresErrorResponse(400, "CATALOG_ALREADY_EXISTS", fmt.Sprintf("Catalog '%s' already exists", catalogID)) } now := nowTime()