Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7801.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Use the Lakebase database catalog or postgres catalog that a create request created despite failing with a transient error, instead of failing the deploy, in the direct deployment engine. ([#7801](https://github.com/databricks/cli/pull/7801))
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
bundle:
name: test-bundle

sync:
paths: []

resources:
database_instances:
my_instance:
name: test-db-instance
capacity: CU_1
database_catalogs:
my_catalog:
database_instance_name: ${resources.database_instances.my_instance.name}
database_name: my_database
name: my_catalog
create_database_if_not_exists: true
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Cloud = false
CloudEnvs.gcp = false
EnvMatrix.DMS = ["", "true"]
EnvMatrix.READPLAN = []
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@

=== The create request times out but the catalog was created: deploy uses the catalog

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created database_catalogs.my_catalog
Created database_instances.my_instance
Files: 0 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //database/catalogs //unity-catalog/catalogs
POST /api/2.0/database/catalogs
GET /api/2.0/database/catalogs/my_catalog
GET /api/2.1/unity-catalog/catalogs/my_catalog

=== The create request times out and the catalog was not created: deploy repeats the create

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created database_instances.my_instance
Warn: deploying resources.database_catalogs.my_catalog: retrying after 504 Gateway Timeout from POST /api/2.0/database/catalogs
Created database_catalogs.my_catalog
Files: 0 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //database/catalogs //unity-catalog/catalogs
POST /api/2.0/database/catalogs
GET /api/2.0/database/catalogs/my_catalog
POST /api/2.0/database/catalogs

=== The create request times out, the catalog shows up only after the create is repeated: deploy uses the catalog

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created database_instances.my_instance
Warn: deploying resources.database_catalogs.my_catalog: retrying after 504 Gateway Timeout from POST /api/2.0/database/catalogs
Created database_catalogs.my_catalog
Files: 0 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //database/catalogs //unity-catalog/catalogs
POST /api/2.0/database/catalogs
GET /api/2.0/database/catalogs/my_catalog
POST /api/2.0/database/catalogs
GET /api/2.0/database/catalogs/my_catalog
GET /api/2.1/unity-catalog/catalogs/my_catalog

=== The catalog cannot be read after the create request times out: deploy fails with the create error

>>> musterr [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created database_instances.my_instance
Warn: deploying resources.database_catalogs.my_catalog: checking whether database catalog my_catalog was created: Fault injected by test.
Error: cannot create resources.database_catalogs.my_catalog: Fault injected by test. (504 DEADLINE_EXCEEDED)

Endpoint: POST [DATABRICKS_URL]/api/2.0/database/catalogs
HTTP Status: 504 Gateway Timeout
API error_code: DEADLINE_EXCEEDED
API message: Fault injected by test.

Files: 0 uploaded, 0 deleted

>>> print_requests.py --get //database/catalogs //unity-catalog/catalogs
POST /api/2.0/database/catalogs
GET /api/2.0/database/catalogs/my_catalog
28 changes: 28 additions & 0 deletions acceptance/bundle/resources/database_catalogs/create_504/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
title "The create request times out but the catalog was created: deploy uses the catalog"
fault.py --after-handler "POST /api/2.0/database/catalogs" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //database/catalogs //unity-catalog/catalogs | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve &> LOG.destroy
rm -f "$OUT_REQUESTS"

title "The create request times out and the catalog was not created: deploy repeats the create"
fault.py "POST /api/2.0/database/catalogs" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //database/catalogs //unity-catalog/catalogs | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve &> LOG.destroy
rm -f "$OUT_REQUESTS"

title "The create request times out, the catalog shows up only after the create is repeated: deploy uses the catalog"
fault.py --after-handler "POST /api/2.0/database/catalogs" 504 0 1 DEADLINE_EXCEEDED
fault.py "GET /api/2.0/database/catalogs/my_catalog" 404 0 1 CATALOG_DOES_NOT_EXIST
trace $CLI bundle deploy
trace print_requests.py --get //database/catalogs //unity-catalog/catalogs | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve &> LOG.destroy
rm -f "$OUT_REQUESTS"

title "The catalog cannot be read after the create request times out: deploy fails with the create error"
fault.py "POST /api/2.0/database/catalogs" 504 0 1 DEADLINE_EXCEEDED
fault.py "GET /api/2.0/database/catalogs/my_catalog" 403 0 1 PERMISSION_DENIED
trace musterr $CLI bundle deploy
trace print_requests.py --get //database/catalogs //unity-catalog/catalogs | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve &> LOG.destroy
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Cloud = false
RecordRequests = true
EnvMatrix.READPLAN = []
Env.DATABRICKS_BUNDLE_ENGINE = "direct"
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
bundle:
name: test-bundle

sync:
paths: []

resources:
postgres_projects:
my_project:
project_id: test-pg-proj
display_name: "Test Project for Catalog"
pg_version: 16

postgres_catalogs:
my_catalog:
catalog_id: lakebase_test
branch: ${resources.postgres_projects.my_project.id}/branches/production
postgres_database: appdb
create_database_if_missing: true
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
EnvMatrix.READPLAN = []
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@

=== The create request fails with 500 but the catalog was created: deploy uses the catalog

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_catalogs.my_catalog
Created postgres_projects.my_project
Files: 0 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //postgres/catalogs
POST /api/2.0/postgres/catalogs
GET /api/2.0/postgres/catalogs/lakebase_test

=== The create request times out and the catalog was not created: deploy repeats the create

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_projects.my_project
Warn: deploying resources.postgres_catalogs.my_catalog: retrying after 504 Gateway Timeout from POST /api/2.0/postgres/catalogs
Created postgres_catalogs.my_catalog
Files: 0 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //postgres/catalogs
POST /api/2.0/postgres/catalogs
GET /api/2.0/postgres/catalogs/lakebase_test
POST /api/2.0/postgres/catalogs
GET /api/2.0/postgres/catalogs/lakebase_test/operations/[UUID]

=== The catalog cannot be read after the create request fails: deploy fails with the create error

>>> musterr [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_projects.my_project
Warn: deploying resources.postgres_catalogs.my_catalog: checking whether catalogs/lakebase_test was created: Fault injected by test.
Error: cannot create resources.postgres_catalogs.my_catalog: Fault injected by test. (500 INTERNAL_ERROR)

Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/catalogs?catalog_id=lakebase_test
HTTP Status: 500 Internal Server Error
API error_code: INTERNAL_ERROR
API message: Fault injected by test.

Files: 0 uploaded, 0 deleted

>>> print_requests.py --get //postgres/catalogs
POST /api/2.0/postgres/catalogs
GET /api/2.0/postgres/catalogs/lakebase_test
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
title "The create request fails with 500 but the catalog was created: deploy uses the catalog"
fault.py --after-handler "POST /api/2.0/postgres/catalogs" 500 0 1 INTERNAL_ERROR
trace $CLI bundle deploy
trace print_requests.py --get //postgres/catalogs | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve &> LOG.destroy
rm -f "$OUT_REQUESTS"

title "The create request times out and the catalog was not created: deploy repeats the create"
fault.py "POST /api/2.0/postgres/catalogs" 504 0 1 DEADLINE_EXCEEDED
trace $CLI bundle deploy
trace print_requests.py --get //postgres/catalogs | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve &> LOG.destroy
rm -f "$OUT_REQUESTS"

title "The catalog cannot be read after the create request fails: deploy fails with the create error"
fault.py "POST /api/2.0/postgres/catalogs" 500 0 1 INTERNAL_ERROR
fault.py "GET /api/2.0/postgres/catalogs/lakebase_test" 403 0 1 PERMISSION_DENIED
trace musterr $CLI bundle deploy
trace print_requests.py --get //postgres/catalogs | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve &> LOG.destroy
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Cloud = false
RecordRequests = true
EnvMatrix.READPLAN = []
Env.DATABRICKS_BUNDLE_ENGINE = "direct"
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
10 changes: 7 additions & 3 deletions bundle/direct/dresources/create_recovery.go
Original file line number Diff line number Diff line change
Expand Up @@ -145,8 +145,12 @@ func (e *recoveryError) Unwrap() error { return e.err }

// isAlreadyExists reports whether a create request failed because the resource exists. Features,
// dashboards, vector search endpoints and AI Gateway services (checked for model provider services)
// answer with 409 ALREADY_EXISTS, serving endpoints with
// 400 RESOURCE_ALREADY_EXISTS (observed on AWS, 2026-11).
// answer with 409 ALREADY_EXISTS, serving endpoints with 400 RESOURCE_ALREADY_EXISTS and database
// catalogs with 400 CATALOG_ALREADY_EXISTS (observed on AWS, 2026-11).
func isAlreadyExists(err error) bool {
return errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists)
if errors.Is(err, apierr.ErrAlreadyExists) || errors.Is(err, apierr.ErrResourceAlreadyExists) {
return true
}
apiErr, ok := errors.AsType[*apierr.APIError](err)
return ok && apiErr.ErrorCode == "CATALOG_ALREADY_EXISTS"
}
65 changes: 60 additions & 5 deletions bundle/direct/dresources/database_catalog.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@ package dresources

import (
"context"
"time"

"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/service/database"
)

Expand All @@ -25,13 +27,66 @@ func (r *ResourceDatabaseCatalog) DoRead(ctx context.Context, id string) (*datab
}

func (r *ResourceDatabaseCatalog) DoCreate(ctx context.Context, config *database.DatabaseCatalog) (string, *database.DatabaseCatalog, error) {
result, err := r.client.Database.CreateDatabaseCatalog(ctx, database.CreateDatabaseCatalogRequest{
Catalog: *config,
})
create := func(ctx context.Context) (*foundDatabaseCatalog, error) {
result, err := r.client.Database.CreateDatabaseCatalog(ctx, database.CreateDatabaseCatalogRequest{
Catalog: *config,
})
if err != nil {
return nil, err
}
return &foundDatabaseCatalog{catalog: result, createdAt: 0}, nil
}
start := time.Now()
found, err := create(ctx)
if err != nil {
return "", nil, err
found, err = r.recoverFailedCreate(ctx, config.Name, start, create, err)
if err != nil {
return "", nil, err
}
}
return result.Name, nil, nil
return found.catalog.Name, nil, nil
}

// foundDatabaseCatalog is a database catalog with the create time of the Unity Catalog catalog it
// registers, which the Database API does not return. createdAt is only set on a catalog that the
// lookup after a failed create found.
type foundDatabaseCatalog struct {
catalog *database.DatabaseCatalog
createdAt int64
}

// recoverFailedCreate handles a create request that failed; see recoverCreate. Catalog creates
// are answered with 504 DEADLINE_EXCEEDED now and then (observed on Azure, 2026-11), and the
// request may have created the catalog anyway. The catalog name is unique in the metastore, so it
// names the catalog to look up.
func (r *ResourceDatabaseCatalog) recoverFailedCreate(ctx context.Context, name string, start time.Time, create func(context.Context) (*foundDatabaseCatalog, error), createErr error) (*foundDatabaseCatalog, error) {
return recoverCreate(ctx, createRecovery[foundDatabaseCatalog]{
name: "database catalog " + name,
start: start,
get: func(ctx context.Context) (*foundDatabaseCatalog, error) {
dbCatalog, err := r.client.Database.GetDatabaseCatalogByName(ctx, name)
if err != nil {
return nil, err
}
ucCatalog, err := r.client.Catalogs.GetByName(ctx, name)
if apierr.IsMissing(err) {
// The database catalog exists, so repeating the create would only fail. Without
// a create time it is not known to be ours, which recoverCreate reports.
return &foundDatabaseCatalog{catalog: dbCatalog, createdAt: 0}, nil
}
if err != nil {
return nil, err
}
return &foundDatabaseCatalog{catalog: dbCatalog, createdAt: ucCatalog.CreatedAt}, nil
},
createTime: func(found *foundDatabaseCatalog) (time.Time, bool) {
if found.createdAt == 0 {
return time.Time{}, false
}
return time.UnixMilli(found.createdAt), true
},
create: create,
}, createErr)
}

func (r *ResourceDatabaseCatalog) DoDelete(ctx context.Context, id string, _ *database.DatabaseCatalog) error {
Expand Down
Loading
Loading