From 946838d58d6b6e2c4a6cfd0a36fc5598eb100083 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 21:33:56 +0100 Subject: [PATCH 01/15] direct: find a catalog renamed outside the bundle by its id UC returns a catalog id (a UUID kept across renames, new on recreate) that the API spec and SDK do not declare. The catalog adapter now calls the catalogs API directly so it sees the id, records it in the state, and names it as the internal ID: the plan finds a catalog renamed out of band like it finds schemas and volumes. The test server returns and keeps the id too. Co-authored-by: Isaac --- .../catalogs/rename_all_out_of_band/script | 17 +-- .../catalogs/rename_out_of_band/script | 17 +-- .../resources/grants/catalogs/out.plan2.json | 9 ++ bundle/direct/dresources/catalog.go | 103 ++++++++++++++---- bundle/direct/dresources/configs/catalogs.yml | 3 + libs/testserver/catalogs.go | 24 +++- libs/testserver/catalogs_test.go | 21 +++- libs/testserver/fake_workspace.go | 4 +- 8 files changed, 153 insertions(+), 45 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script index aa51aadefb6..26fed081e3c 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script @@ -5,8 +5,6 @@ NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve - # The renamed catalog is not where the state looks for it, so destroy cannot remove it. - trace $CLI catalogs delete "$NEW_CATALOG" --force # Consume the destroy's requests so none are left recorded. print_requests.py //unity-catalog &> LOG.destroy_requests @@ -21,6 +19,8 @@ plan() { title "Deploy a catalog with a schema and a volume in it" trace $CLI bundle deploy print_requests.py //unity-catalog &> LOG.deploy_requests +# The SDK drops the catalog's id, so read it with a raw GET. +add_repl "$(MSYS_NO_PATHCONV=1 $CLI api get "/api/2.1/unity-catalog/catalogs/$OLD_CATALOG" | jq -r .id)" CATALOG_ID add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID @@ -34,11 +34,14 @@ title "Rename all three in the config to match" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ trace update_file.py databricks.yml "name: myschema" "name: myschema2" trace update_file.py databricks.yml "name: myvolume" "name: myvolume2" -NEW_SCHEMA="$NEW_CATALOG.myschema2" -NEW_VOLUME="$NEW_SCHEMA.myvolume2" -title "The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again" +title "The plan finds the catalog, the schema and the volume at their new names by their ids" plan -title "The catalog's create fails because the renamed catalog has its name" -trace musterr $CLI bundle deploy +title "Deploy only records the new names in the state" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_out_of_band/script index 1c39637d66b..d94cb0278ff 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/script @@ -5,8 +5,6 @@ NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve - # The renamed catalog is not where the state looks for it, so destroy cannot remove it. - trace $CLI catalogs delete "$NEW_CATALOG" --force # Consume the destroy's requests so none are left recorded. print_requests.py //unity-catalog &> LOG.destroy_requests @@ -21,6 +19,8 @@ plan() { title "Deploy a catalog with a schema and a volume in it" trace $CLI bundle deploy print_requests.py //unity-catalog &> LOG.deploy_requests +# The SDK drops the catalog's id, so read it with a raw GET. +add_repl "$(MSYS_NO_PATHCONV=1 $CLI api get "/api/2.1/unity-catalog/catalogs/$OLD_CATALOG" | jq -r .id)" CATALOG_ID add_repl "$($CLI schemas get "$OLD_CATALOG.myschema" | jq -r .schema_id)" SCHEMA_ID add_repl "$($CLI volumes read "$OLD_CATALOG.myschema.myvolume" | jq -r .volume_id)" VOLUME_ID @@ -30,11 +30,14 @@ print_requests.py //unity-catalog &> LOG.rename_requests title "Rename the catalog in the config to match" trace update_file.py databricks.yml test_catalog_ test_catalog_renamed_ -NEW_SCHEMA="$NEW_CATALOG.myschema" -NEW_VOLUME="$NEW_SCHEMA.myvolume" -title "The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again" +title "The plan finds the catalog, the schema and the volume at their new names by their ids" plan -title "The catalog's create fails because the renamed catalog has its name" -trace musterr $CLI bundle deploy +title "Deploy only records the new names in the state" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' + +title "Redeploy is a no-op" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.json index 92d227bc232..9850835017c 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.json @@ -10,8 +10,17 @@ "catalog_type": "MANAGED_CATALOG", "comment": "Test catalog for grants", "full_name": "catalog_grants_[UNIQUE_NAME]", + "id": "[UUID]", "name": "catalog_grants_[UNIQUE_NAME]", "owner": "[USERNAME]" + }, + "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + } } }, "resources.catalogs.grants_catalog.grants": { diff --git a/bundle/direct/dresources/catalog.go b/bundle/direct/dresources/catalog.go index 53d9b3b23f1..02b808c3386 100644 --- a/bundle/direct/dresources/catalog.go +++ b/bundle/direct/dresources/catalog.go @@ -3,23 +3,52 @@ package dresources import ( "context" "fmt" + "maps" + "net/http" "slices" + "sync" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +const catalogsPath = "/api/2.1/unity-catalog/catalogs" + +// CatalogRemote is CatalogInfo plus the catalog's id, a UUID that UC returns but the API spec does +// not declare, so the SDK drops it. A rename keeps the id and a recreate changes it (verified on +// AWS, Azure and GCP). +type CatalogRemote struct { + catalog.CatalogInfo + + Id string `json:"id,omitempty"` +} + +// Custom marshaler needed because embedded CatalogInfo has its own MarshalJSON +// which would otherwise take over and ignore the additional fields. +func (s *CatalogRemote) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s CatalogRemote) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + // CatalogState is CreateCatalog plus isolation_mode, which the Create API does not -// accept; DoCreate applies it with a follow-up update. +// accept (DoCreate applies it with a follow-up update), and the catalog's id. type CatalogState struct { catalog.CreateCatalog IsolationMode catalog.CatalogIsolationMode `json:"isolation_mode,omitempty"` + + // Recorded by the framework from the remote state (internal_id_field). + Id string `json:"id,omitempty"` } -// Custom marshalers needed because embedded catalog.CreateCatalog has its own MarshalJSON +// Custom marshaler needed because embedded CreateCatalog has its own MarshalJSON // which would otherwise take over and ignore the additional fields. func (s *CatalogState) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, s) @@ -30,22 +59,50 @@ func (s CatalogState) MarshalJSON() ([]byte, error) { } type ResourceCatalog struct { - client *databricks.WorkspaceClient + client *databricks.WorkspaceClient + apiClient func() (*client.DatabricksClient, error) +} + +func (*ResourceCatalog) New(w *databricks.WorkspaceClient) *ResourceCatalog { + return &ResourceCatalog{ + client: w, + apiClient: sync.OnceValues(func() (*client.DatabricksClient, error) { + return client.New(w.Config) + }), + } } -func (*ResourceCatalog) New(client *databricks.WorkspaceClient) *ResourceCatalog { - return &ResourceCatalog{client: client} +// do calls the catalogs API like the SDK does, but decodes the response into CatalogRemote: +// the SDK's CatalogInfo has no id. +func (r *ResourceCatalog) do(ctx context.Context, method, path string, request any) (*CatalogRemote, error) { + apiClient, err := r.apiClient() + if err != nil { + return nil, err + } + headers := map[string]string{"Accept": "application/json"} + if request != nil { + headers["Content-Type"] = "application/json" + } + maps.Copy(headers, auth.WorkspaceIDHeaders(r.client.Config)) + var response CatalogRemote + err = apiClient.Do(ctx, method, path, headers, nil, request, &response) + if err != nil { + return nil, err + } + return &response, nil } func (*ResourceCatalog) PrepareState(input *resources.Catalog) *CatalogState { return &CatalogState{ CreateCatalog: input.CreateCatalog, IsolationMode: input.IsolationMode, + Id: "", // Recorded by the framework from the remote state (internal_id_field). } } -func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *CatalogState { - return &CatalogState{ +func (*ResourceCatalog) RemapState(info *CatalogRemote) *CatalogState { + state := &CatalogState{IsolationMode: info.IsolationMode, Id: info.Id} + state.CreateCatalog = catalog.CreateCatalog{ Comment: info.Comment, ConnectionName: info.ConnectionName, CustomMaxRetentionHours: info.CustomMaxRetentionHours, @@ -57,17 +114,17 @@ func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *CatalogState { ShareName: info.ShareName, StorageRoot: info.StorageRoot, ForceSendFields: utils.FilterFields[catalog.CreateCatalog](info.ForceSendFields), - IsolationMode: info.IsolationMode, } + return state } -func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*catalog.CatalogInfo, error) { - return r.client.Catalogs.GetByName(ctx, id) +func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*CatalogRemote, error) { + return r.do(ctx, http.MethodGet, catalogsPath+"/"+id, nil) } -func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (string, *catalog.CatalogInfo, error) { - response, err := r.client.Catalogs.Create(ctx, config.CreateCatalog) - if err != nil || response == nil { +func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (string, *CatalogRemote, error) { + response, err := r.do(ctx, http.MethodPost, catalogsPath, config.CreateCatalog) + if err != nil { return "", nil, err } if config.IsolationMode == "" { @@ -75,7 +132,7 @@ func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (s } // UC only accepts isolation_mode on update, so a configured value takes a second call. - response, err = r.client.Catalogs.Update(ctx, catalog.UpdateCatalog{ + response, err = r.do(ctx, http.MethodPatch, catalogsPath+"/"+response.Name, catalog.UpdateCatalog{ Comment: "", CustomMaxRetentionHours: 0, EnablePredictiveOptimization: "", @@ -96,6 +153,11 @@ func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (s return response.Name, response, nil } +// IDFromState returns the name, which is the ID. +func (*ResourceCatalog) IDFromState(state *CatalogState) string { + return state.Name +} + // See schemaForceSend. Verified against a real workspace: {"comment": ""} clears it. // // CustomMaxRetentionHours is force-sent here even though schemaForceSend deliberately omits it: @@ -105,7 +167,7 @@ func (r *ResourceCatalog) DoCreate(ctx context.Context, config *CatalogState) (s var catalogForceSend = []string{"Comment", "CustomMaxRetentionHours"} // DoUpdate updates the catalog in place and returns remote state. -func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (*catalog.CatalogInfo, error) { +func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (*CatalogRemote, error) { updateRequest := catalog.UpdateCatalog{ Comment: config.Comment, CustomMaxRetentionHours: config.CustomMaxRetentionHours, @@ -120,16 +182,11 @@ func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *Catal ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](append(slices.Clone(catalogForceSend), config.ForceSendFields...), "EnablePredictiveOptimization", "Owner"), } - response, err := r.client.Catalogs.Update(ctx, updateRequest) - if err != nil { - return nil, err - } - - return response, nil + return r.do(ctx, http.MethodPatch, catalogsPath+"/"+id, updateRequest) } // DoUpdateWithID updates the catalog and returns the new ID if the name changes. -func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (string, *catalog.CatalogInfo, error) { +func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *CatalogState, _ *PlanEntry) (string, *CatalogRemote, error) { updateRequest := catalog.UpdateCatalog{ Comment: config.Comment, CustomMaxRetentionHours: config.CustomMaxRetentionHours, @@ -148,7 +205,7 @@ func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config updateRequest.NewName = config.Name } - response, err := r.client.Catalogs.Update(ctx, updateRequest) + response, err := r.do(ctx, http.MethodPatch, catalogsPath+"/"+id, updateRequest) if err != nil { return "", nil, err } diff --git a/bundle/direct/dresources/configs/catalogs.yml b/bundle/direct/dresources/configs/catalogs.yml index c1e1893af42..346d529795e 100644 --- a/bundle/direct/dresources/configs/catalogs.yml +++ b/bundle/direct/dresources/configs/catalogs.yml @@ -24,3 +24,6 @@ backend_defaults: # value alone (OPEN by default, or ISOLATED when set outside the bundle) rather than # forcing it back to OPEN. - field: isolation_mode +# Assigned by the backend and kept across renames: lets the plan find a catalog renamed outside the +# bundle or by an interrupted deploy (see InternalIDField). +internal_id_field: id diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 53740b3bd89..7479dd0f8d7 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -6,9 +6,26 @@ import ( "net/http" "strings" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +// fakeCatalog is CatalogInfo plus the id UC returns, which its API spec does not declare. The id +// is kept across renames. +type fakeCatalog struct { + catalog.CatalogInfo + + Id string `json:"id,omitempty"` +} + +func (c *fakeCatalog) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c fakeCatalog) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + // catalogNameManagedDefaults scopes the managed-property injection to the // backend-default drift test, keeping unrelated tests free of it. Mirrors // schemaNameManagedDefaults. @@ -85,9 +102,10 @@ func (s *FakeWorkspace) CatalogsCreate(req Request) Response { } } - s.Catalogs[createRequest.Name] = catalogInfo + created := fakeCatalog{CatalogInfo: catalogInfo, Id: nextUUID()} + s.Catalogs[createRequest.Name] = created return Response{ - Body: catalogInfo, + Body: created, } } @@ -115,7 +133,7 @@ func (s *FakeWorkspace) CatalogsUpdate(req Request, name string) Response { } } - applyUpdatedFields(&existing, updateRequest, fields) + applyUpdatedFields(&existing.CatalogInfo, updateRequest, fields) if updateRequest.NewName != "" { existing.Name = updateRequest.NewName existing.FullName = updateRequest.NewName diff --git a/libs/testserver/catalogs_test.go b/libs/testserver/catalogs_test.go index de924c08060..7ea99ee71a2 100644 --- a/libs/testserver/catalogs_test.go +++ b/libs/testserver/catalogs_test.go @@ -38,7 +38,7 @@ func TestCatalogsCreate_AllowsNonEmptyName(t *testing.T) { getResponse := MapGet(workspace, workspace.Catalogs, "my_catalog") require.Equal(t, 0, getResponse.StatusCode) - body, ok := getResponse.Body.(catalog.CatalogInfo) + body, ok := getResponse.Body.(fakeCatalog) require.True(t, ok) assert.Equal(t, "my_catalog", body.Name) assert.Equal(t, "my_catalog", body.FullName) @@ -78,8 +78,9 @@ func TestCatalogsCreate_EchoesEveryAcceptedField(t *testing.T) { response := workspace.CatalogsCreate(Request{Body: []byte(createCatalogRequest)}) assert.Equal(t, 0, response.StatusCode) - info, ok := response.Body.(catalog.CatalogInfo) + body, ok := response.Body.(fakeCatalog) require.True(t, ok) + info := body.CatalogInfo assert.Equal(t, "my_catalog", info.Name) assert.Equal(t, "c", info.Comment) @@ -136,8 +137,9 @@ func TestCatalogsUpdate_AppliesUpdatableFields(t *testing.T) { }`)}, "my_catalog") assert.Equal(t, 0, response.StatusCode) - info, ok := response.Body.(catalog.CatalogInfo) + body, ok := response.Body.(fakeCatalog) require.True(t, ok) + info := body.CatalogInfo assert.Equal(t, "updated", info.Comment) assert.Equal(t, int64(72), info.CustomMaxRetentionHours) @@ -147,3 +149,16 @@ func TestCatalogsUpdate_AppliesUpdatableFields(t *testing.T) { assert.Equal(t, map[string]string{"opt": "3"}, info.Options) assert.Equal(t, map[string]string{"prop": "4"}, info.Properties) } + +func TestCatalogsUpdate_RenameKeepsID(t *testing.T) { + workspace := NewFakeWorkspace("http://test", "dbapi123") + + created, ok := workspace.CatalogsCreate(Request{Body: []byte(`{"name": "my_catalog"}`)}).Body.(fakeCatalog) + require.True(t, ok) + require.NotEmpty(t, created.Id) + + renamed, ok := workspace.CatalogsUpdate(Request{Body: []byte(`{"new_name": "my_catalog2"}`)}, "my_catalog").Body.(fakeCatalog) + require.True(t, ok) + assert.Equal(t, "my_catalog2", renamed.Name) + assert.Equal(t, created.Id, renamed.Id) +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index e4a16f275fa..33f01f8ce00 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -245,7 +245,7 @@ type FakeWorkspace struct { ClusterLibraries map[string][]compute.Library // cluster id -> installed libraries InstancePools map[string]compute.GetInstancePool ClusterPolicies map[string]compute.Policy - Catalogs map[string]catalog.CatalogInfo + Catalogs map[string]fakeCatalog ExternalLocations map[string]catalog.ExternalLocationInfo RegisteredModels map[string]catalog.RegisteredModelInfo ModelServices map[string]catalog.ModelService @@ -544,7 +544,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Monitors: map[string]catalog.MonitorInfo{}, DataQualityMonitors: map[string]dataquality.Monitor{}, Apps: map[string]apps.App{}, - Catalogs: map[string]catalog.CatalogInfo{}, + Catalogs: map[string]fakeCatalog{}, ExternalLocations: map[string]catalog.ExternalLocationInfo{}, Schemas: map[string]catalog.SchemaInfo{}, RegisteredModels: map[string]catalog.RegisteredModelInfo{}, From cf34f2a8d64ff1894f3939cb70b2bdbeeb4df6c7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 21:34:29 +0100 Subject: [PATCH 02/15] acc: drop the catalog out-of-band Badness this fixes Co-authored-by: Isaac --- .../bundle/resources/catalogs/rename_all_out_of_band/test.toml | 1 - .../bundle/resources/catalogs/rename_out_of_band/test.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml index 698c1c6ca49..4272a80cf61 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml @@ -1,4 +1,3 @@ -Badness = "The plan does not look for a catalog renamed outside the bundle, so it creates it again: the create fails with CATALOG_ALREADY_EXISTS on every deploy, and destroy leaves the renamed catalog behind. Same on main." Cloud = true Ignore = [ diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml index 698c1c6ca49..4272a80cf61 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml @@ -1,4 +1,3 @@ -Badness = "The plan does not look for a catalog renamed outside the bundle, so it creates it again: the create fails with CATALOG_ALREADY_EXISTS on every deploy, and destroy leaves the renamed catalog behind. Same on main." Cloud = true Ignore = [ From 75b107f619ff23706ed954f5c260514d29412c6d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 21:36:38 +0100 Subject: [PATCH 03/15] direct: build the catalog state in one literal Co-authored-by: Isaac --- bundle/direct/dresources/catalog.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/catalog.go b/bundle/direct/dresources/catalog.go index 02b808c3386..933c580c2ba 100644 --- a/bundle/direct/dresources/catalog.go +++ b/bundle/direct/dresources/catalog.go @@ -101,8 +101,7 @@ func (*ResourceCatalog) PrepareState(input *resources.Catalog) *CatalogState { } func (*ResourceCatalog) RemapState(info *CatalogRemote) *CatalogState { - state := &CatalogState{IsolationMode: info.IsolationMode, Id: info.Id} - state.CreateCatalog = catalog.CreateCatalog{ + return &CatalogState{ Comment: info.Comment, ConnectionName: info.ConnectionName, CustomMaxRetentionHours: info.CustomMaxRetentionHours, @@ -114,8 +113,9 @@ func (*ResourceCatalog) RemapState(info *CatalogRemote) *CatalogState { ShareName: info.ShareName, StorageRoot: info.StorageRoot, ForceSendFields: utils.FilterFields[catalog.CreateCatalog](info.ForceSendFields), + IsolationMode: info.IsolationMode, + Id: info.Id, } - return state } func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*CatalogRemote, error) { From 0cd7a2c89786b905e7e9e9085a28e14dc171b872 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 21:39:20 +0100 Subject: [PATCH 04/15] Add changelog fragment for the catalog id lookup Co-authored-by: Isaac --- .nextchanges/bundles/catalog-renamed-out-of-band.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/catalog-renamed-out-of-band.md diff --git a/.nextchanges/bundles/catalog-renamed-out-of-band.md b/.nextchanges/bundles/catalog-renamed-out-of-band.md new file mode 100644 index 00000000000..9ca1bd630c8 --- /dev/null +++ b/.nextchanges/bundles/catalog-renamed-out-of-band.md @@ -0,0 +1 @@ +* A catalog renamed outside the bundle is now found by its id, instead of being created again (which failed with `CATALOG_ALREADY_EXISTS`). ([#7418](https://github.com/databricks/cli/pull/7418)) From 0cdde57c4fa027cdd09410a3a7eb479596d83c8e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 9 Nov 2026 22:05:19 +0100 Subject: [PATCH 05/15] acc: regenerate refschema for the catalog id Co-authored-by: Isaac --- acceptance/bundle/refschema/output/catalogs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/refschema/output/catalogs.txt b/acceptance/bundle/refschema/output/catalogs.txt index 42fb0f78ead..df74c0d7910 100644 --- a/acceptance/bundle/refschema/output/catalogs.txt +++ b/acceptance/bundle/refschema/output/catalogs.txt @@ -11,7 +11,7 @@ resources.catalogs.*.effective_predictive_optimization_flag.inherited_from_type resources.catalogs.*.effective_predictive_optimization_flag.value catalog.EnablePredictiveOptimization REMOTE resources.catalogs.*.enable_predictive_optimization catalog.EnablePredictiveOptimization REMOTE resources.catalogs.*.full_name string REMOTE -resources.catalogs.*.id string INPUT +resources.catalogs.*.id string ALL resources.catalogs.*.isolation_mode catalog.CatalogIsolationMode ALL resources.catalogs.*.lifecycle resources.Lifecycle INPUT resources.catalogs.*.lifecycle.depends_on []resources.DependsOn INPUT From 3651c7e527bb47d5f744341b32ce4431065ce37f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 10 Nov 2026 17:05:18 +0100 Subject: [PATCH 06/15] acc: also deploy the catalog out-of-band tests from a saved plan (READPLAN) Co-authored-by: Isaac --- .../catalogs/rename_all_out_of_band/out.test.toml | 1 + .../resources/catalogs/rename_all_out_of_band/script | 4 ++-- .../resources/catalogs/rename_all_out_of_band/test.toml | 7 +++++++ .../resources/catalogs/rename_out_of_band/out.test.toml | 1 + .../bundle/resources/catalogs/rename_out_of_band/script | 4 ++-- .../bundle/resources/catalogs/rename_out_of_band/test.toml | 7 +++++++ 6 files changed, 20 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script index 26fed081e3c..369c17d4a80 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script @@ -13,7 +13,7 @@ trap cleanup EXIT # Drop the remote_state fields that differ between clouds or from the test server. plan() { - trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location + trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location } title "Deploy a catalog with a schema and a volume in it" @@ -39,7 +39,7 @@ title "The plan finds the catalog, the schema and the volume at their new names plan title "Deploy only records the new names in the state" -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml index 4272a80cf61..ebd892fd9df 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/test.toml @@ -1,7 +1,14 @@ Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml index 0f167effef1..7a62f726f25 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_out_of_band/script index d94cb0278ff..4032ce015fc 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/script @@ -13,7 +13,7 @@ trap cleanup EXIT # Drop the remote_state fields that differ between clouds or from the test server. plan() { - trace $CLI bundle plan --output json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location + trace $CLI bundle plan --output json | tee .databricks/plan.json | jq .plan | normalize_uc_payload.py updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type storage_location } title "Deploy a catalog with a schema and a volume in it" @@ -35,7 +35,7 @@ title "The plan finds the catalog, the schema and the volume at their new names plan title "Deploy only records the new names in the state" -trace $CLI bundle deploy +trace $CLI bundle deploy $(readplanarg .databricks/plan.json) trace print_requests.py //unity-catalog trace $CLI bundle summary --output json | jq '.resources | map_values(map_values(.id))' diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml index 4272a80cf61..ebd892fd9df 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/test.toml @@ -1,7 +1,14 @@ Cloud = true +EnvMatrix.READPLAN = ["", "1"] Ignore = [ ".databricks", ".gitignore", "databricks.yml", ] + +# When READPLAN=1, "bundle deploy" is called as "bundle deploy --plan .databricks/plan.json". +# Normalize so both variants produce identical output. +[[Repls]] +Old = ' --plan .databricks/plan.json' +New = '' From e8c809a6fbbbcbbf66ae012b16daea87257d2010 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 10:36:28 +0100 Subject: [PATCH 07/15] Make the changelog entry mirror the PR title Co-authored-by: Isaac --- .nextchanges/bundles/catalog-renamed-out-of-band.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/catalog-renamed-out-of-band.md b/.nextchanges/bundles/catalog-renamed-out-of-band.md index 9ca1bd630c8..1f582e5c5e5 100644 --- a/.nextchanges/bundles/catalog-renamed-out-of-band.md +++ b/.nextchanges/bundles/catalog-renamed-out-of-band.md @@ -1 +1 @@ -* A catalog renamed outside the bundle is now found by its id, instead of being created again (which failed with `CATALOG_ALREADY_EXISTS`). ([#7418](https://github.com/databricks/cli/pull/7418)) +* Find a catalog renamed outside the bundle by its id. ([#7418](https://github.com/databricks/cli/pull/7418)) From 669087d9030747a42264bdc7dcb983690a5650c4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 10:54:33 +0100 Subject: [PATCH 08/15] acc: regenerate the catalog goldens after rebase Co-authored-by: Isaac --- .../catalogs/comment_out_of_band/output.txt | 6 ++ .../drift/managed_properties/output.txt | 6 ++ .../catalogs/isolation_mode/output.txt | 6 ++ .../rename_all_out_of_band/output.txt | 80 ++++++++++++++----- .../catalogs/rename_out_of_band/output.txt | 80 ++++++++++++++----- .../resources/grants/catalogs/out.plan2.json | 8 +- 6 files changed, 142 insertions(+), 44 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt index 5da6c37fe2a..64b652fe1d7 100644 --- a/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/comment_out_of_band/output.txt @@ -17,6 +17,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "action": "update", "remote": "set outside the bundle" }, + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "isolation_mode": { "action": "skip", "reason": "backend_default", diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt b/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt index 886246cec10..cfbb45cb47d 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/output.txt @@ -16,6 +16,12 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan --output json { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "isolation_mode": { "action": "skip", "reason": "backend_default", diff --git a/acceptance/bundle/resources/catalogs/isolation_mode/output.txt b/acceptance/bundle/resources/catalogs/isolation_mode/output.txt index c37befe061b..4f03b1b9c15 100644 --- a/acceptance/bundle/resources/catalogs/isolation_mode/output.txt +++ b/acceptance/bundle/resources/catalogs/isolation_mode/output.txt @@ -80,6 +80,12 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "reason": "empty", "remote": 0 }, + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "isolation_mode": { "action": "skip", "reason": "backend_default", diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt index af5dcff5840..9e9f3e62aa0 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/output.txt @@ -34,23 +34,41 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml name: myvolume name: myvolume2 -=== The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again +=== The plan finds the catalog, the schema and the volume at their new names by their ids >>> [CLI] bundle plan --output json { "resources.catalogs.catalog1": { - "action": "create", + "id": "test_catalog_renamed_[UNIQUE_NAME]", + "action": "update_id", + "state_only": true, "new_state": { "value": { "name": "test_catalog_renamed_[UNIQUE_NAME]" } }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME]", + "id": "[CATALOG_ID]", + "name": "test_catalog_renamed_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[CATALOG_ID]", + "remote": "[CATALOG_ID]" + }, "name": { - "action": "update_id", - "reason": "id_changes", + "action": "skip", + "reason": "remote_already_set", "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]" + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" } } }, @@ -73,7 +91,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "remote_state": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], + "created_at": [UNIX_TIME_MILLIS][1], "created_by": "[USERNAME]", "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2", "name": "myschema2", @@ -127,7 +145,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged }, "remote_state": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", - "created_at": [UNIX_TIME_MILLIS][1], + "created_at": [UNIX_TIME_MILLIS][2], "created_by": "[USERNAME]", "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume2", "name": "myvolume2", @@ -168,26 +186,48 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } } -=== The catalog's create fails because the renamed catalog has its name +=== Deploy only records the new names in the state ->>> musterr [CLI] bundle deploy +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... -Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS) +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 3 unchanged -Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/catalogs -HTTP Status: 400 Bad Request -API error_code: CATALOG_ALREADY_EXISTS -API message: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists +>>> print_requests.py //unity-catalog -Error: cannot update_id resources.schemas.schema1: dependency failed: resources.catalogs.catalog1 +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema2" + }, + "volumes": { + "volume1": "test_catalog_renamed_[UNIQUE_NAME].myschema2.myvolume2" + } +} -Error: cannot update_id resources.volumes.volume1: dependency failed: resources.catalogs.catalog1 +=== Redeploy is a no-op -Files: 3 uploaded, 0 deleted +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged >>> [CLI] bundle destroy --auto-approve -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 -Destroy: 0 deleted +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default ->>> [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt index cf94276e0cd..75997e56b47 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/output.txt @@ -20,23 +20,41 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml test_catalog_ test_catalog_renamed_ -=== The plan finds the schema and the volume at their new names by their ids. A catalog has no such id, so it is created again +=== The plan finds the catalog, the schema and the volume at their new names by their ids >>> [CLI] bundle plan --output json { "resources.catalogs.catalog1": { - "action": "create", + "id": "test_catalog_renamed_[UNIQUE_NAME]", + "action": "update_id", + "state_only": true, "new_state": { "value": { "name": "test_catalog_renamed_[UNIQUE_NAME]" } }, + "remote_state": { + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "full_name": "test_catalog_renamed_[UNIQUE_NAME]", + "id": "[CATALOG_ID]", + "name": "test_catalog_renamed_[UNIQUE_NAME]", + "owner": "[USERNAME]" + }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[CATALOG_ID]", + "remote": "[CATALOG_ID]" + }, "name": { - "action": "update_id", - "reason": "id_changes", + "action": "skip", + "reason": "remote_already_set", "old": "test_catalog_[UNIQUE_NAME]", - "new": "test_catalog_renamed_[UNIQUE_NAME]" + "new": "test_catalog_renamed_[UNIQUE_NAME]", + "remote": "test_catalog_renamed_[UNIQUE_NAME]" } } }, @@ -59,7 +77,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "remote_state": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], + "created_at": [UNIX_TIME_MILLIS][1], "created_by": "[USERNAME]", "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema", "name": "myschema", @@ -106,7 +124,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged }, "remote_state": { "catalog_name": "test_catalog_renamed_[UNIQUE_NAME]", - "created_at": [UNIX_TIME_MILLIS][1], + "created_at": [UNIX_TIME_MILLIS][2], "created_by": "[USERNAME]", "full_name": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume", "name": "myvolume", @@ -133,26 +151,48 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged } } -=== The catalog's create fails because the renamed catalog has its name +=== Deploy only records the new names in the state ->>> musterr [CLI] bundle deploy +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... -Error: cannot create resources.catalogs.catalog1: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists (400 CATALOG_ALREADY_EXISTS) +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 3 unchanged -Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/catalogs -HTTP Status: 400 Bad Request -API error_code: CATALOG_ALREADY_EXISTS -API message: Catalog 'test_catalog_renamed_[UNIQUE_NAME]' already exists +>>> print_requests.py //unity-catalog -Error: cannot update_id resources.schemas.schema1: dependency failed: resources.catalogs.catalog1 +>>> [CLI] bundle summary --output json +{ + "catalogs": { + "catalog1": "test_catalog_renamed_[UNIQUE_NAME]" + }, + "schemas": { + "schema1": "test_catalog_renamed_[UNIQUE_NAME].myschema" + }, + "volumes": { + "volume1": "test_catalog_renamed_[UNIQUE_NAME].myschema.myvolume" + } +} -Error: cannot update_id resources.volumes.volume1: dependency failed: resources.catalogs.catalog1 +=== Redeploy is a no-op -Files: 3 uploaded, 0 deleted +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged >>> [CLI] bundle destroy --auto-approve -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default +The following resources will be deleted: + delete resources.catalogs.catalog1 + delete resources.schemas.schema1 + delete resources.volumes.volume1 -Destroy: 0 deleted +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.schema1 + +This action will result in the deletion of the following volumes. +For managed volumes, the files stored in the volume are also deleted from your +cloud tenant within 30 days. For external volumes, the metadata about the volume +is removed from the catalog, but the underlying files are not deleted: + delete resources.volumes.volume1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default ->>> [CLI] catalogs delete test_catalog_renamed_[UNIQUE_NAME] --force +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.json index 9850835017c..49cd77a9705 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.json @@ -1,7 +1,7 @@ { "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", + "lineage": "[UUID][0]", "serial": 1, "plan": { "resources.catalogs.grants_catalog": { @@ -10,7 +10,7 @@ "catalog_type": "MANAGED_CATALOG", "comment": "Test catalog for grants", "full_name": "catalog_grants_[UNIQUE_NAME]", - "id": "[UUID]", + "id": "[UUID][1]", "name": "catalog_grants_[UNIQUE_NAME]", "owner": "[USERNAME]" }, @@ -18,8 +18,8 @@ "id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, From 30393f225ebf6eefc6ff02eaf6ebd8a4241fd076 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 10:59:10 +0100 Subject: [PATCH 09/15] testserver: the isolation_mode test reads the created catalog as fakeCatalog Co-authored-by: Isaac --- libs/testserver/catalogs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/testserver/catalogs_test.go b/libs/testserver/catalogs_test.go index 7ea99ee71a2..ebd071dc9f0 100644 --- a/libs/testserver/catalogs_test.go +++ b/libs/testserver/catalogs_test.go @@ -115,7 +115,7 @@ func TestCatalogsCreate_DefaultsIsolationModeToOpen(t *testing.T) { response := workspace.CatalogsCreate(Request{Body: []byte(createCatalogRequest)}) require.Equal(t, 0, response.StatusCode) - info, ok := response.Body.(catalog.CatalogInfo) + info, ok := response.Body.(fakeCatalog) require.True(t, ok) assert.Equal(t, catalog.CatalogIsolationModeOpen, info.IsolationMode) } From f321322f5a0d1f0ec4d0f168369505b5f8a0f94c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 11:11:41 +0100 Subject: [PATCH 10/15] acc: show the catalog id in the model, secret and volume rename tests Co-authored-by: Isaac --- .../secrets/parent_rename/output.txt | 28 +++++++++++++++---- .../recreate_in_renamed_schema/output.txt | 15 ++++++++-- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/acceptance/bundle/resources/secrets/parent_rename/output.txt b/acceptance/bundle/resources/secrets/parent_rename/output.txt index 6cac5f13385..3d886f935f3 100644 --- a/acceptance/bundle/resources/secrets/parent_rename/output.txt +++ b/acceptance/bundle/resources/secrets/parent_rename/output.txt @@ -49,8 +49,17 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID][0]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" + }, + "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][0]", + "remote": "[UUID][0]" + } } }, "resources.schemas.schema1": { @@ -74,7 +83,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema", "name": "myschema", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[UUID][1]" }, "changes": { "name": { @@ -87,8 +96,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, @@ -225,10 +234,17 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID][0]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][0]", + "remote": "[UUID][0]" + }, "name": { "action": "update_id", "reason": "id_changes", @@ -262,7 +278,7 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema2", "name": "myschema2", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[UUID][1]" }, "changes": { "catalog_name": { @@ -285,8 +301,8 @@ Resources: 0 created, 1 changed, 0 deleted, 2 unchanged "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, diff --git a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt index c292b659090..1968278597b 100644 --- a/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt +++ b/acceptance/bundle/resources/secrets/recreate_in_renamed_schema/output.txt @@ -23,8 +23,17 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID][0]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" + }, + "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][0]", + "remote": "[UUID][0]" + } } }, "resources.schemas.schema1": { @@ -48,7 +57,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema", "name": "myschema", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[UUID][1]" }, "changes": { "name": { @@ -61,8 +70,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } }, From a3707cd5f08bf8cb29af577a489e413398ea5667 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 14:27:54 +0100 Subject: [PATCH 11/15] acc: show the catalog id in the catalog rename tests that now have a model Co-authored-by: Isaac --- acceptance/bundle/resources/catalogs/rename/output.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/acceptance/bundle/resources/catalogs/rename/output.txt b/acceptance/bundle/resources/catalogs/rename/output.txt index 5bb6c4b2655..12894db3bbb 100644 --- a/acceptance/bundle/resources/catalogs/rename/output.txt +++ b/acceptance/bundle/resources/catalogs/rename/output.txt @@ -65,10 +65,17 @@ Resources: 4 created, 0 changed, 0 deleted, 0 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "name": { "action": "update_id", "reason": "id_changes", From 3a64940d0a7a4c3f09a493ee0bcc44165b142ce8 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 16 Nov 2026 16:43:16 +0100 Subject: [PATCH 12/15] acc: show the catalog id in the catalog and schema rename tests after rebase Co-authored-by: Isaac --- .../rename_catalog_and_schema/output.txt | 13 ++++++++++--- .../volumes/rename_catalog_and_schema/output.txt | 7 +++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt index 895e3dce8a8..5e38a51622e 100644 --- a/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt +++ b/acceptance/bundle/resources/registered_models/rename_catalog_and_schema/output.txt @@ -30,10 +30,17 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "catalog_type": "MANAGED_CATALOG", "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID][0]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID][0]", + "remote": "[UUID][0]" + }, "name": { "action": "update_id", "reason": "id_changes", @@ -124,7 +131,7 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "full_name": "test_catalog_[UNIQUE_NAME].myschema", "name": "myschema", "owner": "[USERNAME]", - "schema_id": "[UUID]" + "schema_id": "[UUID][1]" }, "changes": { "catalog_name": { @@ -144,8 +151,8 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "schema_id": { "action": "skip", "reason": "internal_id", - "old": "[UUID]", - "remote": "[UUID]" + "old": "[UUID][1]", + "remote": "[UUID][1]" } } } diff --git a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt index f5d7df1a699..621819f3180 100644 --- a/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt +++ b/acceptance/bundle/resources/volumes/rename_catalog_and_schema/output.txt @@ -31,10 +31,17 @@ Resources: 3 created, 0 changed, 0 deleted, 0 unchanged "created_at": [UNIX_TIME_MILLIS][0], "created_by": "[USERNAME]", "full_name": "test_catalog_[UNIQUE_NAME]", + "id": "[UUID]", "name": "test_catalog_[UNIQUE_NAME]", "owner": "[USERNAME]" }, "changes": { + "id": { + "action": "skip", + "reason": "internal_id", + "old": "[UUID]", + "remote": "[UUID]" + }, "name": { "action": "update_id", "reason": "id_changes", From 454b4ab4e98ece21fad4ed323c194551a2126d6a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 17 Nov 2026 14:15:10 +0100 Subject: [PATCH 13/15] acc: restore the cleanup that deletes a leaked renamed catalog (Fable review) If a cloud run fails before the plan finds the renamed catalog, destroy looks at the old name and the catalog, with its schema and volume, would leak into the shared workspace. Co-authored-by: Isaac --- .../bundle/resources/catalogs/rename_all_out_of_band/script | 3 +++ acceptance/bundle/resources/catalogs/rename_out_of_band/script | 3 +++ 2 files changed, 6 insertions(+) diff --git a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script index 369c17d4a80..a0ce8b2754c 100644 --- a/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_all_out_of_band/script @@ -5,6 +5,9 @@ NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve + # A run that fails before the plan finds the renamed catalog leaves it where destroy does not + # look. Normally destroy has deleted it already, so this fails quietly. + errcode $CLI catalogs delete "$NEW_CATALOG" --force &> LOG.cleanup_catalog # Consume the destroy's requests so none are left recorded. print_requests.py //unity-catalog &> LOG.destroy_requests diff --git a/acceptance/bundle/resources/catalogs/rename_out_of_band/script b/acceptance/bundle/resources/catalogs/rename_out_of_band/script index 4032ce015fc..ad34b43a367 100644 --- a/acceptance/bundle/resources/catalogs/rename_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/rename_out_of_band/script @@ -5,6 +5,9 @@ NEW_CATALOG="test_catalog_renamed_${UNIQUE_NAME}" cleanup() { trace $CLI bundle destroy --auto-approve + # A run that fails before the plan finds the renamed catalog leaves it where destroy does not + # look. Normally destroy has deleted it already, so this fails quietly. + errcode $CLI catalogs delete "$NEW_CATALOG" --force &> LOG.cleanup_catalog # Consume the destroy's requests so none are left recorded. print_requests.py //unity-catalog &> LOG.destroy_requests From 2b6ac69be1ce34ecc34661a594ad72cac3e0ac64 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 20 Nov 2026 13:48:10 +0100 Subject: [PATCH 14/15] testserver: give a database catalog's UC catalog an id too Main now registers database catalogs in the UC catalog map, which holds fakeCatalog on this branch. Co-authored-by: Isaac --- libs/testserver/database_catalogs.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/libs/testserver/database_catalogs.go b/libs/testserver/database_catalogs.go index 474b4c7caa0..7706818a938 100644 --- a/libs/testserver/database_catalogs.go +++ b/libs/testserver/database_catalogs.go @@ -55,16 +55,19 @@ func (s *FakeWorkspace) DatabaseCatalogCreate(req Request) Response { // The catalog is registered in Unity Catalog, which is the only place its create time shows. now := nowMilli() - s.Catalogs[databaseCatalog.Name] = catalog.CatalogInfo{ - Name: databaseCatalog.Name, - FullName: databaseCatalog.Name, - CreatedAt: now, - UpdatedAt: now, - CreatedBy: s.CurrentUser().UserName, - UpdatedBy: s.CurrentUser().UserName, - Owner: s.CurrentUser().UserName, - MetastoreId: nextUUID(), - CatalogType: catalog.CatalogTypeManagedOnlineCatalog, + s.Catalogs[databaseCatalog.Name] = fakeCatalog{ + CatalogInfo: catalog.CatalogInfo{ + Name: databaseCatalog.Name, + FullName: databaseCatalog.Name, + CreatedAt: now, + UpdatedAt: now, + CreatedBy: s.CurrentUser().UserName, + UpdatedBy: s.CurrentUser().UserName, + Owner: s.CurrentUser().UserName, + MetastoreId: nextUUID(), + CatalogType: catalog.CatalogTypeManagedOnlineCatalog, + }, + Id: nextUUID(), } return Response{ From 49f29b4b690f8ed00486ddcf6139419f043e3f52 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 20 Nov 2026 14:06:26 +0100 Subject: [PATCH 15/15] testserver: drop the embedded type name from the database catalog literal (lint) Co-authored-by: Isaac --- libs/testserver/database_catalogs.go | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/libs/testserver/database_catalogs.go b/libs/testserver/database_catalogs.go index 7706818a938..6d1963c7a5c 100644 --- a/libs/testserver/database_catalogs.go +++ b/libs/testserver/database_catalogs.go @@ -56,18 +56,16 @@ func (s *FakeWorkspace) DatabaseCatalogCreate(req Request) Response { // The catalog is registered in Unity Catalog, which is the only place its create time shows. now := nowMilli() s.Catalogs[databaseCatalog.Name] = fakeCatalog{ - CatalogInfo: catalog.CatalogInfo{ - Name: databaseCatalog.Name, - FullName: databaseCatalog.Name, - CreatedAt: now, - UpdatedAt: now, - CreatedBy: s.CurrentUser().UserName, - UpdatedBy: s.CurrentUser().UserName, - Owner: s.CurrentUser().UserName, - MetastoreId: nextUUID(), - CatalogType: catalog.CatalogTypeManagedOnlineCatalog, - }, - Id: nextUUID(), + Name: databaseCatalog.Name, + FullName: databaseCatalog.Name, + CreatedAt: now, + UpdatedAt: now, + CreatedBy: s.CurrentUser().UserName, + UpdatedBy: s.CurrentUser().UserName, + Owner: s.CurrentUser().UserName, + MetastoreId: nextUUID(), + CatalogType: catalog.CatalogTypeManagedOnlineCatalog, + Id: nextUUID(), } return Response{