diff --git a/acceptance/bundle/config-remote-sync/job_fields/script b/acceptance/bundle/config-remote-sync/job_fields/script index 27b7c8a3abc..e854123cfb9 100755 --- a/acceptance/bundle/config-remote-sync/job_fields/script +++ b/acceptance/bundle/config-remote-sync/job_fields/script @@ -40,21 +40,6 @@ if "tags" not in r: r["tags"]["team"] = "data" EOF -# TODO add support for permissions -# title "Modify job permissions remotely" -# current_user=$($CLI current-user me --output json | jq -r .userName) -# cat > permissions.json < permissions.json <>> [CLI] bundle config-remote-sync --select-ids jobs:[MY_JOB_ID] -o json +{ + "status": "success", + "skippedChanges": null +} + +>>> [CLI] bundle config-remote-sync -o json +{ + "status": "partial", + "changes": [ + "resources.jobs.my_job" + ], + "skippedChanges": [ + { + "resourceKey": "resources.pipelines.my_pipeline", + "reason": "permissions_permission_denied", + "error": "Fault injected by test." + } + ] +} + +=== Detect and save changes +Detected changes in 2 resource(s): + +Resource: resources.jobs.my_job + permissions: add + +Resource: resources.pipelines.my_pipeline + permissions: add + + + +=== Configuration changes + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -14,4 +14,9 @@ + num_workers: 1 + ++ permissions: ++ - level: CAN_MANAGE ++ user_name: viewer@example.com ++ - level: CAN_MANAGE_RUN ++ user_name: admin@example.com + pipelines: + my_pipeline: +@@ -20,4 +25,9 @@ + - notebook: + path: /Users/{{workspace_user_name}}/notebook ++ permissions: ++ - level: IS_OWNER ++ user_name: owner@example.com ++ - group_name: users ++ level: CAN_VIEW + + targets: + +=== Sync again: the permissions are now in the config +No changes detected. + + +=== Deploy keeps the remote owner +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created jobs.my_job.permissions +Created pipelines.my_pipeline.permissions +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 2 unchanged + +>>> [CLI] permissions get pipelines [MY_PIPELINE_ID] +[{"user_name":"owner@example.com","group_name":null,"level":"IS_OWNER"},{"user_name":null,"group_name":"users","level":"CAN_VIEW"},{"user_name":"[USERNAME]","group_name":null,"level":"CAN_MANAGE"}] + +=== Change a level, add and remove principals remotely + +=== Detect and save changes +Detected changes in 1 resource(s): + +Resource: resources.jobs.my_job + permissions[group_name='data-team']: add + permissions[user_name='admin@example.com']: remove + permissions[user_name='viewer@example.com'].level: replace + + + +=== Configuration changes + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -15,8 +15,8 @@ + + permissions: +- - level: CAN_MANAGE ++ - level: CAN_VIEW + user_name: viewer@example.com +- - level: CAN_MANAGE_RUN +- user_name: admin@example.com ++ - group_name: data-team ++ level: CAN_MANAGE_RUN + pipelines: + my_pipeline: + +=== Deploy and sync again +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 4 unchanged +No changes detected. + + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.my_job + delete resources.pipelines.my_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.my_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/config-remote-sync/unmanaged_permissions/script b/acceptance/bundle/config-remote-sync/unmanaged_permissions/script new file mode 100644 index 00000000000..908089fe90b --- /dev/null +++ b/acceptance/bundle/config-remote-sync/unmanaged_permissions/script @@ -0,0 +1,65 @@ +#!/bin/bash + +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +$CLI bundle deploy +job_id="$(read_id.py my_job)" +pipeline_id="$(read_id.py my_pipeline)" + +title "Add permissions remotely to resources that declare none" +# The current user's owner entry is managed by deploy, so it must not be written to the config. +$CLI permissions set jobs $job_id --json '{"access_control_list":[ + {"permission_level":"IS_OWNER","user_name":"'"$CURRENT_USER_NAME"'"}, + {"permission_level":"CAN_MANAGE","user_name":"viewer@example.com"}, + {"permission_level":"CAN_MANAGE_RUN","user_name":"admin@example.com"} +]}' > /dev/null +# An owner transferred to another user is kept. +$CLI permissions set pipelines $pipeline_id --json '{"access_control_list":[ + {"permission_level":"IS_OWNER","user_name":"owner@example.com"}, + {"permission_level":"CAN_MANAGE","user_name":"'"$CURRENT_USER_NAME"'"}, + {"permission_level":"CAN_VIEW","group_name":"users"} +]}' > /dev/null + +title "An ACL that cannot be read is reported as skipped" +fault.py "GET /api/2.0/permissions/pipelines/$pipeline_id" 403 0 2 PERMISSION_DENIED +trace $CLI bundle config-remote-sync --select-ids "jobs:$job_id" -o json | jq '{status, skippedChanges}' +trace $CLI bundle config-remote-sync -o json | jq '{status, changes: (.changes | keys), skippedChanges}' + +title "Detect and save changes" +cp databricks.yml databricks.yml.backup +$CLI bundle config-remote-sync --save + +title "Configuration changes" +trace diff.py databricks.yml.backup databricks.yml +rm databricks.yml.backup + +title "Sync again: the permissions are now in the config" +$CLI bundle config-remote-sync + +title "Deploy keeps the remote owner" +$CLI bundle deploy +trace $CLI permissions get pipelines $pipeline_id | jq -c '[.access_control_list[] | {user_name, group_name, level: .all_permissions[0].permission_level}]' + +title "Change a level, add and remove principals remotely" +$CLI permissions set jobs $job_id --json '{"access_control_list":[ + {"permission_level":"IS_OWNER","user_name":"'"$CURRENT_USER_NAME"'"}, + {"permission_level":"CAN_VIEW","user_name":"viewer@example.com"}, + {"permission_level":"CAN_MANAGE_RUN","group_name":"data-team"} +]}' > /dev/null + +title "Detect and save changes" +cp databricks.yml databricks.yml.backup +$CLI bundle config-remote-sync --save + +title "Configuration changes" +trace diff.py databricks.yml.backup databricks.yml +rm databricks.yml.backup + +title "Deploy and sync again" +$CLI bundle deploy +$CLI bundle config-remote-sync diff --git a/acceptance/bundle/config-remote-sync/unmanaged_permissions/test.toml b/acceptance/bundle/config-remote-sync/unmanaged_permissions/test.toml new file mode 100644 index 00000000000..a96be3a4c44 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/unmanaged_permissions/test.toml @@ -0,0 +1,4 @@ +RecordRequests = false +Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"] + +Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true" diff --git a/bundle/config/mutator/resourcemutator/fix_permissions.go b/bundle/config/mutator/resourcemutator/fix_permissions.go index 9e6c785b0d9..453cf905432 100644 --- a/bundle/config/mutator/resourcemutator/fix_permissions.go +++ b/bundle/config/mutator/resourcemutator/fix_permissions.go @@ -2,6 +2,7 @@ package resourcemutator import ( "context" + "reflect" "strings" "github.com/databricks/cli/bundle" @@ -155,7 +156,7 @@ func useMaximumLevel(permissions dyn.Value, resourceType string) (dyn.Value, err } levelPerPrincipal := make(map[string]string) - principalIndex := make(map[string]int) + maxPermission := make(map[string]dyn.Value) var principals []string for _, permission := range permissionArray { @@ -168,19 +169,26 @@ func useMaximumLevel(permissions dyn.Value, resourceType string) (dyn.Value, err if principal == "" { continue } - _, ok = principalIndex[principal] + _, ok = maxPermission[principal] if !ok { - ind := len(principalIndex) - principalIndex[principal] = ind principals = append(principals, principal) } - levelPerPrincipal[principal] = resources.GetMaxLevel(levelPerPrincipal[principal], level) + maxLevel := resources.GetMaxLevel(levelPerPrincipal[principal], level) + if maxLevel != levelPerPrincipal[principal] { + maxPermission[principal] = permission + } + levelPerPrincipal[principal] = maxLevel } var newPermissions []dyn.Value for _, principal := range principals { - newPermissions = append(newPermissions, createPermissionFromPrincipal(principal, levelPerPrincipal[principal])) + permission := createPermissionFromPrincipal(principal, levelPerPrincipal[principal]) + // Keep the entry that sets the level as is: its source location lets config-remote-sync write it back. + if original := maxPermission[principal]; reflect.DeepEqual(original.AsAny(), permission.AsAny()) { + permission = original + } + newPermissions = append(newPermissions, permission) } return dyn.V(newPermissions), nil diff --git a/bundle/config/mutator/resourcemutator/fix_permissions_test.go b/bundle/config/mutator/resourcemutator/fix_permissions_test.go index f5488777ec4..10cdb8a4d83 100644 --- a/bundle/config/mutator/resourcemutator/fix_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/fix_permissions_test.go @@ -4,7 +4,10 @@ import ( "testing" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestGetLevelScore(t *testing.T) { @@ -28,3 +31,20 @@ func TestGetMaxLevel(t *testing.T) { assert.Equal(t, "UNKNOWN_B", resources.GetMaxLevel("UNKNOWN_A", "UNKNOWN_B")) assert.Equal(t, "UNKNOWN_B", resources.GetMaxLevel("UNKNOWN_B", "UNKNOWN_A")) } + +func TestUseMaximumLevelKeepsDeclaredEntries(t *testing.T) { + viewerLoc := []diag.Location{{File: "databricks.yml", Line: 10}} + managerLoc := []diag.Location{{File: "databricks.yml", Line: 14}} + viewer := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_VIEW"), "user_name": dyn.V("viewer@example.com")}, viewerLoc) + lower := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_VIEW"), "group_name": dyn.V("users")}, viewerLoc) + higher := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_MANAGE"), "group_name": dyn.V("users")}, managerLoc) + + result, err := useMaximumLevel(dyn.V([]dyn.Value{viewer, lower, higher}), "jobs") + require.NoError(t, err) + + permissions := result.MustSequence() + require.Len(t, permissions, 2) + assert.Equal(t, viewerLoc, permissions[0].Locations()) + assert.Equal(t, managerLoc, permissions[1].Locations()) + assert.Equal(t, "CAN_MANAGE", permissions[1].Get("level").MustString()) +} diff --git a/bundle/configsync/cluster_policy_test.go b/bundle/configsync/cluster_policy_test.go index ee864d0e9a0..2f495ca5734 100644 --- a/bundle/configsync/cluster_policy_test.go +++ b/bundle/configsync/cluster_policy_test.go @@ -699,7 +699,7 @@ func policyTestBundle(t *testing.T, resourceKey string, config map[string]any) * func filterPolicyTestChanges(t *testing.T, b *bundle.Bundle, plan *deployplan.Plan) (configsync.Changes, []configsync.SkippedChange, error) { t.Helper() - changes, err := configsync.ExtractChanges(t.Context(), b, plan) + changes, _, err := configsync.ExtractChanges(t.Context(), b, plan) require.NoError(t, err) inputs := []any{plan, changes} for _, entry := range plan.Plan { diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index 4e59f178e13..ddc6cd1bfc9 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -138,17 +138,17 @@ func isPermissionsOrGrantsSubResource(resourceKey string) bool { } // ExtractChanges extracts the map of remote-vs-config changes from a deploy -// plan. -func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (Changes, error) { +// plan, and the resources whose remote permissions could not be read. +func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (Changes, []SkippedChange, error) { changes := make(Changes) for resourceKey, entry := range plan.Plan { // permissions and grants are emitted as their own plan keys // ("resources...permissions" / ".grants"; see splitResourcePath - // in bundle/direct/bundle_plan.go). config-remote-sync cannot write them back - // to YAML: their fields (e.g. object_id) are server-populated with no source - // location, and bundle-level permissions have no per-resource YAML node at all, - // so resolving them would fail the whole sync. Skip them instead. + // in bundle/direct/bundle_plan.go). Their fields (e.g. object_id) are + // server-populated with no source location, so they cannot be written back + // as is; permissions are added below under their parent resource instead, + // and grants are skipped. if isPermissionsOrGrantsSubResource(resourceKey) { continue } @@ -157,34 +157,13 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan if entry.Changes != nil { for path, changeDesc := range entry.Changes { - fullPath := resourceKey + "." + path - if changeDesc.Action == deployplan.Skip && changeDesc.Reason != deployplan.ReasonRemoteAddition && - !keepSkippedRunAsChange(fullPath, changeDesc) { - continue - } - synced, err := applyLifecycle(resourceKey, path, changeDesc) - if err != nil { - return nil, fmt.Errorf("failed to apply lifecycle rules for path %s: %w", path, err) - } - if synced == nil { - continue - } - - change, err := convertChangeDesc(fullPath, synced) + change, err := convertPlanChange(resourceKey, path, changeDesc, b.Config.Presets.NamePrefix) if err != nil { - return nil, fmt.Errorf("failed to compute config change for path %s: %w", path, err) + return nil, nil, fmt.Errorf("failed to compute config change for path %s: %w", path, err) } - if change.Operation == OperationSkip { - continue + if change != nil { + resourceChanges[path] = change } - // The state snapshot holds real per-field values, so New != Old - // means the local config diverged from the last deploy and this - // change overwrites that pending edit. - if !structdiff.IsEqual(changeDesc.Old, changeDesc.New) { - change.LocalEdit = true - } - change.Value = stripNamePrefix(fullPath, change.Value, b.Config.Presets.NamePrefix) - resourceChanges[path] = change } } @@ -195,5 +174,43 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan log.Debugf(ctx, "Resource %s has %d changes", resourceKey, len(resourceChanges)) } - return changes, nil + skipped, err := addPermissionsChanges(ctx, changes, b, plan) + if err != nil { + return nil, nil, err + } + + return changes, skipped, nil +} + +// convertPlanChange converts a plan change at path in resourceKey, returning nil when there is +// nothing to write back. +func convertPlanChange(resourceKey, path string, changeDesc *deployplan.ChangeDesc, namePrefix string) (*ConfigChangeDesc, error) { + fullPath := resourceKey + "." + path + if changeDesc.Action == deployplan.Skip && changeDesc.Reason != deployplan.ReasonRemoteAddition && + !keepSkippedRunAsChange(fullPath, changeDesc) { + return nil, nil + } + synced, err := applyLifecycle(resourceKey, path, changeDesc) + if err != nil { + return nil, fmt.Errorf("failed to apply lifecycle rules: %w", err) + } + if synced == nil { + return nil, nil + } + + change, err := convertChangeDesc(fullPath, synced) + if err != nil { + return nil, err + } + if change.Operation == OperationSkip { + return nil, nil + } + // The state snapshot holds real per-field values, so New != Old + // means the local config diverged from the last deploy and this + // change overwrites that pending edit. + if !structdiff.IsEqual(changeDesc.Old, changeDesc.New) { + change.LocalEdit = true + } + change.Value = stripNamePrefix(fullPath, change.Value, namePrefix) + return change, nil } diff --git a/bundle/configsync/diff_test.go b/bundle/configsync/diff_test.go index 817c638b3ff..780bc52ab8b 100644 --- a/bundle/configsync/diff_test.go +++ b/bundle/configsync/diff_test.go @@ -1,10 +1,19 @@ package configsync import ( + "errors" + "maps" + "slices" + "strings" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/yamlloader" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/serving" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -250,10 +259,11 @@ func TestMatchPattern(t *testing.T) { } } -// ExtractChanges must skip permissions/grants sub-resources: they are emitted as -// their own plan keys but cannot be written back to YAML, and resolving them -// (e.g. the server-populated object_id field, or a bundle-level permissions entry -// with no per-resource YAML node) otherwise fails the whole sync. +// ExtractChanges must not return permissions/grants sub-resources under their own +// plan keys: their fields (e.g. the server-populated object_id) have no source +// location, so resolving them would fail the whole sync. Permissions are written +// under their parent resource instead (see TestExtractChangesDeclaredPermissions), +// and grants are skipped. func TestExtractChangesSkipsPermissionsAndGrants(t *testing.T) { ctx := t.Context() @@ -283,7 +293,7 @@ func TestExtractChangesSkipsPermissionsAndGrants(t *testing.T) { }, }} - changes, err := ExtractChanges(ctx, &bundle.Bundle{}, plan) + changes, _, err := ExtractChanges(ctx, &bundle.Bundle{}, plan) require.NoError(t, err) _, hasPerms := changes["resources.jobs.my_job.permissions"] @@ -300,6 +310,167 @@ func TestExtractChangesSkipsPermissionsAndGrants(t *testing.T) { assert.Contains(t, namedPerms, "description") } +func TestExtractChangesDeclaredPermissions(t *testing.T) { + value, err := yamlloader.LoadYAML("databricks.yml", strings.NewReader(` +resources: + jobs: + my_job: + permissions: + - level: CAN_VIEW + user_name: viewer@example.com + - level: CAN_VIEW + user_name: gone@example.com +`)) + require.NoError(t, err) + // FixPermissions adds the current user's entry, with no location. + value, err = dyn.Set(value, "resources.jobs.my_job.permissions", dyn.V(append( + value.Get("resources").Get("jobs").Get("my_job").Get("permissions").MustSequence(), + dyn.V(map[string]dyn.Value{"level": dyn.V("IS_OWNER"), "user_name": dyn.V("me@example.com")}), + ))) + require.NoError(t, err) + b := &bundle.Bundle{} + require.NoError(t, b.Config.Mutate(func(dyn.Value) (dyn.Value, error) { return value, nil })) + + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.my_job": {}, + "resources.jobs.my_job.permissions": { + Action: deployplan.Update, + RemoteState: &dresources.PermissionsState{ObjectID: "/jobs/123"}, + Changes: deployplan.Changes{ + "[user_name='viewer@example.com'].level": { + Action: deployplan.Update, Old: "CAN_VIEW", New: "CAN_VIEW", Remote: "CAN_MANAGE", + }, + "[group_name='data-team']": { + Action: deployplan.Update, + Remote: dresources.StatePermission{Level: iam.PermissionLevelCanView, GroupName: "data-team"}, + }, + "[user_name='gone@example.com']": { + Action: deployplan.Update, + Old: dresources.StatePermission{Level: iam.PermissionLevelCanView, UserName: "gone@example.com"}, + New: dresources.StatePermission{Level: iam.PermissionLevelCanView, UserName: "gone@example.com"}, + }, + "[user_name='me@example.com'].level": { + Action: deployplan.Update, Old: "IS_OWNER", New: "IS_OWNER", Remote: "CAN_MANAGE", + }, + "object_id": {Action: deployplan.Update, Remote: "/jobs/456"}, + }, + }, + }} + + changes, _, err := ExtractChanges(t.Context(), b, plan) + require.NoError(t, err) + assert.Equal(t, Changes{ + "resources.jobs.my_job": { + "permissions[user_name='viewer@example.com'].level": {Operation: OperationReplace, Value: "CAN_MANAGE"}, + "permissions[group_name='data-team']": { + Operation: OperationAdd, + Value: map[string]any{"level": "CAN_VIEW", "group_name": "data-team"}, + }, + "permissions[user_name='gone@example.com']": {Operation: OperationRemove}, + }, + }, changes) +} + +func TestWithoutCurrentUserManage(t *testing.T) { + permissions := []dresources.StatePermission{ + {Level: iam.PermissionLevelIsOwner, UserName: "me@example.com"}, + {Level: iam.PermissionLevelCanManage, ServicePrincipalName: "me@example.com"}, + {Level: iam.PermissionLevelIsOwner, UserName: "owner@example.com"}, + {Level: iam.PermissionLevelCanManage, UserName: "other@example.com"}, + {Level: iam.PermissionLevelCanView, GroupName: "users"}, + } + + assert.Equal(t, []dresources.StatePermission{ + {Level: iam.PermissionLevelIsOwner, UserName: "owner@example.com"}, + {Level: iam.PermissionLevelCanManage, UserName: "other@example.com"}, + {Level: iam.PermissionLevelCanView, GroupName: "users"}, + }, withoutCurrentUserManage(permissions, "me@example.com")) +} + +func TestPermissionsReadSkip(t *testing.T) { + denied := &apierr.APIError{StatusCode: 403, ErrorCode: "PERMISSION_DENIED", Message: "no access"} + assert.Equal(t, SkippedChange{ + ResourceKey: "resources.jobs.my_job", + Reason: SkipReasonPermissionsPermissionDenied, + Error: "no access", + }, permissionsReadSkip("resources.jobs.my_job", denied)) + + assert.Equal(t, SkippedChange{ + ResourceKey: "resources.jobs.my_job", + Reason: SkipReasonPermissionsReadFailed, + Error: "unexpected object_id", + }, permissionsReadSkip("resources.jobs.my_job", errors.New("unexpected object_id"))) +} + +func TestUnmanagedPermissionsAdapters(t *testing.T) { + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + b := &bundle.Bundle{} + b.DeploymentBundle.Adapters = adapters + + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.unmanaged": {Action: deployplan.Skip}, + "resources.jobs.declared": {Action: deployplan.Skip}, + "resources.jobs.declared.permissions": {Action: deployplan.Skip}, + "resources.jobs.removed": {Action: deployplan.Update}, + "resources.jobs.removed.permissions": {Action: deployplan.Delete}, + "resources.jobs.not_deployed": {Action: deployplan.Create}, + "resources.pipelines.unmanaged": {Action: deployplan.Skip}, + "resources.secret_scopes.scope": {Action: deployplan.Skip}, + "resources.schemas.schema": {Action: deployplan.Skip}, + "resources.schemas.schema.grants": {Action: deployplan.Skip}, + }} + + assert.ElementsMatch(t, + []string{"resources.jobs.unmanaged", "resources.jobs.removed", "resources.pipelines.unmanaged"}, + slices.Collect(maps.Keys(unmanagedPermissionsAdapters(b, plan))), + ) +} + +func TestLookupDeployedFieldReadsRemoteState(t *testing.T) { + b := &bundle.Bundle{} + b.DeploymentBundle.RemoteStateCache.Store("resources.model_serving_endpoints.my_endpoint", + &dresources.ModelServingEndpointRemote{EndpointId: "abc123"}) + + value, err := lookupDeployedField(b, "resources.model_serving_endpoints.my_endpoint", "endpoint_id") + require.NoError(t, err) + assert.Equal(t, "abc123", value) +} + +func TestExtractChangesEmptyRemotePermissions(t *testing.T) { + value, err := yamlloader.LoadYAML("databricks.yml", strings.NewReader(` +resources: + apps: + my_app: + permissions: + - level: CAN_USE + group_name: users +`)) + require.NoError(t, err) + b := &bundle.Bundle{} + require.NoError(t, b.Config.Mutate(func(dyn.Value) (dyn.Value, error) { return value, nil })) + + declared := []dresources.StatePermission{{Level: "CAN_USE", GroupName: "users"}} + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.apps.my_app": {}, + "resources.apps.my_app.permissions": { + Action: deployplan.Update, + RemoteState: &dresources.PermissionsState{ObjectID: "/apps/my_app"}, + Changes: deployplan.Changes{ + "": {Action: deployplan.Update, Old: declared, New: declared}, + }, + }, + }} + + changes, _, err := ExtractChanges(t.Context(), b, plan) + require.NoError(t, err) + assert.Equal(t, Changes{ + "resources.apps.my_app": { + "permissions[group_name='users']": {Operation: OperationRemove}, + }, + }, changes) +} + func TestExtractChangesSkipsLifecycleFields(t *testing.T) { plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ "resources.genie_spaces.my_space": { @@ -358,7 +529,7 @@ func TestExtractChangesSkipsLifecycleFields(t *testing.T) { }, }} - changes, err := ExtractChanges(t.Context(), &bundle.Bundle{}, plan) + changes, _, err := ExtractChanges(t.Context(), &bundle.Bundle{}, plan) require.NoError(t, err) assert.Equal(t, ResourceChanges{ diff --git a/bundle/configsync/output.go b/bundle/configsync/output.go index 196a26db704..2e61e618b4e 100644 --- a/bundle/configsync/output.go +++ b/bundle/configsync/output.go @@ -38,14 +38,16 @@ const ( SkipReasonClusterPolicyNotFound = "cluster_policy_not_found" SkipReasonClusterPolicyPermissionDenied = "cluster_policy_permission_denied" SkipReasonClusterPolicyReadFailed = "cluster_policy_read_failed" + SkipReasonPermissionsPermissionDenied = "permissions_permission_denied" + SkipReasonPermissionsReadFailed = "permissions_read_failed" ) // SkippedChange identifies a resource left unchanged because its remote changes -// could not be classified safely. +// could not be classified safely, or whose remote permissions could not be read. type SkippedChange struct { ResourceKey string `json:"resourceKey"` Reason string `json:"reason"` - PolicyID string `json:"policyId"` + PolicyID string `json:"policyId,omitempty"` Error string `json:"error,omitempty"` } @@ -55,6 +57,10 @@ func (s SkippedChange) describe() string { return fmt.Sprintf("Left %s unchanged: cluster policy %s was not found.", s.ResourceKey, s.PolicyID) case SkipReasonClusterPolicyPermissionDenied: return fmt.Sprintf("Left %s unchanged: no permission to read cluster policy %s.", s.ResourceKey, s.PolicyID) + case SkipReasonPermissionsPermissionDenied: + return fmt.Sprintf("Left the permissions of %s unchanged: no permission to read them.", s.ResourceKey) + case SkipReasonPermissionsReadFailed: + return fmt.Sprintf("Left the permissions of %s unchanged: they could not be read: %s", s.ResourceKey, s.Error) default: return fmt.Sprintf("Left %s unchanged: cluster policy %s could not be read: %s", s.ResourceKey, s.PolicyID, s.Error) } diff --git a/bundle/configsync/permissions.go b/bundle/configsync/permissions.go new file mode 100644 index 00000000000..87eca95a0b8 --- /dev/null +++ b/bundle/configsync/permissions.go @@ -0,0 +1,252 @@ +package configsync + +import ( + "context" + "errors" + "fmt" + "reflect" + "slices" + "strings" + "sync" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/iam" + "golang.org/x/sync/errgroup" +) + +const permissionsReadLimit = 10 + +// addPermissionsChanges adds remote permission changes to the permissions field of their +// resource: per principal where the config declares permissions, and as a whole for deployed +// resources that declare none. It returns the resources whose permissions could not be read. +func addPermissionsChanges(ctx context.Context, changes Changes, b *bundle.Bundle, plan *deployplan.Plan) ([]SkippedChange, error) { + for resourceKey, entry := range plan.Plan { + if err := addDeclaredPermissionsChanges(changes, b, resourceKey, entry); err != nil { + return nil, err + } + } + + unmanaged, skipped := readUnmanagedPermissions(ctx, b, plan) + for resourceKey, permissions := range unmanaged { + value, err := normalizeValue(permissions) + if err != nil { + return nil, fmt.Errorf("failed to compute config change for %s: %w", resourceKey, err) + } + changes.set(resourceKey, "permissions", &ConfigChangeDesc{ + Operation: OperationAdd, + Value: value, + }) + } + return skipped, nil +} + +// addDeclaredPermissionsChanges turns the plan's changes to a resource's permissions into +// changes of its permissions field, e.g. "[user_name='a'].level" becomes +// "permissions[user_name='a'].level". Only entries written in the resource's config are changed: +// the ones FixPermissions or bundle-level permissions add have no YAML node to write to. +func addDeclaredPermissionsChanges(changes Changes, b *bundle.Bundle, resourceKey string, entry *deployplan.PlanEntry) error { + if _, ok := entry.RemoteState.(*dresources.PermissionsState); !ok { + return nil + } + declared := declaredPrincipals(b, resourceKey) + if len(declared) == 0 { + return nil + } + parentKey := strings.TrimSuffix(resourceKey, ".permissions") + + for path, changeDesc := range entry.Changes { + node, err := structpath.ParsePath(path) + if err != nil { + return fmt.Errorf("failed to parse path %s: %w", path, err) + } + nodes := node.AsSlice() + if len(nodes) == 0 { + // The remote ACL has no direct entries left, so the change covers the whole list. + if changeDesc.Action != deployplan.Skip { + for selector := range declared { + changes.set(parentKey, "permissions"+selector, &ConfigChangeDesc{Operation: OperationRemove}) + } + } + continue + } + if _, _, ok := nodes[0].KeyValue(); !ok || (changeDesc.New != nil && !declared[nodes[0].String()]) { + continue + } + + fieldPath := "permissions" + path + change, err := convertPlanChange(parentKey, fieldPath, changeDesc, b.Config.Presets.NamePrefix) + if err != nil { + return fmt.Errorf("failed to compute config change for path %s: %w", resourceKey+"."+path, err) + } + if change != nil { + changes.set(parentKey, fieldPath, change) + } + } + return nil +} + +// declaredPrincipals returns the permissions written in the resource's config as the plan's +// selectors, e.g. "[user_name='a']". Entries added by mutators have no location. +func declaredPrincipals(b *bundle.Bundle, resourceKey string) map[string]bool { + path, err := dyn.NewPathFromString(resourceKey) + if err != nil { + return nil + } + permissions, _ := dyn.GetByPath(b.Config.Value(), path) + sequence, _ := permissions.AsSequence() + result := make(map[string]bool) + for _, permission := range sequence { + if permission.Location().File == "" { + continue + } + for _, key := range []string{"user_name", "service_principal_name", "group_name"} { + if value, ok := permission.Get(key).AsString(); ok && value != "" { + result[structpath.NewKeyValue(nil, key, value).String()] = true + } + } + } + return result +} + +func (c Changes) set(resourceKey, fieldPath string, change *ConfigChangeDesc) { + if c[resourceKey] == nil { + c[resourceKey] = make(ResourceChanges) + } + c[resourceKey][fieldPath] = change +} + +// readUnmanagedPermissions reads the remote permissions of deployed resources whose config +// declares none, keyed by resource. They are imported as a whole: the current ACL cannot tell +// grants made before the deployment from the ones made since. A resource whose ACL cannot be +// read is returned as skipped, so the rest of the sync still goes through. +func readUnmanagedPermissions(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (map[string][]dresources.StatePermission, []SkippedChange) { + var mu sync.Mutex + var g errgroup.Group + g.SetLimit(permissionsReadLimit) + result := make(map[string][]dresources.StatePermission) + var skipped []SkippedChange + currentUser := currentUserName(b) + + for resourceKey, adapter := range unmanagedPermissionsAdapters(b, plan) { + g.Go(func() error { + permissions, err := readPermissions(ctx, b, adapter, resourceKey) + mu.Lock() + defer mu.Unlock() + if err != nil { + skipped = append(skipped, permissionsReadSkip(resourceKey, err)) + return nil + } + permissions = withoutCurrentUserManage(permissions, currentUser) + if len(permissions) > 0 { + result[resourceKey] = permissions + } + return nil + }) + } + + _ = g.Wait() + slices.SortFunc(skipped, func(a, b SkippedChange) int { return strings.Compare(a.ResourceKey, b.ResourceKey) }) + return result, skipped +} + +// permissionsReadSkip reports a resource whose ACL could not be read. A user who can deploy a +// resource is not necessarily allowed to read its ACL. +func permissionsReadSkip(resourceKey string, err error) SkippedChange { + reason := SkipReasonPermissionsReadFailed + if errors.Is(err, apierr.ErrPermissionDenied) { + reason = SkipReasonPermissionsPermissionDenied + } + return SkippedChange{ResourceKey: resourceKey, Reason: reason, Error: err.Error()} +} + +// unmanagedPermissionsAdapters returns the permissions adapter of every deployed resource whose +// config declares no permissions. A state-only permissions entry is deleted on the next deploy +// without changing the ACL, so it does not count. +func unmanagedPermissionsAdapters(b *bundle.Bundle, plan *deployplan.Plan) map[string]*dresources.Adapter { + result := make(map[string]*dresources.Adapter) + for resourceKey, entry := range plan.Plan { + if isPermissionsOrGrantsSubResource(resourceKey) || !entry.Action.KeepsID() { + continue + } + permissionsKey := resourceKey + ".permissions" + if permissions := plan.Plan[permissionsKey]; permissions != nil && permissions.Action != deployplan.Delete { + continue + } + adapter := b.DeploymentBundle.Adapters[config.GetResourceTypeFromKey(permissionsKey)] + if adapter != nil && adapter.RemoteType() == reflect.TypeFor[*dresources.PermissionsState]() { + result[resourceKey] = adapter + } + } + return result +} + +func readPermissions(ctx context.Context, b *bundle.Bundle, adapter *dresources.Adapter, resourceKey string) ([]dresources.StatePermission, error) { + sv, err := adapter.PrepareInputConfig([]dresources.StatePermission{}, resourceKey+".permissions") + if err != nil { + return nil, err + } + // The object ID references a field of the resource, e.g. "/jobs/${resources.jobs.foo.id}". It is + // resolved here by key, since a key with e.g. spaces does not parse as a reference. + objectID := sv.Refs["object_id"] + objectType, field, ok := strings.Cut(objectID, "${"+resourceKey+".") + field, hasSuffix := strings.CutSuffix(field, "}") + if !ok || !hasSuffix { + return nil, fmt.Errorf("unexpected object_id %q", objectID) + } + value, err := lookupDeployedField(b, resourceKey, field) + if err != nil { + return nil, fmt.Errorf("resolving %s.%s: %w", resourceKey, field, err) + } + + id := objectType + fmt.Sprint(value) + remote, err := retry.With(ctx, retry.IsTransientRead, func() (any, error) { + return adapter.DoRead(ctx, id) + }) + if err != nil { + return nil, err + } + return remote.(*dresources.PermissionsState).EmbeddedSlice, nil +} + +// lookupDeployedField reads a field of a deployed resource, such as the ID its permissions are +// keyed by. The plan drops the new state of unchanged resources, so other fields come from the +// remote state it read. +func lookupDeployedField(b *bundle.Bundle, resourceKey, field string) (any, error) { + if field == "id" { + return b.DeploymentBundle.StateDB.GetResourceID(resourceKey), nil + } + path, err := structpath.ParsePath(field) + if err != nil { + return nil, err + } + remote, ok := b.DeploymentBundle.RemoteStateCache.Load(resourceKey) + if !ok { + return nil, fmt.Errorf("no remote state for %s", resourceKey) + } + return structaccess.Get(remote, path) +} + +// withoutCurrentUserManage drops the current user's IS_OWNER and CAN_MANAGE, one of which +// FixPermissions adds on deploy. Another principal's IS_OWNER is kept, so a transferred owner +// stays the owner after the next deploy. +func withoutCurrentUserManage(permissions []dresources.StatePermission, currentUser string) []dresources.StatePermission { + return slices.DeleteFunc(permissions, func(p dresources.StatePermission) bool { + isCurrentUser := currentUser != "" && (p.UserName == currentUser || p.ServicePrincipalName == currentUser) + return isCurrentUser && (p.Level == iam.PermissionLevelIsOwner || p.Level == iam.PermissionLevelCanManage) + }) +} + +func currentUserName(b *bundle.Bundle) string { + if b.Config.Workspace.CurrentUser == nil { + return "" + } + return b.Config.Workspace.CurrentUser.UserName +} diff --git a/cmd/bundle/config_remote_sync.go b/cmd/bundle/config_remote_sync.go index 83344855e5c..972810d6f0f 100644 --- a/cmd/bundle/config_remote_sync.go +++ b/cmd/bundle/config_remote_sync.go @@ -96,7 +96,7 @@ Examples: return fmt.Errorf("failed to detect changes: %w", err) } - detected, err := configsync.ExtractChanges(ctx, b, plan) + detected, permissionsSkipped, err := configsync.ExtractChanges(ctx, b, plan) if err != nil { stats.ErrorCategory = protos.BundleConfigRemoteSyncErrorCategoryDetectChangesFailed return fmt.Errorf("failed to extract changes: %w", err) @@ -129,6 +129,9 @@ Examples: return fmt.Errorf("deployed resources missing from the bundle configuration (renamed or removed since the last deploy): %s", strings.Join(missing, ", ")) } detected = configsync.FilterChanges(detected, selected) + permissionsSkipped = slices.DeleteFunc(permissionsSkipped, func(skip configsync.SkippedChange) bool { + return !slices.Contains(selected, skip.ResourceKey) + }) } detected, policySkipped, err := configsync.FilterClusterPolicyValues(ctx, b, plan, detected) @@ -170,7 +173,7 @@ Examples: files = applied changes = detected - skippedChanges = policySkipped + skippedChanges = append(policySkipped, permissionsSkipped...) return nil }, })