Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,12 @@ Resource: resources.cluster_policies.policy
>>> [CLI] bundle validate -o json
[{"principal":"users","level":"CAN_USE"}]

=== Sync again before deploy

>>> [CLI] bundle config-remote-sync
No changes detected.


=== Deploy applies the synced ACL

>>> [CLI] bundle deploy
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ trace $CLI bundle config-remote-sync --save
# The deployer is a user locally and a service principal on cloud, so print principals only.
trace $CLI bundle validate -o json | jq -c '[.resources.cluster_policies.policy.permissions[] | {principal: (.user_name // .service_principal_name // .group_name), level}] | sort_by(.principal)'

title "Sync again before deploy"
trace $CLI bundle config-remote-sync

title "Deploy applies the synced ACL"
trace $CLI bundle deploy
trace $CLI permissions get cluster-policies "$policy_id" | jq -c '[.access_control_list[] | select(.all_permissions[0].inherited | not) | {principal: (.user_name // .service_principal_name // .group_name), level: .all_permissions[0].permission_level}] | sort_by(.principal)'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,38 @@ Resource: resources.pipelines.my_pipeline
No changes detected.


=== Deploy keeps the remote owner
=== Grant access remotely before the saved permissions are deployed

=== Detect and save changes
Detected changes in 1 resource(s):

Resource: resources.pipelines.my_pipeline
permissions[user_name='late@example.com']: add



=== Configuration changes

>>> diff.py databricks.yml.backup databricks.yml
--- databricks.yml.backup
+++ databricks.yml
@@ -30,4 +30,6 @@
- group_name: users
level: CAN_VIEW
+ - level: CAN_RUN

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it correct from a bundle perspective that they are merged?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[AI] Yes, I think merging is right here. A resource's permissions block is the whole ACL the bundle manages: deploy sends it with Permissions.Set (a PUT; DoCreate goes through DoUpdate) plus the deploying user's entry, so any grant missing from the block is removed. Without the late@example.com entry, the "Deploy keeps the remote owner and grant" step would drop the grant made in the UI, which is what sync is there to prevent.

It is also the same merge sync already does for a deployed block: per principal, keyed by user_name/group_name/service_principal_name, so a remote grant becomes an add, a changed level a replace and a principal removed remotely a remove (the job step further down in this test). The deploying user's own entry is skipped.

The trade-off: with nothing deployed yet there is no state to compare against, so sync cannot tell a hand-written entry from a remote change, and a hand-written block that was never deployed is reverted to the remote ACL. That is the same remote-wins sync applies to other local edits that are not deployed yet (the plan marks them Update and sync writes the remote value; that is what LocalEdit counts). If you would rather keep hand-written entries, the undeployed path can be limited to adds, so remote grants are picked up and nothing declared is replaced or removed; that is a small change.

🤖 Factory-generated comment. I still review the code and double-check every claim myself.

+ user_name: late@example.com

targets:

=== Deploy keeps the remote owner and grant
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
Created jobs.my_job.permissions
Created pipelines.my_pipeline.permissions
Files: 1 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 2 unchanged

>>> [CLI] permissions get pipelines [MY_PIPELINE_ID]
[{"user_name":"owner@example.com","group_name":null,"level":"IS_OWNER"},{"user_name":null,"group_name":"users","level":"CAN_VIEW"},{"user_name":"[USERNAME]","group_name":null,"level":"CAN_MANAGE"}]
[{"user_name":"owner@example.com","group_name":null,"level":"IS_OWNER"},{"user_name":null,"group_name":"users","level":"CAN_VIEW"},{"user_name":"late@example.com","group_name":null,"level":"CAN_RUN"},{"user_name":"[USERNAME]","group_name":null,"level":"CAN_MANAGE"}]

=== Change a level, add and remove principals remotely

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,23 @@ rm databricks.yml.backup
title "Sync again: the permissions are now in the config"
$CLI bundle config-remote-sync

title "Deploy keeps the remote owner"
title "Grant access remotely before the saved permissions are deployed"
$CLI permissions set pipelines $pipeline_id --json '{"access_control_list":[
{"permission_level":"IS_OWNER","user_name":"owner@example.com"},
{"permission_level":"CAN_MANAGE","user_name":"'"$CURRENT_USER_NAME"'"},
{"permission_level":"CAN_VIEW","group_name":"users"},
{"permission_level":"CAN_RUN","user_name":"late@example.com"}
]}' > /dev/null

title "Detect and save changes"
cp databricks.yml databricks.yml.backup
$CLI bundle config-remote-sync --save

title "Configuration changes"
trace diff.py databricks.yml.backup databricks.yml
rm databricks.yml.backup

title "Deploy keeps the remote owner and grant"
$CLI bundle deploy
trace $CLI permissions get pipelines $pipeline_id | jq -c '[.access_control_list[] | {user_name, group_name, level: .all_permissions[0].permission_level}]'

Expand Down
6 changes: 3 additions & 3 deletions bundle/configsync/diff.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,8 +142,8 @@ func isPermissionsOrGrantsSubResource(resourceKey string) bool {
func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (Changes, []SkippedChange, error) {
changes := make(Changes)
// Read first: a job's ACL names the owner that skipJobRunAs compares run_as to.
unmanaged, skipped := readUnmanagedPermissions(ctx, b, plan)
owners := jobOwners(ctx, b, plan, unmanaged)
remote, skipped := readRemotePermissions(ctx, b, plan)
owners := jobOwners(ctx, plan, remote)

for resourceKey, entry := range plan.Plan {
// permissions and grants are emitted as their own plan keys
Expand Down Expand Up @@ -177,7 +177,7 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan
log.Debugf(ctx, "Resource %s has %d changes", resourceKey, len(resourceChanges))
}

if err := addPermissionsChanges(changes, b, plan, unmanaged); err != nil {
if err := addPermissionsChanges(changes, b, plan, remote); err != nil {
return nil, nil, err
}

Expand Down
84 changes: 71 additions & 13 deletions bundle/configsync/diff_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/databricks/cli/bundle/direct/dresources"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/yamlloader"
"github.com/databricks/cli/libs/structs/structvar"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/service/iam"
"github.com/databricks/databricks-sdk-go/service/serving"
Expand Down Expand Up @@ -420,31 +421,88 @@ func TestPermissionsReadSkip(t *testing.T) {
}, permissionsReadSkip("resources.jobs.my_job", errors.New("unexpected object_id")))
}

func TestUnmanagedPermissionsAdapters(t *testing.T) {
func TestUnreadPermissionsAdapters(t *testing.T) {
adapters, err := dresources.InitAll(nil)
require.NoError(t, err)
b := &bundle.Bundle{}
b.DeploymentBundle.Adapters = adapters

plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{
"resources.jobs.unmanaged": {Action: deployplan.Skip},
"resources.jobs.declared": {Action: deployplan.Skip},
"resources.jobs.declared.permissions": {Action: deployplan.Skip},
"resources.jobs.removed": {Action: deployplan.Update},
"resources.jobs.removed.permissions": {Action: deployplan.Delete},
"resources.jobs.not_deployed": {Action: deployplan.Create},
"resources.pipelines.unmanaged": {Action: deployplan.Skip},
"resources.secret_scopes.scope": {Action: deployplan.Skip},
"resources.schemas.schema": {Action: deployplan.Skip},
"resources.schemas.schema.grants": {Action: deployplan.Skip},
"resources.jobs.unmanaged": {Action: deployplan.Skip},
"resources.jobs.declared": {Action: deployplan.Skip},
"resources.jobs.declared.permissions": {Action: deployplan.Skip},
"resources.jobs.removed": {Action: deployplan.Update},
"resources.jobs.removed.permissions": {Action: deployplan.Delete},
"resources.jobs.undeployed": {Action: deployplan.Skip},
"resources.jobs.undeployed.permissions": {Action: deployplan.Create},
"resources.jobs.not_deployed": {Action: deployplan.Create},
"resources.jobs.not_deployed.permissions": {Action: deployplan.Create},
"resources.pipelines.unmanaged": {Action: deployplan.Skip},
"resources.secret_scopes.scope": {Action: deployplan.Skip},
"resources.schemas.schema": {Action: deployplan.Skip},
"resources.schemas.schema.grants": {Action: deployplan.Skip},
}}

assert.ElementsMatch(t,
[]string{"resources.jobs.unmanaged", "resources.jobs.removed", "resources.pipelines.unmanaged"},
slices.Collect(maps.Keys(unmanagedPermissionsAdapters(b, plan))),
[]string{"resources.jobs.unmanaged", "resources.jobs.removed", "resources.jobs.undeployed", "resources.pipelines.unmanaged"},
slices.Collect(maps.Keys(unreadPermissionsAdapters(b, plan))),
)
}

func TestAddUndeployedPermissionsChanges(t *testing.T) {
value, err := yamlloader.LoadYAML("databricks.yml", strings.NewReader(`
resources:
jobs:
my_job:
permissions:
- level: CAN_MANAGE
user_name: viewer@example.com
- level: CAN_VIEW
user_name: gone@example.com
- level: CAN_VIEW
group_name: users
`))
require.NoError(t, err)
// FixPermissions adds the current user's entry, with no location.
value, err = dyn.Set(value, "resources.jobs.my_job.permissions", dyn.V(append(
value.Get("resources").Get("jobs").Get("my_job").Get("permissions").MustSequence(),
dyn.V(map[string]dyn.Value{"level": dyn.V("IS_OWNER"), "user_name": dyn.V("me@example.com")}),
)))
require.NoError(t, err)
adapters, err := dresources.InitAll(nil)
require.NoError(t, err)
b := &bundle.Bundle{}
require.NoError(t, b.Config.Mutate(func(dyn.Value) (dyn.Value, error) { return value, nil }))
b.DeploymentBundle.Adapters = adapters
b.DeploymentBundle.StateCache.Store("resources.jobs.my_job.permissions", &structvar.StructVar{
Value: &dresources.PermissionsState{EmbeddedSlice: []dresources.StatePermission{
{Level: iam.PermissionLevelCanManage, UserName: "viewer@example.com"},
{Level: iam.PermissionLevelCanView, UserName: "gone@example.com"},
{Level: iam.PermissionLevelCanView, GroupName: "users"},
{Level: iam.PermissionLevelIsOwner, UserName: "me@example.com"},
}},
})

changes := make(Changes)
err = addUndeployedPermissionsChanges(changes, b, "resources.jobs.my_job", []dresources.StatePermission{
{Level: iam.PermissionLevelCanManage, UserName: "me@example.com"},
{Level: iam.PermissionLevelCanView, UserName: "viewer@example.com"},
{Level: iam.PermissionLevelCanView, GroupName: "users"},
{Level: iam.PermissionLevelCanManageRun, UserName: "late@example.com"},
})
require.NoError(t, err)
assert.Equal(t, Changes{
"resources.jobs.my_job": {
"permissions[user_name='viewer@example.com'].level": {Operation: OperationReplace, Value: "CAN_VIEW", LocalEdit: true},
"permissions[user_name='gone@example.com']": {Operation: OperationRemove, LocalEdit: true},
"permissions[user_name='late@example.com']": {
Operation: OperationAdd,
Value: map[string]any{"level": "CAN_MANAGE_RUN", "user_name": "late@example.com"},
},
},
}, changes)
}

func TestLookupDeployedFieldReadsRemoteState(t *testing.T) {
b := &bundle.Bundle{}
b.DeploymentBundle.RemoteStateCache.Store("resources.model_serving_endpoints.my_endpoint",
Expand Down
67 changes: 53 additions & 14 deletions bundle/configsync/permissions.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/cli/libs/structs/structaccess"
"github.com/databricks/cli/libs/structs/structdiff"
"github.com/databricks/cli/libs/structs/structpath"
"github.com/databricks/databricks-sdk-go/apierr"
"github.com/databricks/databricks-sdk-go/service/iam"
Expand All @@ -26,16 +27,23 @@ const permissionsReadLimit = 10

// addPermissionsChanges adds remote permission changes to the permissions field of their
// resource: per principal where the config declares permissions, and as a whole for deployed
// resources that declare none. unmanaged holds the ACLs read by readUnmanagedPermissions.
func addPermissionsChanges(changes Changes, b *bundle.Bundle, plan *deployplan.Plan, unmanaged map[string][]dresources.StatePermission) error {
// resources that declare none. remote holds the ACLs read by readRemotePermissions.
func addPermissionsChanges(changes Changes, b *bundle.Bundle, plan *deployplan.Plan, remote map[string][]dresources.StatePermission) error {
for resourceKey, entry := range plan.Plan {
if err := addDeclaredPermissionsChanges(changes, b, resourceKey, entry); err != nil {
return err
}
}

currentUser := currentUserName(b)
for resourceKey, permissions := range unmanaged {
for resourceKey, permissions := range remote {
if entry := plan.Plan[resourceKey+".permissions"]; entry != nil && entry.Action == deployplan.Create {
if err := addUndeployedPermissionsChanges(changes, b, resourceKey, permissions); err != nil {
return err
}
continue
}

permissions = withoutCurrentUserDefaults(slices.Clone(permissions), currentUser, config.GetResourceTypeFromKey(resourceKey))
if len(permissions) == 0 {
continue
Expand Down Expand Up @@ -97,6 +105,36 @@ func addDeclaredPermissionsChanges(changes Changes, b *bundle.Bundle, resourceKe
return nil
}

// addUndeployedPermissionsChanges compares the remote ACL of a deployed resource with the
// permissions its config declares but that were never deployed, e.g. after an import was saved.
// The plan creates those without reading the ACL, and creating them replaces it, so a grant made
// since would be dropped by the next deploy.
func addUndeployedPermissionsChanges(changes Changes, b *bundle.Bundle, resourceKey string, remote []dresources.StatePermission) error {
permissionsKey := resourceKey + ".permissions"
sv, ok := b.DeploymentBundle.StateCache.Load(permissionsKey)
if !ok {
return fmt.Errorf("no config state for %s", permissionsKey)
}
local := sv.Value.(*dresources.PermissionsState)
// Without an entry for the current user, their default grants would be imported as new principals.
currentUser := currentUserName(b)
if !slices.ContainsFunc(local.EmbeddedSlice, func(p dresources.StatePermission) bool { return isCurrentUserPermission(p, currentUser) }) {
remote = withoutCurrentUserDefaults(slices.Clone(remote), currentUser, config.GetResourceTypeFromKey(resourceKey))
}
remoteState := &dresources.PermissionsState{ObjectID: local.ObjectID, EmbeddedSlice: remote}
adapter := b.DeploymentBundle.Adapters[config.GetResourceTypeFromKey(permissionsKey)]
diff, err := structdiff.GetStructDiff(remoteState, local, adapter.KeyedSlices())
if err != nil {
return fmt.Errorf("failed to compare permissions of %s: %w", resourceKey, err)
}

entry := &deployplan.PlanEntry{RemoteState: remoteState, Changes: make(deployplan.Changes)}
for _, ch := range diff {
entry.Changes[ch.Path.String()] = &deployplan.ChangeDesc{Action: deployplan.Update, New: ch.New, Remote: ch.Old}
}
return addDeclaredPermissionsChanges(changes, b, permissionsKey, entry)
}

// declaredPrincipals returns the permissions written in the resource's config as the plan's
// selectors, e.g. "[user_name='a']". Entries added by mutators have no location.
func declaredPrincipals(b *bundle.Bundle, resourceKey string) map[string]bool {
Expand Down Expand Up @@ -127,18 +165,19 @@ func (c Changes) set(resourceKey, fieldPath string, change *ConfigChangeDesc) {
c[resourceKey][fieldPath] = change
}

// readUnmanagedPermissions reads the remote permissions of deployed resources whose config
// declares none, keyed by resource. They are imported as a whole: the current ACL cannot tell
// grants made before the deployment from the ones made since. A resource whose ACL cannot be
// read is returned as skipped, so the rest of the sync still goes through.
func readUnmanagedPermissions(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (map[string][]dresources.StatePermission, []SkippedChange) {
// readRemotePermissions reads the remote permissions of deployed resources whose ACL the plan
// does not read, keyed by resource. Those that declare no permissions are imported as a whole:
// the current ACL cannot tell grants made before the deployment from the ones made since. A
// resource whose ACL cannot be read is returned as skipped, so the rest of the sync still goes
// through.
func readRemotePermissions(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (map[string][]dresources.StatePermission, []SkippedChange) {
var mu sync.Mutex
var g errgroup.Group
g.SetLimit(permissionsReadLimit)
result := make(map[string][]dresources.StatePermission)
var skipped []SkippedChange

for resourceKey, adapter := range unmanagedPermissionsAdapters(b, plan) {
for resourceKey, adapter := range unreadPermissionsAdapters(b, plan) {
g.Go(func() error {
permissions, err := readPermissions(ctx, b, adapter, resourceKey)
mu.Lock()
Expand Down Expand Up @@ -167,17 +206,17 @@ func permissionsReadSkip(resourceKey string, err error) SkippedChange {
return SkippedChange{ResourceKey: resourceKey, Reason: reason, Error: err.Error()}
}

// unmanagedPermissionsAdapters returns the permissions adapter of every deployed resource whose
// config declares no permissions. A state-only permissions entry is deleted on the next deploy
// without changing the ACL, so it does not count.
func unmanagedPermissionsAdapters(b *bundle.Bundle, plan *deployplan.Plan) map[string]*dresources.Adapter {
// unreadPermissionsAdapters returns the permissions adapter of every deployed resource whose
// config declares no permissions, or declares permissions that were never deployed. A state-only
// permissions entry is deleted on the next deploy without changing the ACL, so it does not count.
func unreadPermissionsAdapters(b *bundle.Bundle, plan *deployplan.Plan) map[string]*dresources.Adapter {
result := make(map[string]*dresources.Adapter)
for resourceKey, entry := range plan.Plan {
if isPermissionsOrGrantsSubResource(resourceKey) || !entry.Action.KeepsID() {
continue
}
permissionsKey := resourceKey + ".permissions"
if permissions := plan.Plan[permissionsKey]; permissions != nil && permissions.Action != deployplan.Delete {
if permissions := plan.Plan[permissionsKey]; permissions != nil && permissions.Action != deployplan.Delete && permissions.Action != deployplan.Create {
continue
}
adapter := b.DeploymentBundle.Adapters[config.GetResourceTypeFromKey(permissionsKey)]
Expand Down
Loading
Loading