Skip to content
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,48 @@ bundle:

resources:
jobs:
# run_as is not set in config, so the backend defaults it to the deployer.
no_run_as:
name: no-run-as-$UNIQUE_NAME
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook

# The remote run_as names the owner, which is what no run_as means.
run_as_to_owner:
name: run-as-to-owner-$UNIQUE_NAME
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook

# Ownership moves to another user, who becomes the default run_as.
transferred_owner:
name: transferred-owner-$UNIQUE_NAME
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook

# run_as is removed from config and deployed; the job keeps it remotely.
removed_run_as:
name: removed-run-as-$UNIQUE_NAME
run_as:
user_name: first-user@example.com
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook

# run_as is added to config locally and not deployed.
pending_add:
name: pending-add-$UNIQUE_NAME
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook

user_to_user:
name: user-to-user-$UNIQUE_NAME
run_as:
Expand Down
83 changes: 76 additions & 7 deletions acceptance/bundle/config-remote-sync/job_run_as/output.txt
Original file line number Diff line number Diff line change
@@ -1,18 +1,52 @@

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
Created jobs.no_run_as
Created jobs.pending_add
Created jobs.removed_run_as
Created jobs.run_as_to_owner
Created jobs.transferred_owner
Created jobs.user_to_creator
Created jobs.user_to_sp
Created jobs.user_to_user
Files: 2 uploaded, 0 deleted
Resources: 3 created, 0 changed, 0 deleted, 0 unchanged
Resources: 8 created, 0 changed, 0 deleted, 0 unchanged

=== Remove run_as from config and deploy

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
Updated jobs.removed_run_as
Files: 1 uploaded, 0 deleted
Resources: 0 created, 1 changed, 0 deleted, 7 unchanged

>>> [CLI] jobs get [REMOVED_RUN_AS_ID]
{
"user_name": "first-user@example.com"
}

=== Add run_as to config without deploying

=== Change run_as remotely

>>> [CLI] jobs set-permissions [TRANSFERRED_OWNER_ID] --json {"access_control_list": [{"user_name": "new-owner@example.com", "permission_level": "IS_OWNER"}]}

=== Detect and save changes

>>> [CLI] bundle config-remote-sync --save
Detected changes in 3 resource(s):
Detected changes in 7 resource(s):

Resource: resources.jobs.no_run_as
run_as: add

Resource: resources.jobs.pending_add
run_as.user_name: replace

Resource: resources.jobs.removed_run_as
run_as: add

Resource: resources.jobs.transferred_owner
permissions: add

Resource: resources.jobs.user_to_creator
run_as.user_name: replace
Expand All @@ -29,21 +63,49 @@ Resource: resources.jobs.user_to_user
>>> diff.py databricks.yml.backup databricks.yml
--- databricks.yml.backup
+++ databricks.yml
@@ -7,5 +7,5 @@
@@ -12,4 +12,6 @@
notebook_path: /Users/{{workspace_user_name}}/notebook

+ run_as:
+ service_principal_name: [UUID]
# The remote run_as names the owner, which is what no run_as means.
run_as_to_owner:
@@ -28,4 +30,7 @@
notebook_path: /Users/{{workspace_user_name}}/notebook

+ permissions:
+ - level: IS_OWNER
+ user_name: new-owner@example.com
# run_as is removed from config and deployed; the job keeps it remotely.
removed_run_as:
@@ -36,9 +41,11 @@
notebook_path: /Users/{{workspace_user_name}}/notebook

+ run_as:
+ user_name: first-user@example.com
# run_as is added to config locally and not deployed.
pending_add:
name: pending-add-[UNIQUE_NAME]
run_as:
- user_name: local-user@example.com
+ user_name: remote-user@example.com
tasks:
- task_key: main
@@ -49,5 +56,5 @@
name: user-to-user-[UNIQUE_NAME]
run_as:
- user_name: first-user@example.com
+ user_name: second-user@example.com
tasks:
- task_key: main
@@ -16,5 +16,5 @@
@@ -58,5 +65,5 @@
name: user-to-creator-[UNIQUE_NAME]
run_as:
- user_name: first-user@example.com
+ user_name: [USERNAME]
tasks:
- task_key: main
@@ -25,5 +25,5 @@
@@ -67,5 +74,5 @@
name: user-to-sp-[UNIQUE_NAME]
run_as:
- user_name: first-user@example.com
Expand All @@ -60,14 +122,21 @@ No changes detected.
=== Plan after sync

>>> [CLI] bundle plan
Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged
create jobs.transferred_owner.permissions

Plan: 1 to add, 0 to change, 0 to delete, 8 unchanged

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.jobs.no_run_as
delete resources.jobs.pending_add
delete resources.jobs.removed_run_as
delete resources.jobs.run_as_to_owner
delete resources.jobs.transferred_owner
delete resources.jobs.user_to_creator
delete resources.jobs.user_to_sp
delete resources.jobs.user_to_user

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default

Destroy: 3 deleted
Destroy: 8 deleted
32 changes: 32 additions & 0 deletions acceptance/bundle/config-remote-sync/job_run_as/script
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,37 @@ trap cleanup EXIT

trace $CLI bundle deploy

title "Remove run_as from config and deploy"
update_file.py databricks.yml " name: removed-run-as-$UNIQUE_NAME
run_as:
user_name: first-user@example.com
" " name: removed-run-as-$UNIQUE_NAME
"
trace $CLI bundle deploy
trace $CLI jobs get "$(read_id.py removed_run_as)" | jq .settings.run_as

title "Add run_as to config without deploying"
update_file.py databricks.yml " name: pending-add-$UNIQUE_NAME
" " name: pending-add-$UNIQUE_NAME
run_as:
user_name: local-user@example.com
"

title "Change run_as remotely"
edit_resource.py jobs "$(read_id.py no_run_as)" <<'PY'
r["run_as"] = {"service_principal_name": "11111111-2222-3333-4444-555555555555"}
PY
edit_resource.py jobs "$(read_id.py run_as_to_owner)" <<PY
r["run_as"] = {"user_name": "$CURRENT_USER_NAME"}
PY
# The sync also imports the new owner into the config as permissions, which the plan then creates.
trace $CLI jobs set-permissions "$(read_id.py transferred_owner)" --json '{"access_control_list": [{"user_name": "new-owner@example.com", "permission_level": "IS_OWNER"}]}' > /dev/null
edit_resource.py jobs "$(read_id.py transferred_owner)" <<'PY'
r["run_as"] = {"user_name": "new-owner@example.com"}
PY
edit_resource.py jobs "$(read_id.py pending_add)" <<'PY'
r["run_as"] = {"user_name": "remote-user@example.com"}
PY
edit_resource.py jobs "$(read_id.py user_to_user)" <<'PY'
r["run_as"] = {"user_name": "second-user@example.com"}
PY
Expand All @@ -18,6 +48,8 @@ edit_resource.py jobs "$(read_id.py user_to_sp)" <<'PY'
r["run_as"] = {"service_principal_name": "11111111-2222-3333-4444-555555555555"}
PY

# A run_as the config does not set is synced unless it names the job's owner:
# no_run_as and removed_run_as get one, run_as_to_owner and transferred_owner do not.
title "Detect and save changes"
cp databricks.yml databricks.yml.backup
trace $CLI bundle config-remote-sync --save
Expand Down
3 changes: 3 additions & 0 deletions acceptance/bundle/config-remote-sync/terraform_state/script
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,11 @@ title "Replay a terraform deployment of the bundle (captured from a real terrafo
trace replay_tfstate.py terraform-requests.json terraform.tfstate
job_id="$(read_id.py test_job)"

# Jobs updated by the terraform engine carry an explicit run_as naming the job's owner. That is what
# a config without run_as means, so only max_concurrent_runs must be detected.
edit_resource.py jobs $job_id <<EOF2
r["max_concurrent_runs"] = 3
r["run_as"] = {"user_name": "$CURRENT_USER_NAME"}
EOF2

title "config-remote-sync migrates the terraform state in memory and detects the remote change"
Expand Down
3 changes: 0 additions & 3 deletions bundle/configsync/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,6 @@ var serverSideDefaults = map[string]any{
"resources.sql_warehouses.*.min_num_clusters": int64(1),
"resources.sql_warehouses.*.warehouse_type": "CLASSIC",

// Terraform defaults
"resources.jobs.*.run_as": alwaysSkip,

// deployment.* is CLI-managed: metadata.AnnotateJobs / AnnotatePipelines write
// kind + metadata_file_path on every deploy, and the Deployment Metadata Service
// sets deployment_id + version_id. None is user-authored (validate.ValidateDeploymentFields
Expand Down
16 changes: 11 additions & 5 deletions bundle/configsync/diff.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,9 @@ func isPermissionsOrGrantsSubResource(resourceKey string) bool {
// plan, and the resources whose remote permissions could not be read.
func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (Changes, []SkippedChange, error) {
changes := make(Changes)
// Read first: a job's ACL names the owner that skipJobRunAs compares run_as to.
unmanaged, skipped := readUnmanagedPermissions(ctx, b, plan)
owners := jobOwners(ctx, b, plan, unmanaged)

for resourceKey, entry := range plan.Plan {
// permissions and grants are emitted as their own plan keys
Expand All @@ -157,7 +160,7 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan

if entry.Changes != nil {
for path, changeDesc := range entry.Changes {
change, err := convertPlanChange(resourceKey, path, changeDesc, b.Config.Presets.NamePrefix)
change, err := convertPlanChange(resourceKey, path, changeDesc, owners[resourceKey], b.Config.Presets.NamePrefix)
if err != nil {
return nil, nil, fmt.Errorf("failed to compute config change for path %s: %w", path, err)
}
Expand All @@ -174,22 +177,25 @@ func ExtractChanges(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan
log.Debugf(ctx, "Resource %s has %d changes", resourceKey, len(resourceChanges))
}

skipped, err := addPermissionsChanges(ctx, changes, b, plan)
if err != nil {
if err := addPermissionsChanges(changes, b, plan, unmanaged); err != nil {
return nil, nil, err
}

return changes, skipped, nil
}

// convertPlanChange converts a plan change at path in resourceKey, returning nil when there is
// nothing to write back.
func convertPlanChange(resourceKey, path string, changeDesc *deployplan.ChangeDesc, namePrefix string) (*ConfigChangeDesc, error) {
// nothing to write back. owner is the principal the resource runs as by default, if it is a job.
func convertPlanChange(resourceKey, path string, changeDesc *deployplan.ChangeDesc, owner, namePrefix string) (*ConfigChangeDesc, error) {
fullPath := resourceKey + "." + path
if changeDesc.Action == deployplan.Skip && changeDesc.Reason != deployplan.ReasonRemoteAddition &&
!keepSkippedRunAsChange(fullPath, changeDesc) {
return nil, nil
}
// Runs before applyLifecycle, which lets every whole run_as through.
if skipJobRunAs(fullPath, changeDesc, owner) {
return nil, nil
}
synced, err := applyLifecycle(resourceKey, path, changeDesc)
if err != nil {
return nil, fmt.Errorf("failed to apply lifecycle rules: %w", err)
Expand Down
2 changes: 1 addition & 1 deletion bundle/configsync/lifecycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ func applyLifecycle(resourceKey, fieldPath string, cd *deployplan.ChangeDesc) (*
return nil, err
}
resourceType := config.GetResourceTypeFromKey(resourceKey)
backendDefault := cd.New == nil && cd.Remote != nil && !matchPattern(jobRunAsChildPattern, resourceKey+"."+fieldPath)
backendDefault := cd.New == nil && cd.Remote != nil && !isJobRunAsPath(resourceKey+"."+fieldPath)

var f nestedFieldFilter
for _, rc := range []*dresources.ResourceLifecycleConfig{
Expand Down
24 changes: 12 additions & 12 deletions bundle/configsync/permissions.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,26 +26,30 @@ const permissionsReadLimit = 10

// addPermissionsChanges adds remote permission changes to the permissions field of their
// resource: per principal where the config declares permissions, and as a whole for deployed
// resources that declare none. It returns the resources whose permissions could not be read.
func addPermissionsChanges(ctx context.Context, changes Changes, b *bundle.Bundle, plan *deployplan.Plan) ([]SkippedChange, error) {
// resources that declare none. unmanaged holds the ACLs read by readUnmanagedPermissions.
func addPermissionsChanges(changes Changes, b *bundle.Bundle, plan *deployplan.Plan, unmanaged map[string][]dresources.StatePermission) error {
for resourceKey, entry := range plan.Plan {
if err := addDeclaredPermissionsChanges(changes, b, resourceKey, entry); err != nil {
return nil, err
return err
}
}

unmanaged, skipped := readUnmanagedPermissions(ctx, b, plan)
currentUser := currentUserName(b)
for resourceKey, permissions := range unmanaged {
permissions = withoutCurrentUserDefaults(slices.Clone(permissions), currentUser, config.GetResourceTypeFromKey(resourceKey))
if len(permissions) == 0 {
continue
}
value, err := normalizeValue(permissions)
if err != nil {
return nil, fmt.Errorf("failed to compute config change for %s: %w", resourceKey, err)
return fmt.Errorf("failed to compute config change for %s: %w", resourceKey, err)
}
changes.set(resourceKey, "permissions", &ConfigChangeDesc{
Operation: OperationAdd,
Value: value,
})
}
return skipped, nil
return nil
}

// addDeclaredPermissionsChanges turns the plan's changes to a resource's permissions into
Expand Down Expand Up @@ -82,7 +86,7 @@ func addDeclaredPermissionsChanges(changes Changes, b *bundle.Bundle, resourceKe
}

fieldPath := "permissions" + path
change, err := convertPlanChange(parentKey, fieldPath, changeDesc, b.Config.Presets.NamePrefix)
change, err := convertPlanChange(parentKey, fieldPath, changeDesc, "", b.Config.Presets.NamePrefix)
if err != nil {
return fmt.Errorf("failed to compute config change for path %s: %w", resourceKey+"."+path, err)
}
Expand Down Expand Up @@ -133,7 +137,6 @@ func readUnmanagedPermissions(ctx context.Context, b *bundle.Bundle, plan *deplo
g.SetLimit(permissionsReadLimit)
result := make(map[string][]dresources.StatePermission)
var skipped []SkippedChange
currentUser := currentUserName(b)

for resourceKey, adapter := range unmanagedPermissionsAdapters(b, plan) {
g.Go(func() error {
Expand All @@ -144,10 +147,7 @@ func readUnmanagedPermissions(ctx context.Context, b *bundle.Bundle, plan *deplo
skipped = append(skipped, permissionsReadSkip(resourceKey, err))
return nil
}
permissions = withoutCurrentUserDefaults(permissions, currentUser, config.GetResourceTypeFromKey(resourceKey))
if len(permissions) > 0 {
result[resourceKey] = permissions
}
result[resourceKey] = permissions
return nil
})
}
Expand Down
Loading
Loading