diff --git a/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl index c655e36c9c2..9c28e400732 100644 --- a/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl +++ b/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl @@ -3,6 +3,48 @@ bundle: resources: jobs: + # run_as is not set in config, so the backend defaults it to the deployer. + no_run_as: + name: no-run-as-$UNIQUE_NAME + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + + # The remote run_as names the owner, which is what no run_as means. + run_as_to_owner: + name: run-as-to-owner-$UNIQUE_NAME + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + + # Ownership moves to another user, who becomes the default run_as. + transferred_owner: + name: transferred-owner-$UNIQUE_NAME + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + + # run_as is removed from config and deployed; the job keeps it remotely. + removed_run_as: + name: removed-run-as-$UNIQUE_NAME + run_as: + user_name: first-user@example.com + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + + # run_as is added to config locally and not deployed. + pending_add: + name: pending-add-$UNIQUE_NAME + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + user_to_user: name: user-to-user-$UNIQUE_NAME run_as: diff --git a/acceptance/bundle/config-remote-sync/job_run_as/output.txt b/acceptance/bundle/config-remote-sync/job_run_as/output.txt index 424cac25200..20e51889880 100644 --- a/acceptance/bundle/config-remote-sync/job_run_as/output.txt +++ b/acceptance/bundle/config-remote-sync/job_run_as/output.txt @@ -1,18 +1,52 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created jobs.no_run_as +Created jobs.pending_add +Created jobs.removed_run_as +Created jobs.run_as_to_owner +Created jobs.transferred_owner Created jobs.user_to_creator Created jobs.user_to_sp Created jobs.user_to_user Files: 2 uploaded, 0 deleted -Resources: 3 created, 0 changed, 0 deleted, 0 unchanged +Resources: 8 created, 0 changed, 0 deleted, 0 unchanged + +=== Remove run_as from config and deploy + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated jobs.removed_run_as +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 7 unchanged + +>>> [CLI] jobs get [REMOVED_RUN_AS_ID] +{ + "user_name": "first-user@example.com" +} + +=== Add run_as to config without deploying === Change run_as remotely +>>> [CLI] jobs set-permissions [TRANSFERRED_OWNER_ID] --json {"access_control_list": [{"user_name": "new-owner@example.com", "permission_level": "IS_OWNER"}]} + === Detect and save changes >>> [CLI] bundle config-remote-sync --save -Detected changes in 3 resource(s): +Detected changes in 7 resource(s): + +Resource: resources.jobs.no_run_as + run_as: add + +Resource: resources.jobs.pending_add + run_as.user_name: replace + +Resource: resources.jobs.removed_run_as + run_as: add + +Resource: resources.jobs.transferred_owner + permissions: add Resource: resources.jobs.user_to_creator run_as.user_name: replace @@ -29,21 +63,49 @@ Resource: resources.jobs.user_to_user >>> diff.py databricks.yml.backup databricks.yml --- databricks.yml.backup +++ databricks.yml -@@ -7,5 +7,5 @@ +@@ -12,4 +12,6 @@ + notebook_path: /Users/{{workspace_user_name}}/notebook + ++ run_as: ++ service_principal_name: [UUID] + # The remote run_as names the owner, which is what no run_as means. + run_as_to_owner: +@@ -28,4 +30,7 @@ + notebook_path: /Users/{{workspace_user_name}}/notebook + ++ permissions: ++ - level: IS_OWNER ++ user_name: new-owner@example.com + # run_as is removed from config and deployed; the job keeps it remotely. + removed_run_as: +@@ -36,9 +41,11 @@ + notebook_path: /Users/{{workspace_user_name}}/notebook + ++ run_as: ++ user_name: first-user@example.com + # run_as is added to config locally and not deployed. + pending_add: + name: pending-add-[UNIQUE_NAME] + run_as: +- user_name: local-user@example.com ++ user_name: remote-user@example.com + tasks: + - task_key: main +@@ -49,5 +56,5 @@ name: user-to-user-[UNIQUE_NAME] run_as: - user_name: first-user@example.com + user_name: second-user@example.com tasks: - task_key: main -@@ -16,5 +16,5 @@ +@@ -58,5 +65,5 @@ name: user-to-creator-[UNIQUE_NAME] run_as: - user_name: first-user@example.com + user_name: [USERNAME] tasks: - task_key: main -@@ -25,5 +25,5 @@ +@@ -67,5 +74,5 @@ name: user-to-sp-[UNIQUE_NAME] run_as: - user_name: first-user@example.com @@ -60,14 +122,21 @@ No changes detected. === Plan after sync >>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged +create jobs.transferred_owner.permissions + +Plan: 1 to add, 0 to change, 0 to delete, 8 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: + delete resources.jobs.no_run_as + delete resources.jobs.pending_add + delete resources.jobs.removed_run_as + delete resources.jobs.run_as_to_owner + delete resources.jobs.transferred_owner delete resources.jobs.user_to_creator delete resources.jobs.user_to_sp delete resources.jobs.user_to_user All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default -Destroy: 3 deleted +Destroy: 8 deleted diff --git a/acceptance/bundle/config-remote-sync/job_run_as/script b/acceptance/bundle/config-remote-sync/job_run_as/script index 5beb36780a5..ce0deed718f 100644 --- a/acceptance/bundle/config-remote-sync/job_run_as/script +++ b/acceptance/bundle/config-remote-sync/job_run_as/script @@ -7,7 +7,37 @@ trap cleanup EXIT trace $CLI bundle deploy +title "Remove run_as from config and deploy" +update_file.py databricks.yml " name: removed-run-as-$UNIQUE_NAME + run_as: + user_name: first-user@example.com +" " name: removed-run-as-$UNIQUE_NAME +" +trace $CLI bundle deploy +trace $CLI jobs get "$(read_id.py removed_run_as)" | jq .settings.run_as + +title "Add run_as to config without deploying" +update_file.py databricks.yml " name: pending-add-$UNIQUE_NAME +" " name: pending-add-$UNIQUE_NAME + run_as: + user_name: local-user@example.com +" + title "Change run_as remotely" +edit_resource.py jobs "$(read_id.py no_run_as)" <<'PY' +r["run_as"] = {"service_principal_name": "11111111-2222-3333-4444-555555555555"} +PY +edit_resource.py jobs "$(read_id.py run_as_to_owner)" < /dev/null +edit_resource.py jobs "$(read_id.py transferred_owner)" <<'PY' +r["run_as"] = {"user_name": "new-owner@example.com"} +PY +edit_resource.py jobs "$(read_id.py pending_add)" <<'PY' +r["run_as"] = {"user_name": "remote-user@example.com"} +PY edit_resource.py jobs "$(read_id.py user_to_user)" <<'PY' r["run_as"] = {"user_name": "second-user@example.com"} PY @@ -18,6 +48,8 @@ edit_resource.py jobs "$(read_id.py user_to_sp)" <<'PY' r["run_as"] = {"service_principal_name": "11111111-2222-3333-4444-555555555555"} PY +# A run_as the config does not set is synced unless it names the job's owner: +# no_run_as and removed_run_as get one, run_as_to_owner and transferred_owner do not. title "Detect and save changes" cp databricks.yml databricks.yml.backup trace $CLI bundle config-remote-sync --save diff --git a/acceptance/bundle/config-remote-sync/terraform_state/script b/acceptance/bundle/config-remote-sync/terraform_state/script index 31d7b70bb7f..12de0a54152 100644 --- a/acceptance/bundle/config-remote-sync/terraform_state/script +++ b/acceptance/bundle/config-remote-sync/terraform_state/script @@ -8,8 +8,11 @@ title "Replay a terraform deployment of the bundle (captured from a real terrafo trace replay_tfstate.py terraform-requests.json terraform.tfstate job_id="$(read_id.py test_job)" +# Jobs updated by the terraform engine carry an explicit run_as naming the job's owner. That is what +# a config without run_as means, so only max_concurrent_runs must be detected. edit_resource.py jobs $job_id < 0 { - result[resourceKey] = permissions - } + result[resourceKey] = permissions return nil }) } diff --git a/bundle/configsync/runas.go b/bundle/configsync/runas.go index 3faea944f41..3e25328c4fa 100644 --- a/bundle/configsync/runas.go +++ b/bundle/configsync/runas.go @@ -1,15 +1,137 @@ package configsync -import "github.com/databricks/cli/bundle/deployplan" +import ( + "cmp" + "context" + "errors" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/databricks/databricks-sdk-go/service/jobs" +) // The plan skips a job's run_as keys that the config lacks as backend defaults. // run_as holds a single principal key, so when the remote switches kind (for // example user_name to service_principal_name) the new key is the edit itself, // and dropping it while removing the old key leaves "run_as: {}". -const jobRunAsChildPattern = "resources.jobs.*.run_as.*" +const ( + runAsField = "run_as" + jobRunAsPattern = "resources.jobs.*." + runAsField + jobRunAsChildPattern = jobRunAsPattern + ".*" +) + +// errPermissionsNotRead reports that a job's ACL was not read, so its owner is unknown. +var errPermissionsNotRead = errors.New("permissions were not read") + +// isJobRunAsPath reports whether path is a job's run_as or one of its keys. +// Unlike other backend defaults, these are synced when the config lacks them: +// the keys always, the whole object only as decided by skipJobRunAs. +func isJobRunAsPath(path string) bool { + return matchPattern(jobRunAsPattern, path) || matchPattern(jobRunAsChildPattern, path) +} // keepSkippedRunAsChange reports whether a run_as change the plan skipped as a // backend default should still be considered for syncing. func keepSkippedRunAsChange(path string, cd *deployplan.ChangeDesc) bool { - return cd.Reason == deployplan.ReasonBackendDefault && matchPattern(jobRunAsChildPattern, path) + return cd.Reason == deployplan.ReasonBackendDefault && isJobRunAsPath(path) +} + +// skipJobRunAs reports whether a change to the run_as object as a whole must +// not be synced. Changes to its keys are synced like any other field. +// +// A config without run_as means the job runs as its owner, and the Jobs UI +// shows that owner until a principal is picked. So a remote run_as naming the +// owner says what the config already says and is not synced. Any other remote +// run_as is the identity the job runs as, whether it was set in the Jobs UI or +// kept by the server after run_as was removed from the config (jobs/reset +// without run_as keeps the previous one), so it is synced. +func skipJobRunAs(path string, cd *deployplan.ChangeDesc, owner string) bool { + if !matchPattern(jobRunAsPattern, path) { + return false + } + // The config sets run_as, so the plan reports the remote edit key by key. + if cd.New != nil { + return true + } + runAs, ok := remoteRunAs(cd.Remote) + // The Jobs UI always submits a principal, so a removed remote run_as is not + // an edit to follow. + if !ok { + return true + } + // Without a known owner, a remote run_as can't be told apart from the default. + return owner == "" || cmp.Or(runAs.UserName, runAs.ServicePrincipalName, runAs.GroupName) == owner +} + +func remoteRunAs(remote any) (jobs.JobRunAs, bool) { + switch v := remote.(type) { + case jobs.JobRunAs: + return v, true + case *jobs.JobRunAs: + if v != nil { + return *v, true + } + } + return jobs.JobRunAs{}, false +} + +// jobOwners returns, keyed by plan key, the owner of each job with a remote +// run_as the config does not set: the principal the job runs as without one. +// A job whose ACL could not be read is left out. unmanaged holds the ACLs read +// by readUnmanagedPermissions. +func jobOwners(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, unmanaged map[string][]dresources.StatePermission) map[string]string { + owners := make(map[string]string) + for resourceKey, entry := range plan.Plan { + job, ok := entry.RemoteState.(*dresources.JobRemote) + cd := entry.Changes[runAsField] + if !ok || job == nil || cd == nil || cd.New != nil || cd.Remote == nil { + continue + } + acl, err := jobACL(ctx, b, plan, resourceKey, unmanaged) + if err != nil { + log.Debugf(ctx, "config-remote-sync: not syncing the run_as of %s, its owner is unknown: %v", resourceKey, err) + continue + } + owners[resourceKey] = jobOwner(acl, job.CreatorUserName) + } + return owners +} + +// jobACL returns a job's remote ACL. Permissions that the config declares but +// were never deployed have no remote state in the plan, so they are read here. +func jobACL(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, resourceKey string, unmanaged map[string][]dresources.StatePermission) ([]dresources.StatePermission, error) { + permissionsKey := resourceKey + ".permissions" + entry := plan.Plan[permissionsKey] + if entry == nil { + // Read with the unmanaged permissions; missing if that read failed. + acl, ok := unmanaged[resourceKey] + if !ok { + return nil, errPermissionsNotRead + } + return acl, nil + } + if state, ok := entry.RemoteState.(*dresources.PermissionsState); ok && state != nil { + return state.EmbeddedSlice, nil + } + adapter := b.DeploymentBundle.Adapters[config.GetResourceTypeFromKey(permissionsKey)] + if adapter == nil { + return nil, errPermissionsNotRead + } + return readPermissions(ctx, b, adapter, resourceKey) +} + +// jobOwner returns the principal a job without run_as runs as: the IS_OWNER +// entry of its ACL or, when job access control is off, its creator. See +// run_as_user_name in the Jobs API. +func jobOwner(acl []dresources.StatePermission, creator string) string { + for _, p := range acl { + if p.Level == iam.PermissionLevelIsOwner { + return cmp.Or(p.UserName, p.ServicePrincipalName) + } + } + return creator } diff --git a/bundle/configsync/runas_test.go b/bundle/configsync/runas_test.go index 3c1cfbf0f61..e81a2701d3b 100644 --- a/bundle/configsync/runas_test.go +++ b/bundle/configsync/runas_test.go @@ -3,11 +3,18 @@ package configsync import ( "testing" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" ) -const testRunAsPath = "resources.jobs.my_job.run_as" +const ( + testRunAsPath = "resources.jobs.my_job.run_as" + testSPID = "11111111-2222-3333-4444-555555555555" +) func TestKeepSkippedRunAsChange(t *testing.T) { backendDefault := &deployplan.ChangeDesc{Action: deployplan.Skip, Reason: deployplan.ReasonBackendDefault} @@ -20,7 +27,7 @@ func TestKeepSkippedRunAsChange(t *testing.T) { {"service_principal_name backend default", testRunAsPath + ".service_principal_name", backendDefault, true}, {"user_name backend default", testRunAsPath + ".user_name", backendDefault, true}, {"group_name backend default", testRunAsPath + ".group_name", backendDefault, true}, - {"whole run_as backend default", testRunAsPath, backendDefault, false}, + {"whole run_as backend default", testRunAsPath, backendDefault, true}, {"run_as key already matches the remote", testRunAsPath + ".user_name", &deployplan.ChangeDesc{Action: deployplan.Skip, Reason: deployplan.ReasonRemoteAlreadySet}, false}, {"other job field", "resources.jobs.my_job.tasks[task_key='main'].new_cluster.enable_elastic_disk", backendDefault, false}, {"pipeline run_as key", "resources.pipelines.my_pipeline.run_as.user_name", backendDefault, false}, @@ -31,3 +38,103 @@ func TestKeepSkippedRunAsChange(t *testing.T) { }) } } + +func TestSkipJobRunAs(t *testing.T) { + sp := jobs.JobRunAs{ServicePrincipalName: testSPID} + user := jobs.JobRunAs{UserName: "user@example.com"} + owner := "owner@example.com" + + tests := []struct { + name string + path string + cd *deployplan.ChangeDesc + want bool + }{ + {"set remotely to another principal", testRunAsPath, &deployplan.ChangeDesc{Remote: sp}, false}, + {"set remotely as a pointer", testRunAsPath, &deployplan.ChangeDesc{Remote: &sp}, false}, + {"set remotely to the owner", testRunAsPath, &deployplan.ChangeDesc{Remote: jobs.JobRunAs{UserName: owner}}, true}, + {"set remotely to the owner as a service principal", testRunAsPath, &deployplan.ChangeDesc{Remote: jobs.JobRunAs{ServicePrincipalName: owner}}, true}, + {"removed from config, kept remotely", testRunAsPath, &deployplan.ChangeDesc{Old: user, Remote: user}, false}, + {"set in config and remotely", testRunAsPath, &deployplan.ChangeDesc{New: user, Remote: sp}, true}, + {"removed remotely", testRunAsPath, &deployplan.ChangeDesc{Old: user, New: user}, true}, + {"removed from config and remotely", testRunAsPath, &deployplan.ChangeDesc{Old: user}, true}, + {"run_as key", testRunAsPath + ".service_principal_name", &deployplan.ChangeDesc{Remote: testSPID}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, skipJobRunAs(tt.path, tt.cd, owner)) + }) + } + + // Without a known owner, no remote run_as is synced. + assert.True(t, skipJobRunAs(testRunAsPath, &deployplan.ChangeDesc{Remote: sp}, "")) +} + +func TestJobOwners(t *testing.T) { + const jobKey = "resources.jobs.my_job" + unsetRunAs := map[string]*deployplan.ChangeDesc{"run_as": {Remote: jobs.JobRunAs{UserName: "user@example.com"}}} + job := &deployplan.PlanEntry{ + RemoteState: &dresources.JobRemote{CreatorUserName: "creator@example.com"}, + Changes: unsetRunAs, + } + acl := []dresources.StatePermission{ + {Level: iam.PermissionLevelCanManage, UserName: "manager@example.com"}, + {Level: iam.PermissionLevelIsOwner, ServicePrincipalName: testSPID}, + } + + tests := []struct { + name string + plan map[string]*deployplan.PlanEntry + unmanaged map[string][]dresources.StatePermission + want map[string]string + }{ + { + name: "owner from the unmanaged ACL", + plan: map[string]*deployplan.PlanEntry{jobKey: job}, + unmanaged: map[string][]dresources.StatePermission{jobKey: acl}, + want: map[string]string{jobKey: testSPID}, + }, + { + name: "owner from the declared permissions", + plan: map[string]*deployplan.PlanEntry{ + jobKey: job, + jobKey + ".permissions": {RemoteState: &dresources.PermissionsState{EmbeddedSlice: acl}}, + }, + want: map[string]string{jobKey: testSPID}, + }, + { + name: "creator when the ACL has no owner", + plan: map[string]*deployplan.PlanEntry{jobKey: job}, + unmanaged: map[string][]dresources.StatePermission{jobKey: acl[:1]}, + want: map[string]string{jobKey: "creator@example.com"}, + }, + { + name: "unknown when the ACL was not read", + plan: map[string]*deployplan.PlanEntry{jobKey: job}, + want: map[string]string{}, + }, + { + name: "unknown when declared permissions can't be read", + plan: map[string]*deployplan.PlanEntry{jobKey: job, jobKey + ".permissions": {}}, + want: map[string]string{}, + }, + { + name: "run_as set in config", + plan: map[string]*deployplan.PlanEntry{jobKey: { + RemoteState: job.RemoteState, + Changes: map[string]*deployplan.ChangeDesc{"run_as": {New: jobs.JobRunAs{UserName: "user@example.com"}, Remote: jobs.JobRunAs{UserName: "user@example.com"}}}, + }}, + want: map[string]string{}, + }, + { + name: "not a job", + plan: map[string]*deployplan.PlanEntry{jobKey: {RemoteState: &dresources.PermissionsState{}, Changes: unsetRunAs}}, + want: map[string]string{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, jobOwners(t.Context(), &bundle.Bundle{}, &deployplan.Plan{Plan: tt.plan}, tt.unmanaged)) + }) + } +}