From fba3d80548184404bfacddfd2a970bb25be7fbf5 Mon Sep 17 00:00:00 2001 From: Petra Postelnicu Date: Wed, 11 Nov 2026 17:39:06 +0000 Subject: [PATCH 1/6] Sync a remote run_as on jobs whose config leaves it unset config-remote-sync skipped a job's run_as entirely, so a run-as change made in the Jobs UI never reached a config without run_as. Replace the blanket skip with a rule for the whole run_as object: sync it unless the remote has none, it is unchanged since the last deploy, or it names the job's creator. Jobs updated by the terraform engine carry an explicit run_as set to the job's identity, which the creator check filters out. Co-authored-by: Isaac --- .../config-remote-sync-run-as-unset.md | 1 + .../job_run_as/databricks.yml.tmpl | 18 ++++ .../config-remote-sync/job_run_as/output.txt | 28 ++++-- .../config-remote-sync/job_run_as/script | 6 ++ .../config-remote-sync/terraform_state/script | 3 + bundle/configsync/defaults.go | 3 - bundle/configsync/diff.go | 3 + bundle/configsync/runas.go | 44 +++++++++- bundle/configsync/runas_test.go | 88 ++++++++++++++++++- 9 files changed, 179 insertions(+), 15 deletions(-) create mode 100644 .nextchanges/bundles/config-remote-sync-run-as-unset.md diff --git a/.nextchanges/bundles/config-remote-sync-run-as-unset.md b/.nextchanges/bundles/config-remote-sync-run-as-unset.md new file mode 100644 index 00000000000..379ce6f0b17 --- /dev/null +++ b/.nextchanges/bundles/config-remote-sync-run-as-unset.md @@ -0,0 +1 @@ +* `bundle config-remote-sync` now syncs a `run_as` set remotely, for example in the Jobs UI, on a job whose config leaves it unset, unless it names the job's creator. diff --git a/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl index c655e36c9c2..cf5653eeec2 100644 --- a/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl +++ b/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl @@ -3,6 +3,23 @@ bundle: resources: jobs: + # run_as is not set in config, so the backend defaults it to the deployer. + no_run_as: + name: no-run-as-$UNIQUE_NAME + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + + # The terraform provider pins run_as to the job's identity on every update, + # so a remote run_as that names the creator is a default, not an edit. + no_run_as_creator: + name: no-run-as-creator-$UNIQUE_NAME + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + user_to_user: name: user-to-user-$UNIQUE_NAME run_as: @@ -12,6 +29,7 @@ resources: notebook_task: notebook_path: /Users/{{workspace_user_name}}/notebook + # The creator default only applies when the config leaves run_as unset. user_to_creator: name: user-to-creator-$UNIQUE_NAME run_as: diff --git a/acceptance/bundle/config-remote-sync/job_run_as/output.txt b/acceptance/bundle/config-remote-sync/job_run_as/output.txt index 424cac25200..6080e872493 100644 --- a/acceptance/bundle/config-remote-sync/job_run_as/output.txt +++ b/acceptance/bundle/config-remote-sync/job_run_as/output.txt @@ -1,18 +1,23 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Created jobs.no_run_as +Created jobs.no_run_as_creator Created jobs.user_to_creator Created jobs.user_to_sp Created jobs.user_to_user Files: 2 uploaded, 0 deleted -Resources: 3 created, 0 changed, 0 deleted, 0 unchanged +Resources: 5 created, 0 changed, 0 deleted, 0 unchanged === Change run_as remotely === Detect and save changes >>> [CLI] bundle config-remote-sync --save -Detected changes in 3 resource(s): +Detected changes in 4 resource(s): + +Resource: resources.jobs.no_run_as + run_as: add Resource: resources.jobs.user_to_creator run_as.user_name: replace @@ -29,21 +34,28 @@ Resource: resources.jobs.user_to_user >>> diff.py databricks.yml.backup databricks.yml --- databricks.yml.backup +++ databricks.yml -@@ -7,5 +7,5 @@ +@@ -12,4 +12,6 @@ + notebook_path: /Users/{{workspace_user_name}}/notebook + ++ run_as: ++ service_principal_name: [UUID] + # The terraform provider pins run_as to the job's identity on every update, + # so a remote run_as that names the creator is a default, not an edit. +@@ -24,5 +26,5 @@ name: user-to-user-[UNIQUE_NAME] run_as: - user_name: first-user@example.com + user_name: second-user@example.com tasks: - task_key: main -@@ -16,5 +16,5 @@ +@@ -34,5 +36,5 @@ name: user-to-creator-[UNIQUE_NAME] run_as: - user_name: first-user@example.com + user_name: [USERNAME] tasks: - task_key: main -@@ -25,5 +25,5 @@ +@@ -43,5 +45,5 @@ name: user-to-sp-[UNIQUE_NAME] run_as: - user_name: first-user@example.com @@ -60,14 +72,16 @@ No changes detected. === Plan after sync >>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: + delete resources.jobs.no_run_as + delete resources.jobs.no_run_as_creator delete resources.jobs.user_to_creator delete resources.jobs.user_to_sp delete resources.jobs.user_to_user All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default -Destroy: 3 deleted +Destroy: 5 deleted diff --git a/acceptance/bundle/config-remote-sync/job_run_as/script b/acceptance/bundle/config-remote-sync/job_run_as/script index 5beb36780a5..ed68fef3637 100644 --- a/acceptance/bundle/config-remote-sync/job_run_as/script +++ b/acceptance/bundle/config-remote-sync/job_run_as/script @@ -8,6 +8,12 @@ trap cleanup EXIT trace $CLI bundle deploy title "Change run_as remotely" +edit_resource.py jobs "$(read_id.py no_run_as)" <<'PY' +r["run_as"] = {"service_principal_name": "11111111-2222-3333-4444-555555555555"} +PY +edit_resource.py jobs "$(read_id.py no_run_as_creator)" < Date: Wed, 11 Nov 2026 17:44:58 +0000 Subject: [PATCH 2/6] Add PR link to changelog entry Co-authored-by: Isaac --- .nextchanges/bundles/config-remote-sync-run-as-unset.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/config-remote-sync-run-as-unset.md b/.nextchanges/bundles/config-remote-sync-run-as-unset.md index 379ce6f0b17..dfb90c282fe 100644 --- a/.nextchanges/bundles/config-remote-sync-run-as-unset.md +++ b/.nextchanges/bundles/config-remote-sync-run-as-unset.md @@ -1 +1 @@ -* `bundle config-remote-sync` now syncs a `run_as` set remotely, for example in the Jobs UI, on a job whose config leaves it unset, unless it names the job's creator. +* `bundle config-remote-sync` now syncs a `run_as` set remotely, for example in the Jobs UI, on a job whose config leaves it unset, unless it names the job's creator. ([#7494](https://github.com/databricks/cli/pull/7494)) From 9d7317e25508ff3cf4e89be8823b92f11fac17e1 Mon Sep 17 00:00:00 2001 From: Petra Postelnicu Date: Fri, 13 Nov 2026 12:15:39 +0000 Subject: [PATCH 3/6] Remove changelog entry config-remote-sync is a hidden command used only by DABs in the workspace; changes to it do not get release notes. Co-authored-by: Isaac --- .nextchanges/bundles/config-remote-sync-run-as-unset.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .nextchanges/bundles/config-remote-sync-run-as-unset.md diff --git a/.nextchanges/bundles/config-remote-sync-run-as-unset.md b/.nextchanges/bundles/config-remote-sync-run-as-unset.md deleted file mode 100644 index dfb90c282fe..00000000000 --- a/.nextchanges/bundles/config-remote-sync-run-as-unset.md +++ /dev/null @@ -1 +0,0 @@ -* `bundle config-remote-sync` now syncs a `run_as` set remotely, for example in the Jobs UI, on a job whose config leaves it unset, unless it names the job's creator. ([#7494](https://github.com/databricks/cli/pull/7494)) From 55b720ede2e48eb699b637006be054ac6331ea45 Mon Sep 17 00:00:00 2001 From: Petra Postelnicu Date: Mon, 16 Nov 2026 10:25:00 +0000 Subject: [PATCH 4/6] Sync an unset run_as only for jobs passed in --sync-run-as-ids A remote run_as on a job whose config has none is either a Jobs UI edit or a value the server kept: jobs/reset without run_as keeps the previous run_as, so a run_as removed from the config and deployed stays on the job. Every sync wrote it back. Only the caller knows which case applies, so a whole-object run_as is now synced only for jobs listed in the new --sync-run-as-ids flag. This replaces the creator heuristic, which also missed jobs whose ownership was transferred. When the config sets run_as, the plan reports the remote edit key by key, so the whole-object change is skipped instead of emitted as a duplicate replace. Co-authored-by: Isaac --- .../job_run_as/databricks.yml.tmpl | 25 +++- .../config-remote-sync/job_run_as/output.txt | 60 +++++++--- .../config-remote-sync/job_run_as/script | 34 +++++- .../terraform_state/output.txt | 27 +++++ .../config-remote-sync/terraform_state/script | 9 +- bundle/configsync/cluster_policy_test.go | 2 +- bundle/configsync/diff.go | 7 +- bundle/configsync/diff_test.go | 2 +- bundle/configsync/runas.go | 69 +++++++---- bundle/configsync/runas_test.go | 110 +++++++++--------- cmd/bundle/config_remote_sync.go | 17 ++- 11 files changed, 256 insertions(+), 106 deletions(-) diff --git a/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl index cf5653eeec2..7bfde32e50e 100644 --- a/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl +++ b/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl @@ -11,10 +11,26 @@ resources: notebook_task: notebook_path: /Users/{{workspace_user_name}}/notebook - # The terraform provider pins run_as to the job's identity on every update, - # so a remote run_as that names the creator is a default, not an edit. - no_run_as_creator: - name: no-run-as-creator-$UNIQUE_NAME + not_selected: + name: not-selected-$UNIQUE_NAME + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + + # run_as is removed from config and deployed; the job keeps it remotely. + removed_run_as: + name: removed-run-as-$UNIQUE_NAME + run_as: + user_name: first-user@example.com + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + + # run_as is added to config locally and not deployed. + pending_add: + name: pending-add-$UNIQUE_NAME tasks: - task_key: main notebook_task: @@ -29,7 +45,6 @@ resources: notebook_task: notebook_path: /Users/{{workspace_user_name}}/notebook - # The creator default only applies when the config leaves run_as unset. user_to_creator: name: user-to-creator-$UNIQUE_NAME run_as: diff --git a/acceptance/bundle/config-remote-sync/job_run_as/output.txt b/acceptance/bundle/config-remote-sync/job_run_as/output.txt index 6080e872493..2ac4035f742 100644 --- a/acceptance/bundle/config-remote-sync/job_run_as/output.txt +++ b/acceptance/bundle/config-remote-sync/job_run_as/output.txt @@ -2,23 +2,43 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created jobs.no_run_as -Created jobs.no_run_as_creator +Created jobs.not_selected +Created jobs.pending_add +Created jobs.removed_run_as Created jobs.user_to_creator Created jobs.user_to_sp Created jobs.user_to_user Files: 2 uploaded, 0 deleted -Resources: 5 created, 0 changed, 0 deleted, 0 unchanged +Resources: 7 created, 0 changed, 0 deleted, 0 unchanged + +=== Remove run_as from config and deploy + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Updated jobs.removed_run_as +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 6 unchanged + +>>> [CLI] jobs get [REMOVED_RUN_AS_ID] +{ + "user_name": "first-user@example.com" +} + +=== Add run_as to config without deploying === Change run_as remotely === Detect and save changes ->>> [CLI] bundle config-remote-sync --save -Detected changes in 4 resource(s): +>>> [CLI] bundle config-remote-sync --save --sync-run-as-ids jobs:[NO_RUN_AS_ID],jobs:[PENDING_ADD_ID] +Detected changes in 5 resource(s): Resource: resources.jobs.no_run_as run_as: add +Resource: resources.jobs.pending_add + run_as.user_name: replace + Resource: resources.jobs.user_to_creator run_as.user_name: replace @@ -39,23 +59,30 @@ Resource: resources.jobs.user_to_user + run_as: + service_principal_name: [UUID] - # The terraform provider pins run_as to the job's identity on every update, - # so a remote run_as that names the creator is a default, not an edit. -@@ -24,5 +26,5 @@ + not_selected: + name: not-selected-[UNIQUE_NAME] +@@ -31,5 +33,5 @@ + name: pending-add-[UNIQUE_NAME] + run_as: +- user_name: local-user@example.com ++ user_name: remote-user@example.com + tasks: + - task_key: main +@@ -40,5 +42,5 @@ name: user-to-user-[UNIQUE_NAME] run_as: - user_name: first-user@example.com + user_name: second-user@example.com tasks: - task_key: main -@@ -34,5 +36,5 @@ +@@ -49,5 +51,5 @@ name: user-to-creator-[UNIQUE_NAME] run_as: - user_name: first-user@example.com + user_name: [USERNAME] tasks: - task_key: main -@@ -43,5 +45,5 @@ +@@ -58,5 +60,5 @@ name: user-to-sp-[UNIQUE_NAME] run_as: - user_name: first-user@example.com @@ -65,23 +92,30 @@ Resource: resources.jobs.user_to_user === Second sync is a no-op ->>> [CLI] bundle config-remote-sync +>>> [CLI] bundle config-remote-sync --sync-run-as-ids jobs:[NO_RUN_AS_ID],jobs:[PENDING_ADD_ID] No changes detected. === Plan after sync >>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 7 unchanged + +=== Invalid --sync-run-as-ids value + +>>> musterr [CLI] bundle config-remote-sync --sync-run-as-ids pipelines:123 +Error: invalid --sync-run-as-ids value "pipelines:123", expected jobs: (e.g. jobs:123456789) >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.no_run_as - delete resources.jobs.no_run_as_creator + delete resources.jobs.not_selected + delete resources.jobs.pending_add + delete resources.jobs.removed_run_as delete resources.jobs.user_to_creator delete resources.jobs.user_to_sp delete resources.jobs.user_to_user All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default -Destroy: 5 deleted +Destroy: 7 deleted diff --git a/acceptance/bundle/config-remote-sync/job_run_as/script b/acceptance/bundle/config-remote-sync/job_run_as/script index ed68fef3637..d6c7ba03f22 100644 --- a/acceptance/bundle/config-remote-sync/job_run_as/script +++ b/acceptance/bundle/config-remote-sync/job_run_as/script @@ -7,12 +7,31 @@ trap cleanup EXIT trace $CLI bundle deploy +title "Remove run_as from config and deploy" +update_file.py databricks.yml " name: removed-run-as-$UNIQUE_NAME + run_as: + user_name: first-user@example.com +" " name: removed-run-as-$UNIQUE_NAME +" +trace $CLI bundle deploy +trace $CLI jobs get "$(read_id.py removed_run_as)" | jq .settings.run_as + +title "Add run_as to config without deploying" +update_file.py databricks.yml " name: pending-add-$UNIQUE_NAME +" " name: pending-add-$UNIQUE_NAME + run_as: + user_name: local-user@example.com +" + title "Change run_as remotely" edit_resource.py jobs "$(read_id.py no_run_as)" <<'PY' r["run_as"] = {"service_principal_name": "11111111-2222-3333-4444-555555555555"} PY -edit_resource.py jobs "$(read_id.py no_run_as_creator)" <>> [CLI] bundle config-remote-sync --save --sync-run-as-ids jobs:[TEST_JOB_ID] +Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). +Detected changes in 1 resource(s): + +Resource: resources.jobs.test_job + run_as: add + + + +>>> cat databricks.yml +bundle: + name: test-bundle + +resources: + jobs: + test_job: + name: test-job + max_concurrent_runs: 3 + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + run_as: + user_name: [USERNAME] + === Cleanup >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/config-remote-sync/terraform_state/script b/acceptance/bundle/config-remote-sync/terraform_state/script index ffa1a1a9452..f4b42c8662b 100644 --- a/acceptance/bundle/config-remote-sync/terraform_state/script +++ b/acceptance/bundle/config-remote-sync/terraform_state/script @@ -8,8 +8,9 @@ title "Replay a terraform deployment of the bundle (captured from a real terrafo trace replay_tfstate.py terraform-requests.json terraform.tfstate job_id="$(read_id.py test_job)" -# Jobs updated by the terraform engine carry an explicit run_as naming the job's identity; that is -# not a remote edit, so only max_concurrent_runs must be detected. +# Jobs updated by the terraform engine carry an explicit run_as naming the job's identity. A run_as +# the config does not set is synced only with --sync-run-as-ids, so only max_concurrent_runs must be +# detected. edit_resource.py jobs $job_id <", whose remote run_as is synced +// even when the config does not set one. +type RunAsSelection []string + +// ParseRunAsSelection validates the --sync-run-as-ids selectors. +func ParseRunAsSelection(selectors []string) (RunAsSelection, error) { + for _, selector := range selectors { + id, ok := strings.CutPrefix(selector, jobsSelectorPrefix) + if !ok || id == "" { + return nil, fmt.Errorf("invalid --sync-run-as-ids value %q, expected jobs: (e.g. jobs:123456789)", selector) + } + } + return selectors, nil +} + +// ResourceKeys maps the selected jobs to their plan keys using the index from +// IndexDeployedResources. A job this bundle has not deployed is ignored: it has +// no changes to sync. +func (s RunAsSelection) ResourceKeys(ctx context.Context, deployed map[string]string) map[string]bool { + keys := make(map[string]bool, len(s)) + for _, selector := range s { + key, ok := deployed[selector] + if !ok { + log.Debugf(ctx, "config-remote-sync: --sync-run-as-ids %s is not deployed by this bundle", selector) + continue + } + keys[key] = true + } + return keys +} + // keepSkippedRunAsChange reports whether a run_as change the plan skipped as a // backend default should still be considered for syncing. func keepSkippedRunAsChange(path string, cd *deployplan.ChangeDesc) bool { @@ -26,28 +58,23 @@ func keepSkippedRunAsChange(path string, cd *deployplan.ChangeDesc) bool { } // skipJobRunAs reports whether a change to the run_as object as a whole must -// not be synced. The plan diffs a run_as set on both sides key by key, so a -// change to the whole object means one side has none. -func skipJobRunAs(path string, cd *deployplan.ChangeDesc, remoteState any) bool { +// not be synced. Changes to its keys are synced like any other field. +// +// A remote run_as the config does not set can be a Jobs UI edit or a value the +// server kept: jobs/reset without run_as keeps the previous one. Only the +// caller can tell them apart, so it is synced only for selected jobs. +func skipJobRunAs(path string, cd *deployplan.ChangeDesc, selected bool) bool { if !matchPattern(jobRunAsPattern, path) { return false } - // A removed remote run_as is not synced, as before: the Jobs UI always - // submits a principal, and removal is unverified on a real workspace. - if cd.Remote == nil { + // The config sets run_as, so the plan reports the remote edit key by key. + if cd.New != nil { return true } - // The config has no run_as. Skip one unchanged since the last deploy. - if structdiff.IsEqual(cd.Old, cd.Remote) { + // The Jobs UI always submits a principal, so a removed remote run_as is not + // an edit to follow. + if cd.Remote == nil { return true } - return isCreatorRunAs(cd.Remote.(jobs.JobRunAs), remoteState.(*dresources.JobRemote)) -} - -// isCreatorRunAs reports whether run_as names the job's creator. Jobs updated by -// the terraform engine before it was removed carry an explicit run_as set to the -// job's identity, usually its creator, which is a default rather than an edit: -// https://github.com/databricks/terraform-provider-databricks/blob/76f95e4/jobs/resource_job.go#L844-L857 -func isCreatorRunAs(runAs jobs.JobRunAs, job *dresources.JobRemote) bool { - return cmp.Or(runAs.UserName, runAs.ServicePrincipalName, runAs.GroupName) == job.CreatorUserName + return !selected } diff --git a/bundle/configsync/runas_test.go b/bundle/configsync/runas_test.go index c72a2bb2b40..ae06e3cfb55 100644 --- a/bundle/configsync/runas_test.go +++ b/bundle/configsync/runas_test.go @@ -4,14 +4,13 @@ import ( "testing" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const ( testRunAsPath = "resources.jobs.my_job.run_as" - testCreator = "creator@example.com" testSPID = "11111111-2222-3333-4444-555555555555" ) @@ -41,77 +40,82 @@ func TestKeepSkippedRunAsChange(t *testing.T) { func TestSkipJobRunAs(t *testing.T) { sp := jobs.JobRunAs{ServicePrincipalName: testSPID} user := jobs.JobRunAs{UserName: "user@example.com"} - creator := jobs.JobRunAs{UserName: testCreator} - group := jobs.JobRunAs{GroupName: "data-engineers"} tests := []struct { - name string - path string - cd *deployplan.ChangeDesc - creator string - want bool + name string + path string + cd *deployplan.ChangeDesc + selected bool + want bool }{ { - name: "service principal set remotely", - path: testRunAsPath, - cd: &deployplan.ChangeDesc{Remote: sp}, - creator: testCreator, - want: false, + name: "set remotely on a selected job", + path: testRunAsPath, + cd: &deployplan.ChangeDesc{Remote: sp}, + selected: true, + want: false, }, { - name: "group set remotely", - path: testRunAsPath, - cd: &deployplan.ChangeDesc{Remote: group}, - creator: testCreator, - want: false, + name: "set remotely on a job that is not selected", + path: testRunAsPath, + cd: &deployplan.ChangeDesc{Remote: sp}, + want: true, }, { - name: "user that is the creator", - path: testRunAsPath, - cd: &deployplan.ChangeDesc{Remote: creator}, - creator: testCreator, - want: true, + name: "removed from config, kept remotely, selected", + path: testRunAsPath, + cd: &deployplan.ChangeDesc{Old: user, Remote: user}, + selected: true, + want: false, }, { - name: "service principal that is the creator", - path: testRunAsPath, - cd: &deployplan.ChangeDesc{Remote: sp}, - creator: testSPID, - want: true, + name: "set in config and remotely", + path: testRunAsPath, + cd: &deployplan.ChangeDesc{New: user, Remote: sp}, + selected: true, + want: true, }, { - name: "unchanged since the last deploy", - path: testRunAsPath, - cd: &deployplan.ChangeDesc{Old: user, Remote: user}, - creator: testCreator, - want: true, + name: "removed remotely", + path: testRunAsPath, + cd: &deployplan.ChangeDesc{Old: user, New: user}, + selected: true, + want: true, }, { - name: "changed since the last deploy", - path: testRunAsPath, - cd: &deployplan.ChangeDesc{Old: user, Remote: sp}, - creator: testCreator, - want: false, + name: "removed from config and remotely", + path: testRunAsPath, + cd: &deployplan.ChangeDesc{Old: user}, + selected: true, + want: true, }, { - name: "removed remotely", - path: testRunAsPath, - cd: &deployplan.ChangeDesc{Old: user, New: user}, - creator: testCreator, - want: true, - }, - { - name: "run_as key", - path: testRunAsPath + ".service_principal_name", - cd: &deployplan.ChangeDesc{Remote: testSPID}, - creator: testSPID, - want: false, + name: "run_as key", + path: testRunAsPath + ".service_principal_name", + cd: &deployplan.ChangeDesc{Remote: testSPID}, + selected: false, + want: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - remoteState := &dresources.JobRemote{CreatorUserName: tt.creator} - assert.Equal(t, tt.want, skipJobRunAs(tt.path, tt.cd, remoteState)) + assert.Equal(t, tt.want, skipJobRunAs(tt.path, tt.cd, tt.selected)) }) } } + +func TestParseRunAsSelection(t *testing.T) { + selection, err := ParseRunAsSelection([]string{"jobs:123", "jobs:456"}) + require.NoError(t, err) + deployed := map[string]string{ + "jobs:123": "resources.jobs.first", + "jobs:789": "resources.jobs.other", + "pipelines:456": "resources.pipelines.pipeline", + } + assert.Equal(t, map[string]bool{"resources.jobs.first": true}, selection.ResourceKeys(t.Context(), deployed)) + + for _, selector := range []string{"123", "jobs:", "pipelines:123"} { + _, err := ParseRunAsSelection([]string{selector}) + assert.ErrorContains(t, err, "invalid --sync-run-as-ids value", selector) + } +} diff --git a/cmd/bundle/config_remote_sync.go b/cmd/bundle/config_remote_sync.go index 83344855e5c..fda27583346 100644 --- a/cmd/bundle/config_remote_sync.go +++ b/cmd/bundle/config_remote_sync.go @@ -25,6 +25,7 @@ import ( func newConfigRemoteSyncCommand() *cobra.Command { var save bool var selectIDs []string + var syncRunAsIDs []string cmd := &cobra.Command{ Use: "config-remote-sync", @@ -44,18 +45,27 @@ Examples: databricks bundle config-remote-sync --save # Restrict the sync to a single resource by its type and deployed resource ID - databricks bundle config-remote-sync --select-ids jobs:123456789 --save`, + databricks bundle config-remote-sync --select-ids jobs:123456789 --save + + # Also sync the job's remote run_as when the configuration does not set one + databricks bundle config-remote-sync --select-ids jobs:123456789 --sync-run-as-ids jobs:123456789 --save`, Hidden: true, // Used by DABs in the Workspace only } cmd.Flags().BoolVar(&save, "save", false, "Write updated config files to disk") cmd.Flags().StringSliceVar(&selectIDs, "select-ids", nil, "Sync only the given resources, each as : (e.g. jobs:123456789). Can be repeated or comma-separated.") + cmd.Flags().StringSliceVar(&syncRunAsIDs, "sync-run-as-ids", nil, "Also sync the remote run_as of the given jobs, each as jobs:, when the configuration does not set one. Pass only jobs whose run_as was just changed remotely; with --select-ids, a job must also be selected there. Can be repeated or comma-separated.") cmd.RunE = func(cmd *cobra.Command, args []string) error { if runtime.GOOS == "windows" { return errors.New("config-remote-sync command is not supported on Windows") } + runAs, err := configsync.ParseRunAsSelection(syncRunAsIDs) + if err != nil { + return err + } + stats := configsync.Stats{Save: save} // Emit telemetry on every exit path, including failures inside @@ -96,7 +106,8 @@ Examples: return fmt.Errorf("failed to detect changes: %w", err) } - detected, err := configsync.ExtractChanges(ctx, b, plan) + deployed := configsync.IndexDeployedResources(&deployBundle.StateDB) + detected, err := configsync.ExtractChanges(ctx, b, plan, runAs.ResourceKeys(ctx, deployed)) if err != nil { stats.ErrorCategory = protos.BundleConfigRemoteSyncErrorCategoryDetectChangesFailed return fmt.Errorf("failed to extract changes: %w", err) @@ -105,7 +116,7 @@ Examples: // Record the ids present in state and the ids requested: on failure // they are what classifies the miss. - stats.CollectStateIDs(slices.Collect(maps.Keys(configsync.IndexDeployedResources(&deployBundle.StateDB)))) + stats.CollectStateIDs(slices.Collect(maps.Keys(deployed))) if len(selectIDs) > 0 { stats.CollectSelectedIDs(selectIDs) From 520ef9754c5f4cca5befd5314125701a9a054dab Mon Sep 17 00:00:00 2001 From: Petra Postelnicu Date: Tue, 17 Nov 2026 14:39:46 +0000 Subject: [PATCH 5/6] Sync an unset job run_as unless it names the job's owner Replace the --sync-run-as-ids flag with a rule config-remote-sync applies on its own. A job without run_as runs as its owner: the IS_OWNER entry of its ACL, or its creator when job access control is off. The Jobs UI always shows and saves a principal, so a remote run_as that names the owner says what the config already says and is skipped. Any other remote run_as is what the job runs as, so it is written to the config. That includes a run_as removed from the config and deployed, which jobs/reset keeps. The owner comes from the ACLs config-remote-sync already reads for unmanaged permissions (now returned unfiltered), from the plan for declared ones, or from a direct read for declared permissions that were never deployed. When the ACL can't be read, run_as is not synced. Co-authored-by: Isaac --- .../job_run_as/databricks.yml.tmpl | 13 +- .../config-remote-sync/job_run_as/output.txt | 59 +++++--- .../config-remote-sync/job_run_as/script | 22 +-- .../json_status_selectors/output.txt | 8 -- .../json_status_selectors/script | 3 - .../terraform_state/output.txt | 27 ---- .../config-remote-sync/terraform_state/script | 9 +- bundle/configsync/cluster_policy_test.go | 2 +- bundle/configsync/diff.go | 19 ++- bundle/configsync/diff_test.go | 8 +- bundle/configsync/permissions.go | 24 ++-- bundle/configsync/runas.go | 134 ++++++++++++------ bundle/configsync/runas_test.go | 133 +++++++++-------- cmd/bundle/config_remote_sync.go | 19 +-- 14 files changed, 261 insertions(+), 219 deletions(-) diff --git a/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl index 7bfde32e50e..9c28e400732 100644 --- a/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl +++ b/acceptance/bundle/config-remote-sync/job_run_as/databricks.yml.tmpl @@ -11,8 +11,17 @@ resources: notebook_task: notebook_path: /Users/{{workspace_user_name}}/notebook - not_selected: - name: not-selected-$UNIQUE_NAME + # The remote run_as names the owner, which is what no run_as means. + run_as_to_owner: + name: run-as-to-owner-$UNIQUE_NAME + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + + # Ownership moves to another user, who becomes the default run_as. + transferred_owner: + name: transferred-owner-$UNIQUE_NAME tasks: - task_key: main notebook_task: diff --git a/acceptance/bundle/config-remote-sync/job_run_as/output.txt b/acceptance/bundle/config-remote-sync/job_run_as/output.txt index 2ac4035f742..20e51889880 100644 --- a/acceptance/bundle/config-remote-sync/job_run_as/output.txt +++ b/acceptance/bundle/config-remote-sync/job_run_as/output.txt @@ -2,14 +2,15 @@ >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Created jobs.no_run_as -Created jobs.not_selected Created jobs.pending_add Created jobs.removed_run_as +Created jobs.run_as_to_owner +Created jobs.transferred_owner Created jobs.user_to_creator Created jobs.user_to_sp Created jobs.user_to_user Files: 2 uploaded, 0 deleted -Resources: 7 created, 0 changed, 0 deleted, 0 unchanged +Resources: 8 created, 0 changed, 0 deleted, 0 unchanged === Remove run_as from config and deploy @@ -17,7 +18,7 @@ Resources: 7 created, 0 changed, 0 deleted, 0 unchanged Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... Updated jobs.removed_run_as Files: 1 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 6 unchanged +Resources: 0 created, 1 changed, 0 deleted, 7 unchanged >>> [CLI] jobs get [REMOVED_RUN_AS_ID] { @@ -28,10 +29,12 @@ Resources: 0 created, 1 changed, 0 deleted, 6 unchanged === Change run_as remotely +>>> [CLI] jobs set-permissions [TRANSFERRED_OWNER_ID] --json {"access_control_list": [{"user_name": "new-owner@example.com", "permission_level": "IS_OWNER"}]} + === Detect and save changes ->>> [CLI] bundle config-remote-sync --save --sync-run-as-ids jobs:[NO_RUN_AS_ID],jobs:[PENDING_ADD_ID] -Detected changes in 5 resource(s): +>>> [CLI] bundle config-remote-sync --save +Detected changes in 7 resource(s): Resource: resources.jobs.no_run_as run_as: add @@ -39,6 +42,12 @@ Resource: resources.jobs.no_run_as Resource: resources.jobs.pending_add run_as.user_name: replace +Resource: resources.jobs.removed_run_as + run_as: add + +Resource: resources.jobs.transferred_owner + permissions: add + Resource: resources.jobs.user_to_creator run_as.user_name: replace @@ -59,30 +68,44 @@ Resource: resources.jobs.user_to_user + run_as: + service_principal_name: [UUID] - not_selected: - name: not-selected-[UNIQUE_NAME] -@@ -31,5 +33,5 @@ + # The remote run_as names the owner, which is what no run_as means. + run_as_to_owner: +@@ -28,4 +30,7 @@ + notebook_path: /Users/{{workspace_user_name}}/notebook + ++ permissions: ++ - level: IS_OWNER ++ user_name: new-owner@example.com + # run_as is removed from config and deployed; the job keeps it remotely. + removed_run_as: +@@ -36,9 +41,11 @@ + notebook_path: /Users/{{workspace_user_name}}/notebook + ++ run_as: ++ user_name: first-user@example.com + # run_as is added to config locally and not deployed. + pending_add: name: pending-add-[UNIQUE_NAME] run_as: - user_name: local-user@example.com + user_name: remote-user@example.com tasks: - task_key: main -@@ -40,5 +42,5 @@ +@@ -49,5 +56,5 @@ name: user-to-user-[UNIQUE_NAME] run_as: - user_name: first-user@example.com + user_name: second-user@example.com tasks: - task_key: main -@@ -49,5 +51,5 @@ +@@ -58,5 +65,5 @@ name: user-to-creator-[UNIQUE_NAME] run_as: - user_name: first-user@example.com + user_name: [USERNAME] tasks: - task_key: main -@@ -58,5 +60,5 @@ +@@ -67,5 +74,5 @@ name: user-to-sp-[UNIQUE_NAME] run_as: - user_name: first-user@example.com @@ -92,30 +115,28 @@ Resource: resources.jobs.user_to_user === Second sync is a no-op ->>> [CLI] bundle config-remote-sync --sync-run-as-ids jobs:[NO_RUN_AS_ID],jobs:[PENDING_ADD_ID] +>>> [CLI] bundle config-remote-sync No changes detected. === Plan after sync >>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 7 unchanged - -=== Invalid --sync-run-as-ids value +create jobs.transferred_owner.permissions ->>> musterr [CLI] bundle config-remote-sync --sync-run-as-ids pipelines:123 -Error: invalid --sync-run-as-ids value "pipelines:123", expected jobs: (e.g. jobs:123456789) +Plan: 1 to add, 0 to change, 0 to delete, 8 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.no_run_as - delete resources.jobs.not_selected delete resources.jobs.pending_add delete resources.jobs.removed_run_as + delete resources.jobs.run_as_to_owner + delete resources.jobs.transferred_owner delete resources.jobs.user_to_creator delete resources.jobs.user_to_sp delete resources.jobs.user_to_user All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default -Destroy: 7 deleted +Destroy: 8 deleted diff --git a/acceptance/bundle/config-remote-sync/job_run_as/script b/acceptance/bundle/config-remote-sync/job_run_as/script index d6c7ba03f22..ce0deed718f 100644 --- a/acceptance/bundle/config-remote-sync/job_run_as/script +++ b/acceptance/bundle/config-remote-sync/job_run_as/script @@ -27,8 +27,13 @@ title "Change run_as remotely" edit_resource.py jobs "$(read_id.py no_run_as)" <<'PY' r["run_as"] = {"service_principal_name": "11111111-2222-3333-4444-555555555555"} PY -edit_resource.py jobs "$(read_id.py not_selected)" <<'PY' -r["run_as"] = {"service_principal_name": "11111111-2222-3333-4444-555555555555"} +edit_resource.py jobs "$(read_id.py run_as_to_owner)" < /dev/null +edit_resource.py jobs "$(read_id.py transferred_owner)" <<'PY' +r["run_as"] = {"user_name": "new-owner@example.com"} PY edit_resource.py jobs "$(read_id.py pending_add)" <<'PY' r["run_as"] = {"user_name": "remote-user@example.com"} @@ -43,21 +48,16 @@ edit_resource.py jobs "$(read_id.py user_to_sp)" <<'PY' r["run_as"] = {"service_principal_name": "11111111-2222-3333-4444-555555555555"} PY -# Only no_run_as and pending_add are selected. A run_as the config does not set is -# synced only for selected jobs; removed_run_as and not_selected keep their config. -sync_run_as_ids="jobs:$(read_id.py no_run_as),jobs:$(read_id.py pending_add)" - +# A run_as the config does not set is synced unless it names the job's owner: +# no_run_as and removed_run_as get one, run_as_to_owner and transferred_owner do not. title "Detect and save changes" cp databricks.yml databricks.yml.backup -trace $CLI bundle config-remote-sync --save --sync-run-as-ids "$sync_run_as_ids" +trace $CLI bundle config-remote-sync --save trace diff.py databricks.yml.backup databricks.yml rm databricks.yml.backup title "Second sync is a no-op" -trace $CLI bundle config-remote-sync --sync-run-as-ids "$sync_run_as_ids" +trace $CLI bundle config-remote-sync title "Plan after sync" trace $CLI bundle plan - -title "Invalid --sync-run-as-ids value" -trace musterr $CLI bundle config-remote-sync --sync-run-as-ids "pipelines:123" diff --git a/acceptance/bundle/config-remote-sync/json_status_selectors/output.txt b/acceptance/bundle/config-remote-sync/json_status_selectors/output.txt index 4c74f07ce6e..2b13979059e 100644 --- a/acceptance/bundle/config-remote-sync/json_status_selectors/output.txt +++ b/acceptance/bundle/config-remote-sync/json_status_selectors/output.txt @@ -19,14 +19,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "changes": {} } -=== A malformed --sync-run-as-ids value is reported as failed, exit 0 -{ - "status": "failed", - "error": "invalid --sync-run-as-ids value \"pipelines:123\", expected jobs:\u003cid\u003e (e.g. jobs:123456789)", - "files": null, - "changes": {} -} - >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.test_job diff --git a/acceptance/bundle/config-remote-sync/json_status_selectors/script b/acceptance/bundle/config-remote-sync/json_status_selectors/script index 87a5c274b8c..17730386ea7 100755 --- a/acceptance/bundle/config-remote-sync/json_status_selectors/script +++ b/acceptance/bundle/config-remote-sync/json_status_selectors/script @@ -19,6 +19,3 @@ errcode $CLI bundle config-remote-sync --select-ids "jobs:no-such-id-999" -o jso title "A malformed selector is reported as failed, exit 0" errcode $CLI bundle config-remote-sync --select-ids "no-such-id-999" -o json - -title "A malformed --sync-run-as-ids value is reported as failed, exit 0" -errcode $CLI bundle config-remote-sync --sync-run-as-ids "pipelines:123" -o json diff --git a/acceptance/bundle/config-remote-sync/terraform_state/output.txt b/acceptance/bundle/config-remote-sync/terraform_state/output.txt index aa3f631c15c..3700a404dcc 100644 --- a/acceptance/bundle/config-remote-sync/terraform_state/output.txt +++ b/acceptance/bundle/config-remote-sync/terraform_state/output.txt @@ -44,33 +44,6 @@ resources: notebook_task: notebook_path: /Users/{{workspace_user_name}}/notebook -=== With --sync-run-as-ids the job's run_as is written too - ->>> [CLI] bundle config-remote-sync --save --sync-run-as-ids jobs:[TEST_JOB_ID] -Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765). -Detected changes in 1 resource(s): - -Resource: resources.jobs.test_job - run_as: add - - - ->>> cat databricks.yml -bundle: - name: test-bundle - -resources: - jobs: - test_job: - name: test-job - max_concurrent_runs: 3 - tasks: - - task_key: main - notebook_task: - notebook_path: /Users/{{workspace_user_name}}/notebook - run_as: - user_name: [USERNAME] - === Cleanup >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/config-remote-sync/terraform_state/script b/acceptance/bundle/config-remote-sync/terraform_state/script index f4b42c8662b..12de0a54152 100644 --- a/acceptance/bundle/config-remote-sync/terraform_state/script +++ b/acceptance/bundle/config-remote-sync/terraform_state/script @@ -8,9 +8,8 @@ title "Replay a terraform deployment of the bundle (captured from a real terrafo trace replay_tfstate.py terraform-requests.json terraform.tfstate job_id="$(read_id.py test_job)" -# Jobs updated by the terraform engine carry an explicit run_as naming the job's identity. A run_as -# the config does not set is synced only with --sync-run-as-ids, so only max_concurrent_runs must be -# detected. +# Jobs updated by the terraform engine carry an explicit run_as naming the job's owner. That is what +# a config without run_as means, so only max_concurrent_runs must be detected. edit_resource.py jobs $job_id < 0 { - result[resourceKey] = permissions - } + result[resourceKey] = permissions return nil }) } diff --git a/bundle/configsync/runas.go b/bundle/configsync/runas.go index c96d21b3ba8..f7ac92a34f4 100644 --- a/bundle/configsync/runas.go +++ b/bundle/configsync/runas.go @@ -1,55 +1,31 @@ package configsync import ( + "cmp" "context" - "fmt" - "strings" + "errors" + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/databricks/databricks-sdk-go/service/jobs" ) // The plan skips a job's run_as keys that the config lacks as backend defaults. // run_as holds a single principal key, so when the remote switches kind (for // example user_name to service_principal_name) the new key is the edit itself, // and dropping it while removing the old key leaves "run_as: {}". +var errPermissionsNotRead = errors.New("permissions were not read") + const ( - jobRunAsPattern = "resources.jobs.*.run_as" - jobRunAsChildPattern = "resources.jobs.*.run_as.*" - jobsSelectorPrefix = "jobs:" + runAsField = "run_as" + jobRunAsPattern = "resources.jobs.*." + runAsField + jobRunAsChildPattern = jobRunAsPattern + ".*" ) -// RunAsSelection holds the jobs, as "jobs:", whose remote run_as is synced -// even when the config does not set one. -type RunAsSelection []string - -// ParseRunAsSelection validates the --sync-run-as-ids selectors. -func ParseRunAsSelection(selectors []string) (RunAsSelection, error) { - for _, selector := range selectors { - id, ok := strings.CutPrefix(selector, jobsSelectorPrefix) - if !ok || id == "" { - return nil, fmt.Errorf("invalid --sync-run-as-ids value %q, expected jobs: (e.g. jobs:123456789)", selector) - } - } - return selectors, nil -} - -// ResourceKeys maps the selected jobs to their plan keys using the index from -// IndexDeployedResources. A job this bundle has not deployed is ignored: it has -// no changes to sync. -func (s RunAsSelection) ResourceKeys(ctx context.Context, deployed map[string]string) map[string]bool { - keys := make(map[string]bool, len(s)) - for _, selector := range s { - key, ok := deployed[selector] - if !ok { - log.Debugf(ctx, "config-remote-sync: --sync-run-as-ids %s is not deployed by this bundle", selector) - continue - } - keys[key] = true - } - return keys -} - // isJobRunAsPath reports whether path is a job's run_as or one of its keys. // Unlike other backend defaults, these are synced when the config lacks them: // the keys always, the whole object only as decided by skipJobRunAs. @@ -66,10 +42,13 @@ func keepSkippedRunAsChange(path string, cd *deployplan.ChangeDesc) bool { // skipJobRunAs reports whether a change to the run_as object as a whole must // not be synced. Changes to its keys are synced like any other field. // -// A remote run_as the config does not set can be a Jobs UI edit or a value the -// server kept: jobs/reset without run_as keeps the previous one. Only the -// caller can tell them apart, so it is synced only for selected jobs. -func skipJobRunAs(path string, cd *deployplan.ChangeDesc, selected bool) bool { +// A config without run_as means the job runs as its owner, and the Jobs UI +// shows that owner until a principal is picked. So a remote run_as naming the +// owner says what the config already says and is not synced. Any other remote +// run_as is the identity the job runs as, whether it was set in the Jobs UI or +// kept by the server after run_as was removed from the config (jobs/reset +// without run_as keeps the previous one), so it is synced. +func skipJobRunAs(path string, cd *deployplan.ChangeDesc, owner string) bool { if !matchPattern(jobRunAsPattern, path) { return false } @@ -77,10 +56,81 @@ func skipJobRunAs(path string, cd *deployplan.ChangeDesc, selected bool) bool { if cd.New != nil { return true } + runAs, ok := remoteRunAs(cd.Remote) // The Jobs UI always submits a principal, so a removed remote run_as is not // an edit to follow. - if cd.Remote == nil { + if !ok { return true } - return !selected + // Without a known owner, a remote run_as can't be told apart from the default. + return owner == "" || cmp.Or(runAs.UserName, runAs.ServicePrincipalName, runAs.GroupName) == owner +} + +func remoteRunAs(remote any) (jobs.JobRunAs, bool) { + switch v := remote.(type) { + case jobs.JobRunAs: + return v, true + case *jobs.JobRunAs: + if v != nil { + return *v, true + } + } + return jobs.JobRunAs{}, false +} + +// jobOwners returns, keyed by plan key, the owner of each job with a remote +// run_as the config does not set: the principal the job runs as without one. +// A job whose ACL could not be read is left out. unmanaged holds the ACLs read +// by readUnmanagedPermissions. +func jobOwners(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, unmanaged map[string][]dresources.StatePermission) map[string]string { + owners := make(map[string]string) + for resourceKey, entry := range plan.Plan { + job, ok := entry.RemoteState.(*dresources.JobRemote) + cd := entry.Changes[runAsField] + if !ok || job == nil || cd == nil || cd.New != nil || cd.Remote == nil { + continue + } + acl, err := jobACL(ctx, b, plan, resourceKey, unmanaged) + if err != nil { + log.Debugf(ctx, "config-remote-sync: not syncing the run_as of %s, its owner is unknown: %v", resourceKey, err) + continue + } + owners[resourceKey] = jobOwner(acl, job.CreatorUserName) + } + return owners +} + +// jobACL returns a job's remote ACL. Permissions that the config declares but +// were never deployed have no remote state in the plan, so they are read here. +func jobACL(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, resourceKey string, unmanaged map[string][]dresources.StatePermission) ([]dresources.StatePermission, error) { + permissionsKey := resourceKey + ".permissions" + entry := plan.Plan[permissionsKey] + if entry == nil { + // Read with the unmanaged permissions; missing if that read failed. + acl, ok := unmanaged[resourceKey] + if !ok { + return nil, errPermissionsNotRead + } + return acl, nil + } + if state, ok := entry.RemoteState.(*dresources.PermissionsState); ok && state != nil { + return state.EmbeddedSlice, nil + } + adapter := b.DeploymentBundle.Adapters[config.GetResourceTypeFromKey(permissionsKey)] + if adapter == nil { + return nil, errPermissionsNotRead + } + return readPermissions(ctx, b, adapter, resourceKey) +} + +// jobOwner returns the principal a job without run_as runs as: the IS_OWNER +// entry of its ACL or, when job access control is off, its creator. See +// run_as_user_name in the Jobs API. +func jobOwner(acl []dresources.StatePermission, creator string) string { + for _, p := range acl { + if p.Level == iam.PermissionLevelIsOwner { + return cmp.Or(p.UserName, p.ServicePrincipalName) + } + } + return creator } diff --git a/bundle/configsync/runas_test.go b/bundle/configsync/runas_test.go index ae06e3cfb55..e81a2701d3b 100644 --- a/bundle/configsync/runas_test.go +++ b/bundle/configsync/runas_test.go @@ -3,10 +3,12 @@ package configsync import ( "testing" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/databricks-sdk-go/service/iam" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) const ( @@ -40,82 +42,99 @@ func TestKeepSkippedRunAsChange(t *testing.T) { func TestSkipJobRunAs(t *testing.T) { sp := jobs.JobRunAs{ServicePrincipalName: testSPID} user := jobs.JobRunAs{UserName: "user@example.com"} + owner := "owner@example.com" tests := []struct { - name string - path string - cd *deployplan.ChangeDesc - selected bool - want bool + name string + path string + cd *deployplan.ChangeDesc + want bool + }{ + {"set remotely to another principal", testRunAsPath, &deployplan.ChangeDesc{Remote: sp}, false}, + {"set remotely as a pointer", testRunAsPath, &deployplan.ChangeDesc{Remote: &sp}, false}, + {"set remotely to the owner", testRunAsPath, &deployplan.ChangeDesc{Remote: jobs.JobRunAs{UserName: owner}}, true}, + {"set remotely to the owner as a service principal", testRunAsPath, &deployplan.ChangeDesc{Remote: jobs.JobRunAs{ServicePrincipalName: owner}}, true}, + {"removed from config, kept remotely", testRunAsPath, &deployplan.ChangeDesc{Old: user, Remote: user}, false}, + {"set in config and remotely", testRunAsPath, &deployplan.ChangeDesc{New: user, Remote: sp}, true}, + {"removed remotely", testRunAsPath, &deployplan.ChangeDesc{Old: user, New: user}, true}, + {"removed from config and remotely", testRunAsPath, &deployplan.ChangeDesc{Old: user}, true}, + {"run_as key", testRunAsPath + ".service_principal_name", &deployplan.ChangeDesc{Remote: testSPID}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, skipJobRunAs(tt.path, tt.cd, owner)) + }) + } + + // Without a known owner, no remote run_as is synced. + assert.True(t, skipJobRunAs(testRunAsPath, &deployplan.ChangeDesc{Remote: sp}, "")) +} + +func TestJobOwners(t *testing.T) { + const jobKey = "resources.jobs.my_job" + unsetRunAs := map[string]*deployplan.ChangeDesc{"run_as": {Remote: jobs.JobRunAs{UserName: "user@example.com"}}} + job := &deployplan.PlanEntry{ + RemoteState: &dresources.JobRemote{CreatorUserName: "creator@example.com"}, + Changes: unsetRunAs, + } + acl := []dresources.StatePermission{ + {Level: iam.PermissionLevelCanManage, UserName: "manager@example.com"}, + {Level: iam.PermissionLevelIsOwner, ServicePrincipalName: testSPID}, + } + + tests := []struct { + name string + plan map[string]*deployplan.PlanEntry + unmanaged map[string][]dresources.StatePermission + want map[string]string }{ { - name: "set remotely on a selected job", - path: testRunAsPath, - cd: &deployplan.ChangeDesc{Remote: sp}, - selected: true, - want: false, + name: "owner from the unmanaged ACL", + plan: map[string]*deployplan.PlanEntry{jobKey: job}, + unmanaged: map[string][]dresources.StatePermission{jobKey: acl}, + want: map[string]string{jobKey: testSPID}, }, { - name: "set remotely on a job that is not selected", - path: testRunAsPath, - cd: &deployplan.ChangeDesc{Remote: sp}, - want: true, + name: "owner from the declared permissions", + plan: map[string]*deployplan.PlanEntry{ + jobKey: job, + jobKey + ".permissions": {RemoteState: &dresources.PermissionsState{EmbeddedSlice: acl}}, + }, + want: map[string]string{jobKey: testSPID}, }, { - name: "removed from config, kept remotely, selected", - path: testRunAsPath, - cd: &deployplan.ChangeDesc{Old: user, Remote: user}, - selected: true, - want: false, + name: "creator when the ACL has no owner", + plan: map[string]*deployplan.PlanEntry{jobKey: job}, + unmanaged: map[string][]dresources.StatePermission{jobKey: acl[:1]}, + want: map[string]string{jobKey: "creator@example.com"}, }, { - name: "set in config and remotely", - path: testRunAsPath, - cd: &deployplan.ChangeDesc{New: user, Remote: sp}, - selected: true, - want: true, + name: "unknown when the ACL was not read", + plan: map[string]*deployplan.PlanEntry{jobKey: job}, + want: map[string]string{}, }, { - name: "removed remotely", - path: testRunAsPath, - cd: &deployplan.ChangeDesc{Old: user, New: user}, - selected: true, - want: true, + name: "unknown when declared permissions can't be read", + plan: map[string]*deployplan.PlanEntry{jobKey: job, jobKey + ".permissions": {}}, + want: map[string]string{}, }, { - name: "removed from config and remotely", - path: testRunAsPath, - cd: &deployplan.ChangeDesc{Old: user}, - selected: true, - want: true, + name: "run_as set in config", + plan: map[string]*deployplan.PlanEntry{jobKey: { + RemoteState: job.RemoteState, + Changes: map[string]*deployplan.ChangeDesc{"run_as": {New: jobs.JobRunAs{UserName: "user@example.com"}, Remote: jobs.JobRunAs{UserName: "user@example.com"}}}, + }}, + want: map[string]string{}, }, { - name: "run_as key", - path: testRunAsPath + ".service_principal_name", - cd: &deployplan.ChangeDesc{Remote: testSPID}, - selected: false, - want: false, + name: "not a job", + plan: map[string]*deployplan.PlanEntry{jobKey: {RemoteState: &dresources.PermissionsState{}, Changes: unsetRunAs}}, + want: map[string]string{}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, skipJobRunAs(tt.path, tt.cd, tt.selected)) + assert.Equal(t, tt.want, jobOwners(t.Context(), &bundle.Bundle{}, &deployplan.Plan{Plan: tt.plan}, tt.unmanaged)) }) } } - -func TestParseRunAsSelection(t *testing.T) { - selection, err := ParseRunAsSelection([]string{"jobs:123", "jobs:456"}) - require.NoError(t, err) - deployed := map[string]string{ - "jobs:123": "resources.jobs.first", - "jobs:789": "resources.jobs.other", - "pipelines:456": "resources.pipelines.pipeline", - } - assert.Equal(t, map[string]bool{"resources.jobs.first": true}, selection.ResourceKeys(t.Context(), deployed)) - - for _, selector := range []string{"123", "jobs:", "pipelines:123"} { - _, err := ParseRunAsSelection([]string{selector}) - assert.ErrorContains(t, err, "invalid --sync-run-as-ids value", selector) - } -} diff --git a/cmd/bundle/config_remote_sync.go b/cmd/bundle/config_remote_sync.go index a5fdaf5e2fd..972810d6f0f 100644 --- a/cmd/bundle/config_remote_sync.go +++ b/cmd/bundle/config_remote_sync.go @@ -25,7 +25,6 @@ import ( func newConfigRemoteSyncCommand() *cobra.Command { var save bool var selectIDs []string - var syncRunAsIDs []string cmd := &cobra.Command{ Use: "config-remote-sync", @@ -45,16 +44,12 @@ Examples: databricks bundle config-remote-sync --save # Restrict the sync to a single resource by its type and deployed resource ID - databricks bundle config-remote-sync --select-ids jobs:123456789 --save - - # Also sync the job's remote run_as when the configuration does not set one - databricks bundle config-remote-sync --select-ids jobs:123456789 --sync-run-as-ids jobs:123456789 --save`, + databricks bundle config-remote-sync --select-ids jobs:123456789 --save`, Hidden: true, // Used by DABs in the Workspace only } cmd.Flags().BoolVar(&save, "save", false, "Write updated config files to disk") cmd.Flags().StringSliceVar(&selectIDs, "select-ids", nil, "Sync only the given resources, each as : (e.g. jobs:123456789). Can be repeated or comma-separated.") - cmd.Flags().StringSliceVar(&syncRunAsIDs, "sync-run-as-ids", nil, "Also sync the remote run_as of the given jobs, each as jobs:, when the configuration does not set one. Pass only jobs whose run_as was just changed remotely; with --select-ids, a job must also be selected there. Can be repeated or comma-separated.") cmd.RunE = func(cmd *cobra.Command, args []string) error { if runtime.GOOS == "windows" { @@ -95,21 +90,13 @@ Examples: // ProcessBundleRet has already opened the direct state for reading. deployBundle := &b.DeploymentBundle - // Validated here, like --select-ids, so an invalid value is reported - // through WriteResult and telemetry. - runAs, err := configsync.ParseRunAsSelection(syncRunAsIDs) - if err != nil { - return err - } - plan, err := deployBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &b.Config) if err != nil { stats.ErrorCategory = protos.BundleConfigRemoteSyncErrorCategoryDetectChangesFailed return fmt.Errorf("failed to detect changes: %w", err) } - deployed := configsync.IndexDeployedResources(&deployBundle.StateDB) - detected, permissionsSkipped, err := configsync.ExtractChanges(ctx, b, plan, runAs.ResourceKeys(ctx, deployed)) + detected, permissionsSkipped, err := configsync.ExtractChanges(ctx, b, plan) if err != nil { stats.ErrorCategory = protos.BundleConfigRemoteSyncErrorCategoryDetectChangesFailed return fmt.Errorf("failed to extract changes: %w", err) @@ -118,7 +105,7 @@ Examples: // Record the ids present in state and the ids requested: on failure // they are what classifies the miss. - stats.CollectStateIDs(slices.Collect(maps.Keys(deployed))) + stats.CollectStateIDs(slices.Collect(maps.Keys(configsync.IndexDeployedResources(&deployBundle.StateDB)))) if len(selectIDs) > 0 { stats.CollectSelectedIDs(selectIDs) From a80e119c6842b8cf43b46cb33fb7890a1c9c1935 Mon Sep 17 00:00:00 2001 From: Petra Postelnicu Date: Tue, 17 Nov 2026 18:40:12 +0000 Subject: [PATCH 6/6] Move the run_as pattern comment back above its const block Co-authored-by: Isaac --- bundle/configsync/runas.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bundle/configsync/runas.go b/bundle/configsync/runas.go index f7ac92a34f4..3e25328c4fa 100644 --- a/bundle/configsync/runas.go +++ b/bundle/configsync/runas.go @@ -18,14 +18,15 @@ import ( // run_as holds a single principal key, so when the remote switches kind (for // example user_name to service_principal_name) the new key is the edit itself, // and dropping it while removing the old key leaves "run_as: {}". -var errPermissionsNotRead = errors.New("permissions were not read") - const ( runAsField = "run_as" jobRunAsPattern = "resources.jobs.*." + runAsField jobRunAsChildPattern = jobRunAsPattern + ".*" ) +// errPermissionsNotRead reports that a job's ACL was not read, so its owner is unknown. +var errPermissionsNotRead = errors.New("permissions were not read") + // isJobRunAsPath reports whether path is a job's run_as or one of its keys. // Unlike other backend defaults, these are synced when the config lacks them: // the keys always, the whole object only as decided by skipJobRunAs.