Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 0 additions & 15 deletions acceptance/bundle/config-remote-sync/job_fields/script
Original file line number Diff line number Diff line change
Expand Up @@ -40,21 +40,6 @@ if "tags" not in r:
r["tags"]["team"] = "data"
EOF

# TODO add support for permissions
# title "Modify job permissions remotely"
# current_user=$($CLI current-user me --output json | jq -r .userName)
# cat > permissions.json <<EOF
# {
# "access_control_list": [
# {"permission_level": "IS_OWNER", "user_name": "$current_user"},
# {"permission_level": "CAN_MANAGE", "user_name": "viewer@example.com"},
# {"permission_level": "CAN_MANAGE_RUN", "user_name": "admin@example.com"}
# ]
# }
# EOF
# $CLI permissions set jobs $job_id --json @permissions.json
# rm permissions.json

title "Detect and save changes"
cp databricks.yml databricks.yml.backup
$CLI bundle config-remote-sync --save
Expand Down
15 changes: 0 additions & 15 deletions acceptance/bundle/config-remote-sync/pipeline_fields/script
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,6 @@ r["tags"]["foo"] = "bar"
r.setdefault("environment", {}).setdefault("dependencies", []).append("--editable /Workspace/bar")
EOF

# TODO add support for permissions
# title "Modify pipeline permissions remotely"
# current_user=$($CLI current-user me --output json | jq -r .userName)
# cat > permissions.json <<EOF
# {
# "access_control_list": [
# {"permission_level": "IS_OWNER", "user_name": "$current_user"},
# {"permission_level": "CAN_MANAGE", "user_name": "viewer@example.com"},
# {"permission_level": "CAN_RUN", "user_name": "admin@example.com"}
# ]
# }
# EOF
# $CLI permissions set pipelines $pipeline_id --json @permissions.json
# rm permissions.json

title "Detect and save changes"
cp databricks.yml databricks.yml.backup
$CLI bundle config-remote-sync --save
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
bundle:
name: test-bundle-$UNIQUE_NAME

resources:
jobs:
my_job:
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/main
new_cluster:
spark_version: $DEFAULT_SPARK_VERSION
node_type_id: $NODE_TYPE_ID
num_workers: 1

pipelines:
my_pipeline:
name: test-pipeline-$UNIQUE_NAME
libraries:
- notebook:
path: /Users/{{workspace_user_name}}/notebook

targets:
default:
mode: development
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
GOOS.windows = false
EnvMatrix.DMS = ["", "true"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
Created jobs.my_job
Created pipelines.my_pipeline
Files: 2 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 0 unchanged

=== Add permissions remotely to resources that declare none

=== An ACL that cannot be read is reported as skipped

>>> [CLI] bundle config-remote-sync --select-ids jobs:[MY_JOB_ID] -o json
{
"status": "success",
"skippedChanges": null
}

>>> [CLI] bundle config-remote-sync -o json
{
"status": "partial",
"changes": [
"resources.jobs.my_job"
],
"skippedChanges": [
{
"resourceKey": "resources.pipelines.my_pipeline",
"reason": "permissions_permission_denied",
"error": "Fault injected by test."
}
]
}

=== Detect and save changes
Detected changes in 2 resource(s):

Resource: resources.jobs.my_job
permissions: add

Resource: resources.pipelines.my_pipeline
permissions: add



=== Configuration changes

>>> diff.py databricks.yml.backup databricks.yml
--- databricks.yml.backup
+++ databricks.yml
@@ -14,4 +14,9 @@
num_workers: 1

+ permissions:
+ - level: CAN_MANAGE
+ user_name: viewer@example.com
+ - level: CAN_MANAGE_RUN
+ user_name: admin@example.com
pipelines:
my_pipeline:
@@ -20,4 +25,9 @@
- notebook:
path: /Users/{{workspace_user_name}}/notebook
+ permissions:
+ - level: IS_OWNER
+ user_name: owner@example.com
+ - group_name: users
+ level: CAN_VIEW

targets:

=== Sync again: the permissions are now in the config
No changes detected.


=== Deploy keeps the remote owner
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
Created jobs.my_job.permissions
Created pipelines.my_pipeline.permissions
Files: 1 uploaded, 0 deleted
Resources: 2 created, 0 changed, 0 deleted, 2 unchanged

>>> [CLI] permissions get pipelines [MY_PIPELINE_ID]
[{"user_name":"owner@example.com","group_name":null,"level":"IS_OWNER"},{"user_name":null,"group_name":"users","level":"CAN_VIEW"},{"user_name":"[USERNAME]","group_name":null,"level":"CAN_MANAGE"}]

=== Change a level, add and remove principals remotely

=== Detect and save changes
Detected changes in 1 resource(s):

Resource: resources.jobs.my_job
permissions[group_name='data-team']: add
permissions[user_name='admin@example.com']: remove
permissions[user_name='viewer@example.com'].level: replace



=== Configuration changes

>>> diff.py databricks.yml.backup databricks.yml
--- databricks.yml.backup
+++ databricks.yml
@@ -15,8 +15,8 @@

permissions:
- - level: CAN_MANAGE
+ - level: CAN_VIEW
user_name: viewer@example.com
- - level: CAN_MANAGE_RUN
- user_name: admin@example.com
+ - group_name: data-team
+ level: CAN_MANAGE_RUN
pipelines:
my_pipeline:

=== Deploy and sync again
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
Files: 1 uploaded, 0 deleted
Resources: 0 created, 0 changed, 0 deleted, 4 unchanged
No changes detected.


>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.jobs.my_job
delete resources.pipelines.my_pipeline

This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:
delete resources.pipelines.my_pipeline

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default

Destroy: 2 deleted
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
#!/bin/bash

envsubst < databricks.yml.tmpl > databricks.yml

cleanup() {
trace $CLI bundle destroy --auto-approve
}
trap cleanup EXIT

$CLI bundle deploy
job_id="$(read_id.py my_job)"
pipeline_id="$(read_id.py my_pipeline)"

title "Add permissions remotely to resources that declare none"
# The current user's owner entry is managed by deploy, so it must not be written to the config.
$CLI permissions set jobs $job_id --json '{"access_control_list":[
{"permission_level":"IS_OWNER","user_name":"'"$CURRENT_USER_NAME"'"},
{"permission_level":"CAN_MANAGE","user_name":"viewer@example.com"},
{"permission_level":"CAN_MANAGE_RUN","user_name":"admin@example.com"}
]}' > /dev/null
# An owner transferred to another user is kept.
$CLI permissions set pipelines $pipeline_id --json '{"access_control_list":[
{"permission_level":"IS_OWNER","user_name":"owner@example.com"},
{"permission_level":"CAN_MANAGE","user_name":"'"$CURRENT_USER_NAME"'"},
{"permission_level":"CAN_VIEW","group_name":"users"}
]}' > /dev/null

title "An ACL that cannot be read is reported as skipped"
fault.py "GET /api/2.0/permissions/pipelines/$pipeline_id" 403 0 2 PERMISSION_DENIED
trace $CLI bundle config-remote-sync --select-ids "jobs:$job_id" -o json | jq '{status, skippedChanges}'
trace $CLI bundle config-remote-sync -o json | jq '{status, changes: (.changes | keys), skippedChanges}'

title "Detect and save changes"
cp databricks.yml databricks.yml.backup
$CLI bundle config-remote-sync --save

title "Configuration changes"
trace diff.py databricks.yml.backup databricks.yml
rm databricks.yml.backup

title "Sync again: the permissions are now in the config"
$CLI bundle config-remote-sync

title "Deploy keeps the remote owner"
$CLI bundle deploy
trace $CLI permissions get pipelines $pipeline_id | jq -c '[.access_control_list[] | {user_name, group_name, level: .all_permissions[0].permission_level}]'

title "Change a level, add and remove principals remotely"
$CLI permissions set jobs $job_id --json '{"access_control_list":[
{"permission_level":"IS_OWNER","user_name":"'"$CURRENT_USER_NAME"'"},
{"permission_level":"CAN_VIEW","user_name":"viewer@example.com"},
{"permission_level":"CAN_MANAGE_RUN","group_name":"data-team"}
]}' > /dev/null

title "Detect and save changes"
cp databricks.yml databricks.yml.backup
$CLI bundle config-remote-sync --save

title "Configuration changes"
trace diff.py databricks.yml.backup databricks.yml
rm databricks.yml.backup

title "Deploy and sync again"
$CLI bundle deploy
$CLI bundle config-remote-sync
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
RecordRequests = false

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unlike skip_permissions, this test has no Cloud = true and uses principals that exist only in the fake server, so the import and redeploy never ran against a real ACL. Can it use built-in groups?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

factory, if this is doable, let's verify with a local cloud run

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[AI Generated] Still open at head: unmanaged_permissions/test.toml has no Cloud = true (out.test.toml shows Cloud = false), and the script uses viewer@, admin@, owner@example.com and data-team, which won't exist in a real workspace. Built-in users covers group grants. IS_OWNER needs a real user or service principal, so create a service principal in the script. I can't push or run cloud tests from this checkout; deco env run on your side would settle it.

review-factory reply (Claude Opus 5.5)

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[AI] Not done yet: a local cloud run is not possible from the factory's machine, which has no test-workspace environment. Making the test cloud-ready also takes more than built-in groups: the transferred pipeline owner needs a real second principal (e.g. the test service principal, since a group cannot be owner), and the raw permissions get output the script prints can carry inherited admin entries and a different entry order on a real workspace, so the golden output needs one cloud run to settle.

Two ways forward: I switch it to Cloud = true with real principals and you run it against a test env, or I switch it and let this PR's integration run verify it, fixing whatever it reports. Your call; it stays local-only until then.

🤖 Factory-generated comment. I still review the code and double-check every claim myself.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[AI] Verified by hand against a real workspace instead, deploying as a service principal (the acceptance test itself stays local-only). A job and a pipeline with no permissions: got a user and a group added remotely: both were detected and saved without the principal's own IS_OWNER or the inherited admins entry. After a deploy, a remote level change, an added group and two removals were written back per principal, and the next deploy and sync were no-ops.

🤖 Factory-generated comment. I still review the code and double-check every claim myself.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[AI Generated] On the open question: the manual run as a service principal checks the behavior I was worried about (owner and inherited admins excluded, per-principal write-back, no-op redeploy), so keeping the test local-only is fine for this PR. CI still won't catch a regression on a real ACL, so making it Cloud = true with real principals is worth a follow-up.

review-factory reply (Claude Opus 5.5)

Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"]

Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
20 changes: 14 additions & 6 deletions bundle/config/mutator/resourcemutator/fix_permissions.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package resourcemutator

import (
"context"
"reflect"
"strings"

"github.com/databricks/cli/bundle"
Expand Down Expand Up @@ -155,7 +156,7 @@ func useMaximumLevel(permissions dyn.Value, resourceType string) (dyn.Value, err
}

levelPerPrincipal := make(map[string]string)
principalIndex := make(map[string]int)
maxPermission := make(map[string]dyn.Value)
var principals []string

for _, permission := range permissionArray {
Expand All @@ -168,19 +169,26 @@ func useMaximumLevel(permissions dyn.Value, resourceType string) (dyn.Value, err
if principal == "" {
continue
}
_, ok = principalIndex[principal]
_, ok = maxPermission[principal]
if !ok {
ind := len(principalIndex)
principalIndex[principal] = ind
principals = append(principals, principal)
}
levelPerPrincipal[principal] = resources.GetMaxLevel(levelPerPrincipal[principal], level)
maxLevel := resources.GetMaxLevel(levelPerPrincipal[principal], level)
if maxLevel != levelPerPrincipal[principal] {
maxPermission[principal] = permission
}
levelPerPrincipal[principal] = maxLevel
}

var newPermissions []dyn.Value

for _, principal := range principals {
newPermissions = append(newPermissions, createPermissionFromPrincipal(principal, levelPerPrincipal[principal]))
permission := createPermissionFromPrincipal(principal, levelPerPrincipal[principal])
// Keep the entry that sets the level as is: its source location lets config-remote-sync write it back.
if original := maxPermission[principal]; reflect.DeepEqual(original.AsAny(), permission.AsAny()) {
permission = original
}
newPermissions = append(newPermissions, permission)
}

return dyn.V(newPermissions), nil
Expand Down
20 changes: 20 additions & 0 deletions bundle/config/mutator/resourcemutator/fix_permissions_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@ import (
"testing"

"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestGetLevelScore(t *testing.T) {
Expand All @@ -28,3 +31,20 @@ func TestGetMaxLevel(t *testing.T) {
assert.Equal(t, "UNKNOWN_B", resources.GetMaxLevel("UNKNOWN_A", "UNKNOWN_B"))
assert.Equal(t, "UNKNOWN_B", resources.GetMaxLevel("UNKNOWN_B", "UNKNOWN_A"))
}

func TestUseMaximumLevelKeepsDeclaredEntries(t *testing.T) {
viewerLoc := []diag.Location{{File: "databricks.yml", Line: 10}}
managerLoc := []diag.Location{{File: "databricks.yml", Line: 14}}
viewer := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_VIEW"), "user_name": dyn.V("viewer@example.com")}, viewerLoc)
lower := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_VIEW"), "group_name": dyn.V("users")}, viewerLoc)
higher := dyn.NewValue(map[string]dyn.Value{"level": dyn.V("CAN_MANAGE"), "group_name": dyn.V("users")}, managerLoc)

result, err := useMaximumLevel(dyn.V([]dyn.Value{viewer, lower, higher}), "jobs")
require.NoError(t, err)

permissions := result.MustSequence()
require.Len(t, permissions, 2)
assert.Equal(t, viewerLoc, permissions[0].Locations())
assert.Equal(t, managerLoc, permissions[1].Locations())
assert.Equal(t, "CAN_MANAGE", permissions[1].Get("level").MustString())
}
2 changes: 1 addition & 1 deletion bundle/configsync/cluster_policy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -699,7 +699,7 @@ func policyTestBundle(t *testing.T, resourceKey string, config map[string]any) *

func filterPolicyTestChanges(t *testing.T, b *bundle.Bundle, plan *deployplan.Plan) (configsync.Changes, []configsync.SkippedChange, error) {
t.Helper()
changes, err := configsync.ExtractChanges(t.Context(), b, plan)
changes, _, err := configsync.ExtractChanges(t.Context(), b, plan)
require.NoError(t, err)
inputs := []any{plan, changes}
for _, entry := range plan.Plan {
Expand Down
Loading
Loading