Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7713.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `bundle plan` no longer shows or saves sensitive variable values, and `bundle deploy --plan` requires them to be supplied again at deploy time (for example with `--var` or `BUNDLE_VAR_<name>`). `deploy --plan` applies the sensitive values supplied at deploy time, which are not checked against the values used at plan time. ([#7713](https://github.com/databricks/cli/pull/7713))
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
bundle:
name: sensitive-variables

# Sensitive variables that no resource uses need no value for a saved plan to apply. A variable used as a
# secret value cannot have a default or a target value, so the other sensitive variables are unused.
variables:
api_credential:
description: Credential for the external API, passed via the variable-overrides file.
sensitive: true
api_header:
description: Sensitive, derived from another sensitive variable, and not used by a resource.
sensitive: true
default: "Bearer ${var.api_credential}"
api_literal:
description: Sensitive with a literal default, and not used by a resource.
sensitive: true
default: dummy-literal
api_token:
description: Sensitive, not used by a resource, and set only for the dev target.
sensitive: true

targets:
dev:
default: true
variables:
api_token: dummy-target

resources:
secrets:
credential_secret:
name: credential_secret
catalog_name: main
schema_name: default
value: ${var.api_credential}
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
25 changes: 25 additions & 0 deletions acceptance/bundle/variables/sensitive/plan-derived/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@

=== bundle plan -o json, saved to a file

>>> jq -S .plan | map_values(.new_state) .databricks/plan.json
{
"resources.secrets.credential_secret": {
"value": {
"catalog_name": "main",
"name": "credential_secret",
"schema_name": "default",
"value": "[redacted]"
}
}
}

=== deploy from the saved plan with the credential in the variable-overrides file

>>> env -u BUNDLE_VAR_api_credential [CLI] bundle deploy --plan .databricks/plan.json
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/dev/files...
Created secrets.credential_secret
Files: 1 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

=== requests carrying a secret literal: path, name and value
{"path":"/api/2.1/unity-catalog/secrets","name":"credential_secret","value":"dummy-credential"}
17 changes: 17 additions & 0 deletions acceptance/bundle/variables/sensitive/plan-derived/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# The secret values must not appear in CLI output or the saved plan, only in the requests.
SECRETS=('!dummy-credential' '!Bearer dummy-credential' '!dummy-literal' '!dummy-target')

title "bundle plan -o json, saved to a file"
mkdir -p .databricks
$CLI bundle plan -o json | contains.py "${SECRETS[@]}" > .databricks/plan.json
trace jq -S '.plan | map_values(.new_state)' .databricks/plan.json

# The variable-overrides file supplies the credential, and the unused derived and literal variables need no value.
title "deploy from the saved plan with the credential in the variable-overrides file"
mkdir -p .databricks/bundle/dev
echo '{"api_credential": "dummy-credential"}' > .databricks/bundle/dev/variable-overrides.json
trace env -u BUNDLE_VAR_api_credential $CLI bundle deploy --plan .databricks/plan.json | contains.py "${SECRETS[@]}"
rm .databricks/bundle/dev/variable-overrides.json

title "requests carrying a secret literal: path, name and value"
jq -c 'select(tostring | contains("dummy-credential")) | {path, name: .body.name, value: .body.value}' < "$OUT_REQUESTS" | sort
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
RecordRequests = true
Cloud = false
EnvMatrix.DMS = ["", "true"]
26 changes: 26 additions & 0 deletions acceptance/bundle/variables/sensitive/plan/databricks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
bundle:
name: sensitive-variables

variables:
api_credential:
description: Credential for the external API, passed via BUNDLE_VAR_api_credential.
sensitive: true
api_header:
description: Header value for the plain secret, passed via BUNDLE_VAR_api_header.
sensitive: true

resources:
catalogs:
audit_catalog:
name: audit_catalog
secrets:
ref_secret:
name: ref_secret
catalog_name: ${resources.catalogs.audit_catalog.name}
schema_name: default
value: ${var.api_credential}
plain_secret:
name: plain_secret
catalog_name: main
schema_name: default
value: ${var.api_header}
2 changes: 2 additions & 0 deletions acceptance/bundle/variables/sensitive/plan/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
103 changes: 103 additions & 0 deletions acceptance/bundle/variables/sensitive/plan/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@

=== bundle plan (text)

>>> [CLI] bundle plan
create catalogs.audit_catalog
create secrets.plain_secret
create secrets.ref_secret

Plan: 3 to add, 0 to change, 0 to delete, 0 unchanged

=== bundle plan -o json, saved to a file

>>> jq -S .plan | map_values(.new_state) .databricks/plan.json
{
"resources.catalogs.audit_catalog": {
"value": {
"name": "audit_catalog"
}
},
"resources.secrets.plain_secret": {
"value": {
"catalog_name": "main",
"name": "plain_secret",
"schema_name": "default",
"value": "[redacted]"
}
},
"resources.secrets.ref_secret": {
"value": {
"catalog_name": "audit_catalog",
"name": "ref_secret",
"schema_name": "default",
"value": "[redacted]"
}
}
}

=== deploy from the saved plan without the sensitive variable

>>> env -u BUNDLE_VAR_api_header [CLI] bundle deploy --plan .databricks/plan.json
Error: no value assigned to required variable api_header. Variables are usually assigned in databricks.yml, and they can be overridden using "--var", the BUNDLE_VAR_api_header environment variable, or .databricks/bundle/<target>/variable-overrides.json


=== deploy from the saved plan with an empty sensitive variable

>>> env BUNDLE_VAR_api_credential= [CLI] bundle deploy --plan .databricks/plan.json
Error: resource resources.secrets.ref_secret: the config has no value for sensitive field value; set the variable it uses with --var or BUNDLE_VAR_<name>

state-changing requests sent by the failed deploy: 0

=== deploy from the saved plan with the sensitive variables supplied

>>> [CLI] bundle deploy --plan .databricks/plan.json
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default/files...
Created catalogs.audit_catalog
Created secrets.plain_secret
Created secrets.ref_secret
Files: 1 uploaded, 0 deleted
Resources: 3 created, 0 changed, 0 deleted, 0 unchanged

=== bundle destroy, then a plain bundle deploy creates the secrets again

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.catalogs.audit_catalog
delete resources.secrets.plain_secret
delete resources.secrets.ref_secret

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default

Destroy: 3 deleted

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default/files...
Created catalogs.audit_catalog
Created secrets.plain_secret
Created secrets.ref_secret
Files: 1 uploaded, 0 deleted
Resources: 3 created, 0 changed, 0 deleted, 0 unchanged

=== bundle plan -o json after deploy: remote_state holds the marker

>>> jq -S .plan | map_values(.remote_state.value) .databricks/plan-after.json
{
"resources.catalogs.audit_catalog": null,
"resources.secrets.plain_secret": "[redacted]",
"resources.secrets.ref_secret": "[redacted]"
}

=== rotate the plain secret: plan and deploy with a new value
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default/files...
Updated secrets.plain_secret
Files: 0 uploaded, 0 deleted
Resources: 0 created, 1 changed, 0 deleted, 2 unchanged

=== requests carrying a secret literal: path, name and value
{"path":"/api/2.1/unity-catalog/secrets","name":"plain_secret","value":"dummy-header"}
{"path":"/api/2.1/unity-catalog/secrets","name":"plain_secret","value":"dummy-header"}
{"path":"/api/2.1/unity-catalog/secrets","name":"ref_secret","value":"dummy-credential"}
{"path":"/api/2.1/unity-catalog/secrets","name":"ref_secret","value":"dummy-credential"}
{"path":"/api/2.1/unity-catalog/secrets/main.default.plain_secret","name":"plain_secret","value":"dummy-rotated"}

=== no secret value in the local state (the dump differs per DMS variant, so it is not shown)
45 changes: 45 additions & 0 deletions acceptance/bundle/variables/sensitive/plan/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# The values of the sensitive variables must not appear in any CLI output, the saved plan or the
# local state. The request log is checked for the real values instead, since the API needs them.
SECRETS=('!dummy-credential' '!dummy-header')
ROTATED=dummy-rotated

title "bundle plan (text)"
trace $CLI bundle plan | contains.py "${SECRETS[@]}"

title "bundle plan -o json, saved to a file"
$CLI bundle plan -o json | contains.py "${SECRETS[@]}" > .databricks/plan.json
trace jq -S '.plan | map_values(.new_state)' .databricks/plan.json

title "deploy from the saved plan without the sensitive variable"
musterr trace env -u BUNDLE_VAR_api_header $CLI bundle deploy --plan .databricks/plan.json

# An empty value counts as missing. The deploy reads the workspace before the check, and the number of
# those reads differs per DMS variant, so only the state-changing requests are counted.
title "deploy from the saved plan with an empty sensitive variable"
before=$(wc -l < "$OUT_REQUESTS")
musterr trace env BUNDLE_VAR_api_credential= $CLI bundle deploy --plan .databricks/plan.json
echo "state-changing requests sent by the failed deploy: $(tail -n +$((before+1)) "$OUT_REQUESTS" | jq -s 'map(select(.method != "GET" and .path != "/telemetry-ext")) | length')"

title "deploy from the saved plan with the sensitive variables supplied"
trace $CLI bundle deploy --plan .databricks/plan.json | contains.py "${SECRETS[@]}"

# A plain deploy sends the values from the state cache, a different path from the saved plan.
title "bundle destroy, then a plain bundle deploy creates the secrets again"
trace $CLI bundle destroy --auto-approve | contains.py "${SECRETS[@]}"
trace $CLI bundle deploy | contains.py "${SECRETS[@]}"

title "bundle plan -o json after deploy: remote_state holds the marker"
$CLI bundle plan -o json | contains.py "${SECRETS[@]}" > .databricks/plan-after.json
trace jq -S '.plan | map_values(.remote_state.value)' .databricks/plan-after.json

# Rotating a value: the saved plan is applied with the new value, which only the PATCH may carry.
title "rotate the plain secret: plan and deploy with a new value"
BUNDLE_VAR_api_header=$ROTATED $CLI bundle plan -o json | contains.py "${SECRETS[@]}" "!$ROTATED" > .databricks/plan-rotate.json
BUNDLE_VAR_api_header=$ROTATED $CLI bundle deploy --plan .databricks/plan-rotate.json | contains.py "${SECRETS[@]}" "!$ROTATED"

title "requests carrying a secret literal: path, name and value"
jq -c --arg rotated "$ROTATED" 'select(tostring | contains("dummy-credential") or contains("dummy-header") or contains($rotated)) | {path, name: .body.name, value: .body.value}' < "$OUT_REQUESTS" | sort

# The state dump differs per DMS variant, so only the match is checked, not the output.
title "no secret value in the local state (the dump differs per DMS variant, so it is not shown)"
find .databricks -type f -exec cat {} + | contains.py "${SECRETS[@]}" "!$ROTATED" > /dev/null
2 changes: 2 additions & 0 deletions acceptance/bundle/variables/sensitive/plan/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
RecordRequests = true
Env.BUNDLE_VAR_api_header = "dummy-header"
6 changes: 3 additions & 3 deletions bundle/config/variable/variable.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@ type Variable struct {

// Marks the variable as holding a secret. Set automatically for a variable used as the
// value of a sensitive resource field (e.g. the value of a UC secret). Its value is
// redacted in the output of `bundle validate -o json` and `bundle summary -o json`, and
// it may only be referenced from sensitive resource fields or from other sensitive
// variables, so that the secret never ends up in plain text.
// redacted in the output of `bundle validate -o json`, `bundle summary -o json` and
// `bundle plan`. It may only be referenced from sensitive resource fields or from other
// sensitive variables, so that the secret never ends up in plain text.
Sensitive *bool `json:"sensitive,omitempty"`

// This field stores the resolved value for the variable. The variable are
Expand Down
Loading
Loading