diff --git a/.nextchanges/bundles/7713.md b/.nextchanges/bundles/7713.md new file mode 100644 index 00000000000..8e4743c6e97 --- /dev/null +++ b/.nextchanges/bundles/7713.md @@ -0,0 +1 @@ +* `bundle plan` no longer shows or saves sensitive variable values, and `bundle deploy --plan` requires them to be supplied again at deploy time (for example with `--var` or `BUNDLE_VAR_`). `deploy --plan` applies the sensitive values supplied at deploy time, which are not checked against the values used at plan time. ([#7713](https://github.com/databricks/cli/pull/7713)) diff --git a/acceptance/bundle/variables/sensitive/plan-derived/databricks.yml b/acceptance/bundle/variables/sensitive/plan-derived/databricks.yml new file mode 100644 index 00000000000..f8bc0bee2d9 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan-derived/databricks.yml @@ -0,0 +1,34 @@ +bundle: + name: sensitive-variables + +# Sensitive variables that no resource uses need no value for a saved plan to apply. A variable used as a +# secret value cannot have a default or a target value, so the other sensitive variables are unused. +variables: + api_credential: + description: Credential for the external API, passed via the variable-overrides file. + sensitive: true + api_header: + description: Sensitive, derived from another sensitive variable, and not used by a resource. + sensitive: true + default: "Bearer ${var.api_credential}" + api_literal: + description: Sensitive with a literal default, and not used by a resource. + sensitive: true + default: dummy-literal + api_token: + description: Sensitive, not used by a resource, and set only for the dev target. + sensitive: true + +targets: + dev: + default: true + variables: + api_token: dummy-target + +resources: + secrets: + credential_secret: + name: credential_secret + catalog_name: main + schema_name: default + value: ${var.api_credential} diff --git a/acceptance/bundle/variables/sensitive/plan-derived/out.test.toml b/acceptance/bundle/variables/sensitive/plan-derived/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan-derived/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/variables/sensitive/plan-derived/output.txt b/acceptance/bundle/variables/sensitive/plan-derived/output.txt new file mode 100644 index 00000000000..2bb715c67f6 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan-derived/output.txt @@ -0,0 +1,25 @@ + +=== bundle plan -o json, saved to a file + +>>> jq -S .plan | map_values(.new_state) .databricks/plan.json +{ + "resources.secrets.credential_secret": { + "value": { + "catalog_name": "main", + "name": "credential_secret", + "schema_name": "default", + "value": "[redacted]" + } + } +} + +=== deploy from the saved plan with the credential in the variable-overrides file + +>>> env -u BUNDLE_VAR_api_credential [CLI] bundle deploy --plan .databricks/plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/dev/files... +Created secrets.credential_secret +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== requests carrying a secret literal: path, name and value +{"path":"/api/2.1/unity-catalog/secrets","name":"credential_secret","value":"dummy-credential"} diff --git a/acceptance/bundle/variables/sensitive/plan-derived/script b/acceptance/bundle/variables/sensitive/plan-derived/script new file mode 100644 index 00000000000..5cc78bd6f57 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan-derived/script @@ -0,0 +1,17 @@ +# The secret values must not appear in CLI output or the saved plan, only in the requests. +SECRETS=('!dummy-credential' '!Bearer dummy-credential' '!dummy-literal' '!dummy-target') + +title "bundle plan -o json, saved to a file" +mkdir -p .databricks +$CLI bundle plan -o json | contains.py "${SECRETS[@]}" > .databricks/plan.json +trace jq -S '.plan | map_values(.new_state)' .databricks/plan.json + +# The variable-overrides file supplies the credential, and the unused derived and literal variables need no value. +title "deploy from the saved plan with the credential in the variable-overrides file" +mkdir -p .databricks/bundle/dev +echo '{"api_credential": "dummy-credential"}' > .databricks/bundle/dev/variable-overrides.json +trace env -u BUNDLE_VAR_api_credential $CLI bundle deploy --plan .databricks/plan.json | contains.py "${SECRETS[@]}" +rm .databricks/bundle/dev/variable-overrides.json + +title "requests carrying a secret literal: path, name and value" +jq -c 'select(tostring | contains("dummy-credential")) | {path, name: .body.name, value: .body.value}' < "$OUT_REQUESTS" | sort diff --git a/acceptance/bundle/variables/sensitive/plan-derived/test.toml b/acceptance/bundle/variables/sensitive/plan-derived/test.toml new file mode 100644 index 00000000000..b9864165e0c --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan-derived/test.toml @@ -0,0 +1,3 @@ +RecordRequests = true +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/variables/sensitive/plan/databricks.yml b/acceptance/bundle/variables/sensitive/plan/databricks.yml new file mode 100644 index 00000000000..bcf245d43f5 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan/databricks.yml @@ -0,0 +1,26 @@ +bundle: + name: sensitive-variables + +variables: + api_credential: + description: Credential for the external API, passed via BUNDLE_VAR_api_credential. + sensitive: true + api_header: + description: Header value for the plain secret, passed via BUNDLE_VAR_api_header. + sensitive: true + +resources: + catalogs: + audit_catalog: + name: audit_catalog + secrets: + ref_secret: + name: ref_secret + catalog_name: ${resources.catalogs.audit_catalog.name} + schema_name: default + value: ${var.api_credential} + plain_secret: + name: plain_secret + catalog_name: main + schema_name: default + value: ${var.api_header} diff --git a/acceptance/bundle/variables/sensitive/plan/out.test.toml b/acceptance/bundle/variables/sensitive/plan/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/variables/sensitive/plan/output.txt b/acceptance/bundle/variables/sensitive/plan/output.txt new file mode 100644 index 00000000000..f190193ef33 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan/output.txt @@ -0,0 +1,103 @@ + +=== bundle plan (text) + +>>> [CLI] bundle plan +create catalogs.audit_catalog +create secrets.plain_secret +create secrets.ref_secret + +Plan: 3 to add, 0 to change, 0 to delete, 0 unchanged + +=== bundle plan -o json, saved to a file + +>>> jq -S .plan | map_values(.new_state) .databricks/plan.json +{ + "resources.catalogs.audit_catalog": { + "value": { + "name": "audit_catalog" + } + }, + "resources.secrets.plain_secret": { + "value": { + "catalog_name": "main", + "name": "plain_secret", + "schema_name": "default", + "value": "[redacted]" + } + }, + "resources.secrets.ref_secret": { + "value": { + "catalog_name": "audit_catalog", + "name": "ref_secret", + "schema_name": "default", + "value": "[redacted]" + } + } +} + +=== deploy from the saved plan without the sensitive variable + +>>> env -u BUNDLE_VAR_api_header [CLI] bundle deploy --plan .databricks/plan.json +Error: no value assigned to required variable api_header. Variables are usually assigned in databricks.yml, and they can be overridden using "--var", the BUNDLE_VAR_api_header environment variable, or .databricks/bundle//variable-overrides.json + + +=== deploy from the saved plan with an empty sensitive variable + +>>> env BUNDLE_VAR_api_credential= [CLI] bundle deploy --plan .databricks/plan.json +Error: resource resources.secrets.ref_secret: the config has no value for sensitive field value; set the variable it uses with --var or BUNDLE_VAR_ + +state-changing requests sent by the failed deploy: 0 + +=== deploy from the saved plan with the sensitive variables supplied + +>>> [CLI] bundle deploy --plan .databricks/plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default/files... +Created catalogs.audit_catalog +Created secrets.plain_secret +Created secrets.ref_secret +Files: 1 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== bundle destroy, then a plain bundle deploy creates the secrets again + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.audit_catalog + delete resources.secrets.plain_secret + delete resources.secrets.ref_secret + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default + +Destroy: 3 deleted + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default/files... +Created catalogs.audit_catalog +Created secrets.plain_secret +Created secrets.ref_secret +Files: 1 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== bundle plan -o json after deploy: remote_state holds the marker + +>>> jq -S .plan | map_values(.remote_state.value) .databricks/plan-after.json +{ + "resources.catalogs.audit_catalog": null, + "resources.secrets.plain_secret": "[redacted]", + "resources.secrets.ref_secret": "[redacted]" +} + +=== rotate the plain secret: plan and deploy with a new value +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default/files... +Updated secrets.plain_secret +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +=== requests carrying a secret literal: path, name and value +{"path":"/api/2.1/unity-catalog/secrets","name":"plain_secret","value":"dummy-header"} +{"path":"/api/2.1/unity-catalog/secrets","name":"plain_secret","value":"dummy-header"} +{"path":"/api/2.1/unity-catalog/secrets","name":"ref_secret","value":"dummy-credential"} +{"path":"/api/2.1/unity-catalog/secrets","name":"ref_secret","value":"dummy-credential"} +{"path":"/api/2.1/unity-catalog/secrets/main.default.plain_secret","name":"plain_secret","value":"dummy-rotated"} + +=== no secret value in the local state (the dump differs per DMS variant, so it is not shown) diff --git a/acceptance/bundle/variables/sensitive/plan/script b/acceptance/bundle/variables/sensitive/plan/script new file mode 100644 index 00000000000..a26543c56e1 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan/script @@ -0,0 +1,45 @@ +# The values of the sensitive variables must not appear in any CLI output, the saved plan or the +# local state. The request log is checked for the real values instead, since the API needs them. +SECRETS=('!dummy-credential' '!dummy-header') +ROTATED=dummy-rotated + +title "bundle plan (text)" +trace $CLI bundle plan | contains.py "${SECRETS[@]}" + +title "bundle plan -o json, saved to a file" +$CLI bundle plan -o json | contains.py "${SECRETS[@]}" > .databricks/plan.json +trace jq -S '.plan | map_values(.new_state)' .databricks/plan.json + +title "deploy from the saved plan without the sensitive variable" +musterr trace env -u BUNDLE_VAR_api_header $CLI bundle deploy --plan .databricks/plan.json + +# An empty value counts as missing. The deploy reads the workspace before the check, and the number of +# those reads differs per DMS variant, so only the state-changing requests are counted. +title "deploy from the saved plan with an empty sensitive variable" +before=$(wc -l < "$OUT_REQUESTS") +musterr trace env BUNDLE_VAR_api_credential= $CLI bundle deploy --plan .databricks/plan.json +echo "state-changing requests sent by the failed deploy: $(tail -n +$((before+1)) "$OUT_REQUESTS" | jq -s 'map(select(.method != "GET" and .path != "/telemetry-ext")) | length')" + +title "deploy from the saved plan with the sensitive variables supplied" +trace $CLI bundle deploy --plan .databricks/plan.json | contains.py "${SECRETS[@]}" + +# A plain deploy sends the values from the state cache, a different path from the saved plan. +title "bundle destroy, then a plain bundle deploy creates the secrets again" +trace $CLI bundle destroy --auto-approve | contains.py "${SECRETS[@]}" +trace $CLI bundle deploy | contains.py "${SECRETS[@]}" + +title "bundle plan -o json after deploy: remote_state holds the marker" +$CLI bundle plan -o json | contains.py "${SECRETS[@]}" > .databricks/plan-after.json +trace jq -S '.plan | map_values(.remote_state.value)' .databricks/plan-after.json + +# Rotating a value: the saved plan is applied with the new value, which only the PATCH may carry. +title "rotate the plain secret: plan and deploy with a new value" +BUNDLE_VAR_api_header=$ROTATED $CLI bundle plan -o json | contains.py "${SECRETS[@]}" "!$ROTATED" > .databricks/plan-rotate.json +BUNDLE_VAR_api_header=$ROTATED $CLI bundle deploy --plan .databricks/plan-rotate.json | contains.py "${SECRETS[@]}" "!$ROTATED" + +title "requests carrying a secret literal: path, name and value" +jq -c --arg rotated "$ROTATED" 'select(tostring | contains("dummy-credential") or contains("dummy-header") or contains($rotated)) | {path, name: .body.name, value: .body.value}' < "$OUT_REQUESTS" | sort + +# The state dump differs per DMS variant, so only the match is checked, not the output. +title "no secret value in the local state (the dump differs per DMS variant, so it is not shown)" +find .databricks -type f -exec cat {} + | contains.py "${SECRETS[@]}" "!$ROTATED" > /dev/null diff --git a/acceptance/bundle/variables/sensitive/plan/test.toml b/acceptance/bundle/variables/sensitive/plan/test.toml new file mode 100644 index 00000000000..54fc177c9f1 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan/test.toml @@ -0,0 +1,2 @@ +RecordRequests = true +Env.BUNDLE_VAR_api_header = "dummy-header" diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index f5a5fd6ee75..a86a6264b9e 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -41,9 +41,9 @@ type Variable struct { // Marks the variable as holding a secret. Set automatically for a variable used as the // value of a sensitive resource field (e.g. the value of a UC secret). Its value is - // redacted in the output of `bundle validate -o json` and `bundle summary -o json`, and - // it may only be referenced from sensitive resource fields or from other sensitive - // variables, so that the secret never ends up in plain text. + // redacted in the output of `bundle validate -o json`, `bundle summary -o json` and + // `bundle plan`. It may only be referenced from sensitive resource fields or from other + // sensitive variables, so that the secret never ends up in plain text. Sensitive *bool `json:"sensitive,omitempty"` // This field stores the resolved value for the variable. The variable are diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 1011a0791d5..b96734bd316 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -69,9 +69,67 @@ func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan. return nil } -// InitForApply initializes the DeploymentBundle for applying a pre-computed plan. +// CheckSensitivePlanValues returns one error per problem that stops a saved plan from being applied: +// a sensitive field the plan sets whose resolved config value is empty. The resolved config holds the +// values from --var, BUNDLE_VAR_, defaults, targets and the variable-overrides file alike, so any of +// them supplies a value. Sensitive variables the plan does not use need no value. +func CheckSensitivePlanValues(root *config.Root, plan *deployplan.Plan) []error { + var problems []error + seen := make(map[string]bool) + addProblem := func(err error) { + if !seen[err.Error()] { + seen[err.Error()] = true + problems = append(problems, err) + } + } + + for _, resourceKey := range slices.Sorted(maps.Keys(plan.Plan)) { + entry := plan.Plan[resourceKey] + if entry.NewState == nil { + continue + } + resourceType := config.GetResourceTypeFromKey(resourceKey) + typedNil, ok := dresources.SupportedResources[resourceType] + if !ok { + addProblem(fmt.Errorf("resource %s: unknown resource type", resourceKey)) + continue + } + rules := dresources.SensitiveFieldRules(resourceType) + if len(rules) == 0 { + continue + } + adapter, err := dresources.NewAdapter(typedNil, resourceType, nil) + if err != nil { + addProblem(fmt.Errorf("resource %s: %w", resourceKey, err)) + continue + } + sv, err := entry.NewState.ToStructVar(adapter.StateType()) + if err != nil { + addProblem(fmt.Errorf("resource %s: %w", resourceKey, err)) + continue + } + paths, err := sensitivePaths(rules) + if err != nil { + addProblem(fmt.Errorf("resource %s: %w", resourceKey, err)) + continue + } + for _, path := range paths { + planned, err := structaccess.Get(sv.Value, path) + if err != nil { + continue + } + if _, err := sensitiveConfigValue(root, resourceKey, path, planned); err != nil { + addProblem(err) + } + } + } + return problems +} + +// InitForApply initializes the DeploymentBundle for applying a pre-computed plan. The sensitive +// values the plan redacted are restored from root, the current config. // StateDB must already be open for write before calling this function. -func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) error { +func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan, root *config.Root) error { b.StateDB.AssertOpenedForWrite() err := b.init(client) @@ -102,6 +160,9 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. if err != nil { return fmt.Errorf("loading plan entry %s: %w", resourceKey, err) } + if err := restoreSensitiveFields(adapter, resourceKey, sv.Value, root); err != nil { + return err + } b.StateCache.Store(resourceKey, sv) } @@ -513,7 +574,7 @@ func prepareChanges(ctx context.Context, adapter *dresources.Adapter, localDiff, entry.Remote = ch.Old if !structdiff.IsEqual(entry.New, ch.New) { // this is not fatal (may result in unexpected drift or undetected change but not incorrect deploy), but good to log this - log.Warnf(ctx, "unexpected local and remote diffs (%T, %T); entry=%v ch=%v", entry.New, ch.New, entry, ch) + log.Warnf(ctx, "unexpected local and remote diffs at %s (%T, %T)", ch.Path, entry.New, ch.New) } } } @@ -1704,8 +1765,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root }) // Store an unredacted copy in the cache so Apply can deploy with the - // actual values. The original newStateConfig is redacted below and used - // only for the plan output. + // actual values. newStateConfig is serialised into the plan and redacted there. stateType := adapter.StateType() cacheCopyPtr := reflect.New(stateType.Elem()) cacheCopyPtr.Elem().Set(reflect.ValueOf(newStateConfig).Elem()) @@ -1714,20 +1774,12 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root Refs: refs, }) - // Redact sensitive fields before serialising. Sensitive values always come - // from bundle variables (enforced by ValidateSecretValueIsVariable), which - // are resolved before plan time, so SyncToJSON (called when cross-resource - // refs are resolved during planNode) will never re-serialise these fields. - if err := redactStruct(adapter, newStateConfig); err != nil { - return nil, fmt.Errorf("%s: cannot redact state: %w", node, err) - } - newState := &structvar.StructVar{ Value: newStateConfig, Refs: refs, } - // Convert to JSON for serialization in plan (values already redacted above). + // Convert to JSON for serialization in plan. newStateJSON, err := newState.ToJSON() if err != nil { return nil, fmt.Errorf("%s: cannot serialize state: %w", node, err) diff --git a/bundle/direct/dresources/redact_config.go b/bundle/direct/dresources/redact_config.go index 185713ea830..b45ea11deb8 100644 --- a/bundle/direct/dresources/redact_config.go +++ b/bundle/direct/dresources/redact_config.go @@ -60,6 +60,11 @@ func redactSensitiveVariables(root *config.Root) { } } +// SensitiveFieldRules returns the sensitive fields of a resource type without building its adapter. +func SensitiveFieldRules(resourceType string) []FieldRule { + return getSensitiveFields()[resourceType] +} + // getSensitiveFields returns the map of resource type to list of sensitive fields. func getSensitiveFields() map[string][]FieldRule { cfg := MustLoadConfig() diff --git a/bundle/direct/redact.go b/bundle/direct/redact.go index b44ab74e7b4..9a14b5fd746 100644 --- a/bundle/direct/redact.go +++ b/bundle/direct/redact.go @@ -4,6 +4,7 @@ import ( "fmt" "slices" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/structs/structaccess" @@ -12,6 +13,20 @@ import ( const sensitiveRedactedValue = "[redacted]" +// sensitivePaths parses the sensitive field rules of a resource type. +func sensitivePaths(rules []dresources.FieldRule) ([]*structpath.PathNode, error) { + var paths []*structpath.PathNode + for _, rule := range rules { + field := rule.Field.String() + path, err := structpath.ParsePath(field) + if err != nil { + return nil, fmt.Errorf("parsing sensitive field path %q: %w", field, err) + } + paths = append(paths, path) + } + return paths, nil +} + // redactSensitiveFields replaces (or zeros) scalar fields in s (a pointer to a typed struct) // that the adapter marks as sensitive. replacement is what to set the field to; pass // sensitiveRedactedValue for plan output display, or "" for state-file storage. @@ -20,13 +35,11 @@ func redactSensitiveFields(adapter *dresources.Adapter, s any, replacement strin return nil } - fields := adapter.GetSensitiveFields() - for _, field := range fields { - path, err := structpath.ParsePath(field) - if err != nil { - return fmt.Errorf("parsing sensitive field path %q: %w", field, err) - } - + paths, err := sensitivePaths(adapter.ResourceConfig().SensitiveFields) + if err != nil { + return err + } + for _, path := range paths { fv, err := structaccess.Get(s, path) if err != nil { continue @@ -76,13 +89,29 @@ func redactChanges(adapter *dresources.Adapter, changes deployplan.Changes) erro return nil } -// redactPlanEntry redacts sensitive fields from remote_state and changes of a plan -// entry. new_state is already redacted before serialization in makePlan. +// redactPlanEntry redacts new_state, remote_state and changes of a plan entry. It runs once planning +// is done: reference resolution re-serializes new_state from the unredacted cache, and apply reads +// that cache, so the plan must not be serialized after this. func redactPlanEntry(adapter *dresources.Adapter, entry *deployplan.PlanEntry) error { if len(adapter.GetSensitiveFields()) == 0 { return nil } + if entry.NewState != nil { + sv, err := entry.NewState.ToStructVar(adapter.StateType()) + if err != nil { + return fmt.Errorf("redacting new_state: %w", err) + } + if err := redactStruct(adapter, sv.Value); err != nil { + return fmt.Errorf("redacting new_state: %w", err) + } + redacted, err := sv.ToJSON() + if err != nil { + return fmt.Errorf("redacting new_state: %w", err) + } + *entry.NewState = *redacted + } + if entry.RemoteState != nil { if err := redactStruct(adapter, entry.RemoteState); err != nil { return fmt.Errorf("redacting remote_state: %w", err) @@ -97,3 +126,53 @@ func redactPlanEntry(adapter *dresources.Adapter, entry *deployplan.PlanEntry) e return nil } + +// sensitiveConfigValue returns the value the config sets for a sensitive field, which a deploy sends. +// The config value is used whenever it is set. An empty config value is an error only when the plan +// holds the field non-zero, since the plan then expects a value the config cannot supply. A field +// that is zero in both is output-only and yields "". +func sensitiveConfigValue(root *config.Root, resourceKey string, path *structpath.PathNode, planned any) (string, error) { + rc, err := root.GetResourceConfig(resourceKey) + if err != nil { + return "", err + } + var value string + if rc != nil { + if v, err := structaccess.Get(rc, path); err == nil { + value, _ = v.(string) + } + } + if value != "" || planned == nil || planned == "" { + return value, nil + } + if rc == nil { + return "", fmt.Errorf("resource %s is not in the config; run 'bundle plan' again", resourceKey) + } + return "", fmt.Errorf("resource %s: the config has no value for sensitive field %s; set the variable it uses with --var or BUNDLE_VAR_", resourceKey, path) +} + +// restoreSensitiveFields replaces the sensitive values in s, a saved plan's state, with the values +// the current config sets. CheckSensitivePlanValues runs the same rule before the plan is approved. +func restoreSensitiveFields(adapter *dresources.Adapter, resourceKey string, s any, root *config.Root) error { + paths, err := sensitivePaths(adapter.ResourceConfig().SensitiveFields) + if err != nil { + return err + } + for _, path := range paths { + planned, err := structaccess.Get(s, path) + if err != nil { + continue + } + value, err := sensitiveConfigValue(root, resourceKey, path, planned) + if err != nil { + return err + } + if value == "" { + continue + } + if err := structaccess.Set(s, path, value); err != nil { + return fmt.Errorf("resource %s: restoring sensitive field %s: %w", resourceKey, path, err) + } + } + return nil +} diff --git a/bundle/direct/redact_test.go b/bundle/direct/redact_test.go new file mode 100644 index 00000000000..8081387392c --- /dev/null +++ b/bundle/direct/redact_test.go @@ -0,0 +1,261 @@ +package direct + +import ( + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/config/variable" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const secretLiteral = "dummy-credential" + +func secretAdapter(t *testing.T) *dresources.Adapter { + t.Helper() + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + adapter, ok := adapters["secrets"] + require.True(t, ok) + return adapter +} + +// syncedRoot gives a Go-literal config the dynamic value that a loaded bundle has, which GetResourceConfig reads. +func syncedRoot(t *testing.T, root *config.Root) *config.Root { + t.Helper() + require.NoError(t, root.Mutate(func(v dyn.Value) (dyn.Value, error) { return v, nil })) + return root +} + +// TestRedactPlanEntryRedactsNewStateAndChanges checks that the literal is absent from the new_state +// and changes of a plan entry, and that the non-sensitive fields are kept. +func TestRedactPlanEntryRedactsNewStateAndChanges(t *testing.T) { + adapter := secretAdapter(t) + + newState := &structvar.StructVarJSON{} + var err error + newState.Value, err = json.Marshal(map[string]any{ + "name": "s", + "catalog_name": "main", + "schema_name": "default", + "value": secretLiteral, + }) + require.NoError(t, err) + + entry := &deployplan.PlanEntry{ + Action: deployplan.Update, + NewState: newState, + Changes: deployplan.Changes{ + "value": {Action: deployplan.Update, Old: "", New: secretLiteral, Remote: secretLiteral}, + "name": {Action: deployplan.Skip, Old: "s", New: "s", Remote: "s"}, + }, + } + + require.NoError(t, redactPlanEntry(adapter, entry)) + + out, err := json.Marshal(entry) + require.NoError(t, err) + assert.NotContains(t, string(out), secretLiteral) + + var redacted struct { + NewState struct { + Value map[string]any `json:"value"` + } `json:"new_state"` + } + require.NoError(t, json.Unmarshal(out, &redacted)) + assert.Equal(t, "[redacted]", redacted.NewState.Value["value"]) + assert.Equal(t, "main", redacted.NewState.Value["catalog_name"]) + assert.Equal(t, "[redacted]", entry.Changes["value"].New) + assert.Equal(t, "s", entry.Changes["name"].New) +} + +func secretRoot(t *testing.T, value string, variables map[string]*variable.Variable) *config.Root { + t.Helper() + root := &config.Root{ + Variables: variables, + Resources: config.Resources{ + Secrets: map[string]*resources.Secret{ + "s": {Secret: catalog.Secret{Name: "s", Value: value}}, + }, + }, + } + return syncedRoot(t, root) +} + +const ( + msgValueMissing = "resource resources.secrets.s: the config has no value for sensitive field value; set the variable it uses with --var or BUNDLE_VAR_" + msgNotInConfig = "resource resources.secrets.gone is not in the config; run 'bundle plan' again" +) + +func TestRestoreSensitiveFields(t *testing.T) { + adapter := secretAdapter(t) + + tests := []struct { + name string + resourceKey string + state *catalog.Secret + configValue string + wantValue string + wantErr string + }{ + { + name: "redaction marker is replaced by the config value", + resourceKey: "resources.secrets.s", + state: &catalog.Secret{Name: "s", Value: sensitiveRedactedValue}, + configValue: secretLiteral, + wantValue: secretLiteral, + }, + { + name: "plaintext from a plan saved by an older CLI is replaced by the config value", + resourceKey: "resources.secrets.s", + state: &catalog.Secret{Name: "s", Value: "stale-plaintext"}, + configValue: secretLiteral, + wantValue: secretLiteral, + }, + { + name: "zero in the plan but set in the config is restored", + resourceKey: "resources.secrets.s", + state: &catalog.Secret{Name: "s"}, + configValue: secretLiteral, + wantValue: secretLiteral, + }, + { + name: "zero in both is skipped", + resourceKey: "resources.secrets.s", + state: &catalog.Secret{Name: "s"}, + configValue: "", + wantValue: "", + }, + { + name: "non-zero in the plan with an empty config value is an error", + resourceKey: "resources.secrets.s", + state: &catalog.Secret{Name: "s", Value: sensitiveRedactedValue}, + configValue: "", + wantErr: msgValueMissing, + }, + { + name: "non-zero in the plan with the resource missing from config is an error", + resourceKey: "resources.secrets.gone", + state: &catalog.Secret{Name: "s", Value: sensitiveRedactedValue}, + configValue: secretLiteral, + wantErr: msgNotInConfig, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + root := secretRoot(t, tc.configValue, nil) + err := restoreSensitiveFields(adapter, tc.resourceKey, tc.state, root) + if tc.wantErr != "" { + assert.EqualError(t, err, tc.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantValue, tc.state.Value) + assert.Empty(t, tc.state.EffectiveValue) + }) + } +} + +// TestSensitiveConfigValueNestedField checks that a dotted sensitive field path is read from the +// resource config, not only a top-level field. +func TestSensitiveConfigValueNestedField(t *testing.T) { + root := syncedRoot(t, &config.Root{Resources: config.Resources{Jobs: map[string]*resources.Job{ + "j": {JobSettings: jobs.JobSettings{Schedule: &jobs.CronSchedule{QuartzCronExpression: "0 0 * * *"}}}, + }}}) + path, err := structpath.ParsePath("schedule.quartz_cron_expression") + require.NoError(t, err) + + value, err := sensitiveConfigValue(root, "resources.jobs.j", path, "planned") + require.NoError(t, err) + assert.Equal(t, "0 0 * * *", value) +} + +func TestCheckSensitivePlanValues(t *testing.T) { + sensitive := true + variables := map[string]*variable.Variable{ + "api_credential": {Sensitive: &sensitive}, + "target_schema": {}, + } + secretPlan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.secrets.s": {NewState: &structvar.StructVarJSON{Value: []byte(`{"name":"s","value":"[redacted]"}`)}}, + }} + zeroPlan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.secrets.s": {NewState: &structvar.StructVarJSON{Value: []byte(`{"name":"s"}`)}}, + }} + jobsPlan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.j": {NewState: &structvar.StructVarJSON{Value: []byte(`{"name":"j"}`)}}, + }} + + tests := []struct { + name string + root *config.Root + plan *deployplan.Plan + wantErr string + }{ + { + name: "plan without sensitive resources needs nothing", + root: secretRoot(t, "", variables), + plan: jobsPlan, + }, + { + name: "sensitive value set in the config passes", + root: secretRoot(t, secretLiteral, variables), + plan: secretPlan, + }, + { + name: "zero in the plan and set in the config passes", + root: secretRoot(t, secretLiteral, variables), + plan: zeroPlan, + }, + { + name: "empty config value is reported", + root: secretRoot(t, "", variables), + plan: secretPlan, + wantErr: msgValueMissing, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + errs := CheckSensitivePlanValues(tc.root, tc.plan) + if tc.wantErr == "" { + assert.Empty(t, errs) + return + } + require.Len(t, errs, 1) + assert.EqualError(t, errs[0], tc.wantErr) + }) + } +} + +// TestCheckSensitivePlanValuesOneErrorPerProblem checks that each resource with a missing value gets its +// own error, and that a resource missing from the config is reported once, not once per sensitive field. +func TestCheckSensitivePlanValuesOneErrorPerProblem(t *testing.T) { + root := syncedRoot(t, &config.Root{Resources: config.Resources{ + Secrets: map[string]*resources.Secret{ + "a": {Secret: catalog.Secret{Name: "a"}}, + "b": {Secret: catalog.Secret{Name: "b"}}, + }, + }}) + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.secrets.a": {NewState: &structvar.StructVarJSON{Value: []byte(`{"name":"a","value":"[redacted]"}`)}}, + "resources.secrets.b": {NewState: &structvar.StructVarJSON{Value: []byte(`{"name":"b","value":"[redacted]"}`)}}, + "resources.secrets.gone": {NewState: &structvar.StructVarJSON{Value: []byte(`{"name":"gone","value":"[redacted]"}`)}}, + }} + + errs := CheckSensitivePlanValues(root, plan) + require.Len(t, errs, 3) + assert.EqualError(t, errs[0], "resource resources.secrets.a: the config has no value for sensitive field value; set the variable it uses with --var or BUNDLE_VAR_") + assert.EqualError(t, errs[1], "resource resources.secrets.b: the config has no value for sensitive field value; set the variable it uses with --var or BUNDLE_VAR_") + assert.EqualError(t, errs[2], msgNotInConfig) +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 1f6831281ea..b460d4ed046 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -310,7 +310,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand if planFromFile { // Initialize DeploymentBundle for applying the loaded plan - err := b.DeploymentBundle.InitForApply(ctx, b.WorkspaceClient(ctx), plan) + err := b.DeploymentBundle.InitForApply(ctx, b.WorkspaceClient(ctx), plan, &b.Config) if err != nil { logdiag.LogError(ctx, err) return diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 423c41f35ae..22059254c50 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -495,6 +495,13 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } + + if errs := direct.CheckSensitivePlanValues(&b.Config, plan); len(errs) > 0 { + for _, err := range errs { + logdiag.LogError(ctx, err) + } + return b, stateDesc, root.ErrAlreadyPrinted + } } // Reject an unknown --select before PreDeployChecks and the predeploy script