From 7292e12c77958fb668747a24ff5c0970b82c2df1 Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Mon, 16 Nov 2026 16:24:02 +0000 Subject: [PATCH 1/5] Redact sensitive variables from bundle plan and saved plans bundle plan no longer prints or saves sensitive values: new_state is redacted once planning is done. A saved plan does not carry them, so deploy --plan requires every sensitive variable of the bundle to be set with --var or BUNDLE_VAR_, and restores each sensitive field from the config when the config sets it. One check runs before approval and reports all missing values in one error. The prepareChanges warning no longer prints the change values, which were visible before redaction. Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 119 ++++++++++++++++++++++++++++++----- bundle/direct/redact.go | 96 +++++++++++++++++++++++++--- bundle/phases/deploy.go | 2 +- cmd/bundle/utils/process.go | 5 ++ 4 files changed, 198 insertions(+), 24 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 145402cdef2..fbf964c1747 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -19,6 +19,7 @@ import ( "github.com/databricks/cli/bundle/terraform_dabs_map" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/retry" @@ -69,9 +70,105 @@ func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan. return nil } -// InitForApply initializes the DeploymentBundle for applying a pre-computed plan. +// CheckSensitivePlanValues rejects a saved plan that cannot be applied until its sensitive values are +// supplied again, and returns one error that lists every problem. When the plan has a sensitive +// resource, every sensitive variable of the bundle must be set with --var or BUNDLE_VAR_, since +// telling used variables from unused ones needs reference resolution. +func CheckSensitivePlanValues(ctx context.Context, root *config.Root, plan *deployplan.Plan, flagVars []string) error { + adapters, err := dresources.InitAll(nil) + if err != nil { + return err + } + + var problems []string + needsValues := false + for _, resourceKey := range slices.Sorted(maps.Keys(plan.Plan)) { + entry := plan.Plan[resourceKey] + if entry.NewState == nil { + continue + } + adapter, ok := adapters[config.GetResourceTypeFromKey(resourceKey)] + if !ok { + problems = append(problems, fmt.Sprintf("resource %s: unknown resource type", resourceKey)) + continue + } + if len(adapter.GetSensitiveFields()) == 0 { + continue + } + needsValues = true + sv, err := entry.NewState.ToStructVar(adapter.StateType()) + if err != nil { + problems = append(problems, fmt.Sprintf("resource %s: %s", resourceKey, err)) + continue + } + paths, err := sensitivePaths(adapter) + if err != nil { + problems = append(problems, fmt.Sprintf("resource %s: %s", resourceKey, err)) + continue + } + for _, path := range paths { + planned, err := structaccess.Get(sv.Value, path) + if err != nil { + continue + } + if _, err := sensitiveConfigValue(root, resourceKey, path, planned); err != nil { + problems = append(problems, err.Error()) + } + } + } + + var missing []string + if needsValues { + for name, v := range root.Variables { + if v == nil || !config.IsExplicitlyEnabled(v.Sensitive) || variableSupplied(ctx, name, flagVars) { + continue + } + missing = append(missing, name) + } + slices.Sort(missing) + } + if len(missing) > 0 { + problems = append(problems, missingVariablesError(missing)) + } + + if len(problems) == 0 { + return nil + } + return errors.New(strings.Join(problems, "; ")) +} + +// missingVariablesError names the sensitive variables that have no --var or BUNDLE_VAR_ value. +func missingVariablesError(missing []string) string { + const overrides = "the variable-overrides file is not read for sensitive variables" + if len(missing) == 1 { + name := missing[0] + return fmt.Sprintf("sensitive variable %s has no value; pass it with --var %s=... or BUNDLE_VAR_%s when deploying from a saved plan (%s)", name, name, name, overrides) + } + return fmt.Sprintf("sensitive variables %s have no value; pass each with --var NAME=... or BUNDLE_VAR_NAME when deploying from a saved plan (%s)", strings.Join(missing, ", "), overrides) +} + +// variableSupplied reports whether a sensitive variable has a value from --var or BUNDLE_VAR_. +// The resolved value also carries values from the variable-overrides file, so the check reads only +// flags and the environment. A --var given with an empty value wins over the environment. +func variableSupplied(ctx context.Context, name string, flagVars []string) bool { + flagged, found := "", false + for _, assignment := range flagVars { + key, value, _ := strings.Cut(assignment, "=") + if key == name { + flagged, found = value, true + } + } + if found { + return flagged != "" + } + value, _ := env.Lookup(ctx, "BUNDLE_VAR_"+name) + return value != "" +} + +// InitForApply initializes the DeploymentBundle for applying a pre-computed plan. The sensitive +// values the plan redacted are restored from root, the current config. // StateDB must already be open for write before calling this function. -func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) error { +func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan, root *config.Root) error { b.StateDB.AssertOpenedForWrite() err := b.init(client) @@ -102,6 +199,9 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. if err != nil { return fmt.Errorf("loading plan entry %s: %w", resourceKey, err) } + if err := restoreSensitiveFields(adapter, resourceKey, sv.Value, root); err != nil { + return err + } b.StateCache.Store(resourceKey, sv) } @@ -510,7 +610,7 @@ func prepareChanges(ctx context.Context, adapter *dresources.Adapter, localDiff, entry.Remote = ch.Old if !structdiff.IsEqual(entry.New, ch.New) { // this is not fatal (may result in unexpected drift or undetected change but not incorrect deploy), but good to log this - log.Warnf(ctx, "unexpected local and remote diffs (%T, %T); entry=%v ch=%v", entry.New, ch.New, entry, ch) + log.Warnf(ctx, "unexpected local and remote diffs at %s (%T, %T)", ch.Path, entry.New, ch.New) } } } @@ -1700,8 +1800,7 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root }) // Store an unredacted copy in the cache so Apply can deploy with the - // actual values. The original newStateConfig is redacted below and used - // only for the plan output. + // actual values. newStateConfig is serialised into the plan and redacted there. stateType := adapter.StateType() cacheCopyPtr := reflect.New(stateType.Elem()) cacheCopyPtr.Elem().Set(reflect.ValueOf(newStateConfig).Elem()) @@ -1710,20 +1809,12 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root Refs: refs, }) - // Redact sensitive fields before serialising. Sensitive values always come - // from bundle variables (enforced by ValidateSecretValueIsVariable), which - // are resolved before plan time, so SyncToJSON (called when cross-resource - // refs are resolved during planNode) will never re-serialise these fields. - if err := redactStruct(adapter, newStateConfig); err != nil { - return nil, fmt.Errorf("%s: cannot redact state: %w", node, err) - } - newState := &structvar.StructVar{ Value: newStateConfig, Refs: refs, } - // Convert to JSON for serialization in plan (values already redacted above). + // Convert to JSON for serialization in plan. newStateJSON, err := newState.ToJSON() if err != nil { return nil, fmt.Errorf("%s: cannot serialize state: %w", node, err) diff --git a/bundle/direct/redact.go b/bundle/direct/redact.go index b44ab74e7b4..ff0c0bc485c 100644 --- a/bundle/direct/redact.go +++ b/bundle/direct/redact.go @@ -4,6 +4,7 @@ import ( "fmt" "slices" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/structs/structaccess" @@ -12,6 +13,19 @@ import ( const sensitiveRedactedValue = "[redacted]" +// sensitivePaths parses the sensitive field paths of the adapter. +func sensitivePaths(adapter *dresources.Adapter) ([]*structpath.PathNode, error) { + var paths []*structpath.PathNode + for _, field := range adapter.GetSensitiveFields() { + path, err := structpath.ParsePath(field) + if err != nil { + return nil, fmt.Errorf("parsing sensitive field path %q: %w", field, err) + } + paths = append(paths, path) + } + return paths, nil +} + // redactSensitiveFields replaces (or zeros) scalar fields in s (a pointer to a typed struct) // that the adapter marks as sensitive. replacement is what to set the field to; pass // sensitiveRedactedValue for plan output display, or "" for state-file storage. @@ -20,13 +34,11 @@ func redactSensitiveFields(adapter *dresources.Adapter, s any, replacement strin return nil } - fields := adapter.GetSensitiveFields() - for _, field := range fields { - path, err := structpath.ParsePath(field) - if err != nil { - return fmt.Errorf("parsing sensitive field path %q: %w", field, err) - } - + paths, err := sensitivePaths(adapter) + if err != nil { + return err + } + for _, path := range paths { fv, err := structaccess.Get(s, path) if err != nil { continue @@ -76,13 +88,29 @@ func redactChanges(adapter *dresources.Adapter, changes deployplan.Changes) erro return nil } -// redactPlanEntry redacts sensitive fields from remote_state and changes of a plan -// entry. new_state is already redacted before serialization in makePlan. +// redactPlanEntry redacts new_state, remote_state and changes of a plan entry. It runs once planning +// is done: reference resolution re-serializes new_state from the unredacted cache, and apply reads +// that cache, so the plan must not be serialized after this. func redactPlanEntry(adapter *dresources.Adapter, entry *deployplan.PlanEntry) error { if len(adapter.GetSensitiveFields()) == 0 { return nil } + if entry.NewState != nil { + sv, err := entry.NewState.ToStructVar(adapter.StateType()) + if err != nil { + return fmt.Errorf("redacting new_state: %w", err) + } + if err := redactStruct(adapter, sv.Value); err != nil { + return fmt.Errorf("redacting new_state: %w", err) + } + redacted, err := sv.ToJSON() + if err != nil { + return fmt.Errorf("redacting new_state: %w", err) + } + *entry.NewState = *redacted + } + if entry.RemoteState != nil { if err := redactStruct(adapter, entry.RemoteState); err != nil { return fmt.Errorf("redacting remote_state: %w", err) @@ -97,3 +125,53 @@ func redactPlanEntry(adapter *dresources.Adapter, entry *deployplan.PlanEntry) e return nil } + +// sensitiveConfigValue returns the value the config sets for a sensitive field, which a deploy sends. +// The config value is used whenever it is set. An empty config value is an error only when the plan +// holds the field non-zero, since the plan then expects a value the config cannot supply. A field +// that is zero in both is output-only and yields "". +func sensitiveConfigValue(root *config.Root, resourceKey string, path *structpath.PathNode, planned any) (string, error) { + rc, err := root.GetResourceConfig(resourceKey) + if err != nil { + return "", err + } + var value string + if rc != nil { + if v, err := structaccess.Get(rc, path); err == nil { + value, _ = v.(string) + } + } + if value != "" || planned == nil || planned == "" { + return value, nil + } + if rc == nil { + return "", fmt.Errorf("resource %s is not in the config; run 'bundle plan' again", resourceKey) + } + return "", fmt.Errorf("resource %s: the config has no value for sensitive field %s; pass the value", resourceKey, path) +} + +// restoreSensitiveFields replaces the sensitive values in s, a saved plan's state, with the values +// the current config sets. CheckSensitivePlanValues runs the same rule before the plan is approved. +func restoreSensitiveFields(adapter *dresources.Adapter, resourceKey string, s any, root *config.Root) error { + paths, err := sensitivePaths(adapter) + if err != nil { + return err + } + for _, path := range paths { + planned, err := structaccess.Get(s, path) + if err != nil { + continue + } + value, err := sensitiveConfigValue(root, resourceKey, path, planned) + if err != nil { + return err + } + if value == "" { + continue + } + if err := structaccess.Set(s, path, value); err != nil { + return fmt.Errorf("resource %s: restoring sensitive field %s: %w", resourceKey, path, err) + } + } + return nil +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 1f6831281ea..b460d4ed046 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -310,7 +310,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand if planFromFile { // Initialize DeploymentBundle for applying the loaded plan - err := b.DeploymentBundle.InitForApply(ctx, b.WorkspaceClient(ctx), plan) + err := b.DeploymentBundle.InitForApply(ctx, b.WorkspaceClient(ctx), plan, &b.Config) if err != nil { logdiag.LogError(ctx, err) return diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 68798a4bb01..49de5da4d62 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -499,6 +499,11 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } + + if err := direct.CheckSensitivePlanValues(ctx, &b.Config, plan, variables); err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } } // Reject an unknown --select before PreDeployChecks and the predeploy script From 2847789972860193d73ce2609bf97d697e99d4b0 Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Mon, 16 Nov 2026 16:24:03 +0000 Subject: [PATCH 2/5] Document sensitive variable redaction in plan and saved plans Co-authored-by: Isaac --- .nextchanges/bundles/7713.md | 1 + bundle/config/variable/variable.go | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 .nextchanges/bundles/7713.md diff --git a/.nextchanges/bundles/7713.md b/.nextchanges/bundles/7713.md new file mode 100644 index 00000000000..59ab5752890 --- /dev/null +++ b/.nextchanges/bundles/7713.md @@ -0,0 +1 @@ +* `bundle plan` no longer shows or saves sensitive variable values, and `bundle deploy --plan` requires them to be passed again with `--var` or `BUNDLE_VAR_`; `deploy --plan` applies the values supplied at deploy time, not those at plan time. ([#7713](https://github.com/databricks/cli/pull/7713)) diff --git a/bundle/config/variable/variable.go b/bundle/config/variable/variable.go index f5a5fd6ee75..a86a6264b9e 100644 --- a/bundle/config/variable/variable.go +++ b/bundle/config/variable/variable.go @@ -41,9 +41,9 @@ type Variable struct { // Marks the variable as holding a secret. Set automatically for a variable used as the // value of a sensitive resource field (e.g. the value of a UC secret). Its value is - // redacted in the output of `bundle validate -o json` and `bundle summary -o json`, and - // it may only be referenced from sensitive resource fields or from other sensitive - // variables, so that the secret never ends up in plain text. + // redacted in the output of `bundle validate -o json`, `bundle summary -o json` and + // `bundle plan`. It may only be referenced from sensitive resource fields or from other + // sensitive variables, so that the secret never ends up in plain text. Sensitive *bool `json:"sensitive,omitempty"` // This field stores the resolved value for the variable. The variable are From f9e47503f7e0f7c22ea29e843e86e43c8033fd45 Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Mon, 16 Nov 2026 16:24:04 +0000 Subject: [PATCH 3/5] Test sensitive value restore and the saved plan check Co-authored-by: Isaac --- bundle/direct/redact_test.go | 274 +++++++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 bundle/direct/redact_test.go diff --git a/bundle/direct/redact_test.go b/bundle/direct/redact_test.go new file mode 100644 index 00000000000..5a806971799 --- /dev/null +++ b/bundle/direct/redact_test.go @@ -0,0 +1,274 @@ +package direct + +import ( + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/config/variable" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/databricks-sdk-go/service/catalog" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const secretLiteral = "dummy-credential" + +func secretAdapter(t *testing.T) *dresources.Adapter { + t.Helper() + adapters, err := dresources.InitAll(nil) + require.NoError(t, err) + adapter, ok := adapters["secrets"] + require.True(t, ok) + return adapter +} + +// syncedRoot gives a Go-literal config the dynamic value that a loaded bundle has, which GetResourceConfig reads. +func syncedRoot(t *testing.T, root *config.Root) *config.Root { + t.Helper() + require.NoError(t, root.Mutate(func(v dyn.Value) (dyn.Value, error) { return v, nil })) + return root +} + +// TestRedactPlanEntryRedactsNewStateAndChanges checks that the literal is absent from the new_state +// and changes of a plan entry, and that the non-sensitive fields are kept. +func TestRedactPlanEntryRedactsNewStateAndChanges(t *testing.T) { + adapter := secretAdapter(t) + + newState := &structvar.StructVarJSON{} + var err error + newState.Value, err = json.Marshal(map[string]any{ + "name": "s", + "catalog_name": "main", + "schema_name": "default", + "value": secretLiteral, + }) + require.NoError(t, err) + + entry := &deployplan.PlanEntry{ + Action: deployplan.Update, + NewState: newState, + Changes: deployplan.Changes{ + "value": {Action: deployplan.Update, Old: "", New: secretLiteral, Remote: secretLiteral}, + "name": {Action: deployplan.Skip, Old: "s", New: "s", Remote: "s"}, + }, + } + + require.NoError(t, redactPlanEntry(adapter, entry)) + + out, err := json.Marshal(entry) + require.NoError(t, err) + assert.NotContains(t, string(out), secretLiteral) + + var redacted struct { + NewState struct { + Value map[string]any `json:"value"` + } `json:"new_state"` + } + require.NoError(t, json.Unmarshal(out, &redacted)) + assert.Equal(t, "[redacted]", redacted.NewState.Value["value"]) + assert.Equal(t, "main", redacted.NewState.Value["catalog_name"]) + assert.Equal(t, "[redacted]", entry.Changes["value"].New) + assert.Equal(t, "s", entry.Changes["name"].New) +} + +func secretRoot(t *testing.T, value string, variables map[string]*variable.Variable) *config.Root { + t.Helper() + root := &config.Root{ + Variables: variables, + Resources: config.Resources{ + Secrets: map[string]*resources.Secret{ + "s": {Secret: catalog.Secret{Name: "s", Value: value}}, + }, + }, + } + return syncedRoot(t, root) +} + +const ( + msgValueMissing = "resource resources.secrets.s: the config has no value for sensitive field value; pass the value" + msgNotInConfig = "resource resources.secrets.gone is not in the config; run 'bundle plan' again" + msgCredentialMissing = "sensitive variable api_credential has no value; pass it with --var api_credential=... or BUNDLE_VAR_api_credential when deploying from a saved plan (the variable-overrides file is not read for sensitive variables)" + msgVarsMissing = "sensitive variables api_credential, api_header have no value; pass each with --var NAME=... or BUNDLE_VAR_NAME when deploying from a saved plan (the variable-overrides file is not read for sensitive variables)" +) + +func TestRestoreSensitiveFields(t *testing.T) { + adapter := secretAdapter(t) + + tests := []struct { + name string + resourceKey string + state *catalog.Secret + configValue string + wantValue string + wantErr string + }{ + { + name: "redaction marker is replaced by the config value", + resourceKey: "resources.secrets.s", + state: &catalog.Secret{Name: "s", Value: sensitiveRedactedValue}, + configValue: secretLiteral, + wantValue: secretLiteral, + }, + { + name: "plaintext from a plan saved by an older CLI is replaced by the config value", + resourceKey: "resources.secrets.s", + state: &catalog.Secret{Name: "s", Value: "stale-plaintext"}, + configValue: secretLiteral, + wantValue: secretLiteral, + }, + { + name: "zero in the plan but set in the config is restored", + resourceKey: "resources.secrets.s", + state: &catalog.Secret{Name: "s"}, + configValue: secretLiteral, + wantValue: secretLiteral, + }, + { + name: "zero in both is skipped", + resourceKey: "resources.secrets.s", + state: &catalog.Secret{Name: "s"}, + configValue: "", + wantValue: "", + }, + { + name: "non-zero in the plan with an empty config value is an error", + resourceKey: "resources.secrets.s", + state: &catalog.Secret{Name: "s", Value: sensitiveRedactedValue}, + configValue: "", + wantErr: msgValueMissing, + }, + { + name: "non-zero in the plan with the resource missing from config is an error", + resourceKey: "resources.secrets.gone", + state: &catalog.Secret{Name: "s", Value: sensitiveRedactedValue}, + configValue: secretLiteral, + wantErr: msgNotInConfig, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + root := secretRoot(t, tc.configValue, nil) + err := restoreSensitiveFields(adapter, tc.resourceKey, tc.state, root) + if tc.wantErr != "" { + assert.EqualError(t, err, tc.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tc.wantValue, tc.state.Value) + assert.Empty(t, tc.state.EffectiveValue) + }) + } +} + +// TestSensitiveConfigValueNestedField checks that a dotted sensitive field path is read from the +// resource config, not only a top-level field. +func TestSensitiveConfigValueNestedField(t *testing.T) { + root := syncedRoot(t, &config.Root{Resources: config.Resources{Jobs: map[string]*resources.Job{ + "j": {JobSettings: jobs.JobSettings{Schedule: &jobs.CronSchedule{QuartzCronExpression: "0 0 * * *"}}}, + }}}) + path, err := structpath.ParsePath("schedule.quartz_cron_expression") + require.NoError(t, err) + + value, err := sensitiveConfigValue(root, "resources.jobs.j", path, "planned") + require.NoError(t, err) + assert.Equal(t, "0 0 * * *", value) +} + +func TestCheckSensitivePlanValues(t *testing.T) { + sensitive := true + variables := func() map[string]*variable.Variable { + return map[string]*variable.Variable{ + "api_credential": {Sensitive: &sensitive}, + "api_header": {Sensitive: &sensitive}, + "target_schema": {}, + } + } + secretPlan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.secrets.s": {NewState: &structvar.StructVarJSON{Value: []byte(`{"name":"s","value":"[redacted]"}`)}}, + }} + zeroPlan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.secrets.s": {NewState: &structvar.StructVarJSON{Value: []byte(`{"name":"s"}`)}}, + }} + jobsPlan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.j": {NewState: &structvar.StructVarJSON{Value: []byte(`{"name":"j"}`)}}, + }} + + tests := []struct { + name string + root *config.Root + plan *deployplan.Plan + flags []string + env map[string]string + wantErr string + }{ + { + name: "plan without sensitive resources needs nothing", + root: secretRoot(t, secretLiteral, variables()), + plan: jobsPlan, + }, + { + name: "missing variables are named", + root: secretRoot(t, secretLiteral, variables()), + plan: secretPlan, + wantErr: msgVarsMissing, + }, + { + name: "empty flag counts as missing even when the environment is set", + root: secretRoot(t, secretLiteral, variables()), + plan: secretPlan, + flags: []string{"api_credential="}, + env: map[string]string{"BUNDLE_VAR_api_credential": "y", "BUNDLE_VAR_api_header": "y"}, + wantErr: msgCredentialMissing, + }, + { + name: "environment supplies both variables", + root: secretRoot(t, secretLiteral, variables()), + plan: secretPlan, + flags: []string{"api_credential=x"}, + env: map[string]string{"BUNDLE_VAR_api_header": "y"}, + }, + { + name: "zero in the plan and set in the config passes", + root: secretRoot(t, secretLiteral, variables()), + plan: zeroPlan, + flags: []string{"api_credential=x", "api_header=y"}, + }, + { + name: "empty config value", + root: secretRoot(t, "", variables()), + plan: secretPlan, + flags: []string{"api_credential=x", "api_header=y"}, + wantErr: msgValueMissing, + }, + { + name: "resource problem comes before the variable problem", + root: secretRoot(t, "", variables()), + plan: secretPlan, + wantErr: msgValueMissing + "; " + msgVarsMissing, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx := t.Context() + for k, v := range tc.env { + ctx = env.Set(ctx, k, v) //nolint:fatcontext // the variables accumulate on one context by design + } + err := CheckSensitivePlanValues(ctx, tc.root, tc.plan, tc.flags) + if tc.wantErr == "" { + assert.NoError(t, err) + return + } + assert.EqualError(t, err, tc.wantErr) + }) + } +} From 2212f0cc4c54c408f4279254d6d107c4978007a2 Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Mon, 16 Nov 2026 16:24:06 +0000 Subject: [PATCH 4/5] Cover sensitive variables in plan and saved plan deploys The acceptance test checks that the literals stay out of plan output, the saved plan and local state, that a saved plan needs the values passed again, that an empty value fails before any state-changing request, and that a rotated value reaches only the PATCH request. Co-authored-by: Isaac --- .../variables/sensitive/plan/databricks.yml | 26 +++++ .../variables/sensitive/plan/out.test.toml | 2 + .../variables/sensitive/plan/output.txt | 103 ++++++++++++++++++ .../bundle/variables/sensitive/plan/script | 49 +++++++++ .../bundle/variables/sensitive/plan/test.toml | 2 + 5 files changed, 182 insertions(+) create mode 100644 acceptance/bundle/variables/sensitive/plan/databricks.yml create mode 100644 acceptance/bundle/variables/sensitive/plan/out.test.toml create mode 100644 acceptance/bundle/variables/sensitive/plan/output.txt create mode 100644 acceptance/bundle/variables/sensitive/plan/script create mode 100644 acceptance/bundle/variables/sensitive/plan/test.toml diff --git a/acceptance/bundle/variables/sensitive/plan/databricks.yml b/acceptance/bundle/variables/sensitive/plan/databricks.yml new file mode 100644 index 00000000000..bcf245d43f5 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan/databricks.yml @@ -0,0 +1,26 @@ +bundle: + name: sensitive-variables + +variables: + api_credential: + description: Credential for the external API, passed via BUNDLE_VAR_api_credential. + sensitive: true + api_header: + description: Header value for the plain secret, passed via BUNDLE_VAR_api_header. + sensitive: true + +resources: + catalogs: + audit_catalog: + name: audit_catalog + secrets: + ref_secret: + name: ref_secret + catalog_name: ${resources.catalogs.audit_catalog.name} + schema_name: default + value: ${var.api_credential} + plain_secret: + name: plain_secret + catalog_name: main + schema_name: default + value: ${var.api_header} diff --git a/acceptance/bundle/variables/sensitive/plan/out.test.toml b/acceptance/bundle/variables/sensitive/plan/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/variables/sensitive/plan/output.txt b/acceptance/bundle/variables/sensitive/plan/output.txt new file mode 100644 index 00000000000..b72d9d38dc5 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan/output.txt @@ -0,0 +1,103 @@ + +=== bundle plan (text) + +>>> [CLI] bundle plan +create catalogs.audit_catalog +create secrets.plain_secret +create secrets.ref_secret + +Plan: 3 to add, 0 to change, 0 to delete, 0 unchanged + +=== bundle plan -o json, saved to a file + +>>> jq -S .plan | map_values(.new_state) .databricks/plan.json +{ + "resources.catalogs.audit_catalog": { + "value": { + "name": "audit_catalog" + } + }, + "resources.secrets.plain_secret": { + "value": { + "catalog_name": "main", + "name": "plain_secret", + "schema_name": "default", + "value": "[redacted]" + } + }, + "resources.secrets.ref_secret": { + "value": { + "catalog_name": "audit_catalog", + "name": "ref_secret", + "schema_name": "default", + "value": "[redacted]" + } + } +} + +=== deploy from the saved plan with the sensitive variable only in a variable file + +>>> env -u BUNDLE_VAR_api_header [CLI] bundle deploy --plan .databricks/plan.json +Error: sensitive variable api_header has no value; pass it with --var api_header=... or BUNDLE_VAR_api_header when deploying from a saved plan (the variable-overrides file is not read for sensitive variables) + + +=== deploy from the saved plan with an empty sensitive variable + +>>> env BUNDLE_VAR_api_credential= [CLI] bundle deploy --plan .databricks/plan.json +Error: resource resources.secrets.ref_secret: the config has no value for sensitive field value; pass the value; sensitive variable api_credential has no value; pass it with --var api_credential=... or BUNDLE_VAR_api_credential when deploying from a saved plan (the variable-overrides file is not read for sensitive variables) + +state-changing requests sent by the failed deploy: 0 + +=== deploy from the saved plan with the sensitive variables supplied + +>>> [CLI] bundle deploy --plan .databricks/plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default/files... +Created catalogs.audit_catalog +Created secrets.plain_secret +Created secrets.ref_secret +Files: 1 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== bundle destroy, then a plain bundle deploy creates the secrets again + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.audit_catalog + delete resources.secrets.plain_secret + delete resources.secrets.ref_secret + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default + +Destroy: 3 deleted + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default/files... +Created catalogs.audit_catalog +Created secrets.plain_secret +Created secrets.ref_secret +Files: 1 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== bundle plan -o json after deploy: remote_state holds the marker + +>>> jq -S .plan | map_values(.remote_state.value) .databricks/plan-after.json +{ + "resources.catalogs.audit_catalog": null, + "resources.secrets.plain_secret": "[redacted]", + "resources.secrets.ref_secret": "[redacted]" +} + +=== rotate the plain secret: plan and deploy with a new value +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/default/files... +Updated secrets.plain_secret +Files: 0 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +=== requests carrying a secret literal: path, name and value +{"path":"/api/2.1/unity-catalog/secrets","name":"plain_secret","value":"dummy-header"} +{"path":"/api/2.1/unity-catalog/secrets","name":"plain_secret","value":"dummy-header"} +{"path":"/api/2.1/unity-catalog/secrets","name":"ref_secret","value":"dummy-credential"} +{"path":"/api/2.1/unity-catalog/secrets","name":"ref_secret","value":"dummy-credential"} +{"path":"/api/2.1/unity-catalog/secrets/main.default.plain_secret","name":"plain_secret","value":"dummy-rotated"} + +=== no secret value in the local state (the dump differs per DMS variant, so it is not shown) diff --git a/acceptance/bundle/variables/sensitive/plan/script b/acceptance/bundle/variables/sensitive/plan/script new file mode 100644 index 00000000000..5c5ed7c6da3 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan/script @@ -0,0 +1,49 @@ +# The values of the sensitive variables must not appear in any CLI output, the saved plan or the +# local state. The request log is checked for the real values instead, since the API needs them. +SECRETS=('!dummy-credential' '!dummy-header') +ROTATED=dummy-rotated + +title "bundle plan (text)" +trace $CLI bundle plan | contains.py "${SECRETS[@]}" + +title "bundle plan -o json, saved to a file" +$CLI bundle plan -o json | contains.py "${SECRETS[@]}" > .databricks/plan.json +trace jq -S '.plan | map_values(.new_state)' .databricks/plan.json + +# A variable file supplies a value, but a saved plan only accepts --var and BUNDLE_VAR_. +title "deploy from the saved plan with the sensitive variable only in a variable file" +mkdir -p .databricks/bundle/default +echo '{"api_header": "dummy-header"}' > .databricks/bundle/default/variable-overrides.json +musterr trace env -u BUNDLE_VAR_api_header $CLI bundle deploy --plan .databricks/plan.json +rm .databricks/bundle/default/variable-overrides.json + +# An empty value counts as missing. The deploy reads the workspace before the check, and the number of +# those reads differs per DMS variant, so only the state-changing requests are counted. +title "deploy from the saved plan with an empty sensitive variable" +before=$(wc -l < "$OUT_REQUESTS") +musterr trace env BUNDLE_VAR_api_credential= $CLI bundle deploy --plan .databricks/plan.json +echo "state-changing requests sent by the failed deploy: $(tail -n +$((before+1)) "$OUT_REQUESTS" | jq -s 'map(select(.method != "GET" and .path != "/telemetry-ext")) | length')" + +title "deploy from the saved plan with the sensitive variables supplied" +trace $CLI bundle deploy --plan .databricks/plan.json | contains.py "${SECRETS[@]}" + +# A plain deploy sends the values from the state cache, a different path from the saved plan. +title "bundle destroy, then a plain bundle deploy creates the secrets again" +trace $CLI bundle destroy --auto-approve | contains.py "${SECRETS[@]}" +trace $CLI bundle deploy | contains.py "${SECRETS[@]}" + +title "bundle plan -o json after deploy: remote_state holds the marker" +$CLI bundle plan -o json | contains.py "${SECRETS[@]}" > .databricks/plan-after.json +trace jq -S '.plan | map_values(.remote_state.value)' .databricks/plan-after.json + +# Rotating a value: the saved plan is applied with the new value, which only the PATCH may carry. +title "rotate the plain secret: plan and deploy with a new value" +BUNDLE_VAR_api_header=$ROTATED $CLI bundle plan -o json | contains.py "${SECRETS[@]}" "!$ROTATED" > .databricks/plan-rotate.json +BUNDLE_VAR_api_header=$ROTATED $CLI bundle deploy --plan .databricks/plan-rotate.json | contains.py "${SECRETS[@]}" "!$ROTATED" + +title "requests carrying a secret literal: path, name and value" +jq -c --arg rotated "$ROTATED" 'select(tostring | contains("dummy-credential") or contains("dummy-header") or contains($rotated)) | {path, name: .body.name, value: .body.value}' < "$OUT_REQUESTS" | sort + +# The state dump differs per DMS variant, so only the match is checked, not the output. +title "no secret value in the local state (the dump differs per DMS variant, so it is not shown)" +find .databricks -type f -exec cat {} + | contains.py "${SECRETS[@]}" "!$ROTATED" > /dev/null diff --git a/acceptance/bundle/variables/sensitive/plan/test.toml b/acceptance/bundle/variables/sensitive/plan/test.toml new file mode 100644 index 00000000000..54fc177c9f1 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan/test.toml @@ -0,0 +1,2 @@ +RecordRequests = true +Env.BUNDLE_VAR_api_header = "dummy-header" From fcecb403f7035aef193689718029765f4f94b459 Mon Sep 17 00:00:00 2001 From: "andrew.nester" Date: Fri, 20 Nov 2026 14:01:02 +0000 Subject: [PATCH 5/5] Check resolved sensitive values for saved plan deploys deploy --plan rejected every sensitive variable that had no --var or BUNDLE_VAR_ value, including variables a resource never uses and variables set by a default, a target or the variable-overrides file. The check now looks only at the resolved config value of each sensitive field the plan sets, so every source of a value counts the same, and the variable-overrides file is no longer a special case. The check reports one error per problem, and it reads the adapter only for resource types with sensitive fields. The acceptance case variables/sensitive/plan-derived covers unused derived, literal-default and target-set sensitive variables. Co-authored-by: Isaac --- .nextchanges/bundles/7713.md | 2 +- .../sensitive/plan-derived/databricks.yml | 34 +++++++ .../sensitive/plan-derived/out.test.toml | 2 + .../sensitive/plan-derived/output.txt | 25 +++++ .../variables/sensitive/plan-derived/script | 17 ++++ .../sensitive/plan-derived/test.toml | 3 + .../variables/sensitive/plan/output.txt | 6 +- .../bundle/variables/sensitive/plan/script | 6 +- bundle/direct/bundle_plan.go | 93 ++++++------------- bundle/direct/dresources/redact_config.go | 5 + bundle/direct/redact.go | 13 +-- bundle/direct/redact_test.go | 93 ++++++++----------- cmd/bundle/utils/process.go | 6 +- 13 files changed, 169 insertions(+), 136 deletions(-) create mode 100644 acceptance/bundle/variables/sensitive/plan-derived/databricks.yml create mode 100644 acceptance/bundle/variables/sensitive/plan-derived/out.test.toml create mode 100644 acceptance/bundle/variables/sensitive/plan-derived/output.txt create mode 100644 acceptance/bundle/variables/sensitive/plan-derived/script create mode 100644 acceptance/bundle/variables/sensitive/plan-derived/test.toml diff --git a/.nextchanges/bundles/7713.md b/.nextchanges/bundles/7713.md index 59ab5752890..8e4743c6e97 100644 --- a/.nextchanges/bundles/7713.md +++ b/.nextchanges/bundles/7713.md @@ -1 +1 @@ -* `bundle plan` no longer shows or saves sensitive variable values, and `bundle deploy --plan` requires them to be passed again with `--var` or `BUNDLE_VAR_`; `deploy --plan` applies the values supplied at deploy time, not those at plan time. ([#7713](https://github.com/databricks/cli/pull/7713)) +* `bundle plan` no longer shows or saves sensitive variable values, and `bundle deploy --plan` requires them to be supplied again at deploy time (for example with `--var` or `BUNDLE_VAR_`). `deploy --plan` applies the sensitive values supplied at deploy time, which are not checked against the values used at plan time. ([#7713](https://github.com/databricks/cli/pull/7713)) diff --git a/acceptance/bundle/variables/sensitive/plan-derived/databricks.yml b/acceptance/bundle/variables/sensitive/plan-derived/databricks.yml new file mode 100644 index 00000000000..f8bc0bee2d9 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan-derived/databricks.yml @@ -0,0 +1,34 @@ +bundle: + name: sensitive-variables + +# Sensitive variables that no resource uses need no value for a saved plan to apply. A variable used as a +# secret value cannot have a default or a target value, so the other sensitive variables are unused. +variables: + api_credential: + description: Credential for the external API, passed via the variable-overrides file. + sensitive: true + api_header: + description: Sensitive, derived from another sensitive variable, and not used by a resource. + sensitive: true + default: "Bearer ${var.api_credential}" + api_literal: + description: Sensitive with a literal default, and not used by a resource. + sensitive: true + default: dummy-literal + api_token: + description: Sensitive, not used by a resource, and set only for the dev target. + sensitive: true + +targets: + dev: + default: true + variables: + api_token: dummy-target + +resources: + secrets: + credential_secret: + name: credential_secret + catalog_name: main + schema_name: default + value: ${var.api_credential} diff --git a/acceptance/bundle/variables/sensitive/plan-derived/out.test.toml b/acceptance/bundle/variables/sensitive/plan-derived/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan-derived/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/variables/sensitive/plan-derived/output.txt b/acceptance/bundle/variables/sensitive/plan-derived/output.txt new file mode 100644 index 00000000000..2bb715c67f6 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan-derived/output.txt @@ -0,0 +1,25 @@ + +=== bundle plan -o json, saved to a file + +>>> jq -S .plan | map_values(.new_state) .databricks/plan.json +{ + "resources.secrets.credential_secret": { + "value": { + "catalog_name": "main", + "name": "credential_secret", + "schema_name": "default", + "value": "[redacted]" + } + } +} + +=== deploy from the saved plan with the credential in the variable-overrides file + +>>> env -u BUNDLE_VAR_api_credential [CLI] bundle deploy --plan .databricks/plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/sensitive-variables/dev/files... +Created secrets.credential_secret +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== requests carrying a secret literal: path, name and value +{"path":"/api/2.1/unity-catalog/secrets","name":"credential_secret","value":"dummy-credential"} diff --git a/acceptance/bundle/variables/sensitive/plan-derived/script b/acceptance/bundle/variables/sensitive/plan-derived/script new file mode 100644 index 00000000000..5cc78bd6f57 --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan-derived/script @@ -0,0 +1,17 @@ +# The secret values must not appear in CLI output or the saved plan, only in the requests. +SECRETS=('!dummy-credential' '!Bearer dummy-credential' '!dummy-literal' '!dummy-target') + +title "bundle plan -o json, saved to a file" +mkdir -p .databricks +$CLI bundle plan -o json | contains.py "${SECRETS[@]}" > .databricks/plan.json +trace jq -S '.plan | map_values(.new_state)' .databricks/plan.json + +# The variable-overrides file supplies the credential, and the unused derived and literal variables need no value. +title "deploy from the saved plan with the credential in the variable-overrides file" +mkdir -p .databricks/bundle/dev +echo '{"api_credential": "dummy-credential"}' > .databricks/bundle/dev/variable-overrides.json +trace env -u BUNDLE_VAR_api_credential $CLI bundle deploy --plan .databricks/plan.json | contains.py "${SECRETS[@]}" +rm .databricks/bundle/dev/variable-overrides.json + +title "requests carrying a secret literal: path, name and value" +jq -c 'select(tostring | contains("dummy-credential")) | {path, name: .body.name, value: .body.value}' < "$OUT_REQUESTS" | sort diff --git a/acceptance/bundle/variables/sensitive/plan-derived/test.toml b/acceptance/bundle/variables/sensitive/plan-derived/test.toml new file mode 100644 index 00000000000..b9864165e0c --- /dev/null +++ b/acceptance/bundle/variables/sensitive/plan-derived/test.toml @@ -0,0 +1,3 @@ +RecordRequests = true +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/variables/sensitive/plan/output.txt b/acceptance/bundle/variables/sensitive/plan/output.txt index b72d9d38dc5..f190193ef33 100644 --- a/acceptance/bundle/variables/sensitive/plan/output.txt +++ b/acceptance/bundle/variables/sensitive/plan/output.txt @@ -35,16 +35,16 @@ Plan: 3 to add, 0 to change, 0 to delete, 0 unchanged } } -=== deploy from the saved plan with the sensitive variable only in a variable file +=== deploy from the saved plan without the sensitive variable >>> env -u BUNDLE_VAR_api_header [CLI] bundle deploy --plan .databricks/plan.json -Error: sensitive variable api_header has no value; pass it with --var api_header=... or BUNDLE_VAR_api_header when deploying from a saved plan (the variable-overrides file is not read for sensitive variables) +Error: no value assigned to required variable api_header. Variables are usually assigned in databricks.yml, and they can be overridden using "--var", the BUNDLE_VAR_api_header environment variable, or .databricks/bundle//variable-overrides.json === deploy from the saved plan with an empty sensitive variable >>> env BUNDLE_VAR_api_credential= [CLI] bundle deploy --plan .databricks/plan.json -Error: resource resources.secrets.ref_secret: the config has no value for sensitive field value; pass the value; sensitive variable api_credential has no value; pass it with --var api_credential=... or BUNDLE_VAR_api_credential when deploying from a saved plan (the variable-overrides file is not read for sensitive variables) +Error: resource resources.secrets.ref_secret: the config has no value for sensitive field value; set the variable it uses with --var or BUNDLE_VAR_ state-changing requests sent by the failed deploy: 0 diff --git a/acceptance/bundle/variables/sensitive/plan/script b/acceptance/bundle/variables/sensitive/plan/script index 5c5ed7c6da3..a26543c56e1 100644 --- a/acceptance/bundle/variables/sensitive/plan/script +++ b/acceptance/bundle/variables/sensitive/plan/script @@ -10,12 +10,8 @@ title "bundle plan -o json, saved to a file" $CLI bundle plan -o json | contains.py "${SECRETS[@]}" > .databricks/plan.json trace jq -S '.plan | map_values(.new_state)' .databricks/plan.json -# A variable file supplies a value, but a saved plan only accepts --var and BUNDLE_VAR_. -title "deploy from the saved plan with the sensitive variable only in a variable file" -mkdir -p .databricks/bundle/default -echo '{"api_header": "dummy-header"}' > .databricks/bundle/default/variable-overrides.json +title "deploy from the saved plan without the sensitive variable" musterr trace env -u BUNDLE_VAR_api_header $CLI bundle deploy --plan .databricks/plan.json -rm .databricks/bundle/default/variable-overrides.json # An empty value counts as missing. The deploy reads the workspace before the check, and the number of # those reads differs per DMS variant, so only the state-changing requests are counted. diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index ab956a8b952..b96734bd316 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -19,7 +19,6 @@ import ( "github.com/databricks/cli/bundle/terraform_dabs_map" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" - "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/retry" @@ -70,40 +69,48 @@ func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan. return nil } -// CheckSensitivePlanValues rejects a saved plan that cannot be applied until its sensitive values are -// supplied again, and returns one error that lists every problem. When the plan has a sensitive -// resource, every sensitive variable of the bundle must be set with --var or BUNDLE_VAR_, since -// telling used variables from unused ones needs reference resolution. -func CheckSensitivePlanValues(ctx context.Context, root *config.Root, plan *deployplan.Plan, flagVars []string) error { - adapters, err := dresources.InitAll(nil) - if err != nil { - return err +// CheckSensitivePlanValues returns one error per problem that stops a saved plan from being applied: +// a sensitive field the plan sets whose resolved config value is empty. The resolved config holds the +// values from --var, BUNDLE_VAR_, defaults, targets and the variable-overrides file alike, so any of +// them supplies a value. Sensitive variables the plan does not use need no value. +func CheckSensitivePlanValues(root *config.Root, plan *deployplan.Plan) []error { + var problems []error + seen := make(map[string]bool) + addProblem := func(err error) { + if !seen[err.Error()] { + seen[err.Error()] = true + problems = append(problems, err) + } } - var problems []string - needsValues := false for _, resourceKey := range slices.Sorted(maps.Keys(plan.Plan)) { entry := plan.Plan[resourceKey] if entry.NewState == nil { continue } - adapter, ok := adapters[config.GetResourceTypeFromKey(resourceKey)] + resourceType := config.GetResourceTypeFromKey(resourceKey) + typedNil, ok := dresources.SupportedResources[resourceType] if !ok { - problems = append(problems, fmt.Sprintf("resource %s: unknown resource type", resourceKey)) + addProblem(fmt.Errorf("resource %s: unknown resource type", resourceKey)) + continue + } + rules := dresources.SensitiveFieldRules(resourceType) + if len(rules) == 0 { continue } - if len(adapter.GetSensitiveFields()) == 0 { + adapter, err := dresources.NewAdapter(typedNil, resourceType, nil) + if err != nil { + addProblem(fmt.Errorf("resource %s: %w", resourceKey, err)) continue } - needsValues = true sv, err := entry.NewState.ToStructVar(adapter.StateType()) if err != nil { - problems = append(problems, fmt.Sprintf("resource %s: %s", resourceKey, err)) + addProblem(fmt.Errorf("resource %s: %w", resourceKey, err)) continue } - paths, err := sensitivePaths(adapter) + paths, err := sensitivePaths(rules) if err != nil { - problems = append(problems, fmt.Sprintf("resource %s: %s", resourceKey, err)) + addProblem(fmt.Errorf("resource %s: %w", resourceKey, err)) continue } for _, path := range paths { @@ -112,57 +119,11 @@ func CheckSensitivePlanValues(ctx context.Context, root *config.Root, plan *depl continue } if _, err := sensitiveConfigValue(root, resourceKey, path, planned); err != nil { - problems = append(problems, err.Error()) - } - } - } - - var missing []string - if needsValues { - for name, v := range root.Variables { - if v == nil || !config.IsExplicitlyEnabled(v.Sensitive) || variableSupplied(ctx, name, flagVars) { - continue + addProblem(err) } - missing = append(missing, name) } - slices.Sort(missing) - } - if len(missing) > 0 { - problems = append(problems, missingVariablesError(missing)) - } - - if len(problems) == 0 { - return nil - } - return errors.New(strings.Join(problems, "; ")) -} - -// missingVariablesError names the sensitive variables that have no --var or BUNDLE_VAR_ value. -func missingVariablesError(missing []string) string { - const overrides = "the variable-overrides file is not read for sensitive variables" - if len(missing) == 1 { - name := missing[0] - return fmt.Sprintf("sensitive variable %s has no value; pass it with --var %s=... or BUNDLE_VAR_%s when deploying from a saved plan (%s)", name, name, name, overrides) - } - return fmt.Sprintf("sensitive variables %s have no value; pass each with --var NAME=... or BUNDLE_VAR_NAME when deploying from a saved plan (%s)", strings.Join(missing, ", "), overrides) -} - -// variableSupplied reports whether a sensitive variable has a value from --var or BUNDLE_VAR_. -// The resolved value also carries values from the variable-overrides file, so the check reads only -// flags and the environment. A --var given with an empty value wins over the environment. -func variableSupplied(ctx context.Context, name string, flagVars []string) bool { - flagged, found := "", false - for _, assignment := range flagVars { - key, value, _ := strings.Cut(assignment, "=") - if key == name { - flagged, found = value, true - } - } - if found { - return flagged != "" } - value, _ := env.Lookup(ctx, "BUNDLE_VAR_"+name) - return value != "" + return problems } // InitForApply initializes the DeploymentBundle for applying a pre-computed plan. The sensitive diff --git a/bundle/direct/dresources/redact_config.go b/bundle/direct/dresources/redact_config.go index 185713ea830..b45ea11deb8 100644 --- a/bundle/direct/dresources/redact_config.go +++ b/bundle/direct/dresources/redact_config.go @@ -60,6 +60,11 @@ func redactSensitiveVariables(root *config.Root) { } } +// SensitiveFieldRules returns the sensitive fields of a resource type without building its adapter. +func SensitiveFieldRules(resourceType string) []FieldRule { + return getSensitiveFields()[resourceType] +} + // getSensitiveFields returns the map of resource type to list of sensitive fields. func getSensitiveFields() map[string][]FieldRule { cfg := MustLoadConfig() diff --git a/bundle/direct/redact.go b/bundle/direct/redact.go index ff0c0bc485c..9a14b5fd746 100644 --- a/bundle/direct/redact.go +++ b/bundle/direct/redact.go @@ -13,10 +13,11 @@ import ( const sensitiveRedactedValue = "[redacted]" -// sensitivePaths parses the sensitive field paths of the adapter. -func sensitivePaths(adapter *dresources.Adapter) ([]*structpath.PathNode, error) { +// sensitivePaths parses the sensitive field rules of a resource type. +func sensitivePaths(rules []dresources.FieldRule) ([]*structpath.PathNode, error) { var paths []*structpath.PathNode - for _, field := range adapter.GetSensitiveFields() { + for _, rule := range rules { + field := rule.Field.String() path, err := structpath.ParsePath(field) if err != nil { return nil, fmt.Errorf("parsing sensitive field path %q: %w", field, err) @@ -34,7 +35,7 @@ func redactSensitiveFields(adapter *dresources.Adapter, s any, replacement strin return nil } - paths, err := sensitivePaths(adapter) + paths, err := sensitivePaths(adapter.ResourceConfig().SensitiveFields) if err != nil { return err } @@ -147,13 +148,13 @@ func sensitiveConfigValue(root *config.Root, resourceKey string, path *structpat if rc == nil { return "", fmt.Errorf("resource %s is not in the config; run 'bundle plan' again", resourceKey) } - return "", fmt.Errorf("resource %s: the config has no value for sensitive field %s; pass the value", resourceKey, path) + return "", fmt.Errorf("resource %s: the config has no value for sensitive field %s; set the variable it uses with --var or BUNDLE_VAR_", resourceKey, path) } // restoreSensitiveFields replaces the sensitive values in s, a saved plan's state, with the values // the current config sets. CheckSensitivePlanValues runs the same rule before the plan is approved. func restoreSensitiveFields(adapter *dresources.Adapter, resourceKey string, s any, root *config.Root) error { - paths, err := sensitivePaths(adapter) + paths, err := sensitivePaths(adapter.ResourceConfig().SensitiveFields) if err != nil { return err } diff --git a/bundle/direct/redact_test.go b/bundle/direct/redact_test.go index 5a806971799..8081387392c 100644 --- a/bundle/direct/redact_test.go +++ b/bundle/direct/redact_test.go @@ -10,7 +10,6 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go/service/catalog" @@ -93,10 +92,8 @@ func secretRoot(t *testing.T, value string, variables map[string]*variable.Varia } const ( - msgValueMissing = "resource resources.secrets.s: the config has no value for sensitive field value; pass the value" - msgNotInConfig = "resource resources.secrets.gone is not in the config; run 'bundle plan' again" - msgCredentialMissing = "sensitive variable api_credential has no value; pass it with --var api_credential=... or BUNDLE_VAR_api_credential when deploying from a saved plan (the variable-overrides file is not read for sensitive variables)" - msgVarsMissing = "sensitive variables api_credential, api_header have no value; pass each with --var NAME=... or BUNDLE_VAR_NAME when deploying from a saved plan (the variable-overrides file is not read for sensitive variables)" + msgValueMissing = "resource resources.secrets.s: the config has no value for sensitive field value; set the variable it uses with --var or BUNDLE_VAR_" + msgNotInConfig = "resource resources.secrets.gone is not in the config; run 'bundle plan' again" ) func TestRestoreSensitiveFields(t *testing.T) { @@ -185,12 +182,9 @@ func TestSensitiveConfigValueNestedField(t *testing.T) { func TestCheckSensitivePlanValues(t *testing.T) { sensitive := true - variables := func() map[string]*variable.Variable { - return map[string]*variable.Variable{ - "api_credential": {Sensitive: &sensitive}, - "api_header": {Sensitive: &sensitive}, - "target_schema": {}, - } + variables := map[string]*variable.Variable{ + "api_credential": {Sensitive: &sensitive}, + "target_schema": {}, } secretPlan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ "resources.secrets.s": {NewState: &structvar.StructVarJSON{Value: []byte(`{"name":"s","value":"[redacted]"}`)}}, @@ -206,69 +200,62 @@ func TestCheckSensitivePlanValues(t *testing.T) { name string root *config.Root plan *deployplan.Plan - flags []string - env map[string]string wantErr string }{ { name: "plan without sensitive resources needs nothing", - root: secretRoot(t, secretLiteral, variables()), + root: secretRoot(t, "", variables), plan: jobsPlan, }, { - name: "missing variables are named", - root: secretRoot(t, secretLiteral, variables()), - plan: secretPlan, - wantErr: msgVarsMissing, - }, - { - name: "empty flag counts as missing even when the environment is set", - root: secretRoot(t, secretLiteral, variables()), - plan: secretPlan, - flags: []string{"api_credential="}, - env: map[string]string{"BUNDLE_VAR_api_credential": "y", "BUNDLE_VAR_api_header": "y"}, - wantErr: msgCredentialMissing, + name: "sensitive value set in the config passes", + root: secretRoot(t, secretLiteral, variables), + plan: secretPlan, }, { - name: "environment supplies both variables", - root: secretRoot(t, secretLiteral, variables()), - plan: secretPlan, - flags: []string{"api_credential=x"}, - env: map[string]string{"BUNDLE_VAR_api_header": "y"}, + name: "zero in the plan and set in the config passes", + root: secretRoot(t, secretLiteral, variables), + plan: zeroPlan, }, { - name: "zero in the plan and set in the config passes", - root: secretRoot(t, secretLiteral, variables()), - plan: zeroPlan, - flags: []string{"api_credential=x", "api_header=y"}, - }, - { - name: "empty config value", - root: secretRoot(t, "", variables()), + name: "empty config value is reported", + root: secretRoot(t, "", variables), plan: secretPlan, - flags: []string{"api_credential=x", "api_header=y"}, wantErr: msgValueMissing, }, - { - name: "resource problem comes before the variable problem", - root: secretRoot(t, "", variables()), - plan: secretPlan, - wantErr: msgValueMissing + "; " + msgVarsMissing, - }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - ctx := t.Context() - for k, v := range tc.env { - ctx = env.Set(ctx, k, v) //nolint:fatcontext // the variables accumulate on one context by design - } - err := CheckSensitivePlanValues(ctx, tc.root, tc.plan, tc.flags) + errs := CheckSensitivePlanValues(tc.root, tc.plan) if tc.wantErr == "" { - assert.NoError(t, err) + assert.Empty(t, errs) return } - assert.EqualError(t, err, tc.wantErr) + require.Len(t, errs, 1) + assert.EqualError(t, errs[0], tc.wantErr) }) } } + +// TestCheckSensitivePlanValuesOneErrorPerProblem checks that each resource with a missing value gets its +// own error, and that a resource missing from the config is reported once, not once per sensitive field. +func TestCheckSensitivePlanValuesOneErrorPerProblem(t *testing.T) { + root := syncedRoot(t, &config.Root{Resources: config.Resources{ + Secrets: map[string]*resources.Secret{ + "a": {Secret: catalog.Secret{Name: "a"}}, + "b": {Secret: catalog.Secret{Name: "b"}}, + }, + }}) + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.secrets.a": {NewState: &structvar.StructVarJSON{Value: []byte(`{"name":"a","value":"[redacted]"}`)}}, + "resources.secrets.b": {NewState: &structvar.StructVarJSON{Value: []byte(`{"name":"b","value":"[redacted]"}`)}}, + "resources.secrets.gone": {NewState: &structvar.StructVarJSON{Value: []byte(`{"name":"gone","value":"[redacted]"}`)}}, + }} + + errs := CheckSensitivePlanValues(root, plan) + require.Len(t, errs, 3) + assert.EqualError(t, errs[0], "resource resources.secrets.a: the config has no value for sensitive field value; set the variable it uses with --var or BUNDLE_VAR_") + assert.EqualError(t, errs[1], "resource resources.secrets.b: the config has no value for sensitive field value; set the variable it uses with --var or BUNDLE_VAR_") + assert.EqualError(t, errs[2], msgNotInConfig) +} diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 5d5b827c944..22059254c50 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -496,8 +496,10 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } - if err := direct.CheckSensitivePlanValues(ctx, &b.Config, plan, variables); err != nil { - logdiag.LogError(ctx, err) + if errs := direct.CheckSensitivePlanValues(&b.Config, plan); len(errs) > 0 { + for _, err := range errs { + logdiag.LogError(ctx, err) + } return b, stateDesc, root.ErrAlreadyPrinted } }