From ee332fabd8a215895c2f886dbefb9086ac515100 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Jun 2026 15:56:04 +0200 Subject: [PATCH 1/8] bundle migrate: add no-API-call migration from TF to direct engine Implement a new `bundle migrate` command that creates the direct state file from local config + Terraform state attributes, without making any API calls. Unlike `bundle deployment migrate` which calls DoRead for each resource, this command reads resolved field values directly from the local TF state file. Cross-resource references (e.g. ${resources.jobs.src.git_source[0].branch}) are resolved using two independent methods and the results reconciled: - Method A: look up the field in the TF state of the resource that contains the reference (e.g. read name from databricks_job.dst TF attributes). - Method B: evaluate the template by reading each ${resources.*} reference from the TF state of the referenced resource and interpolating. If both methods agree, the value is used silently. If only one succeeds, that value is used. If both succeed but disagree, the longer string is used with a warning. If both fail, an error is returned. The bundle/migrate package provides: - ParseTFStateAttrs: parses the full TF state file (all resource attributes) - LookupTFField: looks up a field value using DABsPathToTerraform translation - ResolveFieldRef: reconciles Methods A and B for a single field Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 6 + bundle/migrate/resolve.go | 91 ++++++++ bundle/migrate/tf_state.go | 107 +++++++++ cmd/bundle/bundle.go | 1 + cmd/bundle/deployment/migrate.go | 12 + cmd/bundle/migrate.go | 372 +++++++++++++++++++++++++++++++ 6 files changed, 589 insertions(+) create mode 100644 bundle/migrate/resolve.go create mode 100644 bundle/migrate/tf_state.go create mode 100644 cmd/bundle/migrate.go diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 5591626cd75..79e0b05648a 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -931,6 +931,12 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root return p, nil } +// ExtractReferences extracts all variable references from the config subtree rooted at node. +// Returns a map from structpath string (field path within the resource) to template string. +func ExtractReferences(root dyn.Value, node string) (map[string]string, error) { + return extractReferences(root, node) +} + func extractReferences(root dyn.Value, node string) (map[string]string, error) { nodeType := config.GetResourceTypeFromKey(node) refs := make(map[string]string) diff --git a/bundle/migrate/resolve.go b/bundle/migrate/resolve.go new file mode 100644 index 00000000000..8d5f6bc6b35 --- /dev/null +++ b/bundle/migrate/resolve.go @@ -0,0 +1,91 @@ +package migrate + +import ( + "context" + "fmt" + "strings" + + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" +) + +// evaluateTemplate evaluates a template string like "${resources.pipelines.bar.cluster[0].label}" +// by looking up each ${...} reference from TF state. +func evaluateTemplate(state TFStateAttrs, template string) (string, error) { + ref, ok := dynvar.NewRef(dyn.V(template)) + if !ok { + return template, nil + } + + result := template + for _, pathString := range ref.References() { + path, err := structpath.ParsePath(pathString) + if err != nil { + return "", fmt.Errorf("cannot parse reference path %q: %w", pathString, err) + } + // Expect resources... + if path.Len() < 4 { + return "", fmt.Errorf("unexpected reference format (too short): %q", pathString) + } + // Check first component is "resources" + firstNode := path.Prefix(1) + if firstNode.String() != "resources" { + return "", fmt.Errorf("unexpected reference format (expected resources.*): %q", pathString) + } + + group := path.SkipPrefix(1).Prefix(1).String() + name := path.SkipPrefix(2).Prefix(1).String() + fieldPath := path.SkipPrefix(3) + + value, err := LookupTFField(state, group, name, fieldPath) + if err != nil { + return "", fmt.Errorf("cannot look up %q: %w", pathString, err) + } + + result = strings.ReplaceAll(result, "${"+pathString+"}", fmt.Sprintf("%v", value)) + } + return result, nil +} + +// ResolveFieldRef resolves a single reference for a field in resource (srcGroup, srcName). +// fieldPath is the path of the field within the source resource (in DABs naming, from sv.Refs key). +// refTemplate is the template string for that field, e.g. "${resources.pipelines.bar.cluster[0].label}". +// +// Two methods are tried: +// - Method A: read the field from the source resource's own TF state. +// - Method B: evaluate the template by reading each referenced field from TF state. +// +// Returns the reconciled value or an error if both methods fail. +func ResolveFieldRef(ctx context.Context, state TFStateAttrs, srcGroup, srcName string, fieldPath *structpath.PathNode, refTemplate string) (any, error) { + // Method A: read field from source resource's TF state. + valueA, errA := LookupTFField(state, srcGroup, srcName, fieldPath) + + // Method B: evaluate the template by looking up each reference. + valueB, errB := evaluateTemplate(state, refTemplate) + + switch { + case errA == nil && errB == nil: + aStr := fmt.Sprintf("%v", valueA) + if aStr == valueB { + return valueA, nil + } + // Both succeeded but disagree: prefer longer string and warn. + if len(valueB) > len(aStr) { + log.Warnf(ctx, "resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method B)", + srcGroup, srcName, fieldPath, aStr, valueB) + return valueB, nil + } + log.Warnf(ctx, "resource %s.%s field %s: method A value %q and method B value %q disagree; using longer (method A)", + srcGroup, srcName, fieldPath, aStr, valueB) + return valueA, nil + case errA == nil: + return valueA, nil + case errB == nil: + return valueB, nil + default: + return nil, fmt.Errorf("%s.%s field %s: method A: %w; method B: %w", + srcGroup, srcName, fieldPath, errA, errB) + } +} diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go new file mode 100644 index 00000000000..642f7a26b74 --- /dev/null +++ b/bundle/migrate/tf_state.go @@ -0,0 +1,107 @@ +package migrate + +import ( + "encoding/json" + "fmt" + "os" + "reflect" + "strings" + "sync" + + "github.com/databricks/cli/bundle/deploy/terraform" + tfschema "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/bundle/terraform_dabs_map" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + tfjson "github.com/hashicorp/terraform-json" +) + +// TFStateAttrs maps (tfResourceType → resourceName → raw JSON attributes). +type TFStateAttrs map[string]map[string]json.RawMessage + +// ParseTFStateAttrs parses the terraform state file returning full attribute JSON per resource. +func ParseTFStateAttrs(path string) (TFStateAttrs, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var state struct { + Version int `json:"version"` + Resources []struct { + Type string `json:"type"` + Name string `json:"name"` + Mode tfjson.ResourceMode `json:"mode"` + Instances []struct { + Attributes json.RawMessage `json:"attributes"` + } `json:"instances"` + } `json:"resources"` + } + + if err := json.Unmarshal(raw, &state); err != nil { + return nil, err + } + + result := make(TFStateAttrs) + for _, r := range state.Resources { + if r.Mode != tfjson.ManagedResourceMode || len(r.Instances) == 0 { + continue + } + if result[r.Type] == nil { + result[r.Type] = make(map[string]json.RawMessage) + } + result[r.Type][r.Name] = r.Instances[0].Attributes + } + return result, nil +} + +// tfSchemaTypeMap maps TF resource type name → schema struct type (via AllResources json tags). +var tfSchemaTypeMap = sync.OnceValue(func() map[string]reflect.Type { + t := reflect.TypeOf(tfschema.AllResources{}) + m := make(map[string]reflect.Type, t.NumField()) + for i := range t.NumField() { + f := t.Field(i) + tag := strings.Split(f.Tag.Get("json"), ",")[0] + if tag != "" && tag != "-" { + m[tag] = f.Type + } + } + return m +}) + +// LookupTFField looks up a field from TF state attributes for a bundle resource. +// group is the DABs group (e.g. "pipelines"), name is the resource name. +// fieldPath is the path to the field (may be in DABs or TF naming; both handled by DABsPathToTerraform). +// Returns (nil, nil) for empty/zero fields, error if the resource or field is not found. +func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath.PathNode) (any, error) { + tfType, ok := terraform.GroupToTerraformName[group] + if !ok { + return nil, fmt.Errorf("unknown resource group %q", group) + } + + // Translate field path to TF naming. + // DABsPathToTerraform handles both DABs names (renames) and TF names (pass-through for unknowns). + // Returns error for known DABs-only fields that have no TF equivalent. + tfFieldPath, err := terraform_dabs_map.DABsPathToTerraform(group, fieldPath) + if err != nil { + return nil, err + } + + attrsJSON, ok := state[tfType][name] + if !ok { + return nil, fmt.Errorf("%s.%s not found in TF state", tfType, name) + } + + schemaType, ok := tfSchemaTypeMap()[tfType] + if !ok { + return nil, fmt.Errorf("no schema type registered for %q", tfType) + } + + // Unmarshal attributes into a new instance of the schema struct. + ptr := reflect.New(schemaType) + if err := json.Unmarshal(attrsJSON, ptr.Interface()); err != nil { + return nil, fmt.Errorf("cannot parse TF state for %s.%s: %w", tfType, name, err) + } + + return structaccess.Get(ptr.Interface(), tfFieldPath) +} diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index 7189b1d431d..84666eec689 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -38,6 +38,7 @@ Online documentation: https://docs.databricks.com/en/dev-tools/bundles/index.htm cmd.AddCommand(newDebugCommand()) cmd.AddCommand(newOpenCommand()) cmd.AddCommand(newPlanCommand()) + cmd.AddCommand(newMigrateCommand()) cmd.AddCommand(newConfigRemoteSyncCommand()) // Bundle Metadata Service (DMS) read-only command groups. Only `get` diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 801e46f7918..aa22e5118ac 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -30,6 +30,12 @@ import ( const backupSuffix = ".backup" +// RunPlanCheck runs bundle plan and checks if there are any actions planned. +// Returns error if plan fails or if there are actions planned. +func RunPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { + return runPlanCheck(cmd, extraArgs, extraArgsStr) +} + // runPlanCheck runs bundle plan and checks if there are any actions planned. // Returns error if plan fails or if there are actions planned. func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { @@ -78,6 +84,12 @@ func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) e return nil } +// GetCommonArgs extracts common flags (target, profile, var) from the command into +// argument slices suitable for forwarding to a subprocess. +func GetCommonArgs(cmd *cobra.Command) ([]string, string) { + return getCommonArgs(cmd) +} + func getCommonArgs(cmd *cobra.Command) ([]string, string) { var args []string var quotedArgs []string diff --git a/cmd/bundle/migrate.go b/cmd/bundle/migrate.go new file mode 100644 index 00000000000..3579658d3fe --- /dev/null +++ b/cmd/bundle/migrate.go @@ -0,0 +1,372 @@ +package bundle + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/migrate" + "github.com/databricks/cli/cmd/bundle/deployment" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/spf13/cobra" +) + +func newMigrateCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "migrate", + Short: "Migrate from Terraform to Direct deployment engine (no API calls)", + Long: `Creates a Direct deployment state file from the local config and Terraform state, +without making API calls. Cross-resource references are resolved from TF state.`, + Args: root.NoArgs, + } + + var noPlanCheck bool + cmd.Flags().BoolVar(&noPlanCheck, "noplancheck", false, "Skip running bundle plan before migration.") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + // Clear engine env var: we read TF state and produce a direct state. + cmd.SetContext(env.Set(cmd.Context(), engine.EnvVar, "")) + + opts := utils.ProcessOptions{ + AlwaysPull: true, + FastValidate: true, + Build: true, + PostInitFunc: func(_ context.Context, b *bundle.Bundle) error { + if b.Config.Bundle.Engine == engine.EngineTerraform { + return fmt.Errorf("bundle.engine is set to %q; migration requires \"engine: direct\" or no engine setting", engine.EngineTerraform) + } + return nil + }, + } + + b, stateDesc, err := utils.ProcessBundleRet(cmd, opts) + if err != nil { + return err + } + ctx := cmd.Context() + + if stateDesc.Lineage == "" { + cmdio.LogString(ctx, `Error: no existing state found. To start fresh with direct engine, set "engine: direct".`) + return root.ErrAlreadyPrinted + } + if stateDesc.Engine.IsDirect() { + return fmt.Errorf("already using direct engine: %s", stateDesc.String()) + } + + _, localTerraformPath := b.StateFilenameTerraform(ctx) + if _, err := os.Stat(localTerraformPath); err != nil { + return fmt.Errorf("reading %s: %w", localTerraformPath, err) + } + + // Run plan check unless --noplancheck is set. + if !noPlanCheck { + cmdio.LogString(ctx, "Note: Migration should be done after a full deploy. Running plan now to verify:") + extraArgs, extraArgsStr := deployment.GetCommonArgs(cmd) + if err := deployment.RunPlanCheck(cmd, extraArgs, extraArgsStr); err != nil { + return err + } + } + + // Parse TF state: IDs (for state entries) and full attributes (for ref resolution). + tfResourceIDs, err := terraform.ParseResourcesState(ctx, b) + if err != nil { + return fmt.Errorf("failed to parse terraform state: %w", err) + } + for key, entry := range tfResourceIDs { + if entry.ID == "" { + return fmt.Errorf("missing ID for %s in terraform state", key) + } + } + + cacheDir, err := terraform.Dir(ctx, b) + if err != nil { + return err + } + tfStateFilename, _ := b.StateFilenameTerraform(ctx) + tfStateFullPath := filepath.Join(cacheDir, tfStateFilename) + + tfAttrs, err := migrate.ParseTFStateAttrs(tfStateFullPath) + if err != nil { + return fmt.Errorf("failed to read terraform state attributes: %w", err) + } + + _, localPath := b.StateFilenameDirect(ctx) + tempPath := localPath + ".temp-migration" + + if _, err := os.Stat(tempPath); err == nil { + return fmt.Errorf("temporary state file %s already exists, another migration is in progress or was interrupted. In the latter case, delete the file", tempPath) + } + if _, err := os.Stat(localPath); err == nil { + return fmt.Errorf("state file %s already exists", localPath) + } + + // Apply SecretScopeFixups so the config matches what the direct engine expects. + // This adds MANAGE ACL for the current user to all secret scopes, ensuring + // the migrated state and config agree on .permissions entries. + bundle.ApplyContext(ctx, b, resourcemutator.SecretScopeFixups(engine.EngineDirect)) + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + // Build initial state with IDs and optional ETags. + etags := map[string]string{} + state := make(map[string]dstate.ResourceEntry) + for key, resourceEntry := range tfResourceIDs { + state[key] = dstate.ResourceEntry{ + ID: resourceEntry.ID, + State: json.RawMessage("{}"), + } + if resourceEntry.ETag != "" { + etags[key] = resourceEntry.ETag + } + } + + migratedDB := dstate.NewDatabase(stateDesc.Lineage, stateDesc.Serial+1) + migratedDB.State = state + + var stateDB dstate.DeploymentState + stateDB.OpenWithData(tempPath, migratedDB) + + removeTempPath := true + defer func() { + if removeTempPath { + _ = os.Remove(tempPath) + } + }() + + // Initialize adapters. + adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) + if err != nil { + return err + } + + if err := stateDB.UpgradeToWrite(); err != nil { + return fmt.Errorf("upgrading state for write: %w", err) + } + + // Process each resource: prepare state, resolve refs from TF state, save. + if err := buildStateFromTF(ctx, b, adapters, &stateDB, tfAttrs, tfResourceIDs, etags); err != nil { + return err + } + + if _, err := stateDB.Finalize(ctx); err != nil { + return fmt.Errorf("finalizing state: %w", err) + } + if logdiag.HasError(ctx) { + return errors.New("migration encountered errors") + } + + if err := os.Rename(tempPath, localPath); err != nil { + return fmt.Errorf("renaming %s to %s: %w", tempPath, localPath, err) + } + removeTempPath = false + + localTerraformBackupPath := localTerraformPath + ".backup" + err = os.Rename(localTerraformPath, localTerraformBackupPath) + if err != nil { + // Not fatal since we've already incremented the serial. + logdiag.LogError(ctx, err) + } + + extraArgsStr := "" + if flag := cmd.Flag("target"); flag != nil && flag.Changed { + extraArgsStr = " -t " + flag.Value.String() + } + + cmdio.LogString(ctx, fmt.Sprintf(`Success! Migrated %d resources to direct engine state file: %s + +Validate the migration by running "databricks bundle plan%s", there should be no actions planned. + +The state file is not synchronized to the workspace yet. To finalize the migration, run "bundle deploy%s". + +To undo the migration, remove %s and rename %s to %s +`, len(state), localPath, extraArgsStr, extraArgsStr, localPath, localTerraformBackupPath, localTerraformPath)) + return nil + } + + return cmd +} + +// buildStateFromTF iterates over bundle resources, resolves cross-resource +// references using TF state attributes, and writes each resource's state entry. +func buildStateFromTF( + ctx context.Context, + b *bundle.Bundle, + adapters map[string]*dresources.Adapter, + stateDB *dstate.DeploymentState, + tfAttrs migrate.TFStateAttrs, + tfIDs terraform.ExportedResourcesMap, + etags map[string]string, +) error { + configRoot := &b.Config + + // Collect all resource nodes (same patterns as makePlan). + var nodes []string + patterns := []dyn.Pattern{ + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), + } + for _, pat := range patterns { + _, err := dyn.MapByPattern( + configRoot.Value(), + pat, + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + nodes = append(nodes, p.String()) + return dyn.InvalidValue, nil + }, + ) + if err != nil { + return err + } + } + + for _, node := range nodes { + idEntry, ok := tfIDs[node] + if !ok { + // Resource is in config but not in TF state (new resource); skip. + continue + } + + group := config.GetResourceTypeFromKey(node) + if group == "" { + return fmt.Errorf("cannot determine resource type for %q", node) + } + + adapter, ok := adapters[group] + if !ok { + log.Warnf(ctx, "unsupported resource type %q for %s, skipping", group, node) + continue + } + + inputConfig, err := configRoot.GetResourceConfig(node) + if err != nil { + return fmt.Errorf("%s: getting config: %w", node, err) + } + + baseRefs := map[string]string{} + + switch { + case strings.HasSuffix(node, ".permissions"): + var sv *structvar.StructVar + if strings.HasPrefix(node, "resources.secret_scopes.") { + typedConfig, ok := inputConfig.(*[]resources.SecretScopePermission) + if !ok { + return fmt.Errorf("%s: expected *[]resources.SecretScopePermission, got %T", node, inputConfig) + } + sv, err = dresources.PrepareSecretScopeAclsInputConfig(*typedConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing secret scope ACLs config: %w", node, err) + } + } else { + sv, err = dresources.PreparePermissionsInputConfig(inputConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing permissions config: %w", node, err) + } + } + inputConfig = sv.Value + baseRefs = sv.Refs + + case strings.HasSuffix(node, ".grants"): + sv, err := dresources.PrepareGrantsInputConfig(inputConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing grants config: %w", node, err) + } + inputConfig = sv.Value + baseRefs = sv.Refs + } + + newStateValue, err := adapter.PrepareState(inputConfig) + if err != nil { + return fmt.Errorf("%s: PrepareState: %w", node, err) + } + + refs, err := direct.ExtractReferences(configRoot.Value(), node) + if err != nil { + return fmt.Errorf("%s: extracting references: %w", node, err) + } + maps.Copy(refs, baseRefs) + + sv := structvar.NewStructVar(newStateValue, refs) + + // Resolve each reference using TF state. + // We need to extract the resource name for Method A (looking up in the source resource's TF state). + parts := strings.SplitN(node, ".", 4) + // node format: "resources.." or "resources...permissions" + var srcGroup, srcName string + if len(parts) >= 3 { + srcGroup = parts[1] + srcName = parts[2] + } + + // Collect all field paths that need resolution (avoid modifying map during iteration). + type refEntry struct { + fieldPathStr string + refTemplate string + } + var pendingRefs []refEntry + for fieldPathStr, refTemplate := range sv.Refs { + pendingRefs = append(pendingRefs, refEntry{fieldPathStr, refTemplate}) + } + + for _, pending := range pendingRefs { + fieldPath, err := structpath.ParsePath(pending.fieldPathStr) + if err != nil { + return fmt.Errorf("%s: parsing field path %q: %w", node, pending.fieldPathStr, err) + } + + // ResolveFieldRef returns the fully resolved value for this field, + // using either Method A (TF state lookup) or Method B (template evaluation). + value, err := migrate.ResolveFieldRef(ctx, tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate) + if err != nil { + return fmt.Errorf("%s: cannot resolve field %q (template %q): %w", node, pending.fieldPathStr, pending.refTemplate, err) + } + + // Set the resolved value directly and remove the ref entry. + if err := structaccess.Set(sv.Value, fieldPath, value); err != nil { + return fmt.Errorf("%s: cannot set resolved value for field %q: %w", node, pending.fieldPathStr, err) + } + delete(sv.Refs, pending.fieldPathStr) + } + + if len(sv.Refs) > 0 { + return fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) + } + + // Handle etag for dashboards. + if etag := etags[node]; etag != "" { + if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { + return fmt.Errorf("%s: cannot set etag: %w", node, err) + } + } + + if err := stateDB.SaveState(node, idEntry.ID, sv.Value, nil); err != nil { + return fmt.Errorf("%s: SaveState: %w", node, err) + } + } + + return nil +} From effb66baac1be7f4cc487370ca4e76c7641f4c65 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Jun 2026 16:12:10 +0200 Subject: [PATCH 2/8] bundle migrate: remove plan check, make --noplancheck a no-op The new migrate command reads only from the local TF state file and never invokes the Terraform engine, so a pre-migration plan check has no place here. The --noplancheck flag is kept but ignored to avoid breaking callers. Co-authored-by: Isaac --- cmd/bundle/migrate.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/cmd/bundle/migrate.go b/cmd/bundle/migrate.go index 3579658d3fe..ffc79ddda18 100644 --- a/cmd/bundle/migrate.go +++ b/cmd/bundle/migrate.go @@ -20,7 +20,6 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/migrate" - "github.com/databricks/cli/cmd/bundle/deployment" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" @@ -43,8 +42,9 @@ without making API calls. Cross-resource references are resolved from TF state.` Args: root.NoArgs, } - var noPlanCheck bool - cmd.Flags().BoolVar(&noPlanCheck, "noplancheck", false, "Skip running bundle plan before migration.") + // --noplancheck is kept for compatibility but has no effect: this command reads + // only from the local TF state file and never invokes the Terraform engine. + cmd.Flags().Bool("noplancheck", false, "No-op (kept for compatibility).") cmd.RunE = func(cmd *cobra.Command, args []string) error { // Clear engine env var: we read TF state and produce a direct state. @@ -81,15 +81,6 @@ without making API calls. Cross-resource references are resolved from TF state.` return fmt.Errorf("reading %s: %w", localTerraformPath, err) } - // Run plan check unless --noplancheck is set. - if !noPlanCheck { - cmdio.LogString(ctx, "Note: Migration should be done after a full deploy. Running plan now to verify:") - extraArgs, extraArgsStr := deployment.GetCommonArgs(cmd) - if err := deployment.RunPlanCheck(cmd, extraArgs, extraArgsStr); err != nil { - return err - } - } - // Parse TF state: IDs (for state entries) and full attributes (for ref resolution). tfResourceIDs, err := terraform.ParseResourcesState(ctx, b) if err != nil { From f454a3dfbdc3d1cbf6c8a147eca471f1aecfea99 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Jun 2026 16:31:10 +0200 Subject: [PATCH 3/8] bundle/direct: drop MigrateMode, consolidate state-building in bundle/migrate MigrateMode was a bool parameter on Apply that forked between "save state without deploying" and "actually deploy". The two MigrateMode(true) callers (bundle deployment migrate and upload_state_for_yaml_sync) now use migrate.BuildStateFromTF directly, reading from the local TF state file without any API calls. Changes: - New bundle/migrate/build_state.go: public BuildStateFromTF extracted from cmd/bundle/migrate.go, taking *config.Root so callers can pass an un-interpolated config (needed by upload_state_for_yaml_sync). - bundle/direct/bundle_apply.go: drop MigrateMode type and parameter; Apply now only handles real deployments. - bundle/phases/{deploy,destroy}.go: drop MigrateMode(false) argument. - upload_state_for_yaml_sync.go: replace CalculatePlan+Apply with BuildStateFromTF; keep reverseInterpolate since config is TF-interpolated. - cmd/bundle/deployment/migrate.go: replace CalculatePlan+Apply with BuildStateFromTF; drop exported RunPlanCheck/GetCommonArgs wrappers. - cmd/bundle/migrate.go: delegate to migrate.BuildStateFromTF. Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 25 +-- bundle/migrate/build_state.go | 179 ++++++++++++++++++ bundle/phases/deploy.go | 3 +- bundle/phases/destroy.go | 3 +- .../statemgmt/upload_state_for_yaml_sync.go | 45 ++--- cmd/bundle/deployment/migrate.go | 62 ++---- cmd/bundle/migrate.go | 173 +---------------- 7 files changed, 216 insertions(+), 274 deletions(-) create mode 100644 bundle/migrate/build_state.go diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 16b145f7af8..a5849ffbd1a 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -15,9 +15,7 @@ import ( "github.com/databricks/databricks-sdk-go" ) -type MigrateMode bool - -func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan, migrateMode MigrateMode) { +func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) { if plan == nil { panic("Planning is not done") } @@ -52,9 +50,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa action := entry.Action errorPrefix := fmt.Sprintf("cannot %s %s", action, resourceKey) - if migrateMode { - errorPrefix = "cannot migrate " + resourceKey - } if action == deployplan.Undefined { logdiag.LogError(ctx, fmt.Errorf("cannot deploy %s: unknown action %q", resourceKey, action)) @@ -82,10 +77,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } if action == deployplan.Delete { - if migrateMode { - logdiag.LogError(ctx, fmt.Errorf("%s: Unexpected delete action during migration", errorPrefix)) - return false - } err = d.Destroy(ctx, &b.StateDB) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) @@ -113,18 +104,8 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } - if migrateMode { - // In migration mode we're reading resources in DAG order so that we have fully resolved config snapshots stored - id := b.StateDB.GetResourceID(resourceKey) - if id == "" { - logdiag.LogError(ctx, fmt.Errorf("state entry not found for %q", resourceKey)) - return false - } - err = b.StateDB.SaveState(resourceKey, id, sv.Value, entry.DependsOn) - } else { - // TODO: redo calcDiff to downgrade planned action if possible (?) - err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) - } + // TODO: redo calcDiff to downgrade planned action if possible (?) + err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go new file mode 100644 index 00000000000..af2b3449086 --- /dev/null +++ b/bundle/migrate/build_state.go @@ -0,0 +1,179 @@ +package migrate + +import ( + "context" + "fmt" + "maps" + "strings" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/structs/structvar" +) + +// BuildStateFromTF iterates over bundle resources, resolves cross-resource +// references using TF state attributes, and writes each resource's state entry. +// configRoot should be an un-interpolated config (with ${resources.*} references). +func BuildStateFromTF( + ctx context.Context, + configRoot *config.Root, + adapters map[string]*dresources.Adapter, + stateDB *dstate.DeploymentState, + tfAttrs TFStateAttrs, + tfIDs terraform.ExportedResourcesMap, + etags map[string]string, +) error { + // Collect all resource nodes (same patterns as makePlan). + var nodes []string + patterns := []dyn.Pattern{ + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), + dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), + } + for _, pat := range patterns { + _, err := dyn.MapByPattern( + configRoot.Value(), + pat, + func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + nodes = append(nodes, p.String()) + return dyn.InvalidValue, nil + }, + ) + if err != nil { + return err + } + } + + for _, node := range nodes { + idEntry, ok := tfIDs[node] + if !ok { + // Resource is in config but not in TF state (new resource); skip. + continue + } + + group := config.GetResourceTypeFromKey(node) + if group == "" { + return fmt.Errorf("cannot determine resource type for %q", node) + } + + adapter, ok := adapters[group] + if !ok { + log.Warnf(ctx, "unsupported resource type %q for %s, skipping", group, node) + continue + } + + inputConfig, err := configRoot.GetResourceConfig(node) + if err != nil { + return fmt.Errorf("%s: getting config: %w", node, err) + } + + baseRefs := map[string]string{} + + switch { + case strings.HasSuffix(node, ".permissions"): + var sv *structvar.StructVar + if strings.HasPrefix(node, "resources.secret_scopes.") { + typedConfig, ok := inputConfig.(*[]resources.SecretScopePermission) + if !ok { + return fmt.Errorf("%s: expected *[]resources.SecretScopePermission, got %T", node, inputConfig) + } + sv, err = dresources.PrepareSecretScopeAclsInputConfig(*typedConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing secret scope ACLs config: %w", node, err) + } + } else { + sv, err = dresources.PreparePermissionsInputConfig(inputConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing permissions config: %w", node, err) + } + } + inputConfig = sv.Value + baseRefs = sv.Refs + + case strings.HasSuffix(node, ".grants"): + sv, err := dresources.PrepareGrantsInputConfig(inputConfig, node) + if err != nil { + return fmt.Errorf("%s: preparing grants config: %w", node, err) + } + inputConfig = sv.Value + baseRefs = sv.Refs + } + + newStateValue, err := adapter.PrepareState(inputConfig) + if err != nil { + return fmt.Errorf("%s: PrepareState: %w", node, err) + } + + refs, err := direct.ExtractReferences(configRoot.Value(), node) + if err != nil { + return fmt.Errorf("%s: extracting references: %w", node, err) + } + maps.Copy(refs, baseRefs) + + sv := structvar.NewStructVar(newStateValue, refs) + + // Resolve each reference using TF state. + // node format: "resources.." or "resources...permissions" + parts := strings.SplitN(node, ".", 4) + var srcGroup, srcName string + if len(parts) >= 3 { + srcGroup = parts[1] + srcName = parts[2] + } + + // Collect all field paths that need resolution (avoid modifying map during iteration). + type refEntry struct { + fieldPathStr string + refTemplate string + } + var pendingRefs []refEntry + for fieldPathStr, refTemplate := range sv.Refs { + pendingRefs = append(pendingRefs, refEntry{fieldPathStr, refTemplate}) + } + + for _, pending := range pendingRefs { + fieldPath, err := structpath.ParsePath(pending.fieldPathStr) + if err != nil { + return fmt.Errorf("%s: parsing field path %q: %w", node, pending.fieldPathStr, err) + } + + // ResolveFieldRef returns the fully resolved value for this field, + // using either Method A (TF state lookup) or Method B (template evaluation). + value, err := ResolveFieldRef(ctx, tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate) + if err != nil { + return fmt.Errorf("%s: cannot resolve field %q (template %q): %w", node, pending.fieldPathStr, pending.refTemplate, err) + } + + // Set the resolved value directly and remove the ref entry. + if err := structaccess.Set(sv.Value, fieldPath, value); err != nil { + return fmt.Errorf("%s: cannot set resolved value for field %q: %w", node, pending.fieldPathStr, err) + } + delete(sv.Refs, pending.fieldPathStr) + } + + if len(sv.Refs) > 0 { + return fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) + } + + // Handle etag for dashboards. + if etag := etags[node]; etag != "" { + if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { + return fmt.Errorf("%s: cannot set etag: %w", node, err) + } + } + + if err := stateDB.SaveState(node, idEntry.ID, sv.Value, nil); err != nil { + return fmt.Errorf("%s: SaveState: %w", node, err) + } + } + + return nil +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 3cac322f9e3..78a06ef732e 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -14,7 +14,6 @@ import ( "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/permissions" @@ -81,7 +80,7 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, ta err error ) if targetEngine.IsDirect() { - b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, direct.MigrateMode(false)) + b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) state, err = b.DeploymentBundle.StateDB.Finalize(ctx) } else { bundle.ApplyContext(ctx, b, terraform.Apply()) diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 95eec600dc2..e1196fbc7f4 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -12,7 +12,6 @@ import ( "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" @@ -76,7 +75,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType) { if engine.IsDirect() { - b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, direct.MigrateMode(false)) + b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) } else { // Core destructive mutators for destroy. These require informed user consent. bundle.ApplyContext(ctx, b, terraform.Apply()) diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index 0399c7b31ff..4a6f804789c 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -14,18 +14,16 @@ import ( "github.com/databricks/cli/bundle/config/mutator/resourcemutator" "github.com/databricks/cli/bundle/deploy" "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/bundle/migrate" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" - "github.com/databricks/cli/libs/structs/structaccess" - "github.com/databricks/cli/libs/structs/structpath" ) type uploadStateForYamlSync struct { @@ -117,6 +115,11 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, fmt.Errorf("failed to read terraform state: %w", err) } + tfAttrs, err := migrate.ParseTFStateAttrs(localTerraformPath) + if err != nil { + return false, fmt.Errorf("failed to read terraform state attributes: %w", err) + } + state := make(map[string]dstate.ResourceEntry) etags := map[string]string{} @@ -141,8 +144,8 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun migratedDB := dstate.NewDatabase(tfState.Lineage, tfState.Serial+1) migratedDB.State = state - deploymentBundle := &direct.DeploymentBundle{} - deploymentBundle.StateDB.OpenWithData(snapshotPath, migratedDB) + var stateDB dstate.DeploymentState + stateDB.OpenWithData(snapshotPath, migratedDB) // Apply SecretScopeFixups so the config matches what the direct engine expects. // This adds MANAGE ACL for the current user to all secret scopes, ensuring @@ -152,9 +155,9 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, errors.New("failed to apply secret scope fixups") } - // Get the dynamic value from b.Config and reverse the interpolation. // b.Config has been modified by terraform.Interpolate which converts bundle-style // references (${resources.pipelines.x.id}) to terraform-style (${databricks_pipeline.x.id}). + // BuildStateFromTF expects ${resources.*} references, so reverse the interpolation first. interpolatedRoot := b.Config.Value() uninterpolatedRoot, err := reverseInterpolate(interpolatedRoot) if err != nil { @@ -169,36 +172,20 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, fmt.Errorf("failed to create uninterpolated config: %w", err) } - plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &uninterpolatedConfig) + adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) if err != nil { return false, err } - for _, entry := range plan.Plan { - entry.Action = deployplan.Update - } - - for key := range plan.Plan { - etag := etags[key] - if etag == "" { - continue - } - sv, ok := deploymentBundle.StateCache.Load(key) - if !ok { - continue - } - err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag) - if err != nil { - log.Warnf(ctx, "Failed to set etag on %q: %v", key, err) - } + if err := stateDB.UpgradeToWrite(); err != nil { + return false, fmt.Errorf("upgrading state for apply: %w", err) } - if err := deploymentBundle.StateDB.UpgradeToWrite(); err != nil { - return false, fmt.Errorf("upgrading state for apply: %w", err) + if err := migrate.BuildStateFromTF(ctx, &uninterpolatedConfig, adapters, &stateDB, tfAttrs, terraformResources, etags); err != nil { + return false, err } - deploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, direct.MigrateMode(true)) - if _, err := deploymentBundle.StateDB.Finalize(ctx); err != nil { + if _, err := stateDB.Finalize(ctx); err != nil { return false, err } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index aa22e5118ac..1c59616fcc9 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -14,28 +14,20 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/migrate" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/shellquote" - "github.com/databricks/cli/libs/structs/structaccess" - "github.com/databricks/cli/libs/structs/structpath" "github.com/spf13/cobra" ) const backupSuffix = ".backup" -// RunPlanCheck runs bundle plan and checks if there are any actions planned. -// Returns error if plan fails or if there are actions planned. -func RunPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { - return runPlanCheck(cmd, extraArgs, extraArgsStr) -} - // runPlanCheck runs bundle plan and checks if there are any actions planned. // Returns error if plan fails or if there are actions planned. func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { @@ -84,12 +76,6 @@ func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) e return nil } -// GetCommonArgs extracts common flags (target, profile, var) from the command into -// argument slices suitable for forwarding to a subprocess. -func GetCommonArgs(cmd *cobra.Command) ([]string, string) { - return getCommonArgs(cmd) -} - func getCommonArgs(cmd *cobra.Command) ([]string, string) { var args []string var quotedArgs []string @@ -221,6 +207,11 @@ To start using direct engine, set "engine: direct" under bundle in your databric } } + tfAttrs, err := migrate.ParseTFStateAttrs(localTerraformPath) + if err != nil { + return fmt.Errorf("failed to read terraform state attributes: %w", err) + } + etags := map[string]string{} state := make(map[string]dstate.ResourceEntry) @@ -238,8 +229,8 @@ To start using direct engine, set "engine: direct" under bundle in your databric migratedDB := dstate.NewDatabase(stateDesc.Lineage, stateDesc.Serial+1) migratedDB.State = state - deploymentBundle := &direct.DeploymentBundle{} - deploymentBundle.StateDB.OpenWithData(tempStatePath, migratedDB) + var stateDB dstate.DeploymentState + stateDB.OpenWithData(tempStatePath, migratedDB) tempStatePathAutoRemove := true @@ -257,43 +248,20 @@ To start using direct engine, set "engine: direct" under bundle in your databric return root.ErrAlreadyPrinted } - plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &b.Config) + adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) if err != nil { return err } - for _, entry := range plan.Plan { - // Force all actions to be "update" so that deploym below goes through every resource - entry.Action = deployplan.Update - } - - // We need to copy ETag into new state. - // For most resources state consists of fully resolved local config snapshot + id. - // Dashboards are special in that they also store "etag" in state which is not provided by user but - // comes from remote state. If we don't store "etag" in state, we won't detect remote drift, because - // local=nil, remote="" which will be classified as a backend default and skipped. - - for key := range plan.Plan { - etag := etags[key] - if etag == "" { - continue - } - sv, ok := deploymentBundle.StateCache.Load(key) - if !ok { - return fmt.Errorf("failed to read state for %q", key) - } - err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag) - if err != nil { - return fmt.Errorf("failed to set etag on %q: %w", key, err) - } + if err := stateDB.UpgradeToWrite(); err != nil { + return fmt.Errorf("upgrading state for apply: %w", err) } - if err := deploymentBundle.StateDB.UpgradeToWrite(); err != nil { - return fmt.Errorf("upgrading state for apply: %w", err) + if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfAttrs, terraformResources, etags); err != nil { + return err } - deploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan, direct.MigrateMode(true)) - if _, err := deploymentBundle.StateDB.Finalize(ctx); err != nil { + if _, err := stateDB.Finalize(ctx); err != nil { logdiag.LogError(ctx, err) } if logdiag.HasError(ctx) { diff --git a/cmd/bundle/migrate.go b/cmd/bundle/migrate.go index ffc79ddda18..8a33fe85f55 100644 --- a/cmd/bundle/migrate.go +++ b/cmd/bundle/migrate.go @@ -5,31 +5,21 @@ import ( "encoding/json" "errors" "fmt" - "maps" "os" "path/filepath" - "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/migrate" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/env" - "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" - "github.com/databricks/cli/libs/structs/structaccess" - "github.com/databricks/cli/libs/structs/structpath" - "github.com/databricks/cli/libs/structs/structvar" "github.com/spf13/cobra" ) @@ -159,7 +149,7 @@ without making API calls. Cross-resource references are resolved from TF state.` } // Process each resource: prepare state, resolve refs from TF state, save. - if err := buildStateFromTF(ctx, b, adapters, &stateDB, tfAttrs, tfResourceIDs, etags); err != nil { + if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfAttrs, tfResourceIDs, etags); err != nil { return err } @@ -200,164 +190,3 @@ To undo the migration, remove %s and rename %s to %s return cmd } - -// buildStateFromTF iterates over bundle resources, resolves cross-resource -// references using TF state attributes, and writes each resource's state entry. -func buildStateFromTF( - ctx context.Context, - b *bundle.Bundle, - adapters map[string]*dresources.Adapter, - stateDB *dstate.DeploymentState, - tfAttrs migrate.TFStateAttrs, - tfIDs terraform.ExportedResourcesMap, - etags map[string]string, -) error { - configRoot := &b.Config - - // Collect all resource nodes (same patterns as makePlan). - var nodes []string - patterns := []dyn.Pattern{ - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("permissions")), - dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey(), dyn.Key("grants")), - } - for _, pat := range patterns { - _, err := dyn.MapByPattern( - configRoot.Value(), - pat, - func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - nodes = append(nodes, p.String()) - return dyn.InvalidValue, nil - }, - ) - if err != nil { - return err - } - } - - for _, node := range nodes { - idEntry, ok := tfIDs[node] - if !ok { - // Resource is in config but not in TF state (new resource); skip. - continue - } - - group := config.GetResourceTypeFromKey(node) - if group == "" { - return fmt.Errorf("cannot determine resource type for %q", node) - } - - adapter, ok := adapters[group] - if !ok { - log.Warnf(ctx, "unsupported resource type %q for %s, skipping", group, node) - continue - } - - inputConfig, err := configRoot.GetResourceConfig(node) - if err != nil { - return fmt.Errorf("%s: getting config: %w", node, err) - } - - baseRefs := map[string]string{} - - switch { - case strings.HasSuffix(node, ".permissions"): - var sv *structvar.StructVar - if strings.HasPrefix(node, "resources.secret_scopes.") { - typedConfig, ok := inputConfig.(*[]resources.SecretScopePermission) - if !ok { - return fmt.Errorf("%s: expected *[]resources.SecretScopePermission, got %T", node, inputConfig) - } - sv, err = dresources.PrepareSecretScopeAclsInputConfig(*typedConfig, node) - if err != nil { - return fmt.Errorf("%s: preparing secret scope ACLs config: %w", node, err) - } - } else { - sv, err = dresources.PreparePermissionsInputConfig(inputConfig, node) - if err != nil { - return fmt.Errorf("%s: preparing permissions config: %w", node, err) - } - } - inputConfig = sv.Value - baseRefs = sv.Refs - - case strings.HasSuffix(node, ".grants"): - sv, err := dresources.PrepareGrantsInputConfig(inputConfig, node) - if err != nil { - return fmt.Errorf("%s: preparing grants config: %w", node, err) - } - inputConfig = sv.Value - baseRefs = sv.Refs - } - - newStateValue, err := adapter.PrepareState(inputConfig) - if err != nil { - return fmt.Errorf("%s: PrepareState: %w", node, err) - } - - refs, err := direct.ExtractReferences(configRoot.Value(), node) - if err != nil { - return fmt.Errorf("%s: extracting references: %w", node, err) - } - maps.Copy(refs, baseRefs) - - sv := structvar.NewStructVar(newStateValue, refs) - - // Resolve each reference using TF state. - // We need to extract the resource name for Method A (looking up in the source resource's TF state). - parts := strings.SplitN(node, ".", 4) - // node format: "resources.." or "resources...permissions" - var srcGroup, srcName string - if len(parts) >= 3 { - srcGroup = parts[1] - srcName = parts[2] - } - - // Collect all field paths that need resolution (avoid modifying map during iteration). - type refEntry struct { - fieldPathStr string - refTemplate string - } - var pendingRefs []refEntry - for fieldPathStr, refTemplate := range sv.Refs { - pendingRefs = append(pendingRefs, refEntry{fieldPathStr, refTemplate}) - } - - for _, pending := range pendingRefs { - fieldPath, err := structpath.ParsePath(pending.fieldPathStr) - if err != nil { - return fmt.Errorf("%s: parsing field path %q: %w", node, pending.fieldPathStr, err) - } - - // ResolveFieldRef returns the fully resolved value for this field, - // using either Method A (TF state lookup) or Method B (template evaluation). - value, err := migrate.ResolveFieldRef(ctx, tfAttrs, srcGroup, srcName, fieldPath, pending.refTemplate) - if err != nil { - return fmt.Errorf("%s: cannot resolve field %q (template %q): %w", node, pending.fieldPathStr, pending.refTemplate, err) - } - - // Set the resolved value directly and remove the ref entry. - if err := structaccess.Set(sv.Value, fieldPath, value); err != nil { - return fmt.Errorf("%s: cannot set resolved value for field %q: %w", node, pending.fieldPathStr, err) - } - delete(sv.Refs, pending.fieldPathStr) - } - - if len(sv.Refs) > 0 { - return fmt.Errorf("%s: unresolved references: %v", node, sv.Refs) - } - - // Handle etag for dashboards. - if etag := etags[node]; etag != "" { - if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag); err != nil { - return fmt.Errorf("%s: cannot set etag: %w", node, err) - } - } - - if err := stateDB.SaveState(node, idEntry.ID, sv.Value, nil); err != nil { - return fmt.Errorf("%s: SaveState: %w", node, err) - } - } - - return nil -} From a02cb0f6e845b647aac904cdfb70370f38b46087 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Jun 2026 17:30:22 +0200 Subject: [PATCH 4/8] bundle migrate: remove new top-level command, keep deployment migrate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new approach (BuildStateFromTF, no API calls) has been folded into the existing 'bundle deployment migrate' command. The separate 'bundle migrate' command is removed — it was redundant. Also add resolve_test.go: verify that int and bool cross-resource references work correctly when Method B returns a string value. structaccess.Set already handles string→int and string→bool conversion via strconv, so no bug exists. Co-authored-by: Isaac --- bundle/migrate/resolve_test.go | 78 ++++++++++++++ cmd/bundle/bundle.go | 1 - cmd/bundle/migrate.go | 192 --------------------------------- 3 files changed, 78 insertions(+), 193 deletions(-) create mode 100644 bundle/migrate/resolve_test.go delete mode 100644 cmd/bundle/migrate.go diff --git a/bundle/migrate/resolve_test.go b/bundle/migrate/resolve_test.go new file mode 100644 index 00000000000..8a2ab06d13b --- /dev/null +++ b/bundle/migrate/resolve_test.go @@ -0,0 +1,78 @@ +package migrate_test + +import ( + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/migrate" + "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// state with src job having int and bool fields set. +func testState() migrate.TFStateAttrs { + return migrate.TFStateAttrs{ + "databricks_job": { + "src": json.RawMessage(`{ + "id": "111", + "max_concurrent_runs": 4, + "always_running": true + }`), + "dst": json.RawMessage(`{ + "id": "222", + "max_concurrent_runs": 4, + "always_running": true + }`), + }, + } +} + +// TestResolveFieldRefInt proves that when Method B (template evaluation) wins for +// an int field, the returned string value is still usable: structaccess.Set must +// parse it back to int and not error. +func TestResolveFieldRefInt(t *testing.T) { + state := testState() + // Remove dst from state so Method A fails and Method B must be used. + delete(state["databricks_job"], "dst") + + ctx := t.Context() + fieldPath, err := structpath.ParsePath("max_concurrent_runs") + require.NoError(t, err) + + value, err := migrate.ResolveFieldRef(ctx, state, "jobs", "dst", fieldPath, + "${resources.jobs.src.max_concurrent_runs}") + require.NoError(t, err) + + // Method B succeeds: returns string "4". Verify Set converts it to int. + type target struct { + MaxConcurrentRuns int `json:"max_concurrent_runs"` + } + s := &target{} + err = structaccess.Set(s, fieldPath, value) + assert.NoError(t, err, "Set should parse string %q into int field", value) + assert.Equal(t, 4, s.MaxConcurrentRuns) +} + +// TestResolveFieldRefBool is the same for a bool field. +func TestResolveFieldRefBool(t *testing.T) { + state := testState() + delete(state["databricks_job"], "dst") + + ctx := t.Context() + fieldPath, err := structpath.ParsePath("always_running") + require.NoError(t, err) + + value, err := migrate.ResolveFieldRef(ctx, state, "jobs", "dst", fieldPath, + "${resources.jobs.src.always_running}") + require.NoError(t, err) + + type target struct { + AlwaysRunning bool `json:"always_running"` + } + s := &target{} + err = structaccess.Set(s, fieldPath, value) + assert.NoError(t, err, "Set should parse string %q into bool field", value) + assert.Equal(t, true, s.AlwaysRunning) +} diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index 84666eec689..7189b1d431d 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -38,7 +38,6 @@ Online documentation: https://docs.databricks.com/en/dev-tools/bundles/index.htm cmd.AddCommand(newDebugCommand()) cmd.AddCommand(newOpenCommand()) cmd.AddCommand(newPlanCommand()) - cmd.AddCommand(newMigrateCommand()) cmd.AddCommand(newConfigRemoteSyncCommand()) // Bundle Metadata Service (DMS) read-only command groups. Only `get` diff --git a/cmd/bundle/migrate.go b/cmd/bundle/migrate.go deleted file mode 100644 index 8a33fe85f55..00000000000 --- a/cmd/bundle/migrate.go +++ /dev/null @@ -1,192 +0,0 @@ -package bundle - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/engine" - "github.com/databricks/cli/bundle/config/mutator/resourcemutator" - "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/bundle/direct/dresources" - "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/bundle/migrate" - "github.com/databricks/cli/cmd/bundle/utils" - "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/env" - "github.com/databricks/cli/libs/logdiag" - "github.com/spf13/cobra" -) - -func newMigrateCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "migrate", - Short: "Migrate from Terraform to Direct deployment engine (no API calls)", - Long: `Creates a Direct deployment state file from the local config and Terraform state, -without making API calls. Cross-resource references are resolved from TF state.`, - Args: root.NoArgs, - } - - // --noplancheck is kept for compatibility but has no effect: this command reads - // only from the local TF state file and never invokes the Terraform engine. - cmd.Flags().Bool("noplancheck", false, "No-op (kept for compatibility).") - - cmd.RunE = func(cmd *cobra.Command, args []string) error { - // Clear engine env var: we read TF state and produce a direct state. - cmd.SetContext(env.Set(cmd.Context(), engine.EnvVar, "")) - - opts := utils.ProcessOptions{ - AlwaysPull: true, - FastValidate: true, - Build: true, - PostInitFunc: func(_ context.Context, b *bundle.Bundle) error { - if b.Config.Bundle.Engine == engine.EngineTerraform { - return fmt.Errorf("bundle.engine is set to %q; migration requires \"engine: direct\" or no engine setting", engine.EngineTerraform) - } - return nil - }, - } - - b, stateDesc, err := utils.ProcessBundleRet(cmd, opts) - if err != nil { - return err - } - ctx := cmd.Context() - - if stateDesc.Lineage == "" { - cmdio.LogString(ctx, `Error: no existing state found. To start fresh with direct engine, set "engine: direct".`) - return root.ErrAlreadyPrinted - } - if stateDesc.Engine.IsDirect() { - return fmt.Errorf("already using direct engine: %s", stateDesc.String()) - } - - _, localTerraformPath := b.StateFilenameTerraform(ctx) - if _, err := os.Stat(localTerraformPath); err != nil { - return fmt.Errorf("reading %s: %w", localTerraformPath, err) - } - - // Parse TF state: IDs (for state entries) and full attributes (for ref resolution). - tfResourceIDs, err := terraform.ParseResourcesState(ctx, b) - if err != nil { - return fmt.Errorf("failed to parse terraform state: %w", err) - } - for key, entry := range tfResourceIDs { - if entry.ID == "" { - return fmt.Errorf("missing ID for %s in terraform state", key) - } - } - - cacheDir, err := terraform.Dir(ctx, b) - if err != nil { - return err - } - tfStateFilename, _ := b.StateFilenameTerraform(ctx) - tfStateFullPath := filepath.Join(cacheDir, tfStateFilename) - - tfAttrs, err := migrate.ParseTFStateAttrs(tfStateFullPath) - if err != nil { - return fmt.Errorf("failed to read terraform state attributes: %w", err) - } - - _, localPath := b.StateFilenameDirect(ctx) - tempPath := localPath + ".temp-migration" - - if _, err := os.Stat(tempPath); err == nil { - return fmt.Errorf("temporary state file %s already exists, another migration is in progress or was interrupted. In the latter case, delete the file", tempPath) - } - if _, err := os.Stat(localPath); err == nil { - return fmt.Errorf("state file %s already exists", localPath) - } - - // Apply SecretScopeFixups so the config matches what the direct engine expects. - // This adds MANAGE ACL for the current user to all secret scopes, ensuring - // the migrated state and config agree on .permissions entries. - bundle.ApplyContext(ctx, b, resourcemutator.SecretScopeFixups(engine.EngineDirect)) - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted - } - - // Build initial state with IDs and optional ETags. - etags := map[string]string{} - state := make(map[string]dstate.ResourceEntry) - for key, resourceEntry := range tfResourceIDs { - state[key] = dstate.ResourceEntry{ - ID: resourceEntry.ID, - State: json.RawMessage("{}"), - } - if resourceEntry.ETag != "" { - etags[key] = resourceEntry.ETag - } - } - - migratedDB := dstate.NewDatabase(stateDesc.Lineage, stateDesc.Serial+1) - migratedDB.State = state - - var stateDB dstate.DeploymentState - stateDB.OpenWithData(tempPath, migratedDB) - - removeTempPath := true - defer func() { - if removeTempPath { - _ = os.Remove(tempPath) - } - }() - - // Initialize adapters. - adapters, err := dresources.InitAll(b.WorkspaceClient(ctx)) - if err != nil { - return err - } - - if err := stateDB.UpgradeToWrite(); err != nil { - return fmt.Errorf("upgrading state for write: %w", err) - } - - // Process each resource: prepare state, resolve refs from TF state, save. - if err := migrate.BuildStateFromTF(ctx, &b.Config, adapters, &stateDB, tfAttrs, tfResourceIDs, etags); err != nil { - return err - } - - if _, err := stateDB.Finalize(ctx); err != nil { - return fmt.Errorf("finalizing state: %w", err) - } - if logdiag.HasError(ctx) { - return errors.New("migration encountered errors") - } - - if err := os.Rename(tempPath, localPath); err != nil { - return fmt.Errorf("renaming %s to %s: %w", tempPath, localPath, err) - } - removeTempPath = false - - localTerraformBackupPath := localTerraformPath + ".backup" - err = os.Rename(localTerraformPath, localTerraformBackupPath) - if err != nil { - // Not fatal since we've already incremented the serial. - logdiag.LogError(ctx, err) - } - - extraArgsStr := "" - if flag := cmd.Flag("target"); flag != nil && flag.Changed { - extraArgsStr = " -t " + flag.Value.String() - } - - cmdio.LogString(ctx, fmt.Sprintf(`Success! Migrated %d resources to direct engine state file: %s - -Validate the migration by running "databricks bundle plan%s", there should be no actions planned. - -The state file is not synchronized to the workspace yet. To finalize the migration, run "bundle deploy%s". - -To undo the migration, remove %s and rename %s to %s -`, len(state), localPath, extraArgsStr, extraArgsStr, localPath, localTerraformBackupPath, localTerraformPath)) - return nil - } - - return cmd -} From 6f179b78f5e465593d8631d28ab7a96a4c555190 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 2 Jun 2026 14:11:14 +0200 Subject: [PATCH 5/8] deployment migrate: drop plan check and --noplancheck flag The command now reads from the local TF state file without invoking the Terraform binary, so there is nothing for the plan check to verify. Also simplify getCommonArgs to return only the display string (the args slice was only needed to forward to the plan subprocess). Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 1 - bundle/migrate/resolve_test.go | 2 +- bundle/migrate/tf_state.go | 5 +- cmd/bundle/deployment/migrate.go | 99 ++++---------------------------- 4 files changed, 14 insertions(+), 93 deletions(-) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index a5849ffbd1a..a63d70aee13 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -106,7 +106,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // TODO: redo calcDiff to downgrade planned action if possible (?) err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) - if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false diff --git a/bundle/migrate/resolve_test.go b/bundle/migrate/resolve_test.go index 8a2ab06d13b..221c873a73d 100644 --- a/bundle/migrate/resolve_test.go +++ b/bundle/migrate/resolve_test.go @@ -74,5 +74,5 @@ func TestResolveFieldRefBool(t *testing.T) { s := &target{} err = structaccess.Set(s, fieldPath, value) assert.NoError(t, err, "Set should parse string %q into bool field", value) - assert.Equal(t, true, s.AlwaysRunning) + assert.True(t, s.AlwaysRunning) } diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index 642f7a26b74..f4733c152eb 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -57,10 +57,9 @@ func ParseTFStateAttrs(path string) (TFStateAttrs, error) { // tfSchemaTypeMap maps TF resource type name → schema struct type (via AllResources json tags). var tfSchemaTypeMap = sync.OnceValue(func() map[string]reflect.Type { - t := reflect.TypeOf(tfschema.AllResources{}) + t := reflect.TypeFor[tfschema.AllResources]() m := make(map[string]reflect.Type, t.NumField()) - for i := range t.NumField() { - f := t.Field(i) + for f := range t.Fields() { tag := strings.Split(f.Tag.Get("json"), ",")[0] if tag != "" && tag != "-" { m[tag] = f.Type diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 1c59616fcc9..fb2179145d3 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -1,13 +1,11 @@ package deployment import ( - "bytes" "context" "encoding/json" "errors" "fmt" "os" - "os/exec" "strings" "github.com/databricks/cli/bundle" @@ -28,95 +26,31 @@ import ( const backupSuffix = ".backup" -// runPlanCheck runs bundle plan and checks if there are any actions planned. -// Returns error if plan fails or if there are actions planned. -func runPlanCheck(cmd *cobra.Command, extraArgs []string, extraArgsStr string) error { - ctx := cmd.Context() - - executable, err := os.Executable() - if err != nil { - return fmt.Errorf("failed to get executable path: %w", err) - } - - args := []string{"bundle", "plan"} - args = append(args, extraArgs...) - - planCmd := exec.CommandContext(ctx, executable, args...) - var stdout bytes.Buffer - planCmd.Stdout = &stdout - planCmd.Stderr = cmd.ErrOrStderr() - - // Use the engine encoded in the state - planCmd.Env = append(os.Environ(), "DATABRICKS_BUNDLE_ENGINE=terraform") - - err = planCmd.Run() - - // Output the plan stdout as is - output := stdout.String() - fmt.Fprint(cmd.OutOrStdout(), output) - - if err != nil { - msg := "" - if exitErr, ok := errors.AsType[*exec.ExitError](err); ok { - msg = fmt.Sprintf("exit code %d", exitErr.ExitCode()) - } else { - msg = err.Error() - } - return fmt.Errorf("bundle plan failed with %s, aborting migration. To proceed with migration anyway, re-run the command with --noplancheck option", msg) - } - - if !strings.Contains(output, "Plan:") { - return fmt.Errorf("cannot parse 'databricks bundle plan%s' output, aborting migration. Skip plan check with --noplancheck option", extraArgsStr) - } - - if !strings.Contains(output, "Plan: 0 to add, 0 to change, 0 to delete") { - return fmt.Errorf("'databricks bundle plan%s' shows actions planned, aborting migration. Please run 'databricks bundle deploy%s' first to ensure your bundle is up to date, If actions persist after deploy, skip plan check with --noplancheck option", extraArgsStr, extraArgsStr) - } - - return nil -} - -func getCommonArgs(cmd *cobra.Command) ([]string, string) { - var args []string +func getCommonArgs(cmd *cobra.Command) string { var quotedArgs []string if flag := cmd.Flag("target"); flag != nil && flag.Changed { - target := flag.Value.String() - if target != "" { - args = append(args, "-t") - args = append(args, target) - quotedArgs = append(quotedArgs, "-t") - quotedArgs = append(quotedArgs, shellquote.BashArg(target)) + if target := flag.Value.String(); target != "" { + quotedArgs = append(quotedArgs, "-t", shellquote.BashArg(target)) } } if flag := cmd.Flag("profile"); flag != nil && flag.Changed { - profile := flag.Value.String() - if profile != "" { - args = append(args, "-p") - args = append(args, profile) - quotedArgs = append(quotedArgs, "-p") - quotedArgs = append(quotedArgs, shellquote.BashArg(profile)) + if profile := flag.Value.String(); profile != "" { + quotedArgs = append(quotedArgs, "-p", shellquote.BashArg(profile)) } } if flag := cmd.Flag("var"); flag != nil && flag.Changed { - varValues, err := cmd.Flags().GetStringSlice("var") - if err == nil { + if varValues, err := cmd.Flags().GetStringSlice("var"); err == nil { for _, v := range varValues { - args = append(args, "--var") - args = append(args, v) - quotedArgs = append(quotedArgs, "--var") - quotedArgs = append(quotedArgs, shellquote.BashArg(v)) + quotedArgs = append(quotedArgs, "--var", shellquote.BashArg(v)) } } } - argsStr := "" - - if len(quotedArgs) > 0 { - argsStr = " " + strings.Join(quotedArgs, " ") + if len(quotedArgs) == 0 { + return "" } - - return args, argsStr + return " " + strings.Join(quotedArgs, " ") } func newMigrateCommand() *cobra.Command { @@ -134,11 +68,8 @@ to the workspace so that subsequent deploys of this bundle use direct deployment Args: root.NoArgs, } - var noPlanCheck bool - cmd.Flags().BoolVar(&noPlanCheck, "noplancheck", false, "Skip running bundle plan before migration.") - cmd.RunE = func(cmd *cobra.Command, args []string) error { - extraArgs, extraArgsStr := getCommonArgs(cmd) + extraArgsStr := getCommonArgs(cmd) // Clear the engine env var so migrate always uses terraform engine to read existing state, // regardless of what the user may have set in their environment. @@ -199,14 +130,6 @@ To start using direct engine, set "engine: direct" under bundle in your databric return fmt.Errorf("state file %s already exists", localPath) } - // Run plan check unless --noplancheck is set - if !noPlanCheck { - cmdio.LogString(ctx, "Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done:") - if err = runPlanCheck(cmd, extraArgs, extraArgsStr); err != nil { - return err - } - } - tfAttrs, err := migrate.ParseTFStateAttrs(localTerraformPath) if err != nil { return fmt.Errorf("failed to read terraform state attributes: %w", err) From 51e259c92b1369fe027094a3852ab4c2eddfa401 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 11:50:57 +0200 Subject: [PATCH 6/8] acceptance: update bundle-deployment-migrate help output Remove --noplancheck from expected output after dropping that flag. Co-authored-by: Isaac --- acceptance/bundle/help/bundle-deployment-migrate/output.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/acceptance/bundle/help/bundle-deployment-migrate/output.txt b/acceptance/bundle/help/bundle-deployment-migrate/output.txt index da2f95707bc..36ee7d481e7 100644 --- a/acceptance/bundle/help/bundle-deployment-migrate/output.txt +++ b/acceptance/bundle/help/bundle-deployment-migrate/output.txt @@ -12,8 +12,7 @@ Usage: databricks bundle deployment migrate [flags] Flags: - -h, --help help for migrate - --noplancheck Skip running bundle plan before migration. + -h, --help help for migrate Global Flags: --debug enable debug logging From c3ab34ad8f3b8e5202832be04840cdafa26b0e91 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 12:08:36 +0200 Subject: [PATCH 7/8] migrate: fix TF list-block unmarshaling, restore --noplancheck no-op, update outputs TF state stores single-block fields (e.g. continuous, deployment) as single-element arrays [{}], not plain objects. json.Unmarshal into the generated Go schema structs fails with type mismatch. Switch LookupTFField to navigate via map[string]any + custom navigateTFState that auto-unwraps single-element lists when a string-key step follows. Also: - Restore --noplancheck as a no-op flag (backward compat; used by invariant tests for job_with_depends_on config). - Remove the plan-check lines from acceptance test output.txt files. - Update help output to include --noplancheck. Co-authored-by: Isaac --- .../help/bundle-deployment-migrate/output.txt | 3 +- acceptance/bundle/migrate/basic/output.txt | 2 - .../bundle/migrate/dashboards/output.txt | 2 - .../bundle/migrate/default-python/output.txt | 1 - acceptance/bundle/migrate/grants/output.txt | 2 - .../bundle/migrate/permissions/output.txt | 2 - .../bundle/migrate/profile_arg/output.txt | 4 - acceptance/bundle/migrate/runas/output.txt | 1 - acceptance/bundle/migrate/var_arg/output.txt | 4 - bundle/migrate/tf_state.go | 83 ++++++++++++------- cmd/bundle/deployment/migrate.go | 4 + 11 files changed, 61 insertions(+), 47 deletions(-) diff --git a/acceptance/bundle/help/bundle-deployment-migrate/output.txt b/acceptance/bundle/help/bundle-deployment-migrate/output.txt index 36ee7d481e7..9312fb85ffc 100644 --- a/acceptance/bundle/help/bundle-deployment-migrate/output.txt +++ b/acceptance/bundle/help/bundle-deployment-migrate/output.txt @@ -12,7 +12,8 @@ Usage: databricks bundle deployment migrate [flags] Flags: - -h, --help help for migrate + -h, --help help for migrate + --noplancheck No-op (kept for compatibility). Global Flags: --debug enable debug logging diff --git a/acceptance/bundle/migrate/basic/output.txt b/acceptance/bundle/migrate/basic/output.txt index dafa3a4086e..cd45577a1ee 100644 --- a/acceptance/bundle/migrate/basic/output.txt +++ b/acceptance/bundle/migrate/basic/output.txt @@ -11,8 +11,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 3 unchanged Success! Migrated 3 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/acceptance/bundle/migrate/dashboards/output.txt b/acceptance/bundle/migrate/dashboards/output.txt index 19a4f1c7bb5..cfda4350ceb 100644 --- a/acceptance/bundle/migrate/dashboards/output.txt +++ b/acceptance/bundle/migrate/dashboards/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/acceptance/bundle/migrate/default-python/output.txt b/acceptance/bundle/migrate/default-python/output.txt index 15d49805b47..92efb3066d6 100644 --- a/acceptance/bundle/migrate/default-python/output.txt +++ b/acceptance/bundle/migrate/default-python/output.txt @@ -24,7 +24,6 @@ Deployment complete! >>> musterr [CLI] bundle deployment migrate Building python_artifact... -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: Building python_artifact... update jobs.sample_job diff --git a/acceptance/bundle/migrate/grants/output.txt b/acceptance/bundle/migrate/grants/output.txt index 146787d549a..86acb6398f5 100644 --- a/acceptance/bundle/migrate/grants/output.txt +++ b/acceptance/bundle/migrate/grants/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 6 unchanged Success! Migrated 6 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/acceptance/bundle/migrate/permissions/output.txt b/acceptance/bundle/migrate/permissions/output.txt index f85c8d7bdbf..51caca51104 100644 --- a/acceptance/bundle/migrate/permissions/output.txt +++ b/acceptance/bundle/migrate/permissions/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged Success! Migrated 4 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json Validate the migration by running "databricks bundle plan", there should be no actions planned. diff --git a/acceptance/bundle/migrate/profile_arg/output.txt b/acceptance/bundle/migrate/profile_arg/output.txt index a6def38363a..082feee15b1 100644 --- a/acceptance/bundle/migrate/profile_arg/output.txt +++ b/acceptance/bundle/migrate/profile_arg/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -p non_existent321 -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json Validate the migration by running "databricks bundle plan -p non_existent321", there should be no actions planned. @@ -24,8 +22,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate -p non_existent321 -t prod -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged Success! Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/prod/resources.json Validate the migration by running "databricks bundle plan -t prod -p non_existent321", there should be no actions planned. diff --git a/acceptance/bundle/migrate/runas/output.txt b/acceptance/bundle/migrate/runas/output.txt index 74b9a0217f3..f7034d74ec0 100644 --- a/acceptance/bundle/migrate/runas/output.txt +++ b/acceptance/bundle/migrate/runas/output.txt @@ -81,7 +81,6 @@ Consider using a adding a top-level permissions section such as the following: See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. in databricks.yml:5:3 -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. diff --git a/acceptance/bundle/migrate/var_arg/output.txt b/acceptance/bundle/migrate/var_arg/output.txt index a7f8c0e5b2e..3c20819211d 100644 --- a/acceptance/bundle/migrate/var_arg/output.txt +++ b/acceptance/bundle/migrate/var_arg/output.txt @@ -6,8 +6,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate --var=job_name=Custom Job Name -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json Validate the migration by running "databricks bundle plan --var 'job_name=Custom Job Name'", there should be no actions planned. @@ -39,8 +37,6 @@ Updating deployment state... Deployment complete! >>> [CLI] bundle deployment migrate --var job_name=Custom Job Name -Note: Migration should be done after a full deploy. Running plan now to verify that deployment was done: -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Success! Migrated 1 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json Validate the migration by running "databricks bundle plan --var 'job_name=Custom Job Name'", there should be no actions planned. diff --git a/bundle/migrate/tf_state.go b/bundle/migrate/tf_state.go index f4733c152eb..52799dd79b8 100644 --- a/bundle/migrate/tf_state.go +++ b/bundle/migrate/tf_state.go @@ -4,14 +4,9 @@ import ( "encoding/json" "fmt" "os" - "reflect" - "strings" - "sync" "github.com/databricks/cli/bundle/deploy/terraform" - tfschema "github.com/databricks/cli/bundle/internal/tf/schema" "github.com/databricks/cli/bundle/terraform_dabs_map" - "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" tfjson "github.com/hashicorp/terraform-json" ) @@ -55,23 +50,9 @@ func ParseTFStateAttrs(path string) (TFStateAttrs, error) { return result, nil } -// tfSchemaTypeMap maps TF resource type name → schema struct type (via AllResources json tags). -var tfSchemaTypeMap = sync.OnceValue(func() map[string]reflect.Type { - t := reflect.TypeFor[tfschema.AllResources]() - m := make(map[string]reflect.Type, t.NumField()) - for f := range t.Fields() { - tag := strings.Split(f.Tag.Get("json"), ",")[0] - if tag != "" && tag != "-" { - m[tag] = f.Type - } - } - return m -}) - // LookupTFField looks up a field from TF state attributes for a bundle resource. // group is the DABs group (e.g. "pipelines"), name is the resource name. // fieldPath is the path to the field (may be in DABs or TF naming; both handled by DABsPathToTerraform). -// Returns (nil, nil) for empty/zero fields, error if the resource or field is not found. func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath.PathNode) (any, error) { tfType, ok := terraform.GroupToTerraformName[group] if !ok { @@ -91,16 +72,62 @@ func LookupTFField(state TFStateAttrs, group, name string, fieldPath *structpath return nil, fmt.Errorf("%s.%s not found in TF state", tfType, name) } - schemaType, ok := tfSchemaTypeMap()[tfType] - if !ok { - return nil, fmt.Errorf("no schema type registered for %q", tfType) - } - - // Unmarshal attributes into a new instance of the schema struct. - ptr := reflect.New(schemaType) - if err := json.Unmarshal(attrsJSON, ptr.Interface()); err != nil { + // Unmarshal into map[string]any to handle TF list-blocks: in TF state, single-block + // fields are stored as single-element arrays [{"field": "value"}], not as plain objects. + // Navigating via map avoids the json.Unmarshal type mismatch between []T in JSON and + // struct-typed schema fields. + var attrs map[string]any + if err := json.Unmarshal(attrsJSON, &attrs); err != nil { return nil, fmt.Errorf("cannot parse TF state for %s.%s: %w", tfType, name, err) } - return structaccess.Get(ptr.Interface(), tfFieldPath) + return navigateTFState(attrs, tfFieldPath) +} + +// navigateTFState walks the TF state map using the given path. +// TF stores single-block fields as single-element arrays ([{…}]). When a string-key +// step encounters a []any, it auto-descends into element [0] so callers can use plain +// paths like "continuous.pause_status" even though TF stores them as [{"pause_status":…}]. +func navigateTFState(data map[string]any, path *structpath.PathNode) (any, error) { + var current any = data + for _, node := range path.AsSlice() { + if current == nil { + return nil, nil + } + + if key, ok := node.StringKey(); ok { + // Auto-unwrap TF list-blocks: if the current value is a single-element + // array and the next step wants a map key, descend into element 0. + if arr, isArr := current.([]any); isArr { + if len(arr) == 0 { + return nil, nil + } + current = arr[0] + } + m, ok := current.(map[string]any) + if !ok { + return nil, fmt.Errorf("expected map at %q, got %T", key, current) + } + val, ok := m[key] + if !ok { + return nil, fmt.Errorf("%q: key not found", key) + } + current = val + } else if idx, ok := node.Index(); ok { + switch v := current.(type) { + case []any: + if idx < 0 || idx >= len(v) { + return nil, fmt.Errorf("index %d out of range (len %d)", idx, len(v)) + } + current = v[idx] + default: + // TF [0] on a non-slice (already unwrapped) is a no-op. + if idx == 0 { + continue + } + return nil, fmt.Errorf("index %d: not a slice (%T)", idx, current) + } + } + } + return current, nil } diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index fb2179145d3..9b217062fc8 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -68,6 +68,10 @@ to the workspace so that subsequent deploys of this bundle use direct deployment Args: root.NoArgs, } + // --noplancheck kept for backward compatibility; the plan check was removed + // because the command no longer invokes the Terraform engine. + cmd.Flags().Bool("noplancheck", false, "No-op (kept for compatibility).") + cmd.RunE = func(cmd *cobra.Command, args []string) error { extraArgsStr := getCommonArgs(cmd) From f9471b6a977b65cc20ef1738e679300424af68df Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 4 Jun 2026 12:55:02 +0200 Subject: [PATCH 8/8] migrate: compute depends_on from refs before resolving, restore --noplancheck The refs map is shared with sv.Refs and gets mutated (entries deleted) during reference resolution. depends_on must be computed before that loop runs. Also restore --noplancheck as a no-op flag kept for backward compatibility (used by the invariant test suite for job_with_depends_on config). Co-authored-by: Isaac --- bundle/migrate/build_state.go | 44 ++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index af2b3449086..785c325bfad 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -4,15 +4,18 @@ import ( "context" "fmt" "maps" + "slices" "strings" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" @@ -120,6 +123,45 @@ func BuildStateFromTF( sv := structvar.NewStructVar(newStateValue, refs) + // Compute depends_on from cross-resource references before resolving them + // (resolution deletes entries from the refs map). + // Same logic as makePlan in bundle/direct/bundle_plan.go. + var dependsOn []deployplan.DependsOnEntry //nolint:prealloc + for _, refTemplate := range refs { + ref, ok := dynvar.NewRef(dyn.V(refTemplate)) + if !ok { + continue + } + for _, targetPath := range ref.References() { + targetPathParsed, err := dyn.NewPathFromString(targetPath) + if err != nil { + continue + } + targetNodeDP, _ := config.GetNodeAndType(targetPathParsed) + targetNode := targetNodeDP.String() + fullRef := "${" + targetPath + "}" + found := false + for _, dep := range dependsOn { + if dep.Node == targetNode && dep.Label == fullRef { + found = true + break + } + } + if !found { + dependsOn = append(dependsOn, deployplan.DependsOnEntry{ + Node: targetNode, + Label: fullRef, + }) + } + } + } + slices.SortFunc(dependsOn, func(a, b deployplan.DependsOnEntry) int { + if a.Node != b.Node { + return strings.Compare(a.Node, b.Node) + } + return strings.Compare(a.Label, b.Label) + }) + // Resolve each reference using TF state. // node format: "resources.." or "resources...permissions" parts := strings.SplitN(node, ".", 4) @@ -170,7 +212,7 @@ func BuildStateFromTF( } } - if err := stateDB.SaveState(node, idEntry.ID, sv.Value, nil); err != nil { + if err := stateDB.SaveState(node, idEntry.ID, sv.Value, dependsOn); err != nil { return fmt.Errorf("%s: SaveState: %w", node, err) } }