diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.json b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.json index 05614d798bf..74e555c734d 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.json +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.json @@ -2,7 +2,7 @@ "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, + "serial": 1, "state": { "resources.dashboards.dashboard1": { "__id__": "[DASHBOARD_ID]", diff --git a/acceptance/bundle/deployment/bind/job/undeployed-dependency/output.txt b/acceptance/bundle/deployment/bind/job/undeployed-dependency/output.txt index 445d3b40fa5..56a88a4898d 100644 --- a/acceptance/bundle/deployment/bind/job/undeployed-dependency/output.txt +++ b/acceptance/bundle/deployment/bind/job/undeployed-dependency/output.txt @@ -42,7 +42,7 @@ Run 'bundle deploy' to deploy changes to your workspace >>> jq {serial, job_1: .state["resources.jobs.job_1"] | {__id__, depends_on, description: .state.description}} .databricks/bundle/default/resources.json { - "serial": 3, + "serial": 2, "job_1": { "__id__": "[EXTERNAL_JOB_ID]", "depends_on": [ diff --git a/acceptance/bundle/dms_fault/complete_version/output.txt b/acceptance/bundle/dms_fault/complete_version/output.txt index c7d574f4431..bbf04e469ad 100644 --- a/acceptance/bundle/dms_fault/complete_version/output.txt +++ b/acceptance/bundle/dms_fault/complete_version/output.txt @@ -83,7 +83,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-complete_version-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -102,7 +102,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-complete_version-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/delete_deployment/output.txt b/acceptance/bundle/dms_fault/delete_deployment/output.txt index 8efdbc07a0d..34db7ea5d7d 100644 --- a/acceptance/bundle/dms_fault/delete_deployment/output.txt +++ b/acceptance/bundle/dms_fault/delete_deployment/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-delete_deployment-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-delete_deployment-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/get_operation/output.txt b/acceptance/bundle/dms_fault/get_operation/output.txt index 65498809a65..77a30f34331 100644 --- a/acceptance/bundle/dms_fault/get_operation/output.txt +++ b/acceptance/bundle/dms_fault/get_operation/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_operation-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_operation-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/get_resource/output.txt b/acceptance/bundle/dms_fault/get_resource/output.txt index a562c0f5264..1f73acc8286 100644 --- a/acceptance/bundle/dms_fault/get_resource/output.txt +++ b/acceptance/bundle/dms_fault/get_resource/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_resource-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_resource-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/get_version/output.txt b/acceptance/bundle/dms_fault/get_version/output.txt index b4473435fe9..e2207b8defb 100644 --- a/acceptance/bundle/dms_fault/get_version/output.txt +++ b/acceptance/bundle/dms_fault/get_version/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_version-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_version-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/heartbeat/output.txt b/acceptance/bundle/dms_fault/heartbeat/output.txt index 8b50d26db05..bbce26752db 100644 --- a/acceptance/bundle/dms_fault/heartbeat/output.txt +++ b/acceptance/bundle/dms_fault/heartbeat/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-heartbeat-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-heartbeat-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/list_deployments/output.txt b/acceptance/bundle/dms_fault/list_deployments/output.txt index 0710bf48939..907f90146fc 100644 --- a/acceptance/bundle/dms_fault/list_deployments/output.txt +++ b/acceptance/bundle/dms_fault/list_deployments/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_deployments-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_deployments-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/list_operations/output.txt b/acceptance/bundle/dms_fault/list_operations/output.txt index 5764b81d690..37eb784a030 100644 --- a/acceptance/bundle/dms_fault/list_operations/output.txt +++ b/acceptance/bundle/dms_fault/list_operations/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_operations-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_operations-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/list_resources/output.txt b/acceptance/bundle/dms_fault/list_resources/output.txt index 9f45951b03a..ce7a141c620 100644 --- a/acceptance/bundle/dms_fault/list_resources/output.txt +++ b/acceptance/bundle/dms_fault/list_resources/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_resources-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_resources-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/list_versions/output.txt b/acceptance/bundle/dms_fault/list_versions/output.txt index dabee52cbcd..bb5d8e97403 100644 --- a/acceptance/bundle/dms_fault/list_versions/output.txt +++ b/acceptance/bundle/dms_fault/list_versions/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_versions-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_versions-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/update_operation/output.txt b/acceptance/bundle/dms_fault/update_operation/output.txt index 308345c48c4..caf22b6c307 100644 --- a/acceptance/bundle/dms_fault/update_operation/output.txt +++ b/acceptance/bundle/dms_fault/update_operation/output.txt @@ -83,7 +83,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-update_operation-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Warn: Failed to record deployment history: recording operation for [RESOURCE_KEY]: Fault injected by test. Files: 6 uploaded, 0 deleted @@ -102,7 +102,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-update_operation-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Warn: Failed to record deployment history: recording operation for [RESOURCE_KEY]: Fault injected by test. diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 6fcaa333e08..0a0d8ddfb9c 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -126,7 +126,8 @@ func LoadPlanFromFile(path string) (*Plan, error) { type PlanEntry struct { // ID is set when the resource is at another ID than the one in state: a rename moved it there // (see findMoved), or a parent renamed in this plan moves it before it is recreated (see - // addParentMoves). Applying the entry addresses the resource at ID. + // addParentMoves), or a bind records it at the ID it was bound to. Applying the entry addresses + // the resource at ID. ID string `json:"id,omitempty"` DependsOn []DependsOnEntry `json:"depends_on,omitempty"` Action ActionType `json:"action,omitempty"` @@ -148,6 +149,8 @@ type PlanEntry struct { // - an Update of an otherwise unchanged resource whose state lacks its internal ID (recorded // before the ID was, or by bind): applying it records the ID from the remote state. It is // likewise left out of the textual plan and counted as unchanged. + // - an Update of the resource a bind plan records: applying it saves the config as the + // resource's state at ID, which the state does not have yet. StateOnly bool `json:"state_only,omitempty"` NewState *structvar.StructVarJSON `json:"new_state,omitempty"` RemoteState any `json:"remote_state,omitempty"` diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 2683c6c1532..e5a4acfcca1 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -41,13 +41,14 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, } oldID := db.GetResourceID(d.ResourceKey) - if oldID == "" { - return errors.New("state entry not found") - } if planEntry.ID != "" { - // The plan found the resource at the ID its config gives it: a rename moved it. + // The plan found the resource at the ID its config gives it: a rename moved it, or a bind + // records it at the ID it was bound to. oldID = planEntry.ID } + if oldID == "" { + return errors.New("state entry not found") + } switch actionType { case deployplan.Recreate: diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 73852cdde00..cb83d295ca9 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -2,15 +2,18 @@ package direct import ( "context" + "encoding/json" + "errors" "fmt" + "maps" "os" "strings" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" @@ -41,37 +44,39 @@ type BindResult struct { HasChanges bool // Action is the planned action for the bound resource (e.g., "skip", "update", "recreate") Action deployplan.ActionType - // Plan contains the full deployment plan for the bound resource + // Plan is the plan a deploy after the bind would run; it is the one presented to the user. Plan *deployplan.Plan - // TempStatePath is the path to the temporary state file + // BindPlan records the bound resource in the state without touching the backend. Applying it + // to StatePath completes the bind. + BindPlan *deployplan.Plan + // TempStatePath is the path to the temporary state file BindPlan was applied to TempStatePath string // StatePath is the path to the final state file StatePath string } -// Bind adds an existing workspace resource to a temporary state and calculates -// if there will be any changes when deploying. +// Bind adds an existing workspace resource to the state and calculates the changes a deploy would +// make to it afterwards. It reuses the regular plan and apply: // -// Flow: -// 1. Load current state file -// 2. Update id of the target resource -// 3. Update state path to temp file (state file + ".temp-bind") -// 4. Perform plan + update to populate state with resolved config -// 5. Calculate plan again - this is the plan presented to the user +// 1. Plan against the current state with the resource recorded at resourceID and an empty state. +// The bound resource's entry is turned into a state-only update: applying it records the +// resolved config as the resource's state, with no backend call (the bind plan). +// 2. Apply the bind plan to a copy of the state (state file + ".temp-bind") and plan again +// against it. This is the plan presented to the user. // -// Call Finalize to commit the state or Cancel to discard. +// Call FinalizeBind to apply the bind plan to the state, or Cancel to discard. func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { - // Check if the resource is already managed (bound to a different ID) - var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}); err != nil { + err := b.StateDB.Open(ctx, statePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}) + if err != nil { // State that cannot be read is not the same as state without a binding: // the resource may well be managed already, and binding on top of it // would take over a resource whose ownership was never checked. A state // file that does not exist yet opens successfully as an empty one. return nil, fmt.Errorf("cannot check whether %s is already bound: %w", resourceKey, err) } - existingID := checkStateDB.GetResourceID(resourceKey) - if _, err := checkStateDB.Finalize(ctx); err != nil { + existingID := b.StateDB.GetResourceID(resourceKey) + data := b.StateDB.Data + if _, err := b.StateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) } if existingID != "" { @@ -82,6 +87,11 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } + bindPlan, err := b.calculateBindPlan(ctx, client, configRoot, statePath, data, resourceKey, resourceID) + if err != nil { + return nil, err + } + tmpStatePath := statePath + ".temp-bind" // Copy existing state to temp location if it exists @@ -91,113 +101,21 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.ReadingFile, dstate.WithRecovery(false), dstate.OpenDmsArgs{}) - if err != nil { - os.Remove(tmpStatePath) - return nil, err - } - if err := b.StateDB.UpgradeToWrite(); err != nil { - os.Remove(tmpStatePath) - return nil, err - } - - // Save state with ID and empty state (like migrate does) - err = b.StateDB.SaveState(ctx, resourceKey, resourceID, struct{}{}, nil) - if err != nil { - os.Remove(tmpStatePath) - return nil, err - } - - // Finalize to persist temp state to disk - _, err = b.StateDB.Finalize(ctx) - if err != nil { + if err := b.applyBindPlan(ctx, client, tmpStatePath, bindPlan); err != nil { os.Remove(tmpStatePath) return nil, err } log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) - // First plan + update: populate state with resolved config + // Plan against the temp state: this is the plan to present to the user (change between remote + // resource and config). err = b.StateDB.Open(ctx, tmpStatePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}) if err != nil { os.Remove(tmpStatePath) return nil, err } plan, err := b.CalculatePlan(ctx, client, configRoot) - if err != nil { - os.Remove(tmpStatePath) - return nil, err - } - if _, err := b.StateDB.Finalize(ctx); err != nil { - log.Warnf(ctx, "failed to finalize state: %v", err) - } - - // Populate the state with the resolved config - entry := plan.Plan[resourceKey] - sv, ok := b.StateCache.Load(resourceKey) - if ok && sv != nil { - var dependsOn []deployplan.DependsOnEntry - if entry != nil { - dependsOn = entry.DependsOn - } - - // Copy etag from remote state for resources that use etag-based drift - // detection (dashboards and genie spaces). The etag is not provided by the - // user; it comes from remote. If we don't store it in state, we won't - // detect remote drift correctly and the next plan shows a bogus update. - if (strings.Contains(resourceKey, ".dashboards.") || strings.Contains(resourceKey, ".genie_spaces.")) && entry != nil && entry.RemoteState != nil { - etag, err := structaccess.Get(entry.RemoteState, structpath.NewStringKey(nil, "etag")) - if err == nil && etag != nil { - if etagStr, ok := etag.(string); ok && etagStr != "" { - _ = structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etagStr) - } - } - } - - // Compact hashed_fields fields so the persisted state stays small. Not needed for - // correctness — the next plan (CalculatePlan) compacts the saved state on read. - adapter, err := b.getAdapterForKey(resourceKey) - if err != nil { - os.Remove(tmpStatePath) - return nil, err - } - compacted, err := dresources.CompactState(adapter.ResourceConfig(), sv.Value) - if err != nil { - os.Remove(tmpStatePath) - return nil, fmt.Errorf("compacting state: %w", err) - } - - err = b.StateDB.Open(ctx, tmpStatePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}) - if err != nil { - os.Remove(tmpStatePath) - return nil, err - } - if err := b.StateDB.UpgradeToWrite(); err != nil { - os.Remove(tmpStatePath) - return nil, err - } - - err = b.StateDB.SaveState(ctx, resourceKey, resourceID, compacted, dependsOn) - if err != nil { - os.Remove(tmpStatePath) - return nil, err - } - - _, err = b.StateDB.Finalize(ctx) - if err != nil { - os.Remove(tmpStatePath) - return nil, err - } - } - - // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}) - if err != nil { - os.Remove(tmpStatePath) - return nil, err - } - plan, err = b.CalculatePlan(ctx, client, configRoot) if _, ferr := b.StateDB.Finalize(ctx); ferr != nil { log.Warnf(ctx, "failed to finalize state: %v", ferr) } @@ -211,11 +129,12 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac HasChanges: false, Action: deployplan.Skip, Plan: plan, + BindPlan: bindPlan, TempStatePath: tmpStatePath, StatePath: statePath, } - entry = plan.Plan[resourceKey] + entry := plan.Plan[resourceKey] if entry != nil { result.Action = entry.Action // A state-only entry only records something in the state (such as an internal ID a bind @@ -226,12 +145,111 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac return result, nil } -// Finalize completes the bind operation by renaming the temp state to the final location. -func (result *BindResult) Finalize() error { - if result == nil || result.TempStatePath == "" { - return nil +// calculateBindPlan plans against data with resourceKey recorded at resourceID and an empty +// state, and returns the plan that records the resource's resolved config as its state. +// +// The plan holds the bound resource as a state-only update. Its dependencies are kept as skipped +// entries: the graph needs every node an entry depends on, and the dependency edges are saved in +// the state alongside the resource. +func (b *DeploymentBundle) calculateBindPlan(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath string, data dstate.Database, resourceKey, resourceID string) (*deployplan.Plan, error) { + data.State = maps.Clone(data.State) + if data.State == nil { + data.State = make(map[string]dstate.ResourceEntry) + } + // An empty state, like migrate records: the plan fills it in from the config. + data.State[resourceKey] = dstate.ResourceEntry{ID: resourceID, State: json.RawMessage("{}")} + + b.StateDB.OpenWithData(statePath, data) + b.bindResourceKey = resourceKey + plan, err := b.CalculatePlan(ctx, client, configRoot) + b.bindResourceKey = "" + if _, ferr := b.StateDB.Finalize(ctx); ferr != nil { + log.Warnf(ctx, "failed to finalize state: %v", ferr) + } + if err != nil { + return nil, err + } + + if err := plan.FilterToSelected(ctx, []string{strings.TrimPrefix(resourceKey, "resources.")}); err != nil { + return nil, err + } + for key, entry := range plan.Plan { + if key != resourceKey { + // Deployed, and recorded in the state, by the next deploy. + plan.Plan[key] = &deployplan.PlanEntry{DependsOn: entry.DependsOn, Action: deployplan.Skip} + } + } + + entry := plan.Plan[resourceKey] + // References planning could not resolve point at resources whose values are only known after + // they are deployed. The state records them unresolved; the next deploy resolves them. + entry.NewState.Refs = nil + + // Copy etag from remote state for resources that use etag-based drift + // detection (dashboards and genie spaces). The etag is not provided by the + // user; it comes from remote. If we don't store it in state, we won't + // detect remote drift correctly and the next plan shows a bogus update. + if (strings.Contains(resourceKey, ".dashboards.") || strings.Contains(resourceKey, ".genie_spaces.")) && entry.RemoteState != nil { + etag, err := structaccess.Get(entry.RemoteState, structpath.NewStringKey(nil, "etag")) + if err == nil && etag != nil { + if etagStr, ok := etag.(string); ok && etagStr != "" { + if err := b.setNewStateField(entry, resourceKey, "etag", etagStr); err != nil { + return nil, err + } + } + } } - return os.Rename(result.TempStatePath, result.StatePath) + + return plan, nil +} + +// setNewStateField sets field of the entry's new state, which the apply deploys and saves. +func (b *DeploymentBundle) setNewStateField(entry *deployplan.PlanEntry, resourceKey, field string, value any) error { + adapter, err := b.getAdapterForKey(resourceKey) + if err != nil { + return err + } + sv, err := entry.NewState.ToStructVar(adapter.StateType()) + if err != nil { + return fmt.Errorf("%s: loading new state: %w", resourceKey, err) + } + if err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, field), value); err != nil { + return fmt.Errorf("%s: setting %s: %w", resourceKey, field, err) + } + return sv.SyncToJSON(entry.NewState) +} + +// applyBindPlan applies the bind plan to the state at statePath with the regular deploy apply. +func (b *DeploymentBundle) applyBindPlan(ctx context.Context, client *databricks.WorkspaceClient, statePath string, bindPlan *deployplan.Plan) error { + err := b.StateDB.Open(ctx, statePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}) + if err != nil { + return err + } + if err := b.StateDB.UpgradeToWrite(); err != nil { + _, _ = b.StateDB.Finalize(ctx) + return err + } + + err = b.InitForApply(ctx, client, bindPlan) + if err == nil { + b.Apply(ctx, client, bindPlan, false) + } + + // Finalize even if Apply failed, so the state stays consistent with what was applied. + _, ferr := b.StateDB.Finalize(ctx) + if err != nil { + return err + } + if logdiag.HasError(ctx) { + return errors.New("recording the bound resource in the state failed") + } + return ferr +} + +// FinalizeBind completes the bind operation by applying the bind plan to the state. +func (b *DeploymentBundle) FinalizeBind(ctx context.Context, client *databricks.WorkspaceClient, result *BindResult) error { + defer result.Cancel() + return b.applyBindPlan(ctx, client, result.StatePath, result.BindPlan) } // Cancel cleans up temp state without committing changes. diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 60e1b66bb47..b3230827b6e 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -185,6 +185,12 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa needRemoteState := len(g.Adj[resourceKey]) > 0 if needRemoteState { id := b.StateDB.GetResourceID(d.ResourceKey) + if id == "" && action == deployplan.Skip { + // A skipped node that was never deployed has no remote state to read. Only a bind + // plan skips such a node (see calculateBindPlan); its dependents resolve no + // references to it. + return true + } if id == "" { logdiag.LogError(ctx, fmt.Errorf("%s: internal error: missing entry in state after deploy", errorPrefix)) return false diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 1011a0791d5..2f59fc9eb72 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -428,6 +428,14 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks entry.StateOnly = true } } + if resourceKey == b.bindResourceKey { + // The resource being bound: applying records the config as its state at the bound ID, + // which the state the plan is applied to does not have yet. The changes a deploy would + // make are planned afterwards, against the state the bind leaves. + action = deployplan.Update + entry.StateOnly = true + entry.ID = dbentry.ID + } // Note, this unconditionally stores remoteState. However, it may updated post-deploy, so whether // it can be used for variable resolution depends on several factors, see canReadRemoteCache in LookupReferencePreDeploy diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 84990d0883e..afbc9ea3f11 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -51,6 +51,10 @@ type DeploymentBundle struct { RemoteStateCache sync.Map StateCache structvar.Cache + // bindResourceKey is the resource a bind plan records (see calculateBindPlan): planning makes + // its entry a state-only update at the ID in the state, whatever the diff. + bindResourceKey string + // refTypes maps a resource key to the type of the resource each of its pure-reference // fields references, recorded before resolveReferences drops them (see addParentMoves). refTypes sync.Map diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index 8f19f67a7fa..6bc365310a1 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -107,8 +107,8 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *BindOptions, stateDesc *s } } - // Finalize: rename temp state to final location - err = result.Finalize() + // Finalize: record the bound resource in the state + err = b.DeploymentBundle.FinalizeBind(ctx, b.WorkspaceClient(ctx), result) if err != nil { logdiag.LogError(ctx, err) return