From 4307a20ac9332061b3ff9a76fb8eb7ff859cd0e8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 18 Nov 2026 12:03:17 +0000 Subject: [PATCH] Prepare resource bindings in memory before approval --- .../recreation/out.state_after_bind.json | 2 +- .../bind/job/job-abort-bind/output.txt | 2 + .../deployment/bind/job/job-abort-bind/script | 3 + .../deployment/bind/job/noop-job/output.txt | 8 + .../deployment/bind/job/noop-job/script | 2 + .../dms_fault/complete_version/output.txt | 4 +- .../dms_fault/delete_deployment/output.txt | 4 +- .../bundle/dms_fault/get_operation/output.txt | 4 +- .../bundle/dms_fault/get_resource/output.txt | 4 +- .../bundle/dms_fault/get_version/output.txt | 4 +- .../bundle/dms_fault/heartbeat/output.txt | 4 +- .../dms_fault/list_deployments/output.txt | 4 +- .../dms_fault/list_operations/output.txt | 4 +- .../dms_fault/list_resources/output.txt | 4 +- .../bundle/dms_fault/list_versions/output.txt | 4 +- .../dms_fault/update_operation/output.txt | 4 +- bundle/direct/bind.go | 172 ++++-------------- bundle/direct/bind_test.go | 102 +++++++++++ bundle/direct/dstate/database_test.go | 40 ++++ bundle/direct/dstate/state.go | 21 +++ bundle/phases/bind.go | 22 ++- 21 files changed, 255 insertions(+), 163 deletions(-) create mode 100644 bundle/direct/bind_test.go create mode 100644 bundle/direct/dstate/database_test.go diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.json b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.json index 05614d798bf..74e555c734d 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.json +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.json @@ -2,7 +2,7 @@ "state_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 2, + "serial": 1, "state": { "resources.dashboards.dashboard1": { "__id__": "[DASHBOARD_ID]", diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/output.txt b/acceptance/bundle/deployment/bind/job/job-abort-bind/output.txt index 8e212f477d7..02f2cedc4dd 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/output.txt +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/output.txt @@ -9,6 +9,8 @@ To proceed, use --auto-approve after reviewing the plan above. Exit code: 1 +>>> assert_not_exists.py .databricks/bundle/default/resources.json .databricks/bundle/default/resources.json.wal + === Deploy bundle: >>> [CLI] bundle deploy --force-lock diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/script b/acceptance/bundle/deployment/bind/job/job-abort-bind/script index 63aedd7f285..ab971057881 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/script +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/script @@ -38,6 +38,9 @@ title "Expect binding to fail without an auto-approve flag:" trace errcode $CLI bundle deployment bind foo $JOB_ID &> out.bind-result.txt sed -n '/^Error:/,$p' out.bind-result.txt rm out.bind-result.txt +# This bundle starts without state. A rejected bind must leave no committed state +# or pending WAL that a later command could recover. +trace assert_not_exists.py .databricks/bundle/default/resources.json .databricks/bundle/default/resources.json.wal title "Deploy bundle:" trace $CLI bundle deploy --force-lock diff --git a/acceptance/bundle/deployment/bind/job/noop-job/output.txt b/acceptance/bundle/deployment/bind/job/noop-job/output.txt index 956f90977b4..c9a42b9adf8 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/output.txt +++ b/acceptance/bundle/deployment/bind/job/noop-job/output.txt @@ -5,6 +5,14 @@ Successfully bound job with an id '[JOB_ID]' Run 'bundle deploy' to deploy changes to your workspace +>>> jq {serial, resource_id: .state["resources.jobs.job_1"].__id__} .databricks/bundle/default/resources.json +{ + "serial": 1, + "resource_id": "[JOB_ID]" +} + +>>> assert_not_exists.py .databricks/bundle/default/resources.json.wal .databricks/bundle/default/resources.json.temp-bind + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_project/default/files... Updated jobs.job_1 diff --git a/acceptance/bundle/deployment/bind/job/noop-job/script b/acceptance/bundle/deployment/bind/job/noop-job/script index adafb1c4da4..7a2f30eba35 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/script +++ b/acceptance/bundle/deployment/bind/job/noop-job/script @@ -2,6 +2,8 @@ job_id=$(trace $CLI jobs create --json '{"name": "My Job"}' | jq -r '.job_id') add_repl "$job_id" JOB_ID trace $CLI bundle deployment bind job_1 $job_id --auto-approve +trace jq '{serial, resource_id: .state["resources.jobs.job_1"].__id__}' .databricks/bundle/default/resources.json +trace assert_not_exists.py .databricks/bundle/default/resources.json.wal .databricks/bundle/default/resources.json.temp-bind trace $CLI bundle deploy diff --git a/acceptance/bundle/dms_fault/complete_version/output.txt b/acceptance/bundle/dms_fault/complete_version/output.txt index c7d574f4431..bbf04e469ad 100644 --- a/acceptance/bundle/dms_fault/complete_version/output.txt +++ b/acceptance/bundle/dms_fault/complete_version/output.txt @@ -83,7 +83,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-complete_version-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -102,7 +102,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-complete_version-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/delete_deployment/output.txt b/acceptance/bundle/dms_fault/delete_deployment/output.txt index 8efdbc07a0d..34db7ea5d7d 100644 --- a/acceptance/bundle/dms_fault/delete_deployment/output.txt +++ b/acceptance/bundle/dms_fault/delete_deployment/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-delete_deployment-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-delete_deployment-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/get_operation/output.txt b/acceptance/bundle/dms_fault/get_operation/output.txt index 65498809a65..77a30f34331 100644 --- a/acceptance/bundle/dms_fault/get_operation/output.txt +++ b/acceptance/bundle/dms_fault/get_operation/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_operation-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_operation-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/get_resource/output.txt b/acceptance/bundle/dms_fault/get_resource/output.txt index a562c0f5264..1f73acc8286 100644 --- a/acceptance/bundle/dms_fault/get_resource/output.txt +++ b/acceptance/bundle/dms_fault/get_resource/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_resource-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_resource-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/get_version/output.txt b/acceptance/bundle/dms_fault/get_version/output.txt index b4473435fe9..e2207b8defb 100644 --- a/acceptance/bundle/dms_fault/get_version/output.txt +++ b/acceptance/bundle/dms_fault/get_version/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_version-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-get_version-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/heartbeat/output.txt b/acceptance/bundle/dms_fault/heartbeat/output.txt index 8b50d26db05..bbce26752db 100644 --- a/acceptance/bundle/dms_fault/heartbeat/output.txt +++ b/acceptance/bundle/dms_fault/heartbeat/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-heartbeat-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-heartbeat-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/list_deployments/output.txt b/acceptance/bundle/dms_fault/list_deployments/output.txt index 0710bf48939..907f90146fc 100644 --- a/acceptance/bundle/dms_fault/list_deployments/output.txt +++ b/acceptance/bundle/dms_fault/list_deployments/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_deployments-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_deployments-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/list_operations/output.txt b/acceptance/bundle/dms_fault/list_operations/output.txt index 5764b81d690..37eb784a030 100644 --- a/acceptance/bundle/dms_fault/list_operations/output.txt +++ b/acceptance/bundle/dms_fault/list_operations/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_operations-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_operations-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/list_resources/output.txt b/acceptance/bundle/dms_fault/list_resources/output.txt index 9f45951b03a..ce7a141c620 100644 --- a/acceptance/bundle/dms_fault/list_resources/output.txt +++ b/acceptance/bundle/dms_fault/list_resources/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_resources-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_resources-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/list_versions/output.txt b/acceptance/bundle/dms_fault/list_versions/output.txt index dabee52cbcd..bb5d8e97403 100644 --- a/acceptance/bundle/dms_fault/list_versions/output.txt +++ b/acceptance/bundle/dms_fault/list_versions/output.txt @@ -82,7 +82,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_versions-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Files: 6 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -100,7 +100,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-list_versions-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Files: 2 uploaded, 0 deleted diff --git a/acceptance/bundle/dms_fault/update_operation/output.txt b/acceptance/bundle/dms_fault/update_operation/output.txt index 308345c48c4..caf22b6c307 100644 --- a/acceptance/bundle/dms_fault/update_operation/output.txt +++ b/acceptance/bundle/dms_fault/update_operation/output.txt @@ -83,7 +83,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-update_operation-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=7 Updated pipelines.bar Warn: Failed to record deployment history: recording operation for [RESOURCE_KEY]: Fault injected by test. Files: 6 uploaded, 0 deleted @@ -102,7 +102,7 @@ >>> bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-fault-update_operation-commands-[UNIQUE_NAME]/files... - Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=11 + Current Deployment Version: [DATABRICKS_URL]/deployments/[DEPLOYMENT_ID]?version=9 Updated jobs.foo Updated pipelines.bar Warn: Failed to record deployment history: recording operation for [RESOURCE_KEY]: Fault injected by test. diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 73852cdde00..bd0e2cdbdc0 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -2,8 +2,9 @@ package direct import ( "context" + "encoding/json" "fmt" - "os" + "slices" "strings" "github.com/databricks/cli/bundle/config" @@ -35,46 +36,29 @@ func (e ErrResourceAlreadyBound) Error() string { e.ResourceKey, e.ExistingID) } -// BindResult contains the result of a bind operation including any detected changes. -type BindResult struct { - // HasChanges is true if deploying after bind would make changes to the resource +// PreparedBind holds the state to save after approval and the preview of the next deploy. +type PreparedBind struct { + Resource dstate.ResourceEntry + // HasChanges is true if deploying after bind would make changes to the resource. HasChanges bool - // Action is the planned action for the bound resource (e.g., "skip", "update", "recreate") + // Action is the planned action for the bound resource (e.g., "skip", "update", "recreate"). Action deployplan.ActionType - // Plan contains the full deployment plan for the bound resource + // Plan contains the full deployment plan for the bound resource. Plan *deployplan.Plan - // TempStatePath is the path to the temporary state file - TempStatePath string - // StatePath is the path to the final state file - StatePath string } -// Bind adds an existing workspace resource to a temporary state and calculates -// if there will be any changes when deploying. -// -// Flow: -// 1. Load current state file -// 2. Update id of the target resource -// 3. Update state path to temp file (state file + ".temp-bind") -// 4. Perform plan + update to populate state with resolved config -// 5. Calculate plan again - this is the plan presented to the user -// -// Call Finalize to commit the state or Cancel to discard. -func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { +// Bind prepares a binding in memory and previews the next deploy. The actual state stays +// open read-only; the caller must finalize it after committing the prepared resource or aborting. +func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*PreparedBind, error) { // Check if the resource is already managed (bound to a different ID) - var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}); err != nil { + if err := b.StateDB.Open(ctx, statePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}); err != nil { // State that cannot be read is not the same as state without a binding: // the resource may well be managed already, and binding on top of it // would take over a resource whose ownership was never checked. A state // file that does not exist yet opens successfully as an empty one. return nil, fmt.Errorf("cannot check whether %s is already bound: %w", resourceKey, err) } - existingID := checkStateDB.GetResourceID(resourceKey) - if _, err := checkStateDB.Finalize(ctx); err != nil { - log.Warnf(ctx, "failed to finalize state: %v", err) - } - if existingID != "" { + if existingID := b.StateDB.GetResourceID(resourceKey); existingID != "" { return nil, ErrResourceAlreadyBound{ ResourceKey: resourceKey, ExistingID: existingID, @@ -82,64 +66,29 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - tmpStatePath := statePath + ".temp-bind" - - // Copy existing state to temp location if it exists - if data, err := os.ReadFile(statePath); err == nil { - if err := os.WriteFile(tmpStatePath, data, 0o600); err != nil { - return nil, err + // Keep the planner's state, caches, and locks separate from the approved state. + data := b.StateDB.Data.Clone() + data.State[resourceKey] = dstate.ResourceEntry{ID: resourceID, State: json.RawMessage(`{}`)} + // Adapters hold no per-resource state and can be shared. + preview := &DeploymentBundle{Adapters: b.Adapters} + preview.StateDB.OpenWithData(statePath, data) + defer func() { + if _, err := preview.StateDB.Finalize(ctx); err != nil { + log.Warnf(ctx, "failed to finalize binding preview: %v", err) } - } - - // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.ReadingFile, dstate.WithRecovery(false), dstate.OpenDmsArgs{}) - if err != nil { - os.Remove(tmpStatePath) - return nil, err - } - if err := b.StateDB.UpgradeToWrite(); err != nil { - os.Remove(tmpStatePath) - return nil, err - } - - // Save state with ID and empty state (like migrate does) - err = b.StateDB.SaveState(ctx, resourceKey, resourceID, struct{}{}, nil) - if err != nil { - os.Remove(tmpStatePath) - return nil, err - } - - // Finalize to persist temp state to disk - _, err = b.StateDB.Finalize(ctx) - if err != nil { - os.Remove(tmpStatePath) - return nil, err - } - - log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) + }() // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}) - if err != nil { - os.Remove(tmpStatePath) - return nil, err - } - plan, err := b.CalculatePlan(ctx, client, configRoot) + plan, err := preview.CalculatePlan(ctx, client, configRoot) if err != nil { - os.Remove(tmpStatePath) return nil, err } - if _, err := b.StateDB.Finalize(ctx); err != nil { - log.Warnf(ctx, "failed to finalize state: %v", err) - } - - // Populate the state with the resolved config entry := plan.Plan[resourceKey] - sv, ok := b.StateCache.Load(resourceKey) + sv, ok := preview.StateCache.Load(resourceKey) if ok && sv != nil { var dependsOn []deployplan.DependsOnEntry if entry != nil { - dependsOn = entry.DependsOn + dependsOn = slices.Clone(entry.DependsOn) } // Copy etag from remote state for resources that use etag-based drift @@ -157,64 +106,37 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac // Compact hashed_fields fields so the persisted state stays small. Not needed for // correctness — the next plan (CalculatePlan) compacts the saved state on read. - adapter, err := b.getAdapterForKey(resourceKey) + adapter, err := preview.getAdapterForKey(resourceKey) if err != nil { - os.Remove(tmpStatePath) return nil, err } compacted, err := dresources.CompactState(adapter.ResourceConfig(), sv.Value) if err != nil { - os.Remove(tmpStatePath) return nil, fmt.Errorf("compacting state: %w", err) } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}) - if err != nil { - os.Remove(tmpStatePath) - return nil, err - } - if err := b.StateDB.UpgradeToWrite(); err != nil { - os.Remove(tmpStatePath) - return nil, err - } - - err = b.StateDB.SaveState(ctx, resourceKey, resourceID, compacted, dependsOn) + state, err := json.Marshal(compacted) if err != nil { - os.Remove(tmpStatePath) - return nil, err - } - - _, err = b.StateDB.Finalize(ctx) - if err != nil { - os.Remove(tmpStatePath) return nil, err } + data.State[resourceKey] = dstate.ResourceEntry{ID: resourceID, State: state, DependsOn: dependsOn} } - // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}) - if err != nil { - os.Remove(tmpStatePath) + // Capture unredacted state before the second plan replaces cached values. + resource := data.State[resourceKey] + resource.State = slices.Clone(resource.State) + resource.DependsOn = slices.Clone(resource.DependsOn) + if _, err := preview.StateDB.Finalize(ctx); err != nil { return nil, err } - plan, err = b.CalculatePlan(ctx, client, configRoot) - if _, ferr := b.StateDB.Finalize(ctx); ferr != nil { - log.Warnf(ctx, "failed to finalize state: %v", ferr) - } + preview.StateDB.OpenWithData(statePath, data) + + // Second plan: this is the plan to present to the user (change between remote resource and config) + plan, err = preview.CalculatePlan(ctx, client, configRoot) if err != nil { - os.Remove(tmpStatePath) return nil, err } - - // Check if the bound resource has changes - result := &BindResult{ - HasChanges: false, - Action: deployplan.Skip, - Plan: plan, - TempStatePath: tmpStatePath, - StatePath: statePath, - } - + result := &PreparedBind{Resource: resource, Action: deployplan.Skip, Plan: plan} entry = plan.Plan[resourceKey] if entry != nil { result.Action = entry.Action @@ -222,25 +144,9 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac // does not write), so it changes nothing remotely. result.HasChanges = result.Action != deployplan.Skip && result.Action != deployplan.Undefined && !entry.IsStateOnly() } - return result, nil } -// Finalize completes the bind operation by renaming the temp state to the final location. -func (result *BindResult) Finalize() error { - if result == nil || result.TempStatePath == "" { - return nil - } - return os.Rename(result.TempStatePath, result.StatePath) -} - -// Cancel cleans up temp state without committing changes. -func (result *BindResult) Cancel() { - if result != nil && result.TempStatePath != "" { - os.Remove(result.TempStatePath) - } -} - // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { diff --git a/bundle/direct/bind_test.go b/bundle/direct/bind_test.go new file mode 100644 index 00000000000..08a32db807e --- /dev/null +++ b/bundle/direct/bind_test.go @@ -0,0 +1,102 @@ +package direct_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structvar" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestBindPreparesIsolatedState checks that bind resolves a preview without committing it. +func TestBindPreparesIsolatedState(t *testing.T) { + ctx := logdiag.InitContext(t.Context()) + statePath := filepath.Join(t.TempDir(), "resources.json") + // Start with an already-managed job, "kept" (ID 10). Its state must stay unchanged. + data := dstate.NewDatabase("lineage", 4) + data.Features = map[string]struct{}{dstate.FeatureDeploymentHistoryShadow: {}} + data.State["resources.jobs.kept"] = dstate.ResourceEntry{ + ID: "10", State: json.RawMessage(`{"name":"kept"}`), + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.jobs.old"}}, + } + original, err := json.Marshal(data) + require.NoError(t, err) + require.NoError(t, os.WriteFile(statePath, original, 0o600)) + + server := testserver.New(t) + server.Handle("GET", "/api/2.2/jobs/get", func(req testserver.Request) any { + // The planner reads these jobs while preparing the preview. Check the files here + // so a temporary write cannot go unnoticed just because bind deletes it later. + assert.NoFileExists(t, statePath+".wal") + assert.NoFileExists(t, statePath+".temp-bind") + actual, err := os.ReadFile(statePath) + require.NoError(t, err) + assert.Equal(t, original, actual) + if req.URL.Query().Get("job_id") == "10" { + return jobs.Job{JobId: 10, Settings: &jobs.JobSettings{Name: "kept"}} + } + return jobs.Job{JobId: 20, Settings: &jobs.JobSettings{Name: "remote"}} + }) + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(t, err) + // Bind "bound" to job 20. Its tag references kept's ID, so the preview must resolve + // that reference while leaving the real state and caches alone. + root, diags := config.LoadFromBytes("databricks.yml", []byte(`resources: + jobs: + kept: + name: kept + bound: + name: configured + tags: + other: ${resources.jobs.kept.id} +`)) + require.Empty(t, diags) + var b direct.DeploymentBundle + // Seed the real caches so using them for the preview would be detectable. + cached := structvar.NewStructVar(&jobs.JobSettings{Name: "original cache"}, nil) + b.StateCache.Store("resources.jobs.bound", cached) + b.RemoteStateCache.Store("resources.jobs.bound", "original remote cache") + + prepared, err := b.Bind(ctx, client, root, statePath, "resources.jobs.bound", "20") + require.NoError(t, err) + t.Cleanup(func() { + _, err := b.StateDB.Finalize(ctx) + require.NoError(t, err) + }) + require.False(t, logdiag.HasError(ctx)) + + // Before approval, real state still contains only "kept". The proposed "bound" + // entry belongs only to the preview, and no serial or existing state has changed. + assert.Equal(t, data, b.StateDB.Data) + assert.Empty(t, b.StateDB.GetResourceID("resources.jobs.bound")) + assert.Equal(t, "10", b.StateDB.GetResourceID("resources.jobs.kept")) + + // The preview must also leave the real plan and seeded cache entries untouched. + assert.Nil(t, b.Plan) + actualCache, ok := b.StateCache.Load("resources.jobs.bound") + require.True(t, ok) + assert.Same(t, cached, actualCache) + remoteCache, ok := b.RemoteStateCache.Load("resources.jobs.bound") + require.True(t, ok) + assert.Equal(t, "original remote cache", remoteCache) + + // The prepared binding keeps job 20's ID, resolves kept's ID to 10, and records + // the dependency for the state that will be saved after approval. + assert.Equal(t, "20", prepared.Resource.ID) + assert.JSONEq(t, `{"name":"configured","tags":{"other":"10"}}`, string(prepared.Resource.State)) + assert.Contains(t, prepared.Resource.DependsOn, deployplan.DependsOnEntry{Node: "resources.jobs.kept", Label: "${resources.jobs.kept.id}"}) + // A later deploy would change job 20's name from "remote" to "configured". + assert.True(t, prepared.HasChanges) + assert.Equal(t, deployplan.Update, prepared.Action) +} diff --git a/bundle/direct/dstate/database_test.go b/bundle/direct/dstate/database_test.go new file mode 100644 index 00000000000..b66f1a07beb --- /dev/null +++ b/bundle/direct/dstate/database_test.go @@ -0,0 +1,40 @@ +package dstate_test + +import ( + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDatabaseClone(t *testing.T) { + original := dstate.NewDatabase("lineage", 4) + original.Features = map[string]struct{}{dstate.FeatureDeploymentHistoryShadow: {}} + original.State["resources.jobs.foo"] = dstate.ResourceEntry{ + ID: "123", + State: json.RawMessage(`{"name":"foo"}`), + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.jobs.bar", Label: "dependency"}}, + } + before, err := json.Marshal(original) + require.NoError(t, err) + + clone := original.Clone() + require.Equal(t, original, clone) + clone.Serial++ + delete(clone.Features, dstate.FeatureDeploymentHistoryShadow) + clone.State["resources.jobs.foo"].State[0] = '[' + clone.State["resources.jobs.foo"].DependsOn[0].Node = "changed" + delete(clone.State, "resources.jobs.foo") + clone.State["resources.jobs.new"] = dstate.ResourceEntry{ID: "456"} + + after, err := json.Marshal(original) + require.NoError(t, err) + assert.Equal(t, before, after) + + empty := (dstate.Database{}).Clone() + require.NotNil(t, empty.State) + assert.Empty(t, empty.State) +} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index de00e76233a..de3550300c6 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -165,6 +165,27 @@ type Database struct { State map[string]ResourceEntry `json:"state"` } +// Clone returns an independent copy with an initialized resource map. +// New fields must be added here, with mutable data cloned rather than shared. +func (d Database) Clone() Database { + clone := Database{ + StateVersion: d.StateVersion, + CLIVersion: d.CLIVersion, + Lineage: d.Lineage, + Serial: d.Serial, + Features: maps.Clone(d.Features), + State: make(map[string]ResourceEntry, len(d.State)), + } + for key, entry := range d.State { + clone.State[key] = ResourceEntry{ + ID: entry.ID, + State: slices.Clone(entry.State), + DependsOn: slices.Clone(entry.DependsOn), + } + } + return clone +} + type ResourceEntry struct { ID string `json:"__id__"` State json.RawMessage `json:"state"` diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index 8f19f67a7fa..1c0beef2750 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -37,7 +37,11 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *BindOptions, stateDesc *s return } + db := &b.DeploymentBundle.StateDB defer func() { + if _, err := db.Finalize(ctx); err != nil { + logdiag.LogError(ctx, err) + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalBind)) }() @@ -59,7 +63,7 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *BindOptions, stateDesc *s } } - // Direct engine: import into temp state, run plan, check for changes. + // Prepare the binding without writing state until the plan is approved. resourceKey := fmt.Sprintf("resources.%s.%s", opts.Group, opts.ResourceKey) _, statePath := b.StateFilenameDirect(ctx) @@ -89,27 +93,31 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *BindOptions, stateDesc *s } if !cmdio.IsPromptSupported(ctx) { - result.Cancel() logdiag.LogError(ctx, fmt.Errorf("this bind operation requires user confirmation, but the current console does not support prompting.\nTo proceed, use --auto-approve after reviewing the plan above.%s", agent.AgentNotice())) return } ans, err := cmdio.AskYesOrNo(ctx, "Confirm import changes? Changes will be remotely applied only after running 'bundle deploy'.") if err != nil { - result.Cancel() logdiag.LogError(ctx, err) return } if !ans { - result.Cancel() logdiag.LogError(ctx, errors.New("import aborted")) return } } - // Finalize: rename temp state to final location - err = result.Finalize() - if err != nil { + if err := db.UpgradeToWrite(); err != nil { + logdiag.LogError(ctx, err) + return + } + resource := result.Resource + if err := db.SaveState(ctx, resourceKey, resource.ID, resource.State, resource.DependsOn); err != nil { + logdiag.LogError(ctx, err) + return + } + if _, err := db.Finalize(ctx); err != nil { logdiag.LogError(ctx, err) return }