From 008443c2c64fc8653702013ef6df33c633f29073 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 16:44:15 +0000 Subject: [PATCH 1/4] Fail instead of panicking when the terraform state disappears before migration When the pulled state was terraform but the local terraform state file was gone by the time the auto-migration ran (e.g. a concurrent deploy migrated it), Migrate returned "nothing to migrate" and the deploy continued on the terraform engine with no direct state open, then panicked in CalculatePlan. Migrate only runs for a terraform state, so treat the missing file as an error, and drop its bool return, which is now always true on success. Co-authored-by: Isaac --- .../bundle/migrate/user-agent/race/output.txt | 13 +++++---- .../bundle/migrate/user-agent/race/script | 3 +-- bundle/statemgmt/direct_migration.go | 27 ++++++++++--------- cmd/bundle/utils/process.go | 9 +++---- 4 files changed, 24 insertions(+), 28 deletions(-) diff --git a/acceptance/bundle/migrate/user-agent/race/output.txt b/acceptance/bundle/migrate/user-agent/race/output.txt index 648a28750fb..fd5443095b6 100644 --- a/acceptance/bundle/migrate/user-agent/race/output.txt +++ b/acceptance/bundle/migrate/user-agent/race/output.txt @@ -5,12 +5,11 @@ === Deploy where the local terraform state disappears between the pull and the migration +>>> musterr [CLI] bundle deploy +Executing 'postbuild' script +Error: migrating Terraform state to the direct engine: terraform state [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate was removed after it was pulled; if another deploy is running, wait for it to finish and run this command again + + >>> print_engines 2 MISSING GET /api/2.0/workspace/export -3 engine/terraform GET /api/2.0/workspace/export -4 engine/terraform GET /api/2.0/workspace/get-status -1 engine/terraform POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml -1 engine/terraform POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock -1 engine/terraform POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -2 engine/terraform POST /api/2.0/workspace/delete -1 engine/terraform POST /api/2.0/workspace/mkdirs +1 MISSING POST /telemetry-ext diff --git a/acceptance/bundle/migrate/user-agent/race/script b/acceptance/bundle/migrate/user-agent/race/script index 49caea90aa1..69f37e0cfa8 100644 --- a/acceptance/bundle/migrate/user-agent/race/script +++ b/acceptance/bundle/migrate/user-agent/race/script @@ -6,6 +6,5 @@ replace_ids.py rm -f "$OUT_REQUESTS" title "Deploy where the local terraform state disappears between the pull and the migration" -musterr $CLI bundle deploy &> LOG.deploy -contains.py 'Panic Payload: internal error: DeploymentState must be opened first' < LOG.deploy > /dev/null +trace musterr $CLI bundle deploy trace print_engines diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 11a0fc452c1..ca1373c1141 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -39,8 +39,9 @@ const migrationFailedHint = ` The Terraform deployment engine was removed in Databricks CLI v1.20.0. Auto-migration of your bundle failed. Install Databricks CLI v1.19.x to deploy on Terraform, and please report this to dabs-feedback@databricks.com` // Migrate converts the bundle's terraform state to a direct-engine state and opens -// b.DeploymentBundle.StateDB with it in memory. Returns false when there is no terraform state -// to migrate (the caller then opens the direct state normally). +// b.DeploymentBundle.StateDB with it in memory. It is only called when the pulled state is a +// terraform state, which the pull leaves on disk, so a missing local terraform state means it +// changed after the pull (e.g. a concurrent deploy migrated it) and is an error. // // Migrate is reversible: nothing is written to the local state file or the workspace, so a // command that does not commit - plan, run, or a declined deploy - just drops the in-memory @@ -50,15 +51,16 @@ The Terraform deployment engine was removed in Databricks CLI v1.20.0. Auto-migr // The terraform engine was removed in v1.20.0, so any failure (parsing, conversion, the plan // check) is fatal: there is no engine to fall back to. An empty terraform state is not a failure - // the converter writes an empty base state file, so there is no special case here. -func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) { +func Migrate(ctx context.Context, b *bundle.Bundle) error { _, localTerraformPath := b.StateFilenameTerraform(ctx) tfState, err := migrate.ParseTFStateFull(ctx, localTerraformPath) if err != nil { b.Metrics.SetBoolValue(metrics.DirectMigrateError, true) - return false, fmt.Errorf("parsing terraform state: %w%s", err, migrationFailedHint) + return fmt.Errorf("parsing terraform state: %w%s", err, migrationFailedHint) } if tfState == nil { - return false, nil + b.Metrics.SetBoolValue(metrics.DirectMigrateError, true) + return fmt.Errorf("terraform state %s was removed after it was pulled; if another deploy is running, wait for it to finish and run this command again", localTerraformPath) } _, localDirectPath := b.StateFilenameDirect(ctx) @@ -77,7 +79,7 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) { } if err != nil { b.Metrics.SetBoolValue(metrics.DirectMigrateError, true) - return false, fmt.Errorf("converting terraform state to the direct engine: %w%s", err, migrationFailedHint) + return fmt.Errorf("converting terraform state to the direct engine: %w%s", err, migrationFailedHint) } // Plan-check the converted state: validate it can be planned before committing to it, and @@ -85,7 +87,7 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) { plan, err := checkPlanOnTempState(ctx, b, tempStatePath, cfg) if err != nil { b.Metrics.SetBoolValue(metrics.DirectMigratePlanError, true) - return false, fmt.Errorf("the migrated state failed its plan check: %w%s", err, migrationFailedHint) + return fmt.Errorf("the migrated state failed its plan check: %w%s", err, migrationFailedHint) } // Record when the migrated state's first plan would recreate a resource, but still migrate: @@ -102,22 +104,21 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) { // that once the command is approved. raw, err := os.ReadFile(tempStatePath) if err != nil { - return false, fmt.Errorf("reading migrated state: %w", err) + return fmt.Errorf("reading migrated state: %w", err) } var data dstate.Database if err := json.Unmarshal(raw, &data); err != nil { - return false, fmt.Errorf("parsing migrated state: %w", err) + return fmt.Errorf("parsing migrated state: %w", err) } // The terraform state has schema_id but no volume_id (see BuildStateFromTF); the plan check read it. if err := direct.RecordMissingInternalIDs(plan, &data); err != nil { - return false, fmt.Errorf("recording internal ids in migrated state: %w", err) + return fmt.Errorf("recording internal ids in migrated state: %w", err) } b.DeploymentBundle.StateDB.OpenWithData(localDirectPath, data) - // Announce the migration only once the conversion and plan check succeeded, so a run with no - // terraform state to migrate says nothing misleading. + // Announce the migration only once the conversion and plan check succeeded. cmdio.LogString(ctx, "Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).") - return true, nil + return nil } // CleanupTerraformStateAfterMigration backs up the now-superseded terraform state, remote and diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index a39f8096ff3..d89efd0b042 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -589,14 +589,11 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // after approval, destroy commits as part of its teardown. The caller tags the user agent with the // resolved stateDesc.Engine afterwards. func migrateTerraformToDirect(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error { - migrated, err := statemgmt.Migrate(ctx, b) - if err != nil { + if err := statemgmt.Migrate(ctx, b); err != nil { return fmt.Errorf("migrating Terraform state to the direct engine: %w", err) } - if migrated { - stateDesc.Engine = engine.EngineDirect - b.Metrics.StateEngine = engine.EngineDirect - } + stateDesc.Engine = engine.EngineDirect + b.Metrics.StateEngine = engine.EngineDirect return nil } From e3db7648c157b3f32839b73e168e838a98f8422a Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Wed, 18 Nov 2026 16:45:24 +0000 Subject: [PATCH 2/4] Add changelog entry Co-authored-by: Isaac --- .nextchanges/bundles/7778.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 .nextchanges/bundles/7778.md diff --git a/.nextchanges/bundles/7778.md b/.nextchanges/bundles/7778.md new file mode 100644 index 00000000000..6d579f1d687 --- /dev/null +++ b/.nextchanges/bundles/7778.md @@ -0,0 +1 @@ +* Fix a panic in `bundle deploy` when the local Terraform state is removed between pulling the state and migrating it to the direct engine, e.g. by a concurrent deploy. ([#7778](https://github.com/databricks/cli/pull/7778)) From e2ab2d546d116ca46f3e30b4294815193ea295d9 Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 19 Nov 2026 09:30:08 +0000 Subject: [PATCH 3/4] Record a missing terraform state as its own migration metric; assert it direct_migrate_error is documented as "state conversion itself errored"; a state removed after the pull is a race, not a conversion failure, so report it as direct_migrate_state_missing. The race test now requires the failure with musterr and asserts the metric. Co-authored-by: Isaac --- acceptance/bundle/migrate/user-agent/race/output.txt | 3 +++ acceptance/bundle/migrate/user-agent/race/script | 1 + bundle/metrics/metrics.go | 10 +++++++--- bundle/statemgmt/direct_migration.go | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/migrate/user-agent/race/output.txt b/acceptance/bundle/migrate/user-agent/race/output.txt index fd5443095b6..2cf4adc2714 100644 --- a/acceptance/bundle/migrate/user-agent/race/output.txt +++ b/acceptance/bundle/migrate/user-agent/race/output.txt @@ -10,6 +10,9 @@ Executing 'postbuild' script Error: migrating Terraform state to the direct engine: terraform state [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate was removed after it was pulled; if another deploy is running, wait for it to finish and run this command again +>>> print_migration_telemetry +direct_migrate_state_missing true + >>> print_engines 2 MISSING GET /api/2.0/workspace/export 1 MISSING POST /telemetry-ext diff --git a/acceptance/bundle/migrate/user-agent/race/script b/acceptance/bundle/migrate/user-agent/race/script index 69f37e0cfa8..676c12b97ea 100644 --- a/acceptance/bundle/migrate/user-agent/race/script +++ b/acceptance/bundle/migrate/user-agent/race/script @@ -7,4 +7,5 @@ rm -f "$OUT_REQUESTS" title "Deploy where the local terraform state disappears between the pull and the migration" trace musterr $CLI bundle deploy +trace print_migration_telemetry trace print_engines diff --git a/bundle/metrics/metrics.go b/bundle/metrics/metrics.go index f5fcb82fb1e..a00f16b26bf 100644 --- a/bundle/metrics/metrics.go +++ b/bundle/metrics/metrics.go @@ -30,9 +30,13 @@ const ( // - migrate_commit_error: the state was converted, but committing it // (renaming files / pushing to workspace) failed. // - migrate_warnings: the conversion emitted warnings (e.g. resources the direct engine can't represent). - DirectMigrateError = "direct_migrate_error" - DirectMigrateCommitError = "direct_migrate_commit_error" - DirectMigrateWarnings = "direct_migrate_warnings" + // - migrate_state_missing: the terraform state was pulled, but its local file was gone + // when the migration read it (e.g. a concurrent deploy migrated + // it). Nothing was converted, and a retry is expected to succeed. + DirectMigrateError = "direct_migrate_error" + DirectMigrateCommitError = "direct_migrate_commit_error" + DirectMigrateWarnings = "direct_migrate_warnings" + DirectMigrateStateMissing = "direct_migrate_state_missing" // True when the post-convert plan check failed; the migration was not committed. DirectMigratePlanError = "direct_migrate_plan_error" // True when the migrated state's first plan would recreate (destroy + create) an diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index ca1373c1141..8a7b27de818 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -59,7 +59,7 @@ func Migrate(ctx context.Context, b *bundle.Bundle) error { return fmt.Errorf("parsing terraform state: %w%s", err, migrationFailedHint) } if tfState == nil { - b.Metrics.SetBoolValue(metrics.DirectMigrateError, true) + b.Metrics.SetBoolValue(metrics.DirectMigrateStateMissing, true) return fmt.Errorf("terraform state %s was removed after it was pulled; if another deploy is running, wait for it to finish and run this command again", localTerraformPath) } From bd0f4e36018085a0a58fb878408f76120c07a08e Mon Sep 17 00:00:00 2001 From: "denis.bilenko" Date: Thu, 19 Nov 2026 09:44:34 +0000 Subject: [PATCH 4/4] Widen the changelog entry and qualify the retry note The fix covers every command that migrates the state, not only deploy, and a retry only succeeds when a concurrent deploy did the migration. Co-authored-by: Isaac --- .nextchanges/bundles/7778.md | 2 +- bundle/metrics/metrics.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.nextchanges/bundles/7778.md b/.nextchanges/bundles/7778.md index 6d579f1d687..763b5bbc30d 100644 --- a/.nextchanges/bundles/7778.md +++ b/.nextchanges/bundles/7778.md @@ -1 +1 @@ -* Fix a panic in `bundle deploy` when the local Terraform state is removed between pulling the state and migrating it to the direct engine, e.g. by a concurrent deploy. ([#7778](https://github.com/databricks/cli/pull/7778)) +* Fix a panic when the local Terraform state is removed after it is pulled and before it is migrated to the direct engine, for example by a concurrent `bundle deploy`. ([#7778](https://github.com/databricks/cli/pull/7778)) diff --git a/bundle/metrics/metrics.go b/bundle/metrics/metrics.go index a00f16b26bf..b77e795a232 100644 --- a/bundle/metrics/metrics.go +++ b/bundle/metrics/metrics.go @@ -31,8 +31,8 @@ const ( // (renaming files / pushing to workspace) failed. // - migrate_warnings: the conversion emitted warnings (e.g. resources the direct engine can't represent). // - migrate_state_missing: the terraform state was pulled, but its local file was gone - // when the migration read it (e.g. a concurrent deploy migrated - // it). Nothing was converted, and a retry is expected to succeed. + // when the migration read it. Nothing was converted. When a + // concurrent deploy migrated it, a retry picks up the direct state. DirectMigrateError = "direct_migrate_error" DirectMigrateCommitError = "direct_migrate_commit_error" DirectMigrateWarnings = "direct_migrate_warnings"