diff --git a/.nextchanges/bundles/7778.md b/.nextchanges/bundles/7778.md new file mode 100644 index 00000000000..763b5bbc30d --- /dev/null +++ b/.nextchanges/bundles/7778.md @@ -0,0 +1 @@ +* Fix a panic when the local Terraform state is removed after it is pulled and before it is migrated to the direct engine, for example by a concurrent `bundle deploy`. ([#7778](https://github.com/databricks/cli/pull/7778)) diff --git a/acceptance/bundle/migrate/user-agent/race/output.txt b/acceptance/bundle/migrate/user-agent/race/output.txt index 648a28750fb..2cf4adc2714 100644 --- a/acceptance/bundle/migrate/user-agent/race/output.txt +++ b/acceptance/bundle/migrate/user-agent/race/output.txt @@ -5,12 +5,14 @@ === Deploy where the local terraform state disappears between the pull and the migration +>>> musterr [CLI] bundle deploy +Executing 'postbuild' script +Error: migrating Terraform state to the direct engine: terraform state [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate was removed after it was pulled; if another deploy is running, wait for it to finish and run this command again + + +>>> print_migration_telemetry +direct_migrate_state_missing true + >>> print_engines 2 MISSING GET /api/2.0/workspace/export -3 engine/terraform GET /api/2.0/workspace/export -4 engine/terraform GET /api/2.0/workspace/get-status -1 engine/terraform POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml -1 engine/terraform POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock -1 engine/terraform POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json -2 engine/terraform POST /api/2.0/workspace/delete -1 engine/terraform POST /api/2.0/workspace/mkdirs +1 MISSING POST /telemetry-ext diff --git a/acceptance/bundle/migrate/user-agent/race/script b/acceptance/bundle/migrate/user-agent/race/script index 49caea90aa1..676c12b97ea 100644 --- a/acceptance/bundle/migrate/user-agent/race/script +++ b/acceptance/bundle/migrate/user-agent/race/script @@ -6,6 +6,6 @@ replace_ids.py rm -f "$OUT_REQUESTS" title "Deploy where the local terraform state disappears between the pull and the migration" -musterr $CLI bundle deploy &> LOG.deploy -contains.py 'Panic Payload: internal error: DeploymentState must be opened first' < LOG.deploy > /dev/null +trace musterr $CLI bundle deploy +trace print_migration_telemetry trace print_engines diff --git a/bundle/metrics/metrics.go b/bundle/metrics/metrics.go index f5fcb82fb1e..b77e795a232 100644 --- a/bundle/metrics/metrics.go +++ b/bundle/metrics/metrics.go @@ -30,9 +30,13 @@ const ( // - migrate_commit_error: the state was converted, but committing it // (renaming files / pushing to workspace) failed. // - migrate_warnings: the conversion emitted warnings (e.g. resources the direct engine can't represent). - DirectMigrateError = "direct_migrate_error" - DirectMigrateCommitError = "direct_migrate_commit_error" - DirectMigrateWarnings = "direct_migrate_warnings" + // - migrate_state_missing: the terraform state was pulled, but its local file was gone + // when the migration read it. Nothing was converted. When a + // concurrent deploy migrated it, a retry picks up the direct state. + DirectMigrateError = "direct_migrate_error" + DirectMigrateCommitError = "direct_migrate_commit_error" + DirectMigrateWarnings = "direct_migrate_warnings" + DirectMigrateStateMissing = "direct_migrate_state_missing" // True when the post-convert plan check failed; the migration was not committed. DirectMigratePlanError = "direct_migrate_plan_error" // True when the migrated state's first plan would recreate (destroy + create) an diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 11a0fc452c1..8a7b27de818 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -39,8 +39,9 @@ const migrationFailedHint = ` The Terraform deployment engine was removed in Databricks CLI v1.20.0. Auto-migration of your bundle failed. Install Databricks CLI v1.19.x to deploy on Terraform, and please report this to dabs-feedback@databricks.com` // Migrate converts the bundle's terraform state to a direct-engine state and opens -// b.DeploymentBundle.StateDB with it in memory. Returns false when there is no terraform state -// to migrate (the caller then opens the direct state normally). +// b.DeploymentBundle.StateDB with it in memory. It is only called when the pulled state is a +// terraform state, which the pull leaves on disk, so a missing local terraform state means it +// changed after the pull (e.g. a concurrent deploy migrated it) and is an error. // // Migrate is reversible: nothing is written to the local state file or the workspace, so a // command that does not commit - plan, run, or a declined deploy - just drops the in-memory @@ -50,15 +51,16 @@ The Terraform deployment engine was removed in Databricks CLI v1.20.0. Auto-migr // The terraform engine was removed in v1.20.0, so any failure (parsing, conversion, the plan // check) is fatal: there is no engine to fall back to. An empty terraform state is not a failure - // the converter writes an empty base state file, so there is no special case here. -func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) { +func Migrate(ctx context.Context, b *bundle.Bundle) error { _, localTerraformPath := b.StateFilenameTerraform(ctx) tfState, err := migrate.ParseTFStateFull(ctx, localTerraformPath) if err != nil { b.Metrics.SetBoolValue(metrics.DirectMigrateError, true) - return false, fmt.Errorf("parsing terraform state: %w%s", err, migrationFailedHint) + return fmt.Errorf("parsing terraform state: %w%s", err, migrationFailedHint) } if tfState == nil { - return false, nil + b.Metrics.SetBoolValue(metrics.DirectMigrateStateMissing, true) + return fmt.Errorf("terraform state %s was removed after it was pulled; if another deploy is running, wait for it to finish and run this command again", localTerraformPath) } _, localDirectPath := b.StateFilenameDirect(ctx) @@ -77,7 +79,7 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) { } if err != nil { b.Metrics.SetBoolValue(metrics.DirectMigrateError, true) - return false, fmt.Errorf("converting terraform state to the direct engine: %w%s", err, migrationFailedHint) + return fmt.Errorf("converting terraform state to the direct engine: %w%s", err, migrationFailedHint) } // Plan-check the converted state: validate it can be planned before committing to it, and @@ -85,7 +87,7 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) { plan, err := checkPlanOnTempState(ctx, b, tempStatePath, cfg) if err != nil { b.Metrics.SetBoolValue(metrics.DirectMigratePlanError, true) - return false, fmt.Errorf("the migrated state failed its plan check: %w%s", err, migrationFailedHint) + return fmt.Errorf("the migrated state failed its plan check: %w%s", err, migrationFailedHint) } // Record when the migrated state's first plan would recreate a resource, but still migrate: @@ -102,22 +104,21 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) { // that once the command is approved. raw, err := os.ReadFile(tempStatePath) if err != nil { - return false, fmt.Errorf("reading migrated state: %w", err) + return fmt.Errorf("reading migrated state: %w", err) } var data dstate.Database if err := json.Unmarshal(raw, &data); err != nil { - return false, fmt.Errorf("parsing migrated state: %w", err) + return fmt.Errorf("parsing migrated state: %w", err) } // The terraform state has schema_id but no volume_id (see BuildStateFromTF); the plan check read it. if err := direct.RecordMissingInternalIDs(plan, &data); err != nil { - return false, fmt.Errorf("recording internal ids in migrated state: %w", err) + return fmt.Errorf("recording internal ids in migrated state: %w", err) } b.DeploymentBundle.StateDB.OpenWithData(localDirectPath, data) - // Announce the migration only once the conversion and plan check succeeded, so a run with no - // terraform state to migrate says nothing misleading. + // Announce the migration only once the conversion and plan check succeeded. cmdio.LogString(ctx, "Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).") - return true, nil + return nil } // CleanupTerraformStateAfterMigration backs up the now-superseded terraform state, remote and diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index a39f8096ff3..d89efd0b042 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -589,14 +589,11 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // after approval, destroy commits as part of its teardown. The caller tags the user agent with the // resolved stateDesc.Engine afterwards. func migrateTerraformToDirect(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error { - migrated, err := statemgmt.Migrate(ctx, b) - if err != nil { + if err := statemgmt.Migrate(ctx, b); err != nil { return fmt.Errorf("migrating Terraform state to the direct engine: %w", err) } - if migrated { - stateDesc.Engine = engine.EngineDirect - b.Metrics.StateEngine = engine.EngineDirect - } + stateDesc.Engine = engine.EngineDirect + b.Metrics.StateEngine = engine.EngineDirect return nil }