Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7778.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Fix a panic when the local Terraform state is removed after it is pulled and before it is migrated to the direct engine, for example by a concurrent `bundle deploy`. ([#7778](https://github.com/databricks/cli/pull/7778))
16 changes: 9 additions & 7 deletions acceptance/bundle/migrate/user-agent/race/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@

=== Deploy where the local terraform state disappears between the pull and the migration

>>> musterr [CLI] bundle deploy
Executing 'postbuild' script
Error: migrating Terraform state to the direct engine: terraform state [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate was removed after it was pulled; if another deploy is running, wait for it to finish and run this command again


>>> print_migration_telemetry
direct_migrate_state_missing true

>>> print_engines
2 MISSING GET /api/2.0/workspace/export
3 engine/terraform GET /api/2.0/workspace/export
4 engine/terraform GET /api/2.0/workspace/get-status
1 engine/terraform POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml
1 engine/terraform POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock
1 engine/terraform POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deployment.json
2 engine/terraform POST /api/2.0/workspace/delete
1 engine/terraform POST /api/2.0/workspace/mkdirs
1 MISSING POST /telemetry-ext
4 changes: 2 additions & 2 deletions acceptance/bundle/migrate/user-agent/race/script
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ replace_ids.py
rm -f "$OUT_REQUESTS"

title "Deploy where the local terraform state disappears between the pull and the migration"
musterr $CLI bundle deploy &> LOG.deploy
contains.py 'Panic Payload: internal error: DeploymentState must be opened first' < LOG.deploy > /dev/null
trace musterr $CLI bundle deploy
trace print_migration_telemetry
trace print_engines
10 changes: 7 additions & 3 deletions bundle/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,13 @@ const (
// - migrate_commit_error: the state was converted, but committing it
// (renaming files / pushing to workspace) failed.
// - migrate_warnings: the conversion emitted warnings (e.g. resources the direct engine can't represent).
DirectMigrateError = "direct_migrate_error"
DirectMigrateCommitError = "direct_migrate_commit_error"
DirectMigrateWarnings = "direct_migrate_warnings"
// - migrate_state_missing: the terraform state was pulled, but its local file was gone
// when the migration read it. Nothing was converted. When a
// concurrent deploy migrated it, a retry picks up the direct state.
DirectMigrateError = "direct_migrate_error"
DirectMigrateCommitError = "direct_migrate_commit_error"
DirectMigrateWarnings = "direct_migrate_warnings"
DirectMigrateStateMissing = "direct_migrate_state_missing"
// True when the post-convert plan check failed; the migration was not committed.
DirectMigratePlanError = "direct_migrate_plan_error"
// True when the migrated state's first plan would recreate (destroy + create) an
Expand Down
27 changes: 14 additions & 13 deletions bundle/statemgmt/direct_migration.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,9 @@ const migrationFailedHint = `
The Terraform deployment engine was removed in Databricks CLI v1.20.0. Auto-migration of your bundle failed. Install Databricks CLI v1.19.x to deploy on Terraform, and please report this to dabs-feedback@databricks.com`

// Migrate converts the bundle's terraform state to a direct-engine state and opens
// b.DeploymentBundle.StateDB with it in memory. Returns false when there is no terraform state
// to migrate (the caller then opens the direct state normally).
// b.DeploymentBundle.StateDB with it in memory. It is only called when the pulled state is a
// terraform state, which the pull leaves on disk, so a missing local terraform state means it
// changed after the pull (e.g. a concurrent deploy migrated it) and is an error.
//
// Migrate is reversible: nothing is written to the local state file or the workspace, so a
// command that does not commit - plan, run, or a declined deploy - just drops the in-memory
Expand All @@ -50,15 +51,16 @@ The Terraform deployment engine was removed in Databricks CLI v1.20.0. Auto-migr
// The terraform engine was removed in v1.20.0, so any failure (parsing, conversion, the plan
// check) is fatal: there is no engine to fall back to. An empty terraform state is not a failure -
// the converter writes an empty base state file, so there is no special case here.
func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) {
func Migrate(ctx context.Context, b *bundle.Bundle) error {
_, localTerraformPath := b.StateFilenameTerraform(ctx)
tfState, err := migrate.ParseTFStateFull(ctx, localTerraformPath)
if err != nil {
b.Metrics.SetBoolValue(metrics.DirectMigrateError, true)
return false, fmt.Errorf("parsing terraform state: %w%s", err, migrationFailedHint)
return fmt.Errorf("parsing terraform state: %w%s", err, migrationFailedHint)
}
if tfState == nil {
return false, nil
b.Metrics.SetBoolValue(metrics.DirectMigrateStateMissing, true)
return fmt.Errorf("terraform state %s was removed after it was pulled; if another deploy is running, wait for it to finish and run this command again", localTerraformPath)
}

_, localDirectPath := b.StateFilenameDirect(ctx)
Expand All @@ -77,15 +79,15 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) {
}
if err != nil {
b.Metrics.SetBoolValue(metrics.DirectMigrateError, true)
return false, fmt.Errorf("converting terraform state to the direct engine: %w%s", err, migrationFailedHint)
return fmt.Errorf("converting terraform state to the direct engine: %w%s", err, migrationFailedHint)
}

// Plan-check the converted state: validate it can be planned before committing to it, and
// record the recreate metric.
plan, err := checkPlanOnTempState(ctx, b, tempStatePath, cfg)
if err != nil {
b.Metrics.SetBoolValue(metrics.DirectMigratePlanError, true)
return false, fmt.Errorf("the migrated state failed its plan check: %w%s", err, migrationFailedHint)
return fmt.Errorf("the migrated state failed its plan check: %w%s", err, migrationFailedHint)
}

// Record when the migrated state's first plan would recreate a resource, but still migrate:
Expand All @@ -102,22 +104,21 @@ func Migrate(ctx context.Context, b *bundle.Bundle) (bool, error) {
// that once the command is approved.
raw, err := os.ReadFile(tempStatePath)
if err != nil {
return false, fmt.Errorf("reading migrated state: %w", err)
return fmt.Errorf("reading migrated state: %w", err)
}
var data dstate.Database
if err := json.Unmarshal(raw, &data); err != nil {
return false, fmt.Errorf("parsing migrated state: %w", err)
return fmt.Errorf("parsing migrated state: %w", err)
}
// The terraform state has schema_id but no volume_id (see BuildStateFromTF); the plan check read it.
if err := direct.RecordMissingInternalIDs(plan, &data); err != nil {
return false, fmt.Errorf("recording internal ids in migrated state: %w", err)
return fmt.Errorf("recording internal ids in migrated state: %w", err)
}
b.DeploymentBundle.StateDB.OpenWithData(localDirectPath, data)

// Announce the migration only once the conversion and plan check succeeded, so a run with no
// terraform state to migrate says nothing misleading.
// Announce the migration only once the conversion and plan check succeeded.
cmdio.LogString(ctx, "Notice: migrating your bundle to direct deployment engine (https://github.com/databricks/cli/issues/6765).")
return true, nil
return nil
}

// CleanupTerraformStateAfterMigration backs up the now-superseded terraform state, remote and
Expand Down
9 changes: 3 additions & 6 deletions cmd/bundle/utils/process.go
Original file line number Diff line number Diff line change
Expand Up @@ -589,14 +589,11 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle
// after approval, destroy commits as part of its teardown. The caller tags the user agent with the
// resolved stateDesc.Engine afterwards.
func migrateTerraformToDirect(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error {
migrated, err := statemgmt.Migrate(ctx, b)
if err != nil {
if err := statemgmt.Migrate(ctx, b); err != nil {
return fmt.Errorf("migrating Terraform state to the direct engine: %w", err)
}
if migrated {
stateDesc.Engine = engine.EngineDirect
b.Metrics.StateEngine = engine.EngineDirect
}
stateDesc.Engine = engine.EngineDirect
b.Metrics.StateEngine = engine.EngineDirect
return nil
}

Expand Down
Loading