diff --git a/.nextchanges/cli/apps-import-inline-app-yml.md b/.nextchanges/cli/apps-import-inline-app-yml.md new file mode 100644 index 00000000000..eae9a05376f --- /dev/null +++ b/.nextchanges/cli/apps-import-inline-app-yml.md @@ -0,0 +1 @@ +* Fix `databricks apps import` not inlining the app's `app.yml` into `databricks.yml`. ([#7675](https://github.com/databricks/cli/pull/7675)) diff --git a/acceptance/apps/import/app-yml/out.test.toml b/acceptance/apps/import/app-yml/out.test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/apps/import/app-yml/out.test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/apps/import/app-yml/output.txt b/acceptance/apps/import/app-yml/output.txt new file mode 100644 index 00000000000..0baffb149b6 --- /dev/null +++ b/acceptance/apps/import/app-yml/output.txt @@ -0,0 +1,95 @@ + +>>> [CLI] apps import --name myapp --output-dir out +Loading app 'myapp' configuration +Creating bundle configuration +File successfully saved to app.yml +Inlined and removed app.yml +Bundle configuration created at databricks.yml +Binding bundle to existing app +Warning: Use 'value_from' instead of 'valueFrom' + at resources.apps.app.config.env[1] + in databricks.yml:18:13 + +The field 'valueFrom' should be 'value_from' (snake_case). The 'valueFrom' field will be ignored. + +Successfully bound to app 'myapp' +Deploying bundle +Warning: Use 'value_from' instead of 'valueFrom' + at resources.apps.app.config.env[1] + in databricks.yml:18:13 + +The field 'valueFrom' should be 'value_from' (snake_case). The 'valueFrom' field will be ignored. + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/myapp/default/files... +Updated apps.app +Files: 1 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged +Bundle deployed successfully +Starting app +✓ Getting the status of the app myapp +✓ App is in RUNNING state +✓ App compute is in ACTIVE state +✓ Deployment succeeded +You can access the app at myapp-123.cloud.databricksapps.com +App started successfully + +✔ App 'myapp' imported to [TEST_TMP_DIR]/out + +You can now deploy changes with: databricks bundle deploy + +>>> ls out +databricks.yml + +>>> cat out/databricks.yml +bundle: + name: myapp + +workspace: + host: [DATABRICKS_URL] + +resources: + apps: + app: + config: + command: + - python + - app.py + env: + - name: GREETING + value: hello + - name: WAREHOUSE_ID + valueFrom: sql-warehouse + - name: PYTHON_VERSION + value: 3.1 + name: myapp + description: "" + source_code_path: . + +=== App config the bundle deploys (app.yml, if kept, is uploaded with the source) + +>>> [CLI] bundle validate -o json +Warning: Use 'value_from' instead of 'valueFrom' + at resources.apps.app.config.env[1] + in databricks.yml:18:13 + +The field 'valueFrom' should be 'value_from' (snake_case). The 'valueFrom' field will be ignored. + +{ + "command": [ + "python", + "app.py" + ], + "env": [ + { + "name": "GREETING", + "value": "hello" + }, + { + "name": "WAREHOUSE_ID" + }, + { + "name": "PYTHON_VERSION", + "value": "3.1" + } + ] +} diff --git a/acceptance/apps/import/app-yml/script b/acceptance/apps/import/app-yml/script new file mode 100644 index 00000000000..c7446d0ff09 --- /dev/null +++ b/acceptance/apps/import/app-yml/script @@ -0,0 +1,22 @@ +# apps import of an app whose source has an app.yml with a plain env value, a +# valueFrom reference and an unquoted value that YAML reads as a number. +# MSYS_NO_PATHCONV keeps Git Bash on Windows from rewriting the /Workspace path. +APP_PATH="/Workspace/Users/$CURRENT_USER_NAME/myapp" +MSYS_NO_PATHCONV=1 $CLI workspace import-dir src "$APP_PATH" >> LOG.setup 2>&1 +$CLI apps create myapp >> LOG.setup 2>&1 +MSYS_NO_PATHCONV=1 $CLI apps deploy myapp --source-code-path "$APP_PATH" >> LOG.setup 2>&1 + +trace $CLI apps import --name myapp --output-dir out + +trace ls out + +trace cat out/databricks.yml + +if [ -f out/app.yml ]; then + trace cat out/app.yml +fi + +title "App config the bundle deploys (app.yml, if kept, is uploaded with the source)" +withdir out trace $CLI bundle validate -o json | jq '.resources.apps.app.config' + +rm -rf out diff --git a/acceptance/apps/import/app-yml/src/app.yml b/acceptance/apps/import/app-yml/src/app.yml new file mode 100644 index 00000000000..91084cc1155 --- /dev/null +++ b/acceptance/apps/import/app-yml/src/app.yml @@ -0,0 +1,10 @@ +command: + - python + - app.py +env: + - name: GREETING + value: hello + - name: WAREHOUSE_ID + valueFrom: sql-warehouse + - name: PYTHON_VERSION + value: 3.10 diff --git a/acceptance/apps/import/app-yml/test.toml b/acceptance/apps/import/app-yml/test.toml new file mode 100644 index 00000000000..35325bbd652 --- /dev/null +++ b/acceptance/apps/import/app-yml/test.toml @@ -0,0 +1,6 @@ +Badness = "Inlining loses data: app.yml's valueFrom is copied verbatim but bundle config expects value_from, so WAREHOUSE_ID is deployed with no value (only a warning) while app.yml is deleted; the unquoted 3.10 is re-typed as a number and deployed as 3.1." + +# Fix path with reverse slashes in the output for Windows. +[[Repls]] +Old = 'TEST_TMP_DIR]\\out' +New = 'TEST_TMP_DIR]/out' diff --git a/cmd/apps/import.go b/cmd/apps/import.go index 7df83287871..48137fdc2ef 100644 --- a/cmd/apps/import.go +++ b/cmd/apps/import.go @@ -2,6 +2,7 @@ package apps import ( "bufio" + "bytes" "cmp" "context" "errors" @@ -12,8 +13,6 @@ import ( "slices" "strings" - "go.yaml.in/yaml/v3" - "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/generate" "github.com/databricks/cli/bundle/phases" @@ -27,7 +26,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/dyn/yamlloader" "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/logdiag" @@ -439,6 +438,12 @@ func generateAppBundle(ctx context.Context, w *databricks.WorkspaceClient, app * return "", fmt.Errorf("failed to convert app to value: %w", err) } + // Download the app source files before inlining, since app.yml is part of them + err = downloader.FlushToDisk(ctx, false) + if err != nil { + return "", fmt.Errorf("failed to download app source: %w", err) + } + // Check for app.yml or app.yaml and inline its contents appConfigFile, err := inlineAppConfigFile(&v) if err != nil { @@ -473,12 +478,6 @@ func generateAppBundle(ctx context.Context, w *databricks.WorkspaceClient, app * }, []diag.Location{{Line: 20}}), } - // Download the app source files - err = downloader.FlushToDisk(ctx, false) - if err != nil { - return "", fmt.Errorf("failed to download app source: %w", err) - } - // Save databricks.yml databricksYml := filepath.Join(".", "databricks.yml") saver := yamlsaver.NewSaver() @@ -567,9 +566,9 @@ func inlineAppConfigFile(appValue *dyn.Value) (string, error) { return "", nil } - // Parse the app config - var appConfig map[string]any - err = yaml.Unmarshal(appConfigData, &appConfig) + // Load into a dyn.Value rather than map[string]any: the saver orders mapping keys by + // source line, so env entries keep the key order they have in the file. + appConfig, err := yamlloader.LoadYAML(appConfigFile, bytes.NewReader(appConfigData)) if err != nil { return "", fmt.Errorf("failed to parse %s: %w", appConfigFile, err) } @@ -587,43 +586,31 @@ func inlineAppConfigFile(appValue *dyn.Value) (string, error) { newPairs = append(newPairs, appMap.Pairs()...) // Create config section - configMap := make(map[string]dyn.Value) + // Ordered pairs (not a Go map) so the generated YAML is deterministic. + var configPairs []dyn.Pair - // Add command if present - if cmd, ok := appConfig["command"]; ok { - cmdValue, err := convert.FromTyped(cmd, dyn.NilValue) - if err != nil { - return "", fmt.Errorf("failed to convert command: %w", err) - } - configMap["command"] = cmdValue + // The top-level values drop their app.yml locations so that the saver keeps them in + // insertion order; their contents keep the locations that order nested keys. + if cmd := appConfig.Get("command"); cmd.IsValid() { + configPairs = append(configPairs, dyn.Pair{Key: dyn.V("command"), Value: cmd.WithLocations(nil)}) } - - // Add env if present - if env, ok := appConfig["env"]; ok { - envValue, err := convert.FromTyped(env, dyn.NilValue) - if err != nil { - return "", fmt.Errorf("failed to convert env: %w", err) - } - configMap["env"] = envValue + if env := appConfig.Get("env"); env.IsValid() { + configPairs = append(configPairs, dyn.Pair{Key: dyn.V("env"), Value: env.WithLocations(nil)}) } // Add the config section if we have any items - if len(configMap) > 0 { + if len(configPairs) > 0 { newPairs = append(newPairs, dyn.Pair{ Key: dyn.V("config"), - Value: dyn.V(configMap), + Value: dyn.V(dyn.NewMappingFromPairs(configPairs)), }) } // Add resources at top level if present - if resources, ok := appConfig["resources"]; ok { - resourcesValue, err := convert.FromTyped(resources, dyn.NilValue) - if err != nil { - return "", fmt.Errorf("failed to convert resources: %w", err) - } + if resources := appConfig.Get("resources"); resources.IsValid() { newPairs = append(newPairs, dyn.Pair{ Key: dyn.V("resources"), - Value: resourcesValue, + Value: resources.WithLocations(nil), }) }