Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/cli/apps-import-inline-app-yml.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Fix `databricks apps import` not inlining the app's `app.yml` into `databricks.yml`. ([#7675](https://github.com/databricks/cli/pull/7675))
1 change: 1 addition & 0 deletions acceptance/apps/import/app-yml/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
95 changes: 95 additions & 0 deletions acceptance/apps/import/app-yml/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@

>>> [CLI] apps import --name myapp --output-dir out
Loading app 'myapp' configuration
Creating bundle configuration
File successfully saved to app.yml
Inlined and removed app.yml
Bundle configuration created at databricks.yml
Binding bundle to existing app
Warning: Use 'value_from' instead of 'valueFrom'
at resources.apps.app.config.env[1]
in databricks.yml:18:13

The field 'valueFrom' should be 'value_from' (snake_case). The 'valueFrom' field will be ignored.

Successfully bound to app 'myapp'
Deploying bundle
Warning: Use 'value_from' instead of 'valueFrom'
at resources.apps.app.config.env[1]
in databricks.yml:18:13

The field 'valueFrom' should be 'value_from' (snake_case). The 'valueFrom' field will be ignored.

Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/myapp/default/files...
Updated apps.app
Files: 1 uploaded, 0 deleted
Resources: 0 created, 1 changed, 0 deleted, 0 unchanged
Bundle deployed successfully
Starting app
✓ Getting the status of the app myapp
✓ App is in RUNNING state
✓ App compute is in ACTIVE state
✓ Deployment succeeded
You can access the app at myapp-123.cloud.databricksapps.com
App started successfully

✔ App 'myapp' imported to [TEST_TMP_DIR]/out

You can now deploy changes with: databricks bundle deploy

>>> ls out
databricks.yml

>>> cat out/databricks.yml
bundle:
name: myapp

workspace:
host: [DATABRICKS_URL]

resources:
apps:
app:
config:
command:
- python
- app.py
env:
- name: GREETING
value: hello
- name: WAREHOUSE_ID
valueFrom: sql-warehouse
- name: PYTHON_VERSION
value: 3.1
name: myapp
description: ""
source_code_path: .

=== App config the bundle deploys (app.yml, if kept, is uploaded with the source)

>>> [CLI] bundle validate -o json
Warning: Use 'value_from' instead of 'valueFrom'
at resources.apps.app.config.env[1]
in databricks.yml:18:13

The field 'valueFrom' should be 'value_from' (snake_case). The 'valueFrom' field will be ignored.

{
"command": [
"python",
"app.py"
],
"env": [
{
"name": "GREETING",
"value": "hello"
},
{
"name": "WAREHOUSE_ID"
},
{
"name": "PYTHON_VERSION",
"value": "3.1"
}
]
}
22 changes: 22 additions & 0 deletions acceptance/apps/import/app-yml/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# apps import of an app whose source has an app.yml with a plain env value, a
# valueFrom reference and an unquoted value that YAML reads as a number.
# MSYS_NO_PATHCONV keeps Git Bash on Windows from rewriting the /Workspace path.
APP_PATH="/Workspace/Users/$CURRENT_USER_NAME/myapp"
MSYS_NO_PATHCONV=1 $CLI workspace import-dir src "$APP_PATH" >> LOG.setup 2>&1
$CLI apps create myapp >> LOG.setup 2>&1
MSYS_NO_PATHCONV=1 $CLI apps deploy myapp --source-code-path "$APP_PATH" >> LOG.setup 2>&1

trace $CLI apps import --name myapp --output-dir out

trace ls out

trace cat out/databricks.yml

if [ -f out/app.yml ]; then
trace cat out/app.yml
fi

title "App config the bundle deploys (app.yml, if kept, is uploaded with the source)"
withdir out trace $CLI bundle validate -o json | jq '.resources.apps.app.config'

rm -rf out
10 changes: 10 additions & 0 deletions acceptance/apps/import/app-yml/src/app.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
command:
- python
- app.py
env:
- name: GREETING
value: hello
- name: WAREHOUSE_ID
valueFrom: sql-warehouse
- name: PYTHON_VERSION
value: 3.10
6 changes: 6 additions & 0 deletions acceptance/apps/import/app-yml/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Badness = "Inlining loses data: app.yml's valueFrom is copied verbatim but bundle config expects value_from, so WAREHOUSE_ID is deployed with no value (only a warning) while app.yml is deleted; the unquoted 3.10 is re-typed as a number and deployed as 3.1."

# Fix path with reverse slashes in the output for Windows.
[[Repls]]
Old = 'TEST_TMP_DIR]\\out'
New = 'TEST_TMP_DIR]/out'
59 changes: 23 additions & 36 deletions cmd/apps/import.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package apps

import (
"bufio"
"bytes"
"cmp"
"context"
"errors"
Expand All @@ -12,8 +13,6 @@ import (
"slices"
"strings"

"go.yaml.in/yaml/v3"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/generate"
"github.com/databricks/cli/bundle/phases"
Expand All @@ -27,7 +26,7 @@ import (
"github.com/databricks/cli/libs/cmdio"
"github.com/databricks/cli/libs/diag"
"github.com/databricks/cli/libs/dyn"
"github.com/databricks/cli/libs/dyn/convert"
"github.com/databricks/cli/libs/dyn/yamlloader"
"github.com/databricks/cli/libs/dyn/yamlsaver"
"github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/logdiag"
Expand Down Expand Up @@ -439,6 +438,12 @@ func generateAppBundle(ctx context.Context, w *databricks.WorkspaceClient, app *
return "", fmt.Errorf("failed to convert app to value: %w", err)
}

// Download the app source files before inlining, since app.yml is part of them
err = downloader.FlushToDisk(ctx, false)
if err != nil {
return "", fmt.Errorf("failed to download app source: %w", err)
}

// Check for app.yml or app.yaml and inline its contents
appConfigFile, err := inlineAppConfigFile(&v)
if err != nil {
Expand Down Expand Up @@ -473,12 +478,6 @@ func generateAppBundle(ctx context.Context, w *databricks.WorkspaceClient, app *
}, []diag.Location{{Line: 20}}),
}

// Download the app source files
err = downloader.FlushToDisk(ctx, false)
if err != nil {
return "", fmt.Errorf("failed to download app source: %w", err)
}

// Save databricks.yml
databricksYml := filepath.Join(".", "databricks.yml")
saver := yamlsaver.NewSaver()
Expand Down Expand Up @@ -567,9 +566,9 @@ func inlineAppConfigFile(appValue *dyn.Value) (string, error) {
return "", nil
}

// Parse the app config
var appConfig map[string]any
err = yaml.Unmarshal(appConfigData, &appConfig)
// Load into a dyn.Value rather than map[string]any: the saver orders mapping keys by
// source line, so env entries keep the key order they have in the file.
appConfig, err := yamlloader.LoadYAML(appConfigFile, bytes.NewReader(appConfigData))
if err != nil {
return "", fmt.Errorf("failed to parse %s: %w", appConfigFile, err)
}
Expand All @@ -587,43 +586,31 @@ func inlineAppConfigFile(appValue *dyn.Value) (string, error) {
newPairs = append(newPairs, appMap.Pairs()...)

// Create config section
configMap := make(map[string]dyn.Value)
// Ordered pairs (not a Go map) so the generated YAML is deterministic.
var configPairs []dyn.Pair

// Add command if present
if cmd, ok := appConfig["command"]; ok {
cmdValue, err := convert.FromTyped(cmd, dyn.NilValue)
if err != nil {
return "", fmt.Errorf("failed to convert command: %w", err)
}
configMap["command"] = cmdValue
// The top-level values drop their app.yml locations so that the saver keeps them in
// insertion order; their contents keep the locations that order nested keys.
if cmd := appConfig.Get("command"); cmd.IsValid() {
configPairs = append(configPairs, dyn.Pair{Key: dyn.V("command"), Value: cmd.WithLocations(nil)})
}

// Add env if present
if env, ok := appConfig["env"]; ok {
envValue, err := convert.FromTyped(env, dyn.NilValue)
if err != nil {
return "", fmt.Errorf("failed to convert env: %w", err)
}
configMap["env"] = envValue
if env := appConfig.Get("env"); env.IsValid() {
configPairs = append(configPairs, dyn.Pair{Key: dyn.V("env"), Value: env.WithLocations(nil)})
}

// Add the config section if we have any items
if len(configMap) > 0 {
if len(configPairs) > 0 {
newPairs = append(newPairs, dyn.Pair{
Key: dyn.V("config"),
Value: dyn.V(configMap),
Value: dyn.V(dyn.NewMappingFromPairs(configPairs)),
})
}

// Add resources at top level if present
if resources, ok := appConfig["resources"]; ok {
resourcesValue, err := convert.FromTyped(resources, dyn.NilValue)
if err != nil {
return "", fmt.Errorf("failed to convert resources: %w", err)
}
if resources := appConfig.Get("resources"); resources.IsValid() {
newPairs = append(newPairs, dyn.Pair{
Key: dyn.V("resources"),
Value: resourcesValue,
Value: resources.WithLocations(nil),
})
}

Expand Down
Loading