Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/warn-missing-torch.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air run` (including `--dry-run`) now warns before submitting when the `code_source` imports `torch` but the environment is the Standard environment (version 5 or later) without `torch` in `dependencies`, instead of letting the run fail after it waits for a GPU. ([#7650](https://github.com/databricks/cli/pull/7650))
13 changes: 13 additions & 0 deletions acceptance/air/run/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,19 @@ Tip: use the --debug flag to see more details and a trace of this error:

Exit code: 1

=== torch import on the Standard environment warns before submitting

>>> [CLI] air run -f torch-standard.yaml --dry-run
Warning: train.py imports torch, but environment version 6 (Standard) does not include PyTorch, so the run would fail after waiting for a GPU. Set environment.version: databricks_ai_v6, or add torch to environment.dependencies.
Dry run: code_source snapshot would include 1 file(s).
Dry run: configuration for "smoke-test" is valid; not submitting.

=== the Databricks AI environment provides torch, so no warning

>>> [CLI] air run -f torch-databricks-ai.yaml --dry-run
Dry run: code_source snapshot would include 1 file(s).
Dry run: configuration for "smoke-test" is valid; not submitting.

=== git.remote is rejected

>>> [CLI] air run -f git-remote.yaml --dry-run
Expand Down
6 changes: 6 additions & 0 deletions acceptance/air/run/script
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,12 @@ echo 'print(1)' > .scratch/proj/main.py
errcode trace $CLI air run -f ignored-root.yaml --dry-run
rm -fr .git .gitignore .scratch

title "torch import on the Standard environment warns before submitting"
trace $CLI air run -f torch-standard.yaml --dry-run

title "the Databricks AI environment provides torch, so no warning"
trace $CLI air run -f torch-databricks-ai.yaml --dry-run

title "git.remote is rejected"
errcode trace $CLI air run -f git-remote.yaml --dry-run

Expand Down
9 changes: 9 additions & 0 deletions acceptance/air/run/torch-databricks-ai.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
experiment_name: smoke-test
command: python "$CODE_SOURCE_PATH/train.py"
compute:
accelerator_type: GPU_1xA10
num_accelerators: 1
environment:
version: databricks_ai_v6
code_source:
root_path: torch-src
3 changes: 3 additions & 0 deletions acceptance/air/run/torch-src/train.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
import torch

print(torch.cuda.is_available())
7 changes: 7 additions & 0 deletions acceptance/air/run/torch-standard.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
experiment_name: smoke-test
command: python "$CODE_SOURCE_PATH/train.py"
compute:
accelerator_type: GPU_1xA10
num_accelerators: 1
code_source:
root_path: torch-src
4 changes: 4 additions & 0 deletions cmd/air/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,10 @@ field, pass its path to -h as a separate argument:
return err
}

if msg := missingTorchWarning(ctx, cfg, file); msg != "" {
cmdio.LogString(ctx, msg)
}

if dryRun {
// Local and credential-free, so it runs before workspace validation.
var snapshotFileCount *int
Expand Down
104 changes: 104 additions & 0 deletions cmd/air/torchcheck.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
package aircmd

import (
"bufio"
"context"
"fmt"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"

"github.com/databricks/cli/libs/log"
)

// The Standard environment dropped PyTorch in this version.
const firstStandardVersionWithoutTorch = 5

// Bounds keep the check cheap on large code sources; it is a hint, not a guarantee.
const (
torchScanMaxFiles = 200
torchScanMaxFileBytes = 1 << 20
)

var torchImportRe = sync.OnceValue(func() *regexp.Regexp {
return regexp.MustCompile(`^\s*(import\s+torch\b|from\s+torch\b)`)
})

// missingTorchWarning returns a warning when the code_source imports torch but
// the environment can't provide it, or "" when it can or the check doesn't apply.
// A run like that fails only after it waits for a GPU and then retries, so
// catching it locally saves the whole queue-and-retry cycle.
func missingTorchWarning(ctx context.Context, cfg *runConfig, configPath string) string {
if len(cfg.Containers) > 0 || cfg.CodeSource == nil || cfg.CodeSource.Snapshot == nil {
return ""
}
version := "6"
if env := cfg.Environment; env != nil {
if env.UnityCatalogImage != "" {
return ""
}
if env.Version.set {
version = strings.ToLower(env.Version.raw)
}
for _, dep := range env.Dependencies.list {
// A requirements file or an explicit torch entry may provide it.
if strings.HasPrefix(dep, "-") || strings.HasPrefix(strings.ToLower(dep), "torch") {
return ""
}
}
}
if strings.HasPrefix(version, databricksAIPrefix) {
return ""
}
if major, err := strconv.Atoi(version); err != nil || major < firstStandardVersionWithoutTorch {
return ""
}

snap := cfg.CodeSource.Snapshot
repoPath, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath))
if err != nil {
return ""
}
plan, err := resolveSnapshotPlan(ctx, newGitRepo(repoPath), snap.Git, snap.IncludePaths)
if err != nil {
return ""
}
files, err := snapshotFiles(ctx, repoPath, plan.includePaths, plan.isGitRepo)
if err != nil {
log.Debugf(ctx, "air: skipping torch import check: %v", err)
return ""
}
scanned := 0
for _, f := range files {
if !strings.HasSuffix(f.rel, ".py") || f.size > torchScanMaxFileBytes {
continue
}
if scanned++; scanned > torchScanMaxFiles {
break
}
if importsTorch(filepath.Join(repoPath, f.rel)) {
return fmt.Sprintf("Warning: %s imports torch, but environment version %s (Standard) does not include PyTorch, so the run would fail after waiting for a GPU. Set environment.version: %s6, or add torch to environment.dependencies.",
filepath.ToSlash(f.rel), version, databricksAIPrefix)
}
}
return ""
}

// importsTorch reports whether a top-level import line in the file imports torch.
func importsTorch(path string) bool {
f, err := os.Open(path)
if err != nil {
return false
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
if torchImportRe().MatchString(scanner.Text()) {
return true
}
}
return false
}
50 changes: 50 additions & 0 deletions cmd/air/torchcheck_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package aircmd

import (
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestMissingTorchWarning(t *testing.T) {
const torchScript = "import os\nimport torch.nn as nn\n"
cases := []struct {
name string
environment string
script string
want bool
}{
{"default environment", "", torchScript, true},
{"standard 6", "environment:\n version: 6\n", torchScript, true},
{"from torch import", "", "from torch import nn\n", true},
{"other deps only", "environment:\n version: 6\n dependencies:\n - numpy\n", torchScript, true},
{"databricks ai environment", "environment:\n version: databricks_ai_v6\n", torchScript, false},
{"torch dependency", "environment:\n dependencies:\n - torch==2.6.0\n", torchScript, false},
{"requirements file", "environment:\n dependencies:\n - -r requirements.txt\n", torchScript, false},
{"standard 4 includes torch", "environment:\n version: 4\n", torchScript, false},
{"custom image", "environment:\n unity_catalog_image: main.s.img:v1\n", torchScript, false},
{"no torch import", "", "import torchvision_stub\n# import torch\n", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "train.py"), []byte(tc.script), 0o644))
yaml := "experiment_name: e\ncompute:\n num_accelerators: 1\n accelerator_type: GPU_1xA10\ncode_source:\n root_path: .\ncommand: python train.py\n" + tc.environment
path := filepath.Join(dir, "run.yaml")
require.NoError(t, os.WriteFile(path, []byte(yaml), 0o644))
cfg, err := loadRunConfig(path)
require.NoError(t, err)

got := missingTorchWarning(t.Context(), cfg, path)
if tc.want {
assert.Contains(t, got, "train.py imports torch")
assert.Contains(t, got, "environment.version: databricks_ai_v6")
} else {
assert.Empty(t, got)
}
})
}
}
Loading