diff --git a/.nextchanges/air/warn-missing-torch.md b/.nextchanges/air/warn-missing-torch.md new file mode 100644 index 00000000000..21fa5eb832a --- /dev/null +++ b/.nextchanges/air/warn-missing-torch.md @@ -0,0 +1 @@ +* `air run` (including `--dry-run`) now warns before submitting when the `code_source` imports `torch` but the environment is the Standard environment (version 5 or later) without `torch` in `dependencies`, instead of letting the run fail after it waits for a GPU. ([#7650](https://github.com/databricks/cli/pull/7650)) diff --git a/acceptance/air/run/output.txt b/acceptance/air/run/output.txt index 8bb2f5c4cce..76a934e2884 100644 --- a/acceptance/air/run/output.txt +++ b/acceptance/air/run/output.txt @@ -65,6 +65,19 @@ Tip: use the --debug flag to see more details and a trace of this error: Exit code: 1 +=== torch import on the Standard environment warns before submitting + +>>> [CLI] air run -f torch-standard.yaml --dry-run +Warning: train.py imports torch, but environment version 6 (Standard) does not include PyTorch, so the run would fail after waiting for a GPU. Set environment.version: databricks_ai_v6, or add torch to environment.dependencies. +Dry run: code_source snapshot would include 1 file(s). +Dry run: configuration for "smoke-test" is valid; not submitting. + +=== the Databricks AI environment provides torch, so no warning + +>>> [CLI] air run -f torch-databricks-ai.yaml --dry-run +Dry run: code_source snapshot would include 1 file(s). +Dry run: configuration for "smoke-test" is valid; not submitting. + === git.remote is rejected >>> [CLI] air run -f git-remote.yaml --dry-run diff --git a/acceptance/air/run/script b/acceptance/air/run/script index 4b0163bdec2..61144ee0e1e 100644 --- a/acceptance/air/run/script +++ b/acceptance/air/run/script @@ -33,6 +33,12 @@ echo 'print(1)' > .scratch/proj/main.py errcode trace $CLI air run -f ignored-root.yaml --dry-run rm -fr .git .gitignore .scratch +title "torch import on the Standard environment warns before submitting" +trace $CLI air run -f torch-standard.yaml --dry-run + +title "the Databricks AI environment provides torch, so no warning" +trace $CLI air run -f torch-databricks-ai.yaml --dry-run + title "git.remote is rejected" errcode trace $CLI air run -f git-remote.yaml --dry-run diff --git a/acceptance/air/run/torch-databricks-ai.yaml b/acceptance/air/run/torch-databricks-ai.yaml new file mode 100644 index 00000000000..2898a094660 --- /dev/null +++ b/acceptance/air/run/torch-databricks-ai.yaml @@ -0,0 +1,9 @@ +experiment_name: smoke-test +command: python "$CODE_SOURCE_PATH/train.py" +compute: + accelerator_type: GPU_1xA10 + num_accelerators: 1 +environment: + version: databricks_ai_v6 +code_source: + root_path: torch-src diff --git a/acceptance/air/run/torch-src/train.py b/acceptance/air/run/torch-src/train.py new file mode 100644 index 00000000000..8fcea7b65b3 --- /dev/null +++ b/acceptance/air/run/torch-src/train.py @@ -0,0 +1,3 @@ +import torch + +print(torch.cuda.is_available()) diff --git a/acceptance/air/run/torch-standard.yaml b/acceptance/air/run/torch-standard.yaml new file mode 100644 index 00000000000..840b366740d --- /dev/null +++ b/acceptance/air/run/torch-standard.yaml @@ -0,0 +1,7 @@ +experiment_name: smoke-test +command: python "$CODE_SOURCE_PATH/train.py" +compute: + accelerator_type: GPU_1xA10 + num_accelerators: 1 +code_source: + root_path: torch-src diff --git a/cmd/air/run.go b/cmd/air/run.go index 20a7683461f..63098b87eaa 100644 --- a/cmd/air/run.go +++ b/cmd/air/run.go @@ -160,6 +160,10 @@ field, pass its path to -h as a separate argument: return err } + if msg := missingTorchWarning(ctx, cfg, file); msg != "" { + cmdio.LogString(ctx, msg) + } + if dryRun { // Local and credential-free, so it runs before workspace validation. var snapshotFileCount *int diff --git a/cmd/air/torchcheck.go b/cmd/air/torchcheck.go new file mode 100644 index 00000000000..36a061e921a --- /dev/null +++ b/cmd/air/torchcheck.go @@ -0,0 +1,104 @@ +package aircmd + +import ( + "bufio" + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + + "github.com/databricks/cli/libs/log" +) + +// The Standard environment dropped PyTorch in this version. +const firstStandardVersionWithoutTorch = 5 + +// Bounds keep the check cheap on large code sources; it is a hint, not a guarantee. +const ( + torchScanMaxFiles = 200 + torchScanMaxFileBytes = 1 << 20 +) + +var torchImportRe = sync.OnceValue(func() *regexp.Regexp { + return regexp.MustCompile(`^\s*(import\s+torch\b|from\s+torch\b)`) +}) + +// missingTorchWarning returns a warning when the code_source imports torch but +// the environment can't provide it, or "" when it can or the check doesn't apply. +// A run like that fails only after it waits for a GPU and then retries, so +// catching it locally saves the whole queue-and-retry cycle. +func missingTorchWarning(ctx context.Context, cfg *runConfig, configPath string) string { + if len(cfg.Containers) > 0 || cfg.CodeSource == nil || cfg.CodeSource.Snapshot == nil { + return "" + } + version := "6" + if env := cfg.Environment; env != nil { + if env.UnityCatalogImage != "" { + return "" + } + if env.Version.set { + version = strings.ToLower(env.Version.raw) + } + for _, dep := range env.Dependencies.list { + // A requirements file or an explicit torch entry may provide it. + if strings.HasPrefix(dep, "-") || strings.HasPrefix(strings.ToLower(dep), "torch") { + return "" + } + } + } + if strings.HasPrefix(version, databricksAIPrefix) { + return "" + } + if major, err := strconv.Atoi(version); err != nil || major < firstStandardVersionWithoutTorch { + return "" + } + + snap := cfg.CodeSource.Snapshot + repoPath, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath)) + if err != nil { + return "" + } + plan, err := resolveSnapshotPlan(ctx, newGitRepo(repoPath), snap.Git, snap.IncludePaths) + if err != nil { + return "" + } + files, err := snapshotFiles(ctx, repoPath, plan.includePaths, plan.isGitRepo) + if err != nil { + log.Debugf(ctx, "air: skipping torch import check: %v", err) + return "" + } + scanned := 0 + for _, f := range files { + if !strings.HasSuffix(f.rel, ".py") || f.size > torchScanMaxFileBytes { + continue + } + if scanned++; scanned > torchScanMaxFiles { + break + } + if importsTorch(filepath.Join(repoPath, f.rel)) { + return fmt.Sprintf("Warning: %s imports torch, but environment version %s (Standard) does not include PyTorch, so the run would fail after waiting for a GPU. Set environment.version: %s6, or add torch to environment.dependencies.", + filepath.ToSlash(f.rel), version, databricksAIPrefix) + } + } + return "" +} + +// importsTorch reports whether a top-level import line in the file imports torch. +func importsTorch(path string) bool { + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + if torchImportRe().MatchString(scanner.Text()) { + return true + } + } + return false +} diff --git a/cmd/air/torchcheck_test.go b/cmd/air/torchcheck_test.go new file mode 100644 index 00000000000..5e6facc075f --- /dev/null +++ b/cmd/air/torchcheck_test.go @@ -0,0 +1,50 @@ +package aircmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMissingTorchWarning(t *testing.T) { + const torchScript = "import os\nimport torch.nn as nn\n" + cases := []struct { + name string + environment string + script string + want bool + }{ + {"default environment", "", torchScript, true}, + {"standard 6", "environment:\n version: 6\n", torchScript, true}, + {"from torch import", "", "from torch import nn\n", true}, + {"other deps only", "environment:\n version: 6\n dependencies:\n - numpy\n", torchScript, true}, + {"databricks ai environment", "environment:\n version: databricks_ai_v6\n", torchScript, false}, + {"torch dependency", "environment:\n dependencies:\n - torch==2.6.0\n", torchScript, false}, + {"requirements file", "environment:\n dependencies:\n - -r requirements.txt\n", torchScript, false}, + {"standard 4 includes torch", "environment:\n version: 4\n", torchScript, false}, + {"custom image", "environment:\n unity_catalog_image: main.s.img:v1\n", torchScript, false}, + {"no torch import", "", "import torchvision_stub\n# import torch\n", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "train.py"), []byte(tc.script), 0o644)) + yaml := "experiment_name: e\ncompute:\n num_accelerators: 1\n accelerator_type: GPU_1xA10\ncode_source:\n root_path: .\ncommand: python train.py\n" + tc.environment + path := filepath.Join(dir, "run.yaml") + require.NoError(t, os.WriteFile(path, []byte(yaml), 0o644)) + cfg, err := loadRunConfig(path) + require.NoError(t, err) + + got := missingTorchWarning(t.Context(), cfg, path) + if tc.want { + assert.Contains(t, got, "train.py imports torch") + assert.Contains(t, got, "environment.version: databricks_ai_v6") + } else { + assert.Empty(t, got) + } + }) + } +}