diff --git a/.nextchanges/air/warn-missing-torch.md b/.nextchanges/air/warn-missing-torch.md new file mode 100644 index 00000000000..d2dda6ae477 --- /dev/null +++ b/.nextchanges/air/warn-missing-torch.md @@ -0,0 +1 @@ +* `air run` (including `--dry-run`) now warns before submitting when the `code_source` imports `torch` but PyTorch is not declared: the environment is the Standard environment (version 5 or later) with no `torch` dependency, so the run may fail after it waits for a GPU. ([#7656](https://github.com/databricks/cli/pull/7656)) diff --git a/acceptance/air/run/output.txt b/acceptance/air/run/output.txt index 7615cf8f8f0..40f4475a0bc 100644 --- a/acceptance/air/run/output.txt +++ b/acceptance/air/run/output.txt @@ -87,6 +87,19 @@ Exit code: 1 Dry run: code_source snapshot would include 2 file(s). Dry run: configuration for "smoke-test" is valid; not submitting. +=== torch import on the Standard environment warns before submitting + +>>> [CLI] air run -f torch-standard.yaml --dry-run +Warning: train.py contains a torch import, but Standard environment 6 does not include PyTorch and no torch dependency was detected, so this code path may fail. Consider using databricks_ai_v6 or adding torch to dependencies. +Dry run: code_source snapshot would include 1 file(s). +Dry run: configuration for "smoke-test" is valid; not submitting. + +=== the Databricks AI environment provides torch, so no warning + +>>> [CLI] air run -f torch-databricks-ai.yaml --dry-run +Dry run: code_source snapshot would include 1 file(s). +Dry run: configuration for "smoke-test" is valid; not submitting. + === git.remote is rejected >>> [CLI] air run -f git-remote.yaml --dry-run diff --git a/acceptance/air/run/script b/acceptance/air/run/script index 305848f9eb7..29447bb0c13 100644 --- a/acceptance/air/run/script +++ b/acceptance/air/run/script @@ -50,6 +50,12 @@ cp non-git-subdir.yaml proj/sub/workload.yaml (cd proj/sub && trace $CLI air run -f workload.yaml --dry-run) rm -fr proj +title "torch import on the Standard environment warns before submitting" +trace $CLI air run -f torch-standard.yaml --dry-run + +title "the Databricks AI environment provides torch, so no warning" +trace $CLI air run -f torch-databricks-ai.yaml --dry-run + title "git.remote is rejected" errcode trace $CLI air run -f git-remote.yaml --dry-run diff --git a/acceptance/air/run/torch-databricks-ai.yaml b/acceptance/air/run/torch-databricks-ai.yaml new file mode 100644 index 00000000000..2898a094660 --- /dev/null +++ b/acceptance/air/run/torch-databricks-ai.yaml @@ -0,0 +1,9 @@ +experiment_name: smoke-test +command: python "$CODE_SOURCE_PATH/train.py" +compute: + accelerator_type: GPU_1xA10 + num_accelerators: 1 +environment: + version: databricks_ai_v6 +code_source: + root_path: torch-src diff --git a/acceptance/air/run/torch-src/train.py b/acceptance/air/run/torch-src/train.py new file mode 100644 index 00000000000..8fcea7b65b3 --- /dev/null +++ b/acceptance/air/run/torch-src/train.py @@ -0,0 +1,3 @@ +import torch + +print(torch.cuda.is_available()) diff --git a/acceptance/air/run/torch-standard.yaml b/acceptance/air/run/torch-standard.yaml new file mode 100644 index 00000000000..840b366740d --- /dev/null +++ b/acceptance/air/run/torch-standard.yaml @@ -0,0 +1,7 @@ +experiment_name: smoke-test +command: python "$CODE_SOURCE_PATH/train.py" +compute: + accelerator_type: GPU_1xA10 + num_accelerators: 1 +code_source: + root_path: torch-src diff --git a/cmd/air/run.go b/cmd/air/run.go index 7551d1b661a..df2b54c8991 100644 --- a/cmd/air/run.go +++ b/cmd/air/run.go @@ -165,6 +165,10 @@ databricks-ai-runtime skill: return err } + if msg := missingTorchWarning(ctx, cfg, file); msg != "" { + cmdio.LogString(ctx, msg) + } + if dryRun { // Local and credential-free, so it runs before workspace validation. var snapshotFileCount *int diff --git a/cmd/air/torchcheck.go b/cmd/air/torchcheck.go new file mode 100644 index 00000000000..81bd78c4a94 --- /dev/null +++ b/cmd/air/torchcheck.go @@ -0,0 +1,174 @@ +package aircmd + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + "sync" + + "github.com/databricks/cli/libs/log" +) + +// The Standard environment dropped PyTorch in this version. +const firstStandardVersionWithoutTorch = 5 + +// Bounds keep the check cheap on large code sources; it is a hint, not a guarantee. +const ( + torchScanMaxFiles = 200 + torchScanMaxFileBytes = 1 << 20 +) + +var torchImportRe = sync.OnceValue(func() *regexp.Regexp { + return regexp.MustCompile(`^\s*(import\s+torch\b|from\s+torch\b)`) +}) + +// missingTorchWarning returns a warning when the code_source imports torch but +// the environment can't provide it, or "" when it can or the check doesn't apply. +// A run like that fails only after it waits for a GPU and then retries, so +// catching it locally saves the whole queue-and-retry cycle. +func missingTorchWarning(ctx context.Context, cfg *runConfig, configPath string) string { + if len(cfg.Containers) > 0 || cfg.CodeSource == nil || cfg.CodeSource.Snapshot == nil { + return "" + } + if env := cfg.Environment; env != nil { + if env.UnityCatalogImage != "" { + return "" + } + if slices.ContainsFunc(env.Dependencies.list, mayProvideTorch) { + return "" + } + } + // Resolve the version the way submission does, so a DATABRICKS_DL_RUNTIME_IMAGE + // override is honored when environment.version is unset. + cfgVersion, _ := cfg.runtimeVersion() + version := strings.ToLower(dlRuntimeImage(ctx, cfgVersion)) + if strings.HasPrefix(version, databricksAIPrefix) { + return "" + } + if major, err := strconv.Atoi(version); err != nil || major < firstStandardVersionWithoutTorch { + return "" + } + + snap := cfg.CodeSource.Snapshot + repoPath, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath)) + if err != nil { + return "" + } + plan, err := resolveSnapshotPlan(ctx, newGitRepo(repoPath), snap.Git, snap.IncludePaths) + if err != nil { + return "" + } + file, err := findTorchImport(ctx, repoPath, plan) + if err != nil { + log.Debugf(ctx, "air: skipping torch import check: %v", err) + return "" + } + if file == "" { + return "" + } + return fmt.Sprintf("Warning: %s contains a torch import, but Standard environment %s does not include PyTorch and no torch dependency was detected, so this code path may fail. Consider using %s%s or adding torch to dependencies.", + file, version, databricksAIPrefix, version) +} + +// mayProvideTorch reports whether an environment.dependencies entry could install +// PyTorch: torch itself, or a requirements file or editable package that the +// check can't see into. Options that only configure package lookup, such as +// --extra-index-url, install nothing. +func mayProvideTorch(dep string) bool { + dep = strings.TrimSpace(dep) + if !strings.HasPrefix(dep, "-") { + return strings.HasPrefix(strings.ToLower(dep), "torch") + } + opt := strings.Fields(dep)[0] + // Short options may be glued to their value, as in -rrequirements.txt. + if strings.HasPrefix(opt, "-r") || strings.HasPrefix(opt, "-e") { + return true + } + opt, _, _ = strings.Cut(opt, "=") + return opt == "--requirement" || opt == "--editable" +} + +// findTorchImport returns the snapshot-relative path of a .py file that imports +// torch, or "". It reads the same source submission packages: the pinned commit +// for git.commit and git.branch, else the working tree. +func findTorchImport(ctx context.Context, repoPath string, plan snapshotPlan) (string, error) { + if plan.mode == modeGitArchive { + return findTorchImportAtCommit(ctx, newGitRepo(repoPath), plan) + } + files, err := snapshotFiles(ctx, repoPath, plan.includePaths, plan.isGitRepo) + if err != nil { + return "", err + } + scanned := 0 + for _, f := range files { + if !strings.HasSuffix(f.rel, ".py") || f.size > torchScanMaxFileBytes { + continue + } + if scanned++; scanned > torchScanMaxFiles { + break + } + if importsTorch(filepath.Join(repoPath, f.rel)) { + return filepath.ToSlash(f.rel), nil + } + } + return "", nil +} + +// torchImportERE is torchImportRe in POSIX ERE syntax for git grep. +const torchImportERE = `^[[:space:]]*(import[[:space:]]+torch|from[[:space:]]+torch)([^[:alnum:]_]|$)` + +// findTorchImportAtCommit greps the .py files under the snapshot root (and +// include_paths) at plan.commitSHA. +func findTorchImportAtCommit(ctx context.Context, g gitRepo, plan snapshotPlan) (string, error) { + bases := []string{plan.subtreePrefix} + if len(plan.includePaths) > 0 { + bases = nil + for _, p := range plan.includePaths { + bases = append(bases, path.Join(plan.subtreePrefix, filepath.ToSlash(p))) + } + } + args := []string{"grep", "-l", "-I", "--full-name", "-E", torchImportERE, plan.commitSHA, "--"} + for _, b := range bases { + args = append(args, ":(top,glob)"+path.Join(b, "**", "*.py")) + } + out, err := g.run(ctx, args...) + if err != nil { + // git grep exits 1 when nothing matches. + if exitErr, ok := errors.AsType[*exec.ExitError](err); ok && exitErr.ExitCode() == 1 { + return "", nil + } + return "", err + } + first, _, _ := strings.Cut(strings.TrimSpace(out), "\n") + // Output is ":". + _, file, _ := strings.Cut(first, ":") + if rel, ok := strings.CutPrefix(file, plan.subtreePrefix); ok && plan.subtreePrefix != "" { + file = strings.TrimPrefix(rel, "/") + } + return file, nil +} + +// importsTorch reports whether a top-level import line in the file imports torch. +func importsTorch(path string) bool { + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + if torchImportRe().MatchString(scanner.Text()) { + return true + } + } + return false +} diff --git a/cmd/air/torchcheck_test.go b/cmd/air/torchcheck_test.go new file mode 100644 index 00000000000..76e1ef803d9 --- /dev/null +++ b/cmd/air/torchcheck_test.go @@ -0,0 +1,110 @@ +package aircmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMissingTorchWarning(t *testing.T) { + const torchScript = "import os\nimport torch.nn as nn\n" + cases := []struct { + name string + environment string + script string + want bool + }{ + {"default environment", "", torchScript, true}, + {"standard 6", "environment:\n version: 6\n", torchScript, true}, + {"from torch import", "", "from torch import nn\n", true}, + {"other deps only", "environment:\n version: 6\n dependencies:\n - numpy\n", torchScript, true}, + {"databricks ai environment", "environment:\n version: databricks_ai_v6\n", torchScript, false}, + {"torch dependency", "environment:\n dependencies:\n - torch==2.6.0\n", torchScript, false}, + {"requirements file", "environment:\n dependencies:\n - -r requirements.txt\n", torchScript, false}, + {"long requirements option", "environment:\n dependencies:\n - --requirement=requirements.txt\n", torchScript, false}, + {"editable package", "environment:\n dependencies:\n - -e ./mypkg\n", torchScript, false}, + {"index option installs nothing", "environment:\n dependencies:\n - --extra-index-url https://download.pytorch.org/whl/cu128\n - numpy\n", torchScript, true}, + {"standard 4 includes torch", "environment:\n version: 4\n", torchScript, false}, + {"custom image", "environment:\n unity_catalog_image: main.s.img:v1\n", torchScript, false}, + {"no torch import", "", "import torchvision_stub\n# import torch\n", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "train.py"), []byte(tc.script), 0o644)) + yaml := "experiment_name: e\ncompute:\n num_accelerators: 1\n accelerator_type: GPU_1xA10\ncode_source:\n root_path: .\ncommand: python train.py\n" + tc.environment + path := filepath.Join(dir, "run.yaml") + require.NoError(t, os.WriteFile(path, []byte(yaml), 0o644)) + cfg, err := loadRunConfig(path) + require.NoError(t, err) + + got := missingTorchWarning(t.Context(), cfg, path) + if tc.want { + assert.Contains(t, got, "train.py contains a torch import, but Standard environment 6 does not include PyTorch") + assert.Contains(t, got, "may fail") + assert.Contains(t, got, "Consider using databricks_ai_v6") + } else { + assert.Empty(t, got) + } + }) + } +} + +func writeTorchRunConfig(t *testing.T, dir, codeSource string) *runConfig { + t.Helper() + yaml := "experiment_name: e\ncompute:\n num_accelerators: 1\n accelerator_type: GPU_1xA10\ncode_source:\n root_path: .\n" + codeSource + "command: python train.py\n" + path := filepath.Join(dir, "run.yaml") + require.NoError(t, os.WriteFile(path, []byte(yaml), 0o644)) + cfg, err := loadRunConfig(path) + require.NoError(t, err) + return cfg +} + +func TestMissingTorchWarningHonorsRuntimeImageOverride(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "train.py"), []byte("import torch\n"), 0o644)) + cfg := writeTorchRunConfig(t, dir, "") + path := filepath.Join(dir, "run.yaml") + + // Submission uses DATABRICKS_DL_RUNTIME_IMAGE when environment.version is unset. + assert.Empty(t, missingTorchWarning(env.Set(t.Context(), dlRuntimeImageEnv, "databricks_ai_v6"), cfg, path)) + assert.Contains(t, missingTorchWarning(env.Set(t.Context(), dlRuntimeImageEnv, "CLIENT-GPU-6"), cfg, path), "Standard environment 6 does not include PyTorch") +} + +func TestMissingTorchWarningScansPinnedCommit(t *testing.T) { + t.Run("import only in the commit", func(t *testing.T) { + repo := newTestRepo(t) + writeRepoFile(t, repo, "src/train.py", "import torch\n") + sha := commitAll(t, repo, "torch") + // The working tree no longer imports torch, but the pinned commit does. + writeRepoFile(t, repo, "src/train.py", "print('hi')\n") + cfg := writeTorchRunConfig(t, repo, " git:\n commit: "+sha+"\n") + + assert.Contains(t, missingTorchWarning(t.Context(), cfg, filepath.Join(repo, "run.yaml")), "src/train.py contains a torch import") + }) + + t.Run("import only in the working tree", func(t *testing.T) { + repo := newTestRepo(t) + writeRepoFile(t, repo, "src/train.py", "print('hi')\n") + sha := commitAll(t, repo, "no torch") + writeRepoFile(t, repo, "src/train.py", "import torch\n") + cfg := writeTorchRunConfig(t, repo, " git:\n commit: "+sha+"\n") + + assert.Empty(t, missingTorchWarning(t.Context(), cfg, filepath.Join(repo, "run.yaml"))) + }) + + t.Run("subdirectory root reports a root-relative path", func(t *testing.T) { + repo := newTestRepo(t) + writeRepoFile(t, repo, "proj/model/train.py", "from torch import nn\n") + writeRepoFile(t, repo, "other/x.py", "import torch\n") + sha := commitAll(t, repo, "torch") + dir := filepath.Join(repo, "proj") + cfg := writeTorchRunConfig(t, dir, " git:\n commit: "+sha+"\n include_paths:\n - model\n") + + assert.Contains(t, missingTorchWarning(t.Context(), cfg, filepath.Join(dir, "run.yaml")), "Warning: model/train.py contains a torch import") + }) +}