From 1aaed530a3dea8857eb58e3f32ffa8415934703a Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Sun, 15 Nov 2026 05:28:21 +0000 Subject: [PATCH 1/6] air run: warn when code imports torch on the Standard environment The Standard environment has not included PyTorch since version 5. A workload that imports torch there passes --dry-run, then fails only after it waits for a GPU, and the default retries repeat the failure. Agents hit this in most CAJ trials on the released CLI. Scan the code_source's .py files and warn before submitting, pointing at databricks_ai_v6 or a torch dependency. Co-authored-by: Isaac (cherry picked from commit 6d641a4658e301209c20136d4eb09303d4568125) --- .nextchanges/air/warn-missing-torch.md | 1 + acceptance/air/run/output.txt | 13 +++ acceptance/air/run/script | 6 ++ acceptance/air/run/torch-databricks-ai.yaml | 9 ++ acceptance/air/run/torch-src/train.py | 3 + acceptance/air/run/torch-standard.yaml | 7 ++ cmd/air/run.go | 4 + cmd/air/torchcheck.go | 104 ++++++++++++++++++++ cmd/air/torchcheck_test.go | 50 ++++++++++ 9 files changed, 197 insertions(+) create mode 100644 .nextchanges/air/warn-missing-torch.md create mode 100644 acceptance/air/run/torch-databricks-ai.yaml create mode 100644 acceptance/air/run/torch-src/train.py create mode 100644 acceptance/air/run/torch-standard.yaml create mode 100644 cmd/air/torchcheck.go create mode 100644 cmd/air/torchcheck_test.go diff --git a/.nextchanges/air/warn-missing-torch.md b/.nextchanges/air/warn-missing-torch.md new file mode 100644 index 00000000000..99fbc5d79b7 --- /dev/null +++ b/.nextchanges/air/warn-missing-torch.md @@ -0,0 +1 @@ +* `air run` (including `--dry-run`) now warns before submitting when the `code_source` imports `torch` but the environment is the Standard environment (version 5 or later) without `torch` in `dependencies`, instead of letting the run fail after it waits for a GPU. diff --git a/acceptance/air/run/output.txt b/acceptance/air/run/output.txt index 7615cf8f8f0..4002dd2c4a3 100644 --- a/acceptance/air/run/output.txt +++ b/acceptance/air/run/output.txt @@ -87,6 +87,19 @@ Exit code: 1 Dry run: code_source snapshot would include 2 file(s). Dry run: configuration for "smoke-test" is valid; not submitting. +=== torch import on the Standard environment warns before submitting + +>>> [CLI] air run -f torch-standard.yaml --dry-run +Warning: train.py imports torch, but environment version 6 (Standard) does not include PyTorch, so the run would fail after waiting for a GPU. Set environment.version: databricks_ai_v6, or add torch to environment.dependencies. +Dry run: code_source snapshot would include 1 file(s). +Dry run: configuration for "smoke-test" is valid; not submitting. + +=== the Databricks AI environment provides torch, so no warning + +>>> [CLI] air run -f torch-databricks-ai.yaml --dry-run +Dry run: code_source snapshot would include 1 file(s). +Dry run: configuration for "smoke-test" is valid; not submitting. + === git.remote is rejected >>> [CLI] air run -f git-remote.yaml --dry-run diff --git a/acceptance/air/run/script b/acceptance/air/run/script index 305848f9eb7..29447bb0c13 100644 --- a/acceptance/air/run/script +++ b/acceptance/air/run/script @@ -50,6 +50,12 @@ cp non-git-subdir.yaml proj/sub/workload.yaml (cd proj/sub && trace $CLI air run -f workload.yaml --dry-run) rm -fr proj +title "torch import on the Standard environment warns before submitting" +trace $CLI air run -f torch-standard.yaml --dry-run + +title "the Databricks AI environment provides torch, so no warning" +trace $CLI air run -f torch-databricks-ai.yaml --dry-run + title "git.remote is rejected" errcode trace $CLI air run -f git-remote.yaml --dry-run diff --git a/acceptance/air/run/torch-databricks-ai.yaml b/acceptance/air/run/torch-databricks-ai.yaml new file mode 100644 index 00000000000..2898a094660 --- /dev/null +++ b/acceptance/air/run/torch-databricks-ai.yaml @@ -0,0 +1,9 @@ +experiment_name: smoke-test +command: python "$CODE_SOURCE_PATH/train.py" +compute: + accelerator_type: GPU_1xA10 + num_accelerators: 1 +environment: + version: databricks_ai_v6 +code_source: + root_path: torch-src diff --git a/acceptance/air/run/torch-src/train.py b/acceptance/air/run/torch-src/train.py new file mode 100644 index 00000000000..8fcea7b65b3 --- /dev/null +++ b/acceptance/air/run/torch-src/train.py @@ -0,0 +1,3 @@ +import torch + +print(torch.cuda.is_available()) diff --git a/acceptance/air/run/torch-standard.yaml b/acceptance/air/run/torch-standard.yaml new file mode 100644 index 00000000000..840b366740d --- /dev/null +++ b/acceptance/air/run/torch-standard.yaml @@ -0,0 +1,7 @@ +experiment_name: smoke-test +command: python "$CODE_SOURCE_PATH/train.py" +compute: + accelerator_type: GPU_1xA10 + num_accelerators: 1 +code_source: + root_path: torch-src diff --git a/cmd/air/run.go b/cmd/air/run.go index 7551d1b661a..df2b54c8991 100644 --- a/cmd/air/run.go +++ b/cmd/air/run.go @@ -165,6 +165,10 @@ databricks-ai-runtime skill: return err } + if msg := missingTorchWarning(ctx, cfg, file); msg != "" { + cmdio.LogString(ctx, msg) + } + if dryRun { // Local and credential-free, so it runs before workspace validation. var snapshotFileCount *int diff --git a/cmd/air/torchcheck.go b/cmd/air/torchcheck.go new file mode 100644 index 00000000000..36a061e921a --- /dev/null +++ b/cmd/air/torchcheck.go @@ -0,0 +1,104 @@ +package aircmd + +import ( + "bufio" + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + + "github.com/databricks/cli/libs/log" +) + +// The Standard environment dropped PyTorch in this version. +const firstStandardVersionWithoutTorch = 5 + +// Bounds keep the check cheap on large code sources; it is a hint, not a guarantee. +const ( + torchScanMaxFiles = 200 + torchScanMaxFileBytes = 1 << 20 +) + +var torchImportRe = sync.OnceValue(func() *regexp.Regexp { + return regexp.MustCompile(`^\s*(import\s+torch\b|from\s+torch\b)`) +}) + +// missingTorchWarning returns a warning when the code_source imports torch but +// the environment can't provide it, or "" when it can or the check doesn't apply. +// A run like that fails only after it waits for a GPU and then retries, so +// catching it locally saves the whole queue-and-retry cycle. +func missingTorchWarning(ctx context.Context, cfg *runConfig, configPath string) string { + if len(cfg.Containers) > 0 || cfg.CodeSource == nil || cfg.CodeSource.Snapshot == nil { + return "" + } + version := "6" + if env := cfg.Environment; env != nil { + if env.UnityCatalogImage != "" { + return "" + } + if env.Version.set { + version = strings.ToLower(env.Version.raw) + } + for _, dep := range env.Dependencies.list { + // A requirements file or an explicit torch entry may provide it. + if strings.HasPrefix(dep, "-") || strings.HasPrefix(strings.ToLower(dep), "torch") { + return "" + } + } + } + if strings.HasPrefix(version, databricksAIPrefix) { + return "" + } + if major, err := strconv.Atoi(version); err != nil || major < firstStandardVersionWithoutTorch { + return "" + } + + snap := cfg.CodeSource.Snapshot + repoPath, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath)) + if err != nil { + return "" + } + plan, err := resolveSnapshotPlan(ctx, newGitRepo(repoPath), snap.Git, snap.IncludePaths) + if err != nil { + return "" + } + files, err := snapshotFiles(ctx, repoPath, plan.includePaths, plan.isGitRepo) + if err != nil { + log.Debugf(ctx, "air: skipping torch import check: %v", err) + return "" + } + scanned := 0 + for _, f := range files { + if !strings.HasSuffix(f.rel, ".py") || f.size > torchScanMaxFileBytes { + continue + } + if scanned++; scanned > torchScanMaxFiles { + break + } + if importsTorch(filepath.Join(repoPath, f.rel)) { + return fmt.Sprintf("Warning: %s imports torch, but environment version %s (Standard) does not include PyTorch, so the run would fail after waiting for a GPU. Set environment.version: %s6, or add torch to environment.dependencies.", + filepath.ToSlash(f.rel), version, databricksAIPrefix) + } + } + return "" +} + +// importsTorch reports whether a top-level import line in the file imports torch. +func importsTorch(path string) bool { + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + if torchImportRe().MatchString(scanner.Text()) { + return true + } + } + return false +} diff --git a/cmd/air/torchcheck_test.go b/cmd/air/torchcheck_test.go new file mode 100644 index 00000000000..5e6facc075f --- /dev/null +++ b/cmd/air/torchcheck_test.go @@ -0,0 +1,50 @@ +package aircmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMissingTorchWarning(t *testing.T) { + const torchScript = "import os\nimport torch.nn as nn\n" + cases := []struct { + name string + environment string + script string + want bool + }{ + {"default environment", "", torchScript, true}, + {"standard 6", "environment:\n version: 6\n", torchScript, true}, + {"from torch import", "", "from torch import nn\n", true}, + {"other deps only", "environment:\n version: 6\n dependencies:\n - numpy\n", torchScript, true}, + {"databricks ai environment", "environment:\n version: databricks_ai_v6\n", torchScript, false}, + {"torch dependency", "environment:\n dependencies:\n - torch==2.6.0\n", torchScript, false}, + {"requirements file", "environment:\n dependencies:\n - -r requirements.txt\n", torchScript, false}, + {"standard 4 includes torch", "environment:\n version: 4\n", torchScript, false}, + {"custom image", "environment:\n unity_catalog_image: main.s.img:v1\n", torchScript, false}, + {"no torch import", "", "import torchvision_stub\n# import torch\n", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "train.py"), []byte(tc.script), 0o644)) + yaml := "experiment_name: e\ncompute:\n num_accelerators: 1\n accelerator_type: GPU_1xA10\ncode_source:\n root_path: .\ncommand: python train.py\n" + tc.environment + path := filepath.Join(dir, "run.yaml") + require.NoError(t, os.WriteFile(path, []byte(yaml), 0o644)) + cfg, err := loadRunConfig(path) + require.NoError(t, err) + + got := missingTorchWarning(t.Context(), cfg, path) + if tc.want { + assert.Contains(t, got, "train.py imports torch") + assert.Contains(t, got, "environment.version: databricks_ai_v6") + } else { + assert.Empty(t, got) + } + }) + } +} From da55e7c10443bb3e7fc5b079d8d2f6421ecd131e Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 04:26:27 +0000 Subject: [PATCH 2/6] Add PR link to changelog fragment Co-authored-by: Isaac --- .nextchanges/air/warn-missing-torch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/air/warn-missing-torch.md b/.nextchanges/air/warn-missing-torch.md index 99fbc5d79b7..21fa5eb832a 100644 --- a/.nextchanges/air/warn-missing-torch.md +++ b/.nextchanges/air/warn-missing-torch.md @@ -1 +1 @@ -* `air run` (including `--dry-run`) now warns before submitting when the `code_source` imports `torch` but the environment is the Standard environment (version 5 or later) without `torch` in `dependencies`, instead of letting the run fail after it waits for a GPU. +* `air run` (including `--dry-run`) now warns before submitting when the `code_source` imports `torch` but the environment is the Standard environment (version 5 or later) without `torch` in `dependencies`, instead of letting the run fail after it waits for a GPU. ([#7650](https://github.com/databricks/cli/pull/7650)) From 8c50e69939b95f0c411ac925c4ce4f9c65081653 Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 05:02:49 +0000 Subject: [PATCH 3/6] Point changelog fragment at #7656 Co-authored-by: Isaac --- .nextchanges/air/warn-missing-torch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/air/warn-missing-torch.md b/.nextchanges/air/warn-missing-torch.md index 21fa5eb832a..03114204c19 100644 --- a/.nextchanges/air/warn-missing-torch.md +++ b/.nextchanges/air/warn-missing-torch.md @@ -1 +1 @@ -* `air run` (including `--dry-run`) now warns before submitting when the `code_source` imports `torch` but the environment is the Standard environment (version 5 or later) without `torch` in `dependencies`, instead of letting the run fail after it waits for a GPU. ([#7650](https://github.com/databricks/cli/pull/7650)) +* `air run` (including `--dry-run`) now warns before submitting when the `code_source` imports `torch` but the environment is the Standard environment (version 5 or later) without `torch` in `dependencies`, instead of letting the run fail after it waits for a GPU. ([#7656](https://github.com/databricks/cli/pull/7656)) From f0ad5e06823f13497639f2b51e98cbc39a0d1fb4 Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Mon, 16 Nov 2026 21:19:14 +0000 Subject: [PATCH 4/6] air run: match submission when checking for a missing torch Address review: resolve the runtime through dlRuntimeImage so a DATABRICKS_DL_RUNTIME_IMAGE override is honored; scan the pinned commit for git.commit and git.branch instead of the working tree; only treat requirement and editable options as possibly providing torch, not lookup-only options like --extra-index-url; and phrase the warning as a heuristic. Co-authored-by: Isaac --- .nextchanges/air/warn-missing-torch.md | 2 +- acceptance/air/run/output.txt | 2 +- cmd/air/torchcheck.go | 96 ++++++++++++++++++++++---- cmd/air/torchcheck_test.go | 62 ++++++++++++++++- 4 files changed, 146 insertions(+), 16 deletions(-) diff --git a/.nextchanges/air/warn-missing-torch.md b/.nextchanges/air/warn-missing-torch.md index 03114204c19..d2dda6ae477 100644 --- a/.nextchanges/air/warn-missing-torch.md +++ b/.nextchanges/air/warn-missing-torch.md @@ -1 +1 @@ -* `air run` (including `--dry-run`) now warns before submitting when the `code_source` imports `torch` but the environment is the Standard environment (version 5 or later) without `torch` in `dependencies`, instead of letting the run fail after it waits for a GPU. ([#7656](https://github.com/databricks/cli/pull/7656)) +* `air run` (including `--dry-run`) now warns before submitting when the `code_source` imports `torch` but PyTorch is not declared: the environment is the Standard environment (version 5 or later) with no `torch` dependency, so the run may fail after it waits for a GPU. ([#7656](https://github.com/databricks/cli/pull/7656)) diff --git a/acceptance/air/run/output.txt b/acceptance/air/run/output.txt index 4002dd2c4a3..7e751e90f2a 100644 --- a/acceptance/air/run/output.txt +++ b/acceptance/air/run/output.txt @@ -90,7 +90,7 @@ Dry run: configuration for "smoke-test" is valid; not submitting. === torch import on the Standard environment warns before submitting >>> [CLI] air run -f torch-standard.yaml --dry-run -Warning: train.py imports torch, but environment version 6 (Standard) does not include PyTorch, so the run would fail after waiting for a GPU. Set environment.version: databricks_ai_v6, or add torch to environment.dependencies. +Warning: train.py imports torch, but PyTorch is not declared: environment version 6 (Standard) does not include it, so the run may fail after waiting for a GPU. Set environment.version: databricks_ai_v6, or add torch to environment.dependencies. Dry run: code_source snapshot would include 1 file(s). Dry run: configuration for "smoke-test" is valid; not submitting. diff --git a/cmd/air/torchcheck.go b/cmd/air/torchcheck.go index 36a061e921a..4d3340b8078 100644 --- a/cmd/air/torchcheck.go +++ b/cmd/air/torchcheck.go @@ -3,10 +3,14 @@ package aircmd import ( "bufio" "context" + "errors" "fmt" "os" + "os/exec" + "path" "path/filepath" "regexp" + "slices" "strconv" "strings" "sync" @@ -35,21 +39,18 @@ func missingTorchWarning(ctx context.Context, cfg *runConfig, configPath string) if len(cfg.Containers) > 0 || cfg.CodeSource == nil || cfg.CodeSource.Snapshot == nil { return "" } - version := "6" if env := cfg.Environment; env != nil { if env.UnityCatalogImage != "" { return "" } - if env.Version.set { - version = strings.ToLower(env.Version.raw) - } - for _, dep := range env.Dependencies.list { - // A requirements file or an explicit torch entry may provide it. - if strings.HasPrefix(dep, "-") || strings.HasPrefix(strings.ToLower(dep), "torch") { - return "" - } + if slices.ContainsFunc(env.Dependencies.list, mayProvideTorch) { + return "" } } + // Resolve the version the way submission does, so a DATABRICKS_DL_RUNTIME_IMAGE + // override is honored when environment.version is unset. + cfgVersion, _ := cfg.runtimeVersion() + version := strings.ToLower(dlRuntimeImage(ctx, cfgVersion)) if strings.HasPrefix(version, databricksAIPrefix) { return "" } @@ -66,11 +67,47 @@ func missingTorchWarning(ctx context.Context, cfg *runConfig, configPath string) if err != nil { return "" } - files, err := snapshotFiles(ctx, repoPath, plan.includePaths, plan.isGitRepo) + file, err := findTorchImport(ctx, repoPath, plan) if err != nil { log.Debugf(ctx, "air: skipping torch import check: %v", err) return "" } + if file == "" { + return "" + } + return fmt.Sprintf("Warning: %s imports torch, but PyTorch is not declared: environment version %s (Standard) does not include it, so the run may fail after waiting for a GPU. Set environment.version: %s6, or add torch to environment.dependencies.", + file, version, databricksAIPrefix) +} + +// mayProvideTorch reports whether an environment.dependencies entry could install +// PyTorch: torch itself, or a requirements file or editable package that the +// check can't see into. Options that only configure package lookup, such as +// --extra-index-url, install nothing. +func mayProvideTorch(dep string) bool { + dep = strings.TrimSpace(dep) + if !strings.HasPrefix(dep, "-") { + return strings.HasPrefix(strings.ToLower(dep), "torch") + } + opt := strings.Fields(dep)[0] + // Short options may be glued to their value, as in -rrequirements.txt. + if strings.HasPrefix(opt, "-r") || strings.HasPrefix(opt, "-e") { + return true + } + opt, _, _ = strings.Cut(opt, "=") + return opt == "--requirement" || opt == "--editable" +} + +// findTorchImport returns the snapshot-relative path of a .py file that imports +// torch, or "". It reads the same source submission packages: the pinned commit +// for git.commit and git.branch, else the working tree. +func findTorchImport(ctx context.Context, repoPath string, plan snapshotPlan) (string, error) { + if plan.mode == modeGitArchive { + return findTorchImportAtCommit(ctx, newGitRepo(repoPath), plan) + } + files, err := snapshotFiles(ctx, repoPath, plan.includePaths, plan.isGitRepo) + if err != nil { + return "", err + } scanned := 0 for _, f := range files { if !strings.HasSuffix(f.rel, ".py") || f.size > torchScanMaxFileBytes { @@ -80,11 +117,44 @@ func missingTorchWarning(ctx context.Context, cfg *runConfig, configPath string) break } if importsTorch(filepath.Join(repoPath, f.rel)) { - return fmt.Sprintf("Warning: %s imports torch, but environment version %s (Standard) does not include PyTorch, so the run would fail after waiting for a GPU. Set environment.version: %s6, or add torch to environment.dependencies.", - filepath.ToSlash(f.rel), version, databricksAIPrefix) + return filepath.ToSlash(f.rel), nil + } + } + return "", nil +} + +// torchImportERE is torchImportRe in POSIX ERE syntax for git grep. +const torchImportERE = `^[[:space:]]*(import[[:space:]]+torch|from[[:space:]]+torch)([^[:alnum:]_]|$)` + +// findTorchImportAtCommit greps the .py files under the snapshot root (and +// include_paths) at plan.commitSHA. +func findTorchImportAtCommit(ctx context.Context, g gitRepo, plan snapshotPlan) (string, error) { + bases := []string{plan.subtreePrefix} + if len(plan.includePaths) > 0 { + bases = nil + for _, p := range plan.includePaths { + bases = append(bases, path.Join(plan.subtreePrefix, filepath.ToSlash(p))) + } + } + args := []string{"grep", "-l", "-I", "--full-name", "-E", torchImportERE, plan.commitSHA, "--"} + for _, b := range bases { + args = append(args, ":(top,glob)"+path.Join(b, "**", "*.py")) + } + out, err := g.run(ctx, args...) + if err != nil { + // git grep exits 1 when nothing matches. + if exitErr, ok := errors.AsType[*exec.ExitError](err); ok && exitErr.ExitCode() == 1 { + return "", nil } + return "", err + } + first, _, _ := strings.Cut(strings.TrimSpace(out), "\n") + // Output is ":". + _, file, _ := strings.Cut(first, ":") + if rel, ok := strings.CutPrefix(file, plan.subtreePrefix); ok && plan.subtreePrefix != "" { + file = strings.TrimPrefix(rel, "/") } - return "" + return file, nil } // importsTorch reports whether a top-level import line in the file imports torch. diff --git a/cmd/air/torchcheck_test.go b/cmd/air/torchcheck_test.go index 5e6facc075f..90f07ea5942 100644 --- a/cmd/air/torchcheck_test.go +++ b/cmd/air/torchcheck_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "testing" + "github.com/databricks/cli/libs/env" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -24,6 +25,9 @@ func TestMissingTorchWarning(t *testing.T) { {"databricks ai environment", "environment:\n version: databricks_ai_v6\n", torchScript, false}, {"torch dependency", "environment:\n dependencies:\n - torch==2.6.0\n", torchScript, false}, {"requirements file", "environment:\n dependencies:\n - -r requirements.txt\n", torchScript, false}, + {"long requirements option", "environment:\n dependencies:\n - --requirement=requirements.txt\n", torchScript, false}, + {"editable package", "environment:\n dependencies:\n - -e ./mypkg\n", torchScript, false}, + {"index option installs nothing", "environment:\n dependencies:\n - --extra-index-url https://download.pytorch.org/whl/cu128\n - numpy\n", torchScript, true}, {"standard 4 includes torch", "environment:\n version: 4\n", torchScript, false}, {"custom image", "environment:\n unity_catalog_image: main.s.img:v1\n", torchScript, false}, {"no torch import", "", "import torchvision_stub\n# import torch\n", false}, @@ -40,7 +44,8 @@ func TestMissingTorchWarning(t *testing.T) { got := missingTorchWarning(t.Context(), cfg, path) if tc.want { - assert.Contains(t, got, "train.py imports torch") + assert.Contains(t, got, "train.py imports torch, but PyTorch is not declared") + assert.Contains(t, got, "may fail") assert.Contains(t, got, "environment.version: databricks_ai_v6") } else { assert.Empty(t, got) @@ -48,3 +53,58 @@ func TestMissingTorchWarning(t *testing.T) { }) } } + +func writeTorchRunConfig(t *testing.T, dir, codeSource string) *runConfig { + t.Helper() + yaml := "experiment_name: e\ncompute:\n num_accelerators: 1\n accelerator_type: GPU_1xA10\ncode_source:\n root_path: .\n" + codeSource + "command: python train.py\n" + path := filepath.Join(dir, "run.yaml") + require.NoError(t, os.WriteFile(path, []byte(yaml), 0o644)) + cfg, err := loadRunConfig(path) + require.NoError(t, err) + return cfg +} + +func TestMissingTorchWarningHonorsRuntimeImageOverride(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "train.py"), []byte("import torch\n"), 0o644)) + cfg := writeTorchRunConfig(t, dir, "") + path := filepath.Join(dir, "run.yaml") + + // Submission uses DATABRICKS_DL_RUNTIME_IMAGE when environment.version is unset. + assert.Empty(t, missingTorchWarning(env.Set(t.Context(), dlRuntimeImageEnv, "databricks_ai_v6"), cfg, path)) + assert.Contains(t, missingTorchWarning(env.Set(t.Context(), dlRuntimeImageEnv, "CLIENT-GPU-6"), cfg, path), "environment version 6 (Standard)") +} + +func TestMissingTorchWarningScansPinnedCommit(t *testing.T) { + t.Run("import only in the commit", func(t *testing.T) { + repo := newTestRepo(t) + writeRepoFile(t, repo, "src/train.py", "import torch\n") + sha := commitAll(t, repo, "torch") + // The working tree no longer imports torch, but the pinned commit does. + writeRepoFile(t, repo, "src/train.py", "print('hi')\n") + cfg := writeTorchRunConfig(t, repo, " git:\n commit: "+sha+"\n") + + assert.Contains(t, missingTorchWarning(t.Context(), cfg, filepath.Join(repo, "run.yaml")), "src/train.py imports torch") + }) + + t.Run("import only in the working tree", func(t *testing.T) { + repo := newTestRepo(t) + writeRepoFile(t, repo, "src/train.py", "print('hi')\n") + sha := commitAll(t, repo, "no torch") + writeRepoFile(t, repo, "src/train.py", "import torch\n") + cfg := writeTorchRunConfig(t, repo, " git:\n commit: "+sha+"\n") + + assert.Empty(t, missingTorchWarning(t.Context(), cfg, filepath.Join(repo, "run.yaml"))) + }) + + t.Run("subdirectory root reports a root-relative path", func(t *testing.T) { + repo := newTestRepo(t) + writeRepoFile(t, repo, "proj/model/train.py", "from torch import nn\n") + writeRepoFile(t, repo, "other/x.py", "import torch\n") + sha := commitAll(t, repo, "torch") + dir := filepath.Join(repo, "proj") + cfg := writeTorchRunConfig(t, dir, " git:\n commit: "+sha+"\n include_paths:\n - model\n") + + assert.Contains(t, missingTorchWarning(t.Context(), cfg, filepath.Join(dir, "run.yaml")), "Warning: model/train.py imports torch") + }) +} From 8063d0e7cab856c871772b1f0b5c51cf5b894322 Mon Sep 17 00:00:00 2001 From: caroline-db Date: Tue, 17 Nov 2026 00:16:55 -0500 Subject: [PATCH 5/6] Update cmd/air/torchcheck.go --- cmd/air/torchcheck.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/air/torchcheck.go b/cmd/air/torchcheck.go index 4d3340b8078..81bd78c4a94 100644 --- a/cmd/air/torchcheck.go +++ b/cmd/air/torchcheck.go @@ -75,8 +75,8 @@ func missingTorchWarning(ctx context.Context, cfg *runConfig, configPath string) if file == "" { return "" } - return fmt.Sprintf("Warning: %s imports torch, but PyTorch is not declared: environment version %s (Standard) does not include it, so the run may fail after waiting for a GPU. Set environment.version: %s6, or add torch to environment.dependencies.", - file, version, databricksAIPrefix) + return fmt.Sprintf("Warning: %s contains a torch import, but Standard environment %s does not include PyTorch and no torch dependency was detected, so this code path may fail. Consider using %s%s or adding torch to dependencies.", + file, version, databricksAIPrefix, version) } // mayProvideTorch reports whether an environment.dependencies entry could install From 6f6ad3f971d7883ad0b382ef25c0c76d196ea037 Mon Sep 17 00:00:00 2001 From: Anshul Gupta Date: Tue, 17 Nov 2026 05:18:09 +0000 Subject: [PATCH 6/6] air run: update tests for the new torch warning wording Co-authored-by: Isaac --- acceptance/air/run/output.txt | 2 +- cmd/air/torchcheck_test.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acceptance/air/run/output.txt b/acceptance/air/run/output.txt index 7e751e90f2a..40f4475a0bc 100644 --- a/acceptance/air/run/output.txt +++ b/acceptance/air/run/output.txt @@ -90,7 +90,7 @@ Dry run: configuration for "smoke-test" is valid; not submitting. === torch import on the Standard environment warns before submitting >>> [CLI] air run -f torch-standard.yaml --dry-run -Warning: train.py imports torch, but PyTorch is not declared: environment version 6 (Standard) does not include it, so the run may fail after waiting for a GPU. Set environment.version: databricks_ai_v6, or add torch to environment.dependencies. +Warning: train.py contains a torch import, but Standard environment 6 does not include PyTorch and no torch dependency was detected, so this code path may fail. Consider using databricks_ai_v6 or adding torch to dependencies. Dry run: code_source snapshot would include 1 file(s). Dry run: configuration for "smoke-test" is valid; not submitting. diff --git a/cmd/air/torchcheck_test.go b/cmd/air/torchcheck_test.go index 90f07ea5942..76e1ef803d9 100644 --- a/cmd/air/torchcheck_test.go +++ b/cmd/air/torchcheck_test.go @@ -44,9 +44,9 @@ func TestMissingTorchWarning(t *testing.T) { got := missingTorchWarning(t.Context(), cfg, path) if tc.want { - assert.Contains(t, got, "train.py imports torch, but PyTorch is not declared") + assert.Contains(t, got, "train.py contains a torch import, but Standard environment 6 does not include PyTorch") assert.Contains(t, got, "may fail") - assert.Contains(t, got, "environment.version: databricks_ai_v6") + assert.Contains(t, got, "Consider using databricks_ai_v6") } else { assert.Empty(t, got) } @@ -72,7 +72,7 @@ func TestMissingTorchWarningHonorsRuntimeImageOverride(t *testing.T) { // Submission uses DATABRICKS_DL_RUNTIME_IMAGE when environment.version is unset. assert.Empty(t, missingTorchWarning(env.Set(t.Context(), dlRuntimeImageEnv, "databricks_ai_v6"), cfg, path)) - assert.Contains(t, missingTorchWarning(env.Set(t.Context(), dlRuntimeImageEnv, "CLIENT-GPU-6"), cfg, path), "environment version 6 (Standard)") + assert.Contains(t, missingTorchWarning(env.Set(t.Context(), dlRuntimeImageEnv, "CLIENT-GPU-6"), cfg, path), "Standard environment 6 does not include PyTorch") } func TestMissingTorchWarningScansPinnedCommit(t *testing.T) { @@ -84,7 +84,7 @@ func TestMissingTorchWarningScansPinnedCommit(t *testing.T) { writeRepoFile(t, repo, "src/train.py", "print('hi')\n") cfg := writeTorchRunConfig(t, repo, " git:\n commit: "+sha+"\n") - assert.Contains(t, missingTorchWarning(t.Context(), cfg, filepath.Join(repo, "run.yaml")), "src/train.py imports torch") + assert.Contains(t, missingTorchWarning(t.Context(), cfg, filepath.Join(repo, "run.yaml")), "src/train.py contains a torch import") }) t.Run("import only in the working tree", func(t *testing.T) { @@ -105,6 +105,6 @@ func TestMissingTorchWarningScansPinnedCommit(t *testing.T) { dir := filepath.Join(repo, "proj") cfg := writeTorchRunConfig(t, dir, " git:\n commit: "+sha+"\n include_paths:\n - model\n") - assert.Contains(t, missingTorchWarning(t.Context(), cfg, filepath.Join(dir, "run.yaml")), "Warning: model/train.py imports torch") + assert.Contains(t, missingTorchWarning(t.Context(), cfg, filepath.Join(dir, "run.yaml")), "Warning: model/train.py contains a torch import") }) }