Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/warn-missing-torch.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* `air run` (including `--dry-run`) now warns before submitting when the `code_source` imports `torch` but PyTorch is not declared: the environment is the Standard environment (version 5 or later) with no `torch` dependency, so the run may fail after it waits for a GPU. ([#7656](https://github.com/databricks/cli/pull/7656))
13 changes: 13 additions & 0 deletions acceptance/air/run/output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,19 @@ Exit code: 1
Dry run: code_source snapshot would include 2 file(s).
Dry run: configuration for "smoke-test" is valid; not submitting.

=== torch import on the Standard environment warns before submitting

>>> [CLI] air run -f torch-standard.yaml --dry-run
Warning: train.py contains a torch import, but Standard environment 6 does not include PyTorch and no torch dependency was detected, so this code path may fail. Consider using databricks_ai_v6 or adding torch to dependencies.
Dry run: code_source snapshot would include 1 file(s).
Dry run: configuration for "smoke-test" is valid; not submitting.

=== the Databricks AI environment provides torch, so no warning

>>> [CLI] air run -f torch-databricks-ai.yaml --dry-run
Dry run: code_source snapshot would include 1 file(s).
Dry run: configuration for "smoke-test" is valid; not submitting.

=== git.remote is rejected

>>> [CLI] air run -f git-remote.yaml --dry-run
Expand Down
6 changes: 6 additions & 0 deletions acceptance/air/run/script
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,12 @@ cp non-git-subdir.yaml proj/sub/workload.yaml
(cd proj/sub && trace $CLI air run -f workload.yaml --dry-run)
rm -fr proj

title "torch import on the Standard environment warns before submitting"
trace $CLI air run -f torch-standard.yaml --dry-run

title "the Databricks AI environment provides torch, so no warning"
trace $CLI air run -f torch-databricks-ai.yaml --dry-run

title "git.remote is rejected"
errcode trace $CLI air run -f git-remote.yaml --dry-run

Expand Down
9 changes: 9 additions & 0 deletions acceptance/air/run/torch-databricks-ai.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
experiment_name: smoke-test
command: python "$CODE_SOURCE_PATH/train.py"
compute:
accelerator_type: GPU_1xA10
num_accelerators: 1
environment:
version: databricks_ai_v6
code_source:
root_path: torch-src
3 changes: 3 additions & 0 deletions acceptance/air/run/torch-src/train.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
import torch

print(torch.cuda.is_available())
7 changes: 7 additions & 0 deletions acceptance/air/run/torch-standard.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
experiment_name: smoke-test
command: python "$CODE_SOURCE_PATH/train.py"
compute:
accelerator_type: GPU_1xA10
num_accelerators: 1
code_source:
root_path: torch-src
4 changes: 4 additions & 0 deletions cmd/air/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,10 @@ databricks-ai-runtime skill:
return err
}

if msg := missingTorchWarning(ctx, cfg, file); msg != "" {
cmdio.LogString(ctx, msg)
}

if dryRun {
// Local and credential-free, so it runs before workspace validation.
var snapshotFileCount *int
Expand Down
174 changes: 174 additions & 0 deletions cmd/air/torchcheck.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
package aircmd

import (
"bufio"
"context"
"errors"
"fmt"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
"sync"

"github.com/databricks/cli/libs/log"
)

// The Standard environment dropped PyTorch in this version.
const firstStandardVersionWithoutTorch = 5

// Bounds keep the check cheap on large code sources; it is a hint, not a guarantee.
const (
torchScanMaxFiles = 200
torchScanMaxFileBytes = 1 << 20
)

var torchImportRe = sync.OnceValue(func() *regexp.Regexp {
return regexp.MustCompile(`^\s*(import\s+torch\b|from\s+torch\b)`)
})

// missingTorchWarning returns a warning when the code_source imports torch but
// the environment can't provide it, or "" when it can or the check doesn't apply.
// A run like that fails only after it waits for a GPU and then retries, so
// catching it locally saves the whole queue-and-retry cycle.
func missingTorchWarning(ctx context.Context, cfg *runConfig, configPath string) string {
if len(cfg.Containers) > 0 || cfg.CodeSource == nil || cfg.CodeSource.Snapshot == nil {
return ""
}
if env := cfg.Environment; env != nil {
if env.UnityCatalogImage != "" {
return ""
}
if slices.ContainsFunc(env.Dependencies.list, mayProvideTorch) {
return ""
}
}
// Resolve the version the way submission does, so a DATABRICKS_DL_RUNTIME_IMAGE
// override is honored when environment.version is unset.
cfgVersion, _ := cfg.runtimeVersion()
version := strings.ToLower(dlRuntimeImage(ctx, cfgVersion))
if strings.HasPrefix(version, databricksAIPrefix) {
return ""
}
if major, err := strconv.Atoi(version); err != nil || major < firstStandardVersionWithoutTorch {
return ""
}

snap := cfg.CodeSource.Snapshot
repoPath, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath))
if err != nil {
return ""
}
plan, err := resolveSnapshotPlan(ctx, newGitRepo(repoPath), snap.Git, snap.IncludePaths)
if err != nil {
return ""
}
file, err := findTorchImport(ctx, repoPath, plan)
if err != nil {
log.Debugf(ctx, "air: skipping torch import check: %v", err)
return ""
}
if file == "" {
return ""
}
return fmt.Sprintf("Warning: %s contains a torch import, but Standard environment %s does not include PyTorch and no torch dependency was detected, so this code path may fail. Consider using %s%s or adding torch to dependencies.",
file, version, databricksAIPrefix, version)
}

// mayProvideTorch reports whether an environment.dependencies entry could install
// PyTorch: torch itself, or a requirements file or editable package that the
// check can't see into. Options that only configure package lookup, such as
// --extra-index-url, install nothing.
func mayProvideTorch(dep string) bool {
dep = strings.TrimSpace(dep)
if !strings.HasPrefix(dep, "-") {
return strings.HasPrefix(strings.ToLower(dep), "torch")
}
opt := strings.Fields(dep)[0]
// Short options may be glued to their value, as in -rrequirements.txt.
if strings.HasPrefix(opt, "-r") || strings.HasPrefix(opt, "-e") {
return true
}
opt, _, _ = strings.Cut(opt, "=")
return opt == "--requirement" || opt == "--editable"
}

// findTorchImport returns the snapshot-relative path of a .py file that imports
// torch, or "". It reads the same source submission packages: the pinned commit
// for git.commit and git.branch, else the working tree.
func findTorchImport(ctx context.Context, repoPath string, plan snapshotPlan) (string, error) {
if plan.mode == modeGitArchive {
return findTorchImportAtCommit(ctx, newGitRepo(repoPath), plan)
}
files, err := snapshotFiles(ctx, repoPath, plan.includePaths, plan.isGitRepo)
if err != nil {
return "", err
}
scanned := 0
for _, f := range files {
if !strings.HasSuffix(f.rel, ".py") || f.size > torchScanMaxFileBytes {
continue
}
if scanned++; scanned > torchScanMaxFiles {
break
}
if importsTorch(filepath.Join(repoPath, f.rel)) {
return filepath.ToSlash(f.rel), nil
}
}
return "", nil
}

// torchImportERE is torchImportRe in POSIX ERE syntax for git grep.
const torchImportERE = `^[[:space:]]*(import[[:space:]]+torch|from[[:space:]]+torch)([^[:alnum:]_]|$)`

// findTorchImportAtCommit greps the .py files under the snapshot root (and
// include_paths) at plan.commitSHA.
func findTorchImportAtCommit(ctx context.Context, g gitRepo, plan snapshotPlan) (string, error) {
bases := []string{plan.subtreePrefix}
if len(plan.includePaths) > 0 {
bases = nil
for _, p := range plan.includePaths {
bases = append(bases, path.Join(plan.subtreePrefix, filepath.ToSlash(p)))
}
}
args := []string{"grep", "-l", "-I", "--full-name", "-E", torchImportERE, plan.commitSHA, "--"}
for _, b := range bases {
args = append(args, ":(top,glob)"+path.Join(b, "**", "*.py"))
}
out, err := g.run(ctx, args...)
if err != nil {
// git grep exits 1 when nothing matches.
if exitErr, ok := errors.AsType[*exec.ExitError](err); ok && exitErr.ExitCode() == 1 {
return "", nil
}
return "", err
}
first, _, _ := strings.Cut(strings.TrimSpace(out), "\n")
// Output is "<commit>:<repo-relative path>".
_, file, _ := strings.Cut(first, ":")
if rel, ok := strings.CutPrefix(file, plan.subtreePrefix); ok && plan.subtreePrefix != "" {
file = strings.TrimPrefix(rel, "/")
}
return file, nil
}

// importsTorch reports whether a top-level import line in the file imports torch.
func importsTorch(path string) bool {
f, err := os.Open(path)
if err != nil {
return false
}
defer f.Close()
scanner := bufio.NewScanner(f)
for scanner.Scan() {
if torchImportRe().MatchString(scanner.Text()) {
return true
}
}
return false
}
110 changes: 110 additions & 0 deletions cmd/air/torchcheck_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package aircmd

import (
"os"
"path/filepath"
"testing"

"github.com/databricks/cli/libs/env"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestMissingTorchWarning(t *testing.T) {
const torchScript = "import os\nimport torch.nn as nn\n"
cases := []struct {
name string
environment string
script string
want bool
}{
{"default environment", "", torchScript, true},
{"standard 6", "environment:\n version: 6\n", torchScript, true},
{"from torch import", "", "from torch import nn\n", true},
{"other deps only", "environment:\n version: 6\n dependencies:\n - numpy\n", torchScript, true},
{"databricks ai environment", "environment:\n version: databricks_ai_v6\n", torchScript, false},
{"torch dependency", "environment:\n dependencies:\n - torch==2.6.0\n", torchScript, false},
{"requirements file", "environment:\n dependencies:\n - -r requirements.txt\n", torchScript, false},
{"long requirements option", "environment:\n dependencies:\n - --requirement=requirements.txt\n", torchScript, false},
{"editable package", "environment:\n dependencies:\n - -e ./mypkg\n", torchScript, false},
{"index option installs nothing", "environment:\n dependencies:\n - --extra-index-url https://download.pytorch.org/whl/cu128\n - numpy\n", torchScript, true},
{"standard 4 includes torch", "environment:\n version: 4\n", torchScript, false},
{"custom image", "environment:\n unity_catalog_image: main.s.img:v1\n", torchScript, false},
{"no torch import", "", "import torchvision_stub\n# import torch\n", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "train.py"), []byte(tc.script), 0o644))
yaml := "experiment_name: e\ncompute:\n num_accelerators: 1\n accelerator_type: GPU_1xA10\ncode_source:\n root_path: .\ncommand: python train.py\n" + tc.environment
path := filepath.Join(dir, "run.yaml")
require.NoError(t, os.WriteFile(path, []byte(yaml), 0o644))
cfg, err := loadRunConfig(path)
require.NoError(t, err)

got := missingTorchWarning(t.Context(), cfg, path)
if tc.want {
assert.Contains(t, got, "train.py contains a torch import, but Standard environment 6 does not include PyTorch")
assert.Contains(t, got, "may fail")
assert.Contains(t, got, "Consider using databricks_ai_v6")
} else {
assert.Empty(t, got)
}
})
}
}

func writeTorchRunConfig(t *testing.T, dir, codeSource string) *runConfig {
t.Helper()
yaml := "experiment_name: e\ncompute:\n num_accelerators: 1\n accelerator_type: GPU_1xA10\ncode_source:\n root_path: .\n" + codeSource + "command: python train.py\n"
path := filepath.Join(dir, "run.yaml")
require.NoError(t, os.WriteFile(path, []byte(yaml), 0o644))
cfg, err := loadRunConfig(path)
require.NoError(t, err)
return cfg
}

func TestMissingTorchWarningHonorsRuntimeImageOverride(t *testing.T) {
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "train.py"), []byte("import torch\n"), 0o644))
cfg := writeTorchRunConfig(t, dir, "")
path := filepath.Join(dir, "run.yaml")

// Submission uses DATABRICKS_DL_RUNTIME_IMAGE when environment.version is unset.
assert.Empty(t, missingTorchWarning(env.Set(t.Context(), dlRuntimeImageEnv, "databricks_ai_v6"), cfg, path))
assert.Contains(t, missingTorchWarning(env.Set(t.Context(), dlRuntimeImageEnv, "CLIENT-GPU-6"), cfg, path), "Standard environment 6 does not include PyTorch")
}

func TestMissingTorchWarningScansPinnedCommit(t *testing.T) {
t.Run("import only in the commit", func(t *testing.T) {
repo := newTestRepo(t)
writeRepoFile(t, repo, "src/train.py", "import torch\n")
sha := commitAll(t, repo, "torch")
// The working tree no longer imports torch, but the pinned commit does.
writeRepoFile(t, repo, "src/train.py", "print('hi')\n")
cfg := writeTorchRunConfig(t, repo, " git:\n commit: "+sha+"\n")

assert.Contains(t, missingTorchWarning(t.Context(), cfg, filepath.Join(repo, "run.yaml")), "src/train.py contains a torch import")
})

t.Run("import only in the working tree", func(t *testing.T) {
repo := newTestRepo(t)
writeRepoFile(t, repo, "src/train.py", "print('hi')\n")
sha := commitAll(t, repo, "no torch")
writeRepoFile(t, repo, "src/train.py", "import torch\n")
cfg := writeTorchRunConfig(t, repo, " git:\n commit: "+sha+"\n")

assert.Empty(t, missingTorchWarning(t.Context(), cfg, filepath.Join(repo, "run.yaml")))
})

t.Run("subdirectory root reports a root-relative path", func(t *testing.T) {
repo := newTestRepo(t)
writeRepoFile(t, repo, "proj/model/train.py", "from torch import nn\n")
writeRepoFile(t, repo, "other/x.py", "import torch\n")
sha := commitAll(t, repo, "torch")
dir := filepath.Join(repo, "proj")
cfg := writeTorchRunConfig(t, dir, " git:\n commit: "+sha+"\n include_paths:\n - model\n")

assert.Contains(t, missingTorchWarning(t.Context(), cfg, filepath.Join(dir, "run.yaml")), "Warning: model/train.py contains a torch import")
})
}
Loading