Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/air/air-code-source-without-scm.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Package `code_source` directories outside a git repository without a `git` binary on PATH, and fail early with a clear error when the directory is inside a repository or `code_source.git` is set. ([#7625](https://github.com/databricks/cli/pull/7625))
18 changes: 18 additions & 0 deletions cmd/air/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io"
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"time"
Expand All @@ -21,6 +22,20 @@ import (
"github.com/spf13/cobra"
)

// checkCodeSourceGit fails before the config is validated or anything is packaged when
// git is missing and the code_source cannot be packaged without it. Runs for dry runs too.
func checkCodeSourceGit(ctx context.Context, cfg *runConfig, configPath string) error {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: this re-runs resolveRootPath and gitRequiredError, which resolveSnapshotPlan repeats right after. The early check is justified by running before validation, so a short comment saying so would keep the duplication from looking accidental.

if cfg.CodeSource == nil || cfg.CodeSource.Snapshot == nil || gitInstalled() {
return nil
}
snap := cfg.CodeSource.Snapshot
root, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath))
if err != nil {
return err
}
return gitRequiredError(root, snap.Git)
}

// dryRunValidationTimeout bounds only the config:validate request.
const dryRunValidationTimeout = 15 * time.Second

Expand Down Expand Up @@ -159,6 +174,9 @@ field, pass its path to -h as a separate argument:
if err != nil {
return err
}
if err := checkCodeSourceGit(ctx, cfg, file); err != nil {
return err
}

if dryRun {
// Local and credential-free, so it runs before workspace validation.
Expand Down
37 changes: 37 additions & 0 deletions cmd/air/run_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"sync/atomic"
"testing"
Expand Down Expand Up @@ -87,6 +90,40 @@ func TestRunSubmitTextOutput(t *testing.T) {
assert.Zero(t, counts.runGetOutput.Load(), "bare submission must not poll runs/get-output")
}

func TestCheckCodeSourceGit(t *testing.T) {
repo := newTestRepo(t)
sub := filepath.Join(repo, "sub")
require.NoError(t, os.MkdirAll(sub, 0o755))
plain := t.TempDir()
withCodeSource := func(root string, ref *gitRef) *runConfig {
return &runConfig{CodeSource: &codeSourceConfig{Snapshot: &snapshotSourceConfig{RootPath: root, Git: ref}}}
}
withoutGit(t)

err := checkCodeSourceGit(t.Context(), withCodeSource(".", nil), filepath.Join(sub, "workload.yaml"))
require.Error(t, err)
assert.Contains(t, err.Error(), "is in a git repository")
assert.Contains(t, err.Error(), "needs git on PATH")
assert.NotContains(t, err.Error(), "exec:")

err = checkCodeSourceGit(t.Context(), withCodeSource(".", &gitRef{Branch: new("main")}), filepath.Join(plain, "workload.yaml"))
require.Error(t, err)
assert.Contains(t, err.Error(), "code_source.git is set")

require.NoError(t, checkCodeSourceGit(t.Context(), withCodeSource(".", nil), filepath.Join(plain, "workload.yaml")))

// Without code_source the git binary is never looked up, and its absence is not an error.
looked := false
orig := gitLookPath
gitLookPath = func(string) (string, error) {
looked = true
return "", exec.ErrNotFound
}
t.Cleanup(func() { gitLookPath = orig })
require.NoError(t, checkCodeSourceGit(t.Context(), &runConfig{}, filepath.Join(plain, "workload.yaml")))
assert.False(t, looked)
}

func TestWarnExperimentalContainers(t *testing.T) {
var stdout, stderr bytes.Buffer
ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputText, nil, &stdout, &stderr, "", ""))
Expand Down
38 changes: 38 additions & 0 deletions cmd/air/snapshot_git.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ import (
"errors"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
)
Expand Down Expand Up @@ -73,6 +75,42 @@ func (g gitRepo) isRepository(ctx context.Context) bool {
return strings.TrimSpace(out) == "true"
}

// hasGitMetadata reports whether dir holds a repository's `.git` entry: a gitfile, or a
// directory with HEAD. An empty `.git` directory is not a repository, as in git.
func hasGitMetadata(dir string) bool {
info, err := os.Stat(filepath.Join(dir, ".git"))
if err != nil {
return false
}
if info.Mode().IsRegular() {
return true
}
if !info.IsDir() {
return false
}
_, err = os.Stat(filepath.Join(dir, ".git", "HEAD"))
return err == nil
}

// insideGitRepository is the git-free stand-in for isRepository, used only when no git
// binary is available: repository metadata in dir or one of its ancestors marks a work tree.
func insideGitRepository(dir string) bool {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

folders.FindDirWithLeaf(dir, git.GitDirectoryName) already does this upward walk; libs/git/info.go uses it the same way. FindDirWithLeaf treats an empty .git directory as a repository, which is fine here, so hasGitMetadata and its test can go.

abs, err := filepath.Abs(dir)
if err != nil {
return false
}
for {
if hasGitMetadata(abs) {
return true
}
parent := filepath.Dir(abs)
if parent == abs {
return false
}
abs = parent
}
}

// repoRelativePrefix returns the path from the repository root to g.path. Git
// emits a trailing slash for subdirectories and an empty string at the root.
func (g gitRepo) repoRelativePrefix(ctx context.Context) (string, error) {
Expand Down
182 changes: 158 additions & 24 deletions cmd/air/snapshot_package.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,17 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"os/exec"
"path"
"path/filepath"
"slices"
"strings"

libgit "github.com/databricks/cli/libs/git"
"github.com/databricks/cli/libs/tarpack"
"github.com/databricks/cli/libs/vfs"
"github.com/klauspost/pgzip"
)

Expand Down Expand Up @@ -135,35 +140,20 @@ type snapshotFile struct {
}

func snapshotFiles(ctx context.Context, repoPath string, includePaths []string, isGitRepo bool) ([]snapshotFile, error) {
args := []string{"-C", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"}
if !isGitRepo {
gitDir, err := os.MkdirTemp("", "air-snapshot-git-")
if err != nil {
return nil, fmt.Errorf("failed to create temporary git metadata: %w", err)
}
defer os.RemoveAll(gitDir)
cmd := exec.CommandContext(ctx, "git", "--git-dir", gitDir, "--work-tree", repoPath, "init", "--quiet")
if output, err := cmd.CombinedOutput(); err != nil {
return nil, fmt.Errorf("failed to initialize temporary git metadata: %w: %s", err, strings.TrimSpace(string(output)))
}
args = []string{"--git-dir", gitDir, "--work-tree", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"}
var names []string
var err error
if !isGitRepo && !gitInstalled() {
names, err = listFilesWithoutGit(ctx, repoPath, includePaths)
} else {
names, err = listFilesWithGit(ctx, repoPath, includePaths, isGitRepo)
}

if len(includePaths) > 0 {
args = append(args, "--")
args = append(args, includePaths...)
}
output, err := exec.CommandContext(ctx, "git", args...).Output()
if err != nil {
return nil, fmt.Errorf("failed to evaluate git ignore rules: %w", err)
return nil, err
}

var files []snapshotFile
for raw := range bytes.SplitSeq(output, []byte{0}) {
if len(raw) == 0 {
continue
}
name := filepath.ToSlash(string(raw))
for _, raw := range names {
name := filepath.ToSlash(raw)
base := filepath.Base(name)
if name == ".git" || strings.HasPrefix(name, ".git/") || strings.HasPrefix(base, "._") {
continue
Expand Down Expand Up @@ -196,6 +186,150 @@ func snapshotFiles(ctx context.Context, repoPath string, includePaths []string,
return files, nil
}

// listFilesWithGit returns the repo-relative paths `git ls-files` selects for repoPath.
// A directory outside any repository gets a temporary git dir so the same ignore
// rules apply without touching the directory.
func listFilesWithGit(ctx context.Context, repoPath string, includePaths []string, isGitRepo bool) ([]string, error) {
args := []string{"-C", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"}
if !isGitRepo {
gitDir, err := os.MkdirTemp("", "air-snapshot-git-")
if err != nil {
return nil, fmt.Errorf("failed to create temporary git metadata: %w", err)
}
defer os.RemoveAll(gitDir)
cmd := exec.CommandContext(ctx, "git", "--git-dir", gitDir, "--work-tree", repoPath, "init", "--quiet")
if output, err := cmd.CombinedOutput(); err != nil {
return nil, fmt.Errorf("failed to initialize temporary git metadata: %w: %s", err, strings.TrimSpace(string(output)))
}
args = []string{"--git-dir", gitDir, "--work-tree", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"}
}

if len(includePaths) > 0 {
args = append(args, "--")
args = append(args, includePaths...)
}
output, err := exec.CommandContext(ctx, "git", args...).Output()
if err != nil {
return nil, fmt.Errorf("failed to evaluate git ignore rules: %w", err)
}

var names []string
for raw := range bytes.SplitSeq(output, []byte{0}) {
if len(raw) > 0 {
names = append(names, string(raw))
}
}
return names, nil
}

// gitLookPath finds the git binary. Tests replace it to simulate a host without git.
var gitLookPath = exec.LookPath

// gitInstalled reports whether a git binary is on PATH.
func gitInstalled() bool {
_, err := gitLookPath("git")
return err == nil
}

// listFilesWithoutGit is the fallback for a directory outside any repository when git
// is not on PATH. It walks the tree applying libs/git's ignore rules (the root and
// nested .gitignore files and core.excludesFile from config), lists regular files and
// symlinks without following symlinks, and emits a nested repository as one "dir/"
// entry, as git does. libs/git.FileSet is not used because it drops symlinks and walks
// into nested repositories.
//
// Known differences from `git ls-files --others --exclude-standard`, which only matter
// on hosts without git (with git on PATH the git listing is used unchanged):
// - Rule precedence: libs/git returns the first matching rule from the root down, so

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Root *.log plus nested !keep.log silently diverges: git includes nested/keep.log, while Repository.Ignore returns on the first positive match from the root down. The fixture in writeFallbackFixture has no conflicting root rule, so this is untested. Could we evaluate the deepest .gitignore first, or at least log a one-line warning when the fallback runs in a tree with nested .gitignore files, so users know the snapshot may differ from what git would select?

// a root "*.log" excludes nested/keep.log even when nested/.gitignore has "!keep.log".
// - include_paths use path.Match, where "*" does not cross "/"; git's pathspecs do.
func listFilesWithoutGit(ctx context.Context, repoPath string, includePaths []string) ([]string, error) {
repo, err := libgit.NewRepository(ctx, vfs.MustNew(repoPath))
if err != nil {
return nil, fmt.Errorf("failed to read .gitignore rules: %w", err)
}

var names []string
err = filepath.WalkDir(repoPath, func(p string, d fs.DirEntry, err error) error {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If root_path is a symlink to a directory, WalkDir Lstats the root and never descends, so this returns no entries and the user sees the "directory is empty or all of its files are excluded by .gitignore" error. git -C follows the link, so the git path packages it fine. Could we filepath.EvalSymlinks(repoPath) before the walk? The emitted names are relative, so nothing else changes. A test with a symlinked root would pin it.

if err != nil {
return err
}
if p == repoPath {
return nil
}
rel, err := filepath.Rel(repoPath, p)
if err != nil {
return err
}
name := filepath.ToSlash(rel)
if d.Name() == ".git" {
if d.IsDir() {
return fs.SkipDir
}
return nil
}
if d.IsDir() {
ignored, err := repo.Ignore(name + "/")
if err != nil {
return err
}
if ignored {
return fs.SkipDir
}
if hasGitMetadata(p) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On main, snapshotFiles drops directory entries (#7643), so after the rebase this dir/ entry never reaches the archive. Returning fs.SkipDir here is enough.

names = append(names, name+"/")
return fs.SkipDir
}
return nil
}
// git lists only what it can track; sockets and FIFOs would also block packaging.
if !d.Type().IsRegular() && d.Type()&fs.ModeSymlink == 0 {
return nil
}
ignored, err := repo.Ignore(name)
if err != nil {
return err
}
if !ignored {
names = append(names, name)
}
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to list files: %w", err)
}

slices.Sort(names)
if len(includePaths) > 0 {
names = slices.DeleteFunc(names, func(name string) bool {
return !matchesIncludePath(name, includePaths)
})
}
return names, nil
}

// matchesIncludePath reports whether name, or a directory containing it, matches one of
// the include_paths.
func matchesIncludePath(name string, includePaths []string) bool {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Building on Maggie's glob thread on line 320: if the fallback rejects glob include_paths (isPathspecPattern on main already detects them), it can walk each include_paths entry directly, the way libs/fileset.FileSet does. Then this function and the *.py known-difference test can go.

for _, spec := range includePaths {
spec = path.Clean(filepath.ToSlash(strings.TrimSpace(spec)))
if spec == "." {
return true
}
for p := strings.TrimSuffix(name, "/"); p != ""; {
if ok, _ := path.Match(spec, p); ok {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

include_paths: ["src/*.py"] selects src/x/y.py under git, since git's pathspec * crosses /. Here path.Match's * doesn't cross / and the ancestor loop only tries prefixes, so the file is dropped without any message. The doc comment calls this a known difference, but on a packaging path a silent omission is worse than an error. Could we either match git's semantics, or fail in the fallback when include_paths contain glob metacharacters? Either way, a row in TestSnapshotFilesWithoutGit_MatchesGitListing would catch it.

return true
}
i := strings.LastIndex(p, "/")
if i < 0 {
break
}
p = p[:i]
}
}
return false
}

// emptySnapshotError explains why no file was selected for the snapshot. An empty

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: in the non-repo branch below, the hint suggests git check-ignore -v, which the git-free path can't run, and it's shown whether or not git is installed. Could that branch give a git-agnostic hint, such as which .gitignore file to check?

// tarball is accepted by the upload but fails on the node, after compute was
// provisioned, so it is rejected here with the most likely cause.
Expand Down
Loading
Loading