Repository navigation
[air] Package code_source without git on PATH #7625
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| * Package `code_source` directories outside a git repository without a `git` binary on PATH, and fail early with a clear error when the directory is inside a repository or `code_source.git` is set. ([#7625](https://github.com/databricks/cli/pull/7625)) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,7 +7,9 @@ import ( | |
| "errors" | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "os/exec" | ||
| "path/filepath" | ||
| "strings" | ||
| "time" | ||
| ) | ||
|
|
@@ -73,6 +75,42 @@ func (g gitRepo) isRepository(ctx context.Context) bool { | |
| return strings.TrimSpace(out) == "true" | ||
| } | ||
|
|
||
| // hasGitMetadata reports whether dir holds a repository's `.git` entry: a gitfile, or a | ||
| // directory with HEAD. An empty `.git` directory is not a repository, as in git. | ||
| func hasGitMetadata(dir string) bool { | ||
| info, err := os.Stat(filepath.Join(dir, ".git")) | ||
| if err != nil { | ||
| return false | ||
| } | ||
| if info.Mode().IsRegular() { | ||
| return true | ||
| } | ||
| if !info.IsDir() { | ||
| return false | ||
| } | ||
| _, err = os.Stat(filepath.Join(dir, ".git", "HEAD")) | ||
| return err == nil | ||
| } | ||
|
|
||
| // insideGitRepository is the git-free stand-in for isRepository, used only when no git | ||
| // binary is available: repository metadata in dir or one of its ancestors marks a work tree. | ||
| func insideGitRepository(dir string) bool { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| abs, err := filepath.Abs(dir) | ||
| if err != nil { | ||
| return false | ||
| } | ||
| for { | ||
| if hasGitMetadata(abs) { | ||
| return true | ||
| } | ||
| parent := filepath.Dir(abs) | ||
| if parent == abs { | ||
| return false | ||
| } | ||
| abs = parent | ||
| } | ||
| } | ||
|
|
||
| // repoRelativePrefix returns the path from the repository root to g.path. Git | ||
| // emits a trailing slash for subdirectories and an empty string at the root. | ||
| func (g gitRepo) repoRelativePrefix(ctx context.Context) (string, error) { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,12 +6,17 @@ import ( | |
| "errors" | ||
| "fmt" | ||
| "io" | ||
| "io/fs" | ||
| "os" | ||
| "os/exec" | ||
| "path" | ||
| "path/filepath" | ||
| "slices" | ||
| "strings" | ||
|
|
||
| libgit "github.com/databricks/cli/libs/git" | ||
| "github.com/databricks/cli/libs/tarpack" | ||
| "github.com/databricks/cli/libs/vfs" | ||
| "github.com/klauspost/pgzip" | ||
| ) | ||
|
|
||
|
|
@@ -135,35 +140,20 @@ type snapshotFile struct { | |
| } | ||
|
|
||
| func snapshotFiles(ctx context.Context, repoPath string, includePaths []string, isGitRepo bool) ([]snapshotFile, error) { | ||
| args := []string{"-C", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"} | ||
| if !isGitRepo { | ||
| gitDir, err := os.MkdirTemp("", "air-snapshot-git-") | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to create temporary git metadata: %w", err) | ||
| } | ||
| defer os.RemoveAll(gitDir) | ||
| cmd := exec.CommandContext(ctx, "git", "--git-dir", gitDir, "--work-tree", repoPath, "init", "--quiet") | ||
| if output, err := cmd.CombinedOutput(); err != nil { | ||
| return nil, fmt.Errorf("failed to initialize temporary git metadata: %w: %s", err, strings.TrimSpace(string(output))) | ||
| } | ||
| args = []string{"--git-dir", gitDir, "--work-tree", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"} | ||
| var names []string | ||
| var err error | ||
| if !isGitRepo && !gitInstalled() { | ||
| names, err = listFilesWithoutGit(ctx, repoPath, includePaths) | ||
| } else { | ||
| names, err = listFilesWithGit(ctx, repoPath, includePaths, isGitRepo) | ||
| } | ||
|
|
||
| if len(includePaths) > 0 { | ||
| args = append(args, "--") | ||
| args = append(args, includePaths...) | ||
| } | ||
| output, err := exec.CommandContext(ctx, "git", args...).Output() | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to evaluate git ignore rules: %w", err) | ||
| return nil, err | ||
| } | ||
|
|
||
| var files []snapshotFile | ||
| for raw := range bytes.SplitSeq(output, []byte{0}) { | ||
| if len(raw) == 0 { | ||
| continue | ||
| } | ||
| name := filepath.ToSlash(string(raw)) | ||
| for _, raw := range names { | ||
| name := filepath.ToSlash(raw) | ||
| base := filepath.Base(name) | ||
| if name == ".git" || strings.HasPrefix(name, ".git/") || strings.HasPrefix(base, "._") { | ||
| continue | ||
|
|
@@ -196,6 +186,150 @@ func snapshotFiles(ctx context.Context, repoPath string, includePaths []string, | |
| return files, nil | ||
| } | ||
|
|
||
| // listFilesWithGit returns the repo-relative paths `git ls-files` selects for repoPath. | ||
| // A directory outside any repository gets a temporary git dir so the same ignore | ||
| // rules apply without touching the directory. | ||
| func listFilesWithGit(ctx context.Context, repoPath string, includePaths []string, isGitRepo bool) ([]string, error) { | ||
| args := []string{"-C", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"} | ||
| if !isGitRepo { | ||
| gitDir, err := os.MkdirTemp("", "air-snapshot-git-") | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to create temporary git metadata: %w", err) | ||
| } | ||
| defer os.RemoveAll(gitDir) | ||
| cmd := exec.CommandContext(ctx, "git", "--git-dir", gitDir, "--work-tree", repoPath, "init", "--quiet") | ||
| if output, err := cmd.CombinedOutput(); err != nil { | ||
| return nil, fmt.Errorf("failed to initialize temporary git metadata: %w: %s", err, strings.TrimSpace(string(output))) | ||
| } | ||
| args = []string{"--git-dir", gitDir, "--work-tree", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"} | ||
| } | ||
|
|
||
| if len(includePaths) > 0 { | ||
| args = append(args, "--") | ||
| args = append(args, includePaths...) | ||
| } | ||
| output, err := exec.CommandContext(ctx, "git", args...).Output() | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to evaluate git ignore rules: %w", err) | ||
| } | ||
|
|
||
| var names []string | ||
| for raw := range bytes.SplitSeq(output, []byte{0}) { | ||
| if len(raw) > 0 { | ||
| names = append(names, string(raw)) | ||
| } | ||
| } | ||
| return names, nil | ||
| } | ||
|
|
||
| // gitLookPath finds the git binary. Tests replace it to simulate a host without git. | ||
| var gitLookPath = exec.LookPath | ||
|
|
||
| // gitInstalled reports whether a git binary is on PATH. | ||
| func gitInstalled() bool { | ||
| _, err := gitLookPath("git") | ||
| return err == nil | ||
| } | ||
|
|
||
| // listFilesWithoutGit is the fallback for a directory outside any repository when git | ||
| // is not on PATH. It walks the tree applying libs/git's ignore rules (the root and | ||
| // nested .gitignore files and core.excludesFile from config), lists regular files and | ||
| // symlinks without following symlinks, and emits a nested repository as one "dir/" | ||
| // entry, as git does. libs/git.FileSet is not used because it drops symlinks and walks | ||
| // into nested repositories. | ||
| // | ||
| // Known differences from `git ls-files --others --exclude-standard`, which only matter | ||
| // on hosts without git (with git on PATH the git listing is used unchanged): | ||
| // - Rule precedence: libs/git returns the first matching rule from the root down, so | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Root |
||
| // a root "*.log" excludes nested/keep.log even when nested/.gitignore has "!keep.log". | ||
| // - include_paths use path.Match, where "*" does not cross "/"; git's pathspecs do. | ||
| func listFilesWithoutGit(ctx context.Context, repoPath string, includePaths []string) ([]string, error) { | ||
| repo, err := libgit.NewRepository(ctx, vfs.MustNew(repoPath)) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to read .gitignore rules: %w", err) | ||
| } | ||
|
|
||
| var names []string | ||
| err = filepath.WalkDir(repoPath, func(p string, d fs.DirEntry, err error) error { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If |
||
| if err != nil { | ||
| return err | ||
| } | ||
| if p == repoPath { | ||
| return nil | ||
| } | ||
| rel, err := filepath.Rel(repoPath, p) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| name := filepath.ToSlash(rel) | ||
| if d.Name() == ".git" { | ||
| if d.IsDir() { | ||
| return fs.SkipDir | ||
| } | ||
| return nil | ||
| } | ||
| if d.IsDir() { | ||
| ignored, err := repo.Ignore(name + "/") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if ignored { | ||
| return fs.SkipDir | ||
| } | ||
| if hasGitMetadata(p) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. On main, |
||
| names = append(names, name+"/") | ||
| return fs.SkipDir | ||
| } | ||
| return nil | ||
| } | ||
| // git lists only what it can track; sockets and FIFOs would also block packaging. | ||
| if !d.Type().IsRegular() && d.Type()&fs.ModeSymlink == 0 { | ||
| return nil | ||
| } | ||
| ignored, err := repo.Ignore(name) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if !ignored { | ||
| names = append(names, name) | ||
| } | ||
| return nil | ||
| }) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to list files: %w", err) | ||
| } | ||
|
|
||
| slices.Sort(names) | ||
| if len(includePaths) > 0 { | ||
| names = slices.DeleteFunc(names, func(name string) bool { | ||
| return !matchesIncludePath(name, includePaths) | ||
| }) | ||
| } | ||
| return names, nil | ||
| } | ||
|
|
||
| // matchesIncludePath reports whether name, or a directory containing it, matches one of | ||
| // the include_paths. | ||
| func matchesIncludePath(name string, includePaths []string) bool { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Building on Maggie's glob thread on line 320: if the fallback rejects glob |
||
| for _, spec := range includePaths { | ||
| spec = path.Clean(filepath.ToSlash(strings.TrimSpace(spec))) | ||
| if spec == "." { | ||
| return true | ||
| } | ||
| for p := strings.TrimSuffix(name, "/"); p != ""; { | ||
| if ok, _ := path.Match(spec, p); ok { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| return true | ||
| } | ||
| i := strings.LastIndex(p, "/") | ||
| if i < 0 { | ||
| break | ||
| } | ||
| p = p[:i] | ||
| } | ||
| } | ||
| return false | ||
| } | ||
|
|
||
| // emptySnapshotError explains why no file was selected for the snapshot. An empty | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: in the non-repo branch below, the hint suggests |
||
| // tarball is accepted by the upload but fails on the node, after compute was | ||
| // provisioned, so it is rejected here with the most likely cause. | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: this re-runs
resolveRootPathandgitRequiredError, whichresolveSnapshotPlanrepeats right after. The early check is justified by running before validation, so a short comment saying so would keep the duplication from looking accidental.