Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/tgz-git-subdir.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Fix `tgz` artifacts with `git` set producing an empty tarball or failing when the bundle is in a subdirectory of the git repository. ([#7674](https://github.com/databricks/cli/pull/7674))
2 changes: 2 additions & 0 deletions acceptance/bundle/artifacts/tgz_git_subdir/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
22 changes: 22 additions & 0 deletions acceptance/bundle/artifacts/tgz_git_subdir/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@

>>> errcode [CLI] bundle deploy
Building from_git...
Building from_git_include...
Building from_worktree...
Uploading dist/from_git.tgz...
Uploading dist/from_git_include.tgz...
Uploading dist/from_worktree.tgz...
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Files: 3 uploaded, 0 deleted
Resources: 0 created, 0 changed, 0 deleted, 0 unchanged

=== Contents of from_git.tgz
main.py
util.py

=== Contents of from_git_include.tgz
main.py

=== Contents of from_worktree.tgz
main.py
util.py
35 changes: 35 additions & 0 deletions acceptance/bundle/artifacts/tgz_git_subdir/proj/databricks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
bundle:
name: test-bundle

workspace:
artifact_path: /Workspace/foo/bar/artifacts

sync:
exclude:
- dist/**

# The bundle lives in a subdirectory of the git repository (as in a monorepo).
artifacts:
from_git:
type: tgz
path: src
git:
branch: main
files:
- source: dist/from_git.tgz
from_git_include:
type: tgz
path: src
include:
- main.py
git:
branch: main
files:
- source: dist/from_git_include.tgz
from_worktree:
type: tgz
path: src
include:
- .
files:
- source: dist/from_worktree.tgz
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
print("main")
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
print("util")
23 changes: 23 additions & 0 deletions acceptance/bundle/artifacts/tgz_git_subdir/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
git init -qb main
git config core.autocrlf false
git config user.name "Tester"
git config user.email "tester@databricks.com"
git config core.hooksPath no-hooks
# A repo-root src/ unrelated to the bundle's proj/src/, as in a monorepo.
git add proj src
git commit -qm 'Add bundle'

cd proj
trace errcode $CLI bundle deploy

for name in from_git from_git_include from_worktree; do
title "Contents of $name.tgz"
if [ -f dist/$name.tgz ]; then
tar -tzf dist/$name.tgz | sort
else
echo "(not built)"
fi
done

rm -rf ../.git
rm "$OUT_REQUESTS"
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
print("unrelated")
5 changes: 5 additions & 0 deletions acceptance/bundle/artifacts/tgz_git_subdir/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
RecordRequests = true
Ignore = [
'.databricks',
'proj/dist',
]
32 changes: 19 additions & 13 deletions bundle/artifacts/tarball.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,9 +160,8 @@ func tarballFromInclude(ctx context.Context, b *bundle.Bundle, a *config.Artifac
}

// tarballFromGit snapshots a git ref via `git archive`, so the archive reflects the
// committed tree at that ref. Entries are named relative to `path` (the tree at
// `<ref>:<path>`); `include`, when set, scopes the archived pathspecs. Commit wins over
// Branch.
// committed tree at that ref. Entries are named relative to `path`; `include`, when
// set, scopes the archived pathspecs. Commit wins over Branch.
func tarballFromGit(ctx context.Context, b *bundle.Bundle, a *config.Artifact, w io.Writer) error {
ref := a.Git.Commit
if ref == "" {
Expand All @@ -171,18 +170,20 @@ func tarballFromGit(ctx context.Context, b *bundle.Bundle, a *config.Artifact, w
if ref == "" {
return errors.New("git artifact: specify git.commit or git.branch")
}
relBase, err := codeRel(b, a)
if err != nil {
return err
}
treeish := ref
if relBase != "." {
// The tree at <ref>:<path>, so entries come out relative to `path`.
treeish = ref + ":" + relBase
// Run from `path`: git archive then keeps only that directory, with entries and
// include pathspecs relative to it, wherever it sits in the repository. Archiving a
// commit (not a <ref>:<path> tree) also stamps entries with the commit time, which
// keeps the tarball reproducible.
// git archive run from a directory the ref doesn't have (uncommitted, or added
// after an older ref) succeeds with an empty archive, so check that it exists.
// Only once the ref resolves: otherwise git archive below reports git's own error
// (unknown ref, not a git repository).
if gitVerify(ctx, a.Path, ref+"^{commit}") && !gitVerify(ctx, a.Path, ref+":./") {
return fmt.Errorf("git artifact: %s is not in the tree of %s; commit it or pick a ref that contains it", a.Path, ref)
}
// Ask git for an uncompressed tar and gzip it ourselves with pgzip: git's
// --format=tar.gz gzip is single-threaded, whereas pgzip spreads it across cores.
args := []string{"-C", b.SyncRootPath, "archive", "--format=tar", treeish}
args := []string{"-C", a.Path, "archive", "--format=tar", ref}
if len(a.Include) > 0 {
args = append(args, "--")
args = append(args, a.Include...)
Expand All @@ -197,11 +198,16 @@ func tarballFromGit(ctx context.Context, b *bundle.Bundle, a *config.Artifact, w
cmd.Stderr = &stderr
if err := cmd.Run(); err != nil {
gzw.Close()
return fmt.Errorf("git archive %s: %w: %s", treeish, err, stderr.String())
return fmt.Errorf("git archive %s in %s: %w: %s", ref, a.Path, err, stderr.String())
}
return gzw.Close()
}

// gitVerify reports whether rev resolves when git runs in dir.
func gitVerify(ctx context.Context, dir, rev string) bool {
return exec.CommandContext(ctx, "git", "-C", dir, "rev-parse", "--verify", "--quiet", rev).Run() == nil
}

// addFileToTarball writes f (a sync-root-relative file) to the archive under a name
// relative to relBase, so entries are relative to the artifact's `path`.
func addFileToTarball(tw *tar.Writer, root vfs.Path, relBase string, f fileset.File) error {
Expand Down
90 changes: 90 additions & 0 deletions bundle/artifacts/tarball_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"path/filepath"
"strings"
"testing"
"time"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/bundle/config"
Expand Down Expand Up @@ -105,3 +106,92 @@ func TestTarballFromGitIsReproducible(t *testing.T) {
// Round-trips through gzip and carries the packed file.
assert.Equal(t, "print('x')", strings.TrimSpace(tarEntries(t, buf1.Bytes())["train.py"]))
}

// TestTarballFromGitSubdirIsReproducible covers a `path` below the repository root:
// entries are relative to `path` and carry the commit time, not the current time,
// so rebuilding the same commit gives the same bytes.
func TestTarballFromGitSubdirIsReproducible(t *testing.T) {
repo := t.TempDir()
runGit(t, repo, "init", "-q")
runGit(t, repo, "config", "user.email", "t@example.com")
runGit(t, repo, "config", "user.name", "t")
runGit(t, repo, "config", "core.autocrlf", "false")
src := filepath.Join(repo, "proj", "src")
require.NoError(t, os.MkdirAll(src, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(src, "main.py"), []byte("print('x')\n"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(repo, "unrelated.py"), []byte("print('y')\n"), 0o644))
runGit(t, repo, "add", "-A")
commitTime := time.Unix(1700000000, 0)
cmd := exec.Command("git", "commit", "-qm", "init")
cmd.Dir = repo
cmd.Env = append(os.Environ(), "GIT_COMMITTER_DATE=@1700000000 +0000", "GIT_AUTHOR_DATE=@1700000000 +0000")
out, err := cmd.CombinedOutput()
require.NoError(t, err, "git commit: %s", out)

b := &bundle.Bundle{SyncRootPath: filepath.Join(repo, "proj")}
a := &config.Artifact{Path: src, Git: &config.ArtifactGit{Commit: "HEAD"}}

var buf bytes.Buffer
require.NoError(t, tarballFromGit(t.Context(), b, a, &buf))

gzr, err := gzip.NewReader(bytes.NewReader(buf.Bytes()))
require.NoError(t, err)
tr := tar.NewReader(gzr)
var names []string
for {
hdr, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
require.NoError(t, err)
if hdr.Typeflag == tar.TypeXGlobalHeader {
continue
}
names = append(names, hdr.Name)
assert.True(t, commitTime.Equal(hdr.ModTime), "%s: mtime %v, want commit time %v", hdr.Name, hdr.ModTime, commitTime)
}
assert.Equal(t, []string{"main.py"}, names)
}

// TestTarballFromGitRejectsPathMissingFromRef: git archive run from a directory the
// ref doesn't contain succeeds with an empty archive, which must not be uploaded.
func TestTarballFromGitRejectsPathMissingFromRef(t *testing.T) {
repo := t.TempDir()
runGit(t, repo, "init", "-q")
runGit(t, repo, "config", "user.email", "t@example.com")
runGit(t, repo, "config", "user.name", "t")
require.NoError(t, os.WriteFile(filepath.Join(repo, "README"), []byte("x\n"), 0o644))
runGit(t, repo, "add", "-A")
runGit(t, repo, "commit", "-qm", "init")
src := filepath.Join(repo, "src")
require.NoError(t, os.MkdirAll(src, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(src, "main.py"), []byte("print('x')\n"), 0o644))

b := &bundle.Bundle{SyncRootPath: repo}
a := &config.Artifact{Path: src, Git: &config.ArtifactGit{Commit: "HEAD"}}

var buf bytes.Buffer
err := tarballFromGit(t.Context(), b, a, &buf)
assert.ErrorContains(t, err, "is not in the tree of HEAD")
}

// TestTarballFromGitReportsUnknownRef: a ref that doesn't resolve keeps git's own
// error rather than the "not in the tree" one meant for a missing path.
func TestTarballFromGitReportsUnknownRef(t *testing.T) {
repo := t.TempDir()
runGit(t, repo, "init", "-q")
runGit(t, repo, "config", "user.email", "t@example.com")
runGit(t, repo, "config", "user.name", "t")
require.NoError(t, os.WriteFile(filepath.Join(repo, "README"), []byte("x\n"), 0o644))
runGit(t, repo, "add", "-A")
runGit(t, repo, "commit", "-qm", "init")

b := &bundle.Bundle{SyncRootPath: repo}
a := &config.Artifact{Path: repo, Git: &config.ArtifactGit{Branch: "no-such-branch"}}

var buf bytes.Buffer
err := tarballFromGit(t.Context(), b, a, &buf)
require.Error(t, err)
assert.Contains(t, err.Error(), "git archive no-such-branch")
assert.NotContains(t, err.Error(), "is not in the tree")
}
Loading