diff --git a/.nextchanges/bundles/tgz-git-subdir.md b/.nextchanges/bundles/tgz-git-subdir.md new file mode 100644 index 00000000000..c71f460c648 --- /dev/null +++ b/.nextchanges/bundles/tgz-git-subdir.md @@ -0,0 +1 @@ +* Fix `tgz` artifacts with `git` set producing an empty tarball or failing when the bundle is in a subdirectory of the git repository. ([#7674](https://github.com/databricks/cli/pull/7674)) diff --git a/acceptance/bundle/artifacts/tgz_git_subdir/out.test.toml b/acceptance/bundle/artifacts/tgz_git_subdir/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/artifacts/tgz_git_subdir/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/artifacts/tgz_git_subdir/output.txt b/acceptance/bundle/artifacts/tgz_git_subdir/output.txt new file mode 100644 index 00000000000..f001aaa11ec --- /dev/null +++ b/acceptance/bundle/artifacts/tgz_git_subdir/output.txt @@ -0,0 +1,22 @@ + +>>> errcode [CLI] bundle deploy +Building from_git... +Building from_git_include... +Building from_worktree... +Uploading dist/from_git.tgz... +Uploading dist/from_git_include.tgz... +Uploading dist/from_worktree.tgz... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged + +=== Contents of from_git.tgz +main.py +util.py + +=== Contents of from_git_include.tgz +main.py + +=== Contents of from_worktree.tgz +main.py +util.py diff --git a/acceptance/bundle/artifacts/tgz_git_subdir/proj/databricks.yml b/acceptance/bundle/artifacts/tgz_git_subdir/proj/databricks.yml new file mode 100644 index 00000000000..2cb2be94604 --- /dev/null +++ b/acceptance/bundle/artifacts/tgz_git_subdir/proj/databricks.yml @@ -0,0 +1,35 @@ +bundle: + name: test-bundle + +workspace: + artifact_path: /Workspace/foo/bar/artifacts + +sync: + exclude: + - dist/** + +# The bundle lives in a subdirectory of the git repository (as in a monorepo). +artifacts: + from_git: + type: tgz + path: src + git: + branch: main + files: + - source: dist/from_git.tgz + from_git_include: + type: tgz + path: src + include: + - main.py + git: + branch: main + files: + - source: dist/from_git_include.tgz + from_worktree: + type: tgz + path: src + include: + - . + files: + - source: dist/from_worktree.tgz diff --git a/acceptance/bundle/artifacts/tgz_git_subdir/proj/src/main.py b/acceptance/bundle/artifacts/tgz_git_subdir/proj/src/main.py new file mode 100644 index 00000000000..f6c95a92ad7 --- /dev/null +++ b/acceptance/bundle/artifacts/tgz_git_subdir/proj/src/main.py @@ -0,0 +1 @@ +print("main") diff --git a/acceptance/bundle/artifacts/tgz_git_subdir/proj/src/util.py b/acceptance/bundle/artifacts/tgz_git_subdir/proj/src/util.py new file mode 100644 index 00000000000..78c81cd2d2f --- /dev/null +++ b/acceptance/bundle/artifacts/tgz_git_subdir/proj/src/util.py @@ -0,0 +1 @@ +print("util") diff --git a/acceptance/bundle/artifacts/tgz_git_subdir/script b/acceptance/bundle/artifacts/tgz_git_subdir/script new file mode 100644 index 00000000000..29868988636 --- /dev/null +++ b/acceptance/bundle/artifacts/tgz_git_subdir/script @@ -0,0 +1,23 @@ +git init -qb main +git config core.autocrlf false +git config user.name "Tester" +git config user.email "tester@databricks.com" +git config core.hooksPath no-hooks +# A repo-root src/ unrelated to the bundle's proj/src/, as in a monorepo. +git add proj src +git commit -qm 'Add bundle' + +cd proj +trace errcode $CLI bundle deploy + +for name in from_git from_git_include from_worktree; do + title "Contents of $name.tgz" + if [ -f dist/$name.tgz ]; then + tar -tzf dist/$name.tgz | sort + else + echo "(not built)" + fi +done + +rm -rf ../.git +rm "$OUT_REQUESTS" diff --git a/acceptance/bundle/artifacts/tgz_git_subdir/src/unrelated.py b/acceptance/bundle/artifacts/tgz_git_subdir/src/unrelated.py new file mode 100644 index 00000000000..1ff3133a150 --- /dev/null +++ b/acceptance/bundle/artifacts/tgz_git_subdir/src/unrelated.py @@ -0,0 +1 @@ +print("unrelated") diff --git a/acceptance/bundle/artifacts/tgz_git_subdir/test.toml b/acceptance/bundle/artifacts/tgz_git_subdir/test.toml new file mode 100644 index 00000000000..9f75182ac59 --- /dev/null +++ b/acceptance/bundle/artifacts/tgz_git_subdir/test.toml @@ -0,0 +1,5 @@ +RecordRequests = true +Ignore = [ + '.databricks', + 'proj/dist', +] diff --git a/bundle/artifacts/tarball.go b/bundle/artifacts/tarball.go index 85c0124220b..4fc9a6381b9 100644 --- a/bundle/artifacts/tarball.go +++ b/bundle/artifacts/tarball.go @@ -160,9 +160,8 @@ func tarballFromInclude(ctx context.Context, b *bundle.Bundle, a *config.Artifac } // tarballFromGit snapshots a git ref via `git archive`, so the archive reflects the -// committed tree at that ref. Entries are named relative to `path` (the tree at -// `:`); `include`, when set, scopes the archived pathspecs. Commit wins over -// Branch. +// committed tree at that ref. Entries are named relative to `path`; `include`, when +// set, scopes the archived pathspecs. Commit wins over Branch. func tarballFromGit(ctx context.Context, b *bundle.Bundle, a *config.Artifact, w io.Writer) error { ref := a.Git.Commit if ref == "" { @@ -171,18 +170,20 @@ func tarballFromGit(ctx context.Context, b *bundle.Bundle, a *config.Artifact, w if ref == "" { return errors.New("git artifact: specify git.commit or git.branch") } - relBase, err := codeRel(b, a) - if err != nil { - return err - } - treeish := ref - if relBase != "." { - // The tree at :, so entries come out relative to `path`. - treeish = ref + ":" + relBase + // Run from `path`: git archive then keeps only that directory, with entries and + // include pathspecs relative to it, wherever it sits in the repository. Archiving a + // commit (not a : tree) also stamps entries with the commit time, which + // keeps the tarball reproducible. + // git archive run from a directory the ref doesn't have (uncommitted, or added + // after an older ref) succeeds with an empty archive, so check that it exists. + // Only once the ref resolves: otherwise git archive below reports git's own error + // (unknown ref, not a git repository). + if gitVerify(ctx, a.Path, ref+"^{commit}") && !gitVerify(ctx, a.Path, ref+":./") { + return fmt.Errorf("git artifact: %s is not in the tree of %s; commit it or pick a ref that contains it", a.Path, ref) } // Ask git for an uncompressed tar and gzip it ourselves with pgzip: git's // --format=tar.gz gzip is single-threaded, whereas pgzip spreads it across cores. - args := []string{"-C", b.SyncRootPath, "archive", "--format=tar", treeish} + args := []string{"-C", a.Path, "archive", "--format=tar", ref} if len(a.Include) > 0 { args = append(args, "--") args = append(args, a.Include...) @@ -197,11 +198,16 @@ func tarballFromGit(ctx context.Context, b *bundle.Bundle, a *config.Artifact, w cmd.Stderr = &stderr if err := cmd.Run(); err != nil { gzw.Close() - return fmt.Errorf("git archive %s: %w: %s", treeish, err, stderr.String()) + return fmt.Errorf("git archive %s in %s: %w: %s", ref, a.Path, err, stderr.String()) } return gzw.Close() } +// gitVerify reports whether rev resolves when git runs in dir. +func gitVerify(ctx context.Context, dir, rev string) bool { + return exec.CommandContext(ctx, "git", "-C", dir, "rev-parse", "--verify", "--quiet", rev).Run() == nil +} + // addFileToTarball writes f (a sync-root-relative file) to the archive under a name // relative to relBase, so entries are relative to the artifact's `path`. func addFileToTarball(tw *tar.Writer, root vfs.Path, relBase string, f fileset.File) error { diff --git a/bundle/artifacts/tarball_test.go b/bundle/artifacts/tarball_test.go index 49764877ae6..b381d58e2ed 100644 --- a/bundle/artifacts/tarball_test.go +++ b/bundle/artifacts/tarball_test.go @@ -11,6 +11,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" @@ -105,3 +106,92 @@ func TestTarballFromGitIsReproducible(t *testing.T) { // Round-trips through gzip and carries the packed file. assert.Equal(t, "print('x')", strings.TrimSpace(tarEntries(t, buf1.Bytes())["train.py"])) } + +// TestTarballFromGitSubdirIsReproducible covers a `path` below the repository root: +// entries are relative to `path` and carry the commit time, not the current time, +// so rebuilding the same commit gives the same bytes. +func TestTarballFromGitSubdirIsReproducible(t *testing.T) { + repo := t.TempDir() + runGit(t, repo, "init", "-q") + runGit(t, repo, "config", "user.email", "t@example.com") + runGit(t, repo, "config", "user.name", "t") + runGit(t, repo, "config", "core.autocrlf", "false") + src := filepath.Join(repo, "proj", "src") + require.NoError(t, os.MkdirAll(src, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, "main.py"), []byte("print('x')\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(repo, "unrelated.py"), []byte("print('y')\n"), 0o644)) + runGit(t, repo, "add", "-A") + commitTime := time.Unix(1700000000, 0) + cmd := exec.Command("git", "commit", "-qm", "init") + cmd.Dir = repo + cmd.Env = append(os.Environ(), "GIT_COMMITTER_DATE=@1700000000 +0000", "GIT_AUTHOR_DATE=@1700000000 +0000") + out, err := cmd.CombinedOutput() + require.NoError(t, err, "git commit: %s", out) + + b := &bundle.Bundle{SyncRootPath: filepath.Join(repo, "proj")} + a := &config.Artifact{Path: src, Git: &config.ArtifactGit{Commit: "HEAD"}} + + var buf bytes.Buffer + require.NoError(t, tarballFromGit(t.Context(), b, a, &buf)) + + gzr, err := gzip.NewReader(bytes.NewReader(buf.Bytes())) + require.NoError(t, err) + tr := tar.NewReader(gzr) + var names []string + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + require.NoError(t, err) + if hdr.Typeflag == tar.TypeXGlobalHeader { + continue + } + names = append(names, hdr.Name) + assert.True(t, commitTime.Equal(hdr.ModTime), "%s: mtime %v, want commit time %v", hdr.Name, hdr.ModTime, commitTime) + } + assert.Equal(t, []string{"main.py"}, names) +} + +// TestTarballFromGitRejectsPathMissingFromRef: git archive run from a directory the +// ref doesn't contain succeeds with an empty archive, which must not be uploaded. +func TestTarballFromGitRejectsPathMissingFromRef(t *testing.T) { + repo := t.TempDir() + runGit(t, repo, "init", "-q") + runGit(t, repo, "config", "user.email", "t@example.com") + runGit(t, repo, "config", "user.name", "t") + require.NoError(t, os.WriteFile(filepath.Join(repo, "README"), []byte("x\n"), 0o644)) + runGit(t, repo, "add", "-A") + runGit(t, repo, "commit", "-qm", "init") + src := filepath.Join(repo, "src") + require.NoError(t, os.MkdirAll(src, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(src, "main.py"), []byte("print('x')\n"), 0o644)) + + b := &bundle.Bundle{SyncRootPath: repo} + a := &config.Artifact{Path: src, Git: &config.ArtifactGit{Commit: "HEAD"}} + + var buf bytes.Buffer + err := tarballFromGit(t.Context(), b, a, &buf) + assert.ErrorContains(t, err, "is not in the tree of HEAD") +} + +// TestTarballFromGitReportsUnknownRef: a ref that doesn't resolve keeps git's own +// error rather than the "not in the tree" one meant for a missing path. +func TestTarballFromGitReportsUnknownRef(t *testing.T) { + repo := t.TempDir() + runGit(t, repo, "init", "-q") + runGit(t, repo, "config", "user.email", "t@example.com") + runGit(t, repo, "config", "user.name", "t") + require.NoError(t, os.WriteFile(filepath.Join(repo, "README"), []byte("x\n"), 0o644)) + runGit(t, repo, "add", "-A") + runGit(t, repo, "commit", "-qm", "init") + + b := &bundle.Bundle{SyncRootPath: repo} + a := &config.Artifact{Path: repo, Git: &config.ArtifactGit{Branch: "no-such-branch"}} + + var buf bytes.Buffer + err := tarballFromGit(t.Context(), b, a, &buf) + require.Error(t, err) + assert.Contains(t, err.Error(), "git archive no-such-branch") + assert.NotContains(t, err.Error(), "is not in the tree") +}