diff --git a/.github/scripts/coverage-comment.sh b/.github/scripts/coverage-comment.sh new file mode 100755 index 00000000000..c90cdba2b98 --- /dev/null +++ b/.github/scripts/coverage-comment.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# Posts, or updates, the coverage comment of a PR (.github/workflows/coverage-comment.yml): the +# coverage that its build run uploaded, against that of the newest main build its change starts +# from. Exits quietly when the run has no coverage, or its PR has moved on to another commit. +# +# The run's artifacts and their meta come from the PR's author: they are only read as data, by +# `coverpack -compare`, and what they name is checked against git and the GitHub API. +# +# Env: GH_TOKEN, REPO (owner/name), RUN_ID (the PR's build run), HEAD_SHA (the PR commit it +# built), HEAD_OWNER and HEAD_BRANCH (where that commit is). With DRY_RUN=1, prints the comment. +# Run from the root of a full clone of the default branch. +set -euo pipefail + +sha_re='^[0-9a-f]{40}$' +[[ $HEAD_SHA =~ $sha_re ]] || { echo "bad HEAD_SHA" >&2; exit 1; } +[[ $RUN_ID =~ ^[0-9]+$ ]] || { echo "bad RUN_ID" >&2; exit 1; } + +work=$(mktemp -d) +coverpack="$work/coverpack" +go -C tools build -o "$coverpack" ./coverpack + +# The PR whose head is still the commit the run built. +pr=$(gh api -X GET "repos/$REPO/pulls" -f state=open -f head="$HEAD_OWNER:$HEAD_BRANCH" \ + --jq "[.[] | select(.head.sha == \"$HEAD_SHA\") | .number][0] // empty") +if [ -z "$pr" ]; then + echo "No open PR has $HEAD_SHA as its head." + exit 0 +fi + +# cells RUN: the cells of a run's coverage artifacts, sorted; a failure to list them fails the +# script. A run of a PR names its artifacts as it likes: only plain names of a bounded size +# are taken, as coverpack bounds what it reads of each. +cells() { + gh api "repos/$REPO/actions/runs/$1/artifacts?per_page=100" --jq '.artifacts[] + | select(.expired | not) | select(.name | test("^coverage-linux-[a-z0-9]{1,32}$")) + | select(.size_in_bytes < 16777216) | .name | ltrimstr("coverage-linux-")' | sort | sed -n 1,8p +} + +# lines VAR TEXT: the non-empty lines of TEXT as the array VAR. (The functions here run in a +# command substitution, not a process one, so that their failure fails the script.) +lines() { + local -n into=$1 + into=() + [ -z "$2" ] || mapfile -t into <<<"$2" +} + +# download RUN DIR CELL...: the coverage file of each cell, or the script fails. +download() { + local run=$1 dir=$2 + shift 2 + for cell in "$@"; do + gh run download "$run" -R "$REPO" -n "coverage-linux-$cell" -D "$dir/$cell" >/dev/null + [ -f "$dir/$cell/coverage.bin.zst" ] || { echo "Run $run: no coverage.bin.zst in coverage-linux-$cell" >&2; exit 1; } + echo "$dir/$cell/coverage.bin.zst" + done +} + +out=$(cells "$RUN_ID") +lines head_cells "$out" +if [ ${#head_cells[@]} -eq 0 ]; then + echo "Run $RUN_ID has no coverage." + exit 0 +fi +out=$(download "$RUN_ID" "$work/head" "${head_cells[@]}") +lines head_files "$out" + +# The PR's build tests GitHub's merge of the PR into main, so its lines are those of that +# commit: the PR's change is that commit against its first parent. +merge=$("$coverpack" -commit "${head_files[0]}") +git fetch -q --no-tags origin "$merge" +parents=$(git rev-list --parents -n 1 "$merge") +read -r _ parent1 parent2 rest <<<"$parents" +if [ "$parent2" != "$HEAD_SHA" ] || [ -n "${rest:-}" ] || ! git merge-base --is-ancestor "$parent1" origin/main; then + echo "Commit $merge is not a merge of $HEAD_SHA into main." >&2 + exit 1 +fi + +# The base: the newest main build at or before the PR's starting point that has the coverage of +# the same cells (a build whose cell failed early lacks it). +main_runs=$(gh run list -R "$REPO" --workflow push.yml --branch main --event push --limit 200 \ + --json databaseId,headSha,status --jq '.[] | select(.status == "completed") | "\(.databaseId) \(.headSha)"') +base_args=() +base_desc="" +tries=0 +while read -r run sha; do + [[ $sha =~ $sha_re ]] && git merge-base --is-ancestor "$sha" "$parent1" 2>/dev/null || continue + out=$(cells "$run") + lines base_cells "$out" + if [ "${base_cells[*]}" = "${head_cells[*]}" ]; then + out=$(download "$run" "$work/base-$run" "${base_cells[@]}") + lines base_files "$out" + for f in "${base_files[@]}"; do base_args+=(-base "$f"); done + base_desc="main at [\`${sha:0:9}\`](https://github.com/$REPO/commit/$sha) ([run](https://github.com/$REPO/actions/runs/$run))" + behind=$(git rev-list --count --first-parent "$sha..$parent1") + if [ "$behind" -gt 0 ]; then + base_desc+=", $behind commits before the PR's base: the totals include their changes" + fi + break + fi + tries=$((tries + 1)) + [ "$tries" -lt 10 ] || break +done <<<"$main_runs" + +head_args=() +for f in "${head_files[@]}"; do head_args+=(-head "$f"); done +"$coverpack" -compare -repo . -from "$parent1" -to "$merge" "${head_args[@]}" "${base_args[@]}" \ + -head-desc "[\`${HEAD_SHA:0:9}\`](https://github.com/$REPO/commit/$HEAD_SHA) ([run](https://github.com/$REPO/actions/runs/$RUN_ID))" \ + -base-desc "$base_desc" -o "$work/comment.md" + +if [ "${DRY_RUN:-}" = 1 ]; then + echo "PR #$pr:" + cat "$work/comment.md" + exit 0 +fi + +marker=$(head -n 1 "$work/comment.md") +existing=$(gh api --paginate "repos/$REPO/issues/$pr/comments" \ + --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\"))) | .id" | sed -n 1p) +if [ -n "$existing" ]; then + gh api -X PATCH "repos/$REPO/issues/comments/$existing" -F body=@"$work/comment.md" >/dev/null + echo "Updated the comment on PR #$pr." +else + gh api "repos/$REPO/issues/$pr/comments" -F body=@"$work/comment.md" >/dev/null + echo "Commented on PR #$pr." +fi diff --git a/.github/workflows/coverage-comment.yml b/.github/workflows/coverage-comment.yml new file mode 100644 index 00000000000..9089df57fbe --- /dev/null +++ b/.github/workflows/coverage-comment.yml @@ -0,0 +1,60 @@ +name: Coverage comment + +# Comments on a PR with its coverage against main's, once its build (push.yml) has uploaded it: +# on Linux, for a PR whose head commit message has "with-coverage". A new build updates the same +# comment. This runs from the default branch and reads the build's artifacts only as data +# (.github/scripts/coverage-comment.sh), so it can also comment on fork PRs. +on: + workflow_run: + workflows: [build] + types: [completed] + +# A newer build of the same branch supersedes this one, so an old comment never overwrites a new one. +concurrency: + group: coverage-comment-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: true + +permissions: {} + +jobs: + comment: + if: ${{ github.event.workflow_run.event == 'pull_request' }} + + # The org's IP allow list only lets this runner group call the GitHub API. + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + + # Bounded, as the build's artifacts are the PR author's. + timeout-minutes: 15 + + permissions: + actions: read + contents: read + id-token: write # JFrog, for the Go modules of coverpack + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: Post or update the comment + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + HEAD_OWNER: ${{ github.event.workflow_run.head_repository.owner.login }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + run: .github/scripts/coverage-comment.sh diff --git a/tools/coverpack/compare.go b/tools/coverpack/compare.go new file mode 100644 index 00000000000..8441dc9ab15 --- /dev/null +++ b/tools/coverpack/compare.go @@ -0,0 +1,641 @@ +package main + +// Compare mode (-compare): the coverage of a PR's build against that of a main build, as the +// markdown of a PR comment (.github/workflows/coverage-comment.yml). The files of a PR come from +// its author, so they are read as untrusted data: bounded in size, and every name that reaches +// the markdown is reduced to a safe set of characters. + +import ( + "bufio" + "bytes" + "cmp" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "os" + "os/exec" + "path" + "regexp" + "slices" + "strconv" + "strings" + + "github.com/klauspost/compress/zstd" +) + +const ( + maxPacked = 64 << 20 // bytes of a file once decompressed; the CLI's are about 5 MB + maxShownFiles = 15 // files listed with their uncovered changed lines + maxShownRanges = 8 // ranges of lines listed per file + maxShownPackage = 25 // packages in the table of changed packages + commentMarker = "" + maxCells = 8 // files of a build +) + +// The measures of a comment, in the order of its table. Acceptance is the merged profile of +// the acceptance tests (kindAll), unit that of the unit tests, combined either. +const ( + measureCombined = iota + measureAcceptance + measureUnit + measures +) + +var measureNames = [measures]string{"Combined", "Acceptance", "Unit"} + +type compareOptions struct { + base, head []string // the files of the cells of each build + repo string // the git checkout the diff is read from + from, to string // the PR's change: from main to the commit the head build tested + baseDesc string // markdown naming each build (trusted: written by the workflow) + headDesc string + output string +} + +// fileCov is a file's coverage in a build, its cells merged. +type fileCov struct { + sha [20]byte + blocks []block + acc, unit []byte // bitmaps; nil when the file is not part of that build + conflict bool // the cells measured different versions of the file +} + +func (f *fileCov) covered(j int) [measures]bool { + bit := func(bm []byte) bool { return bm != nil && bm[j/8]>>(j%8)&1 != 0 } + acc, unit := bit(f.acc), bit(f.unit) + return [measures]bool{acc || unit, acc, unit} +} + +type buildCov map[string]*fileCov + +func readPacked(p string) (*decoded, error) { + f, err := os.Open(p) + if err != nil { + return nil, err + } + defer f.Close() + r, err := zstd.NewReader(f, zstd.WithDecoderMaxMemory(maxPacked)) + if err != nil { + return nil, err + } + defer r.Close() + data, err := io.ReadAll(io.LimitReader(r, maxPacked+1)) + if err != nil { + return nil, fmt.Errorf("%s: %w", p, err) + } + if len(data) > maxPacked { + return nil, fmt.Errorf("%s: larger than %d bytes", p, maxPacked) + } + d, err := decode(data) + if err != nil { + return nil, fmt.Errorf("%s: %w", p, err) + } + return d, nil +} + +// loadBuild merges the cells of a build: a block is covered if any cell covered it. +func loadBuild(paths []string) (buildCov, []map[string]any, error) { + if len(paths) > maxCells { + return nil, nil, fmt.Errorf("%d files: more than %d cells", len(paths), maxCells) + } + cov := buildCov{} + var metas []map[string]any + for _, p := range paths { + d, err := readPacked(p) + if err != nil { + return nil, nil, err + } + var meta map[string]any + if err := json.Unmarshal(d.meta, &meta); err != nil { + return nil, nil, fmt.Errorf("%s: meta: %w", p, err) + } + metas = append(metas, meta) + for _, prof := range d.profiles { + if prof.kind != kindAll && prof.kind != kindUnit { + continue + } + for i, l := range prof.files { + lay := d.layouts[l] + f := cov[lay.path] + if f == nil { + f = &fileCov{sha: lay.sha, blocks: lay.blocks} + cov[lay.path] = f + } else if f.sha != lay.sha || len(f.blocks) != len(lay.blocks) { + f.conflict = true + continue + } + dst := &f.acc + if prof.kind == kindUnit { + dst = &f.unit + } + bm := d.bitmaps[l][prof.index[i]] + if *dst == nil { + *dst = make([]byte, len(bm)) + } + for k := range bm { + (*dst)[k] |= bm[k] + } + } + } + } + return cov, metas, nil +} + +type counts struct { + stmts int + covered [measures]int +} + +func (c *counts) add(o counts) { + c.stmts += o.stmts + for m := range c.covered { + c.covered[m] += o.covered[m] + } +} + +func (c *counts) pct(m int) float64 { + if c.stmts == 0 { + return 0 + } + return 100 * float64(c.covered[m]) / float64(c.stmts) +} + +// packages counts the statements of each package (a directory of the module) of a build. +func (b buildCov) packages() map[string]counts { + out := map[string]counts{} + for p, f := range b { + var c counts + for j, blk := range f.blocks { + c.stmts += int(blk.stmts) + for m, ok := range f.covered(j) { + if ok { + c.covered[m] += int(blk.stmts) + } + } + } + pkg := out[path.Dir(p)] + pkg.add(c) + out[path.Dir(p)] = pkg + } + return out +} + +func total(pkgs map[string]counts) counts { + var c counts + for _, p := range pkgs { + c.add(p) + } + return c +} + +// addedLines reads a `git diff -U0`: the added lines of each file, by their number in the new +// version, and every file the diff touches. A hunk's body is skipped by its counts, so a line +// of it that looks like a header is not read as one. +func addedLines(diff io.Reader) (map[string][]int, map[string]bool, error) { + added, touched := map[string][]int{}, map[string]bool{} + var file string + body := 0 // lines left in the current hunk + afterOld := false + s := bufio.NewScanner(diff) + s.Buffer(nil, 16<<20) + for s.Scan() { + line := s.Text() + if body > 0 { + if !strings.HasPrefix(line, `\`) { // "\ No newline at end of file" + body-- + } + continue + } + wasOld := afterOld + afterOld = false + switch { + case strings.HasPrefix(line, "--- "): + afterOld = true + if p, ok := strings.CutPrefix(line[4:], "a/"); ok { + touched[p] = true + } + case wasOld && strings.HasPrefix(line, "+++ "): + file = "" + if p, ok := strings.CutPrefix(line[4:], "b/"); ok { + file = p + touched[p] = true + } + case strings.HasPrefix(line, "@@ "): + oldCount, start, count, err := hunk(line) + if err != nil { + return nil, nil, err + } + body = oldCount + count + for l := start; l < start+count && file != ""; l++ { + added[file] = append(added[file], l) + } + } + } + return added, touched, s.Err() +} + +// hunk reads "@@ -a,b +c,d @@": b, c and d (a count is 1 when left out). +func hunk(line string) (oldCount, start, count int, err error) { + fields := strings.Fields(line) + if len(fields) < 3 || !strings.HasPrefix(fields[1], "-") || !strings.HasPrefix(fields[2], "+") { + return 0, 0, 0, fmt.Errorf("hunk %q", line) + } + span := func(spec string) (int, int, error) { + first, n, found := strings.Cut(spec, ",") + if !found { + n = "1" + } + a, err1 := strconv.Atoi(first) + b, err2 := strconv.Atoi(n) + if err := cmp.Or(err1, err2); err != nil || b < 0 || b > 1<<24 { + return 0, 0, fmt.Errorf("hunk %q", line) + } + return a, b, nil + } + _, oldCount, err = span(fields[1][1:]) + if err != nil { + return 0, 0, 0, err + } + start, count, err = span(fields[2][1:]) + return oldCount, start, count, err +} + +// lineStates is, per wanted line that has code, whether any of its code ran, per measure. +func lineStates(f *fileCov, wanted []int) map[int][measures]bool { + want := map[int]bool{} + for _, l := range wanted { + want[l] = true + } + lo, hi := slices.Min(wanted), slices.Max(wanted) + out := map[int][measures]bool{} + for j, blk := range f.blocks { + if blk.stmts == 0 { + continue + } + last := int(blk.el) + if blk.ec <= 1 && blk.el > blk.sl { + last-- // the block ends before the first byte of its last line + } + cov := f.covered(j) + for l := max(int(blk.sl), lo); l <= min(last, hi); l++ { + if !want[l] { + continue + } + s := out[l] + for m := range s { + s[m] = s[m] || cov[m] + } + out[l] = s + } + } + return out +} + +type patchFile struct { + path string + uncovered []int // lines with code that nothing ran (combined) +} + +type patch struct { + lines int // changed lines that have code + covered [measures]int + files []patchFile // with uncovered lines, the most first + skipped int // changed files whose coverage is of another version of them + packages map[string]bool // that the PR changes a Go file of +} + +func git(repo string, args ...string) ([]byte, error) { + cmd := exec.Command("git", append([]string{"-C", repo}, args...)...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return out, nil +} + +// blobs is the git blob sha of each file of a commit. +func blobs(repo, commit string) (map[string][20]byte, error) { + out, err := git(repo, "ls-tree", "-r", "-z", commit) + if err != nil { + return nil, err + } + result := map[string][20]byte{} + for entry := range bytes.SplitSeq(out, []byte{0}) { + // " blob \t" + meta, p, ok := bytes.Cut(entry, []byte{'\t'}) + fields := bytes.Fields(meta) + if !ok || len(fields) != 3 || string(fields[1]) != "blob" { + continue + } + var sha [20]byte + if _, err := hex.Decode(sha[:], fields[2]); err == nil { + result[string(p)] = sha + } + } + return result, nil +} + +func patchCoverage(head buildCov, o compareOptions) (patch, error) { + var p patch + diff, err := git(o.repo, "-c", "core.quotePath=false", "diff", "-U0", "--no-color", "--no-ext-diff", "--no-renames", "--src-prefix=a/", "--dst-prefix=b/", o.from, o.to, "--", "*.go") + if err != nil { + return p, err + } + added, touched, err := addedLines(bytes.NewReader(diff)) + if err != nil { + return p, err + } + p.packages = map[string]bool{} + for file := range touched { + p.packages[path.Dir(file)] = true + } + shas, err := blobs(o.repo, o.to) + if err != nil { + return p, err + } + for _, file := range slices.Sorted(maps.Keys(added)) { + f := head[file] + if f == nil { + continue // not in the build: a test, or not built on this OS + } + if f.conflict || f.sha != shas[file] { + p.skipped++ + continue + } + states := lineStates(f, added[file]) + pf := patchFile{path: file} + for _, l := range added[file] { + s, ok := states[l] + if !ok { + continue + } + p.lines++ + for m, ran := range s { + if ran { + p.covered[m]++ + } + } + if !s[measureCombined] { + pf.uncovered = append(pf.uncovered, l) + } + } + if len(pf.uncovered) > 0 { + p.files = append(p.files, pf) + } + } + slices.SortStableFunc(p.files, func(a, b patchFile) int { return cmp.Compare(len(b.uncovered), len(a.uncovered)) }) + return p, nil +} + +var unsafeName = regexp.MustCompile(`[^A-Za-z0-9_./+=-]`) + +// safeName is a path for inside backticks in markdown. +func safeName(s string) string { + if len(s) > 200 { + s = s[:200] + "..." + } + return unsafeName.ReplaceAllString(s, "?") +} + +func thousands(n int) string { + s := strconv.Itoa(max(n, -n)) + for i := len(s) - 3; i > 0; i -= 3 { + s = s[:i] + "," + s[i:] + } + if n < 0 { + return "-" + s + } + return s +} + +func pct(part, whole int) string { + if whole == 0 { + return "–" + } + return fmt.Sprintf("%.1f%%", 100*float64(part)/float64(whole)) +} + +func delta(pp float64) string { + if pp > -0.005 && pp < 0.005 { + return "0.00 pp" + } + return fmt.Sprintf("%+.2f pp", pp) +} + +// ranges writes lines (ascending) as "3-5, 9". +func ranges(lines []int, limit int) string { + var parts []string + i := 0 + for i < len(lines) { + j := i + for j+1 < len(lines) && lines[j+1] == lines[j]+1 { + j++ + } + if len(parts) == limit { + parts = append(parts, "...") + break + } + if i == j { + parts = append(parts, strconv.Itoa(lines[i])) + } else { + parts = append(parts, fmt.Sprintf("%d-%d", lines[i], lines[j])) + } + i = j + 1 + } + return strings.Join(parts, ", ") +} + +var cellName = regexp.MustCompile(`^[a-z0-9]{1,32}$`) + +// cells is the names of the cells of a build, sorted: "?" for one that is not a plain name. +func cells(metas []map[string]any) []string { + var names []string + for _, m := range metas { + c, _ := m["cell"].(string) + if !cellName.MatchString(c) { + c = "?" + } + names = append(names, c) + } + slices.Sort(names) + return names +} + +func render(w io.Writer, o compareOptions, base, head map[string]counts, metas []map[string]any, p patch) { + fmt.Fprintln(w, commentMarker) + fmt.Fprintln(w, "### Coverage") + fmt.Fprintln(w) + fmt.Fprintf(w, "Linux (`%s`), statements covered by the unit and the acceptance tests. PR: %s.", strings.Join(cells(metas), "`, `"), o.headDesc) + if base != nil { + fmt.Fprintf(w, " Base: %s.", o.baseDesc) + } + fmt.Fprintln(w) + fmt.Fprintln(w) + + ht := total(head) + if base != nil { + bt := total(base) + fmt.Fprintln(w, "| | Changed lines | Total | Δ vs base |") + fmt.Fprintln(w, "|---|---|---|---|") + for m, name := range measureNames { + fmt.Fprintf(w, "| %s | %s | %.1f%% | %s |\n", name, patchCell(p, m), ht.pct(m), delta(ht.pct(m)-bt.pct(m))) + } + } else { + fmt.Fprintln(w, "| | Changed lines | Total |") + fmt.Fprintln(w, "|---|---|---|") + for m, name := range measureNames { + fmt.Fprintf(w, "| %s | %s | %.1f%% |\n", name, patchCell(p, m), ht.pct(m)) + } + } + fmt.Fprintln(w) + if base == nil { + fmt.Fprintln(w, "No coverage of main was found at or before this PR's base, so there is nothing to compare the totals with.") + fmt.Fprintln(w) + } + if p.skipped > 0 { + fmt.Fprintf(w, "%d changed files are left out: the build measured another version of them.\n\n", p.skipped) + } + + if len(p.files) > 0 { + missed := 0 + for _, f := range p.files { + missed += len(f.uncovered) + } + fmt.Fprintf(w, "
%s changed lines that no test ran, in %d files\n\n", thousands(missed), len(p.files)) + for i, f := range p.files { + if i == maxShownFiles { + fmt.Fprintf(w, "- and %d more files\n", len(p.files)-maxShownFiles) + break + } + fmt.Fprintf(w, "- `%s`: %s\n", safeName(f.path), ranges(f.uncovered, maxShownRanges)) + } + fmt.Fprintln(w, "\n
") + fmt.Fprintln(w) + } + + if base != nil { + renderPackages(w, base, head, p.packages) + } +} + +func patchCell(p patch, m int) string { + if p.lines == 0 { + return "–" + } + return fmt.Sprintf("%s (%s of %s)", pct(p.covered[m], p.lines), thousands(p.covered[m]), thousands(p.lines)) +} + +// renderPackages lists the packages that the PR changes and whose statements or combined +// coverage changed, the most newly uncovered statements first. Other packages can differ too, +// from the commits of main that the base build lacks, but that is not the PR's doing. +func renderPackages(w io.Writer, base, head map[string]counts, changed map[string]bool) { + type row struct { + name string + base, head counts + uncoveredChanged int + } + var rows []row + for name := range maps.Keys(head) { + rows = append(rows, row{name: name, base: base[name], head: head[name]}) + } + for name := range maps.Keys(base) { + if _, ok := head[name]; !ok { + rows = append(rows, row{name: name, base: base[name]}) + } + } + rows = slices.DeleteFunc(rows, func(r row) bool { + return !changed[r.name] || r.base.stmts == r.head.stmts && r.base.covered[measureCombined] == r.head.covered[measureCombined] + }) + if len(rows) == 0 { + return + } + for i := range rows { + r := &rows[i] + r.uncoveredChanged = (r.head.stmts - r.head.covered[measureCombined]) - (r.base.stmts - r.base.covered[measureCombined]) + } + slices.SortFunc(rows, func(a, b row) int { + return cmp.Or(cmp.Compare(max(b.uncoveredChanged, -b.uncoveredChanged), max(a.uncoveredChanged, -a.uncoveredChanged)), strings.Compare(a.name, b.name)) + }) + fmt.Fprintf(w, "
%d changed packages\n\n", len(rows)) + fmt.Fprintln(w, "| Package | Statements | Combined | Δ | Uncovered statements |") + fmt.Fprintln(w, "|---|---|---|---|---|") + for i, r := range rows { + if i == maxShownPackage { + fmt.Fprintf(w, "\nand %d more packages\n", len(rows)-maxShownPackage) + break + } + name := r.name + if name == "." { + name = "(root)" + } + combined, d := pct(r.head.covered[measureCombined], r.head.stmts), "new" + switch { + case r.head.stmts == 0: + combined, d = "–", "removed" + case r.base.stmts > 0: + d = delta(r.head.pct(measureCombined) - r.base.pct(measureCombined)) + } + fmt.Fprintf(w, "| `%s` | %s (%+d) | %s | %s | %+d |\n", safeName(name), thousands(r.head.stmts), r.head.stmts-r.base.stmts, combined, d, r.uncoveredChanged) + } + fmt.Fprintln(w, "\n
") +} + +var hexSha = regexp.MustCompile(`^[0-9a-f]{40}$`) + +func compare(o compareOptions) error { + if len(o.head) == 0 { + return errors.New("-head is required") + } + if !hexSha.MatchString(o.from) || !hexSha.MatchString(o.to) { + return errors.New("-from and -to must be full commit shas") + } + head, metas, err := loadBuild(o.head) + if err != nil { + return err + } + var base map[string]counts + if len(o.base) > 0 { + b, baseMetas, err := loadBuild(o.base) + if err != nil { + return err + } + if !slices.Equal(cells(baseMetas), cells(metas)) { + return fmt.Errorf("the base build has cells %v, the PR's %v", cells(baseMetas), cells(metas)) + } + base = b.packages() + } + p, err := patchCoverage(head, o) + if err != nil { + return err + } + var out bytes.Buffer + render(&out, o, base, head.packages(), metas, p) + if o.output == "" { + _, err = os.Stdout.Write(out.Bytes()) + return err + } + return os.WriteFile(o.output, out.Bytes(), 0o644) +} + +// metaCommit is the commit a packed file says its build tested. +func metaCommit(p string) (string, error) { + d, err := readPacked(p) + if err != nil { + return "", err + } + var meta struct { + Commit string `json:"commit"` + } + if err := json.Unmarshal(d.meta, &meta); err != nil { + return "", err + } + if !hexSha.MatchString(meta.Commit) { + return "", errors.New("the file names no commit") + } + return meta.Commit, nil +} diff --git a/tools/coverpack/compare_test.go b/tools/coverpack/compare_test.go new file mode 100644 index 00000000000..d96fbff87a0 --- /dev/null +++ b/tools/coverpack/compare_test.go @@ -0,0 +1,188 @@ +package main + +import ( + "encoding/hex" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const before = `package m + +func f(x int) int { + return x +} +` + +// The PR adds g: its first branch ran in the acceptance tests, the second ran in nothing. +const after = `package m + +func f(x int) int { + return x +} + +func g(x int) int { + if x > 0 { + return 1 + } + return 2 +} +` + +func commitFile(t *testing.T, repo, content string) string { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(repo, "m", "a.go"), []byte(content), 0o644)) + git := []string{"GIT_CONFIG_GLOBAL=/dev/null", "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t"} + run(t, repo, git, "git", "add", ".") + run(t, repo, git, "git", "commit", "-q", "-m", "change") + out, err := exec.Command("git", "-C", repo, "rev-parse", "HEAD").Output() + require.NoError(t, err) + return strings.TrimSpace(string(out)) +} + +// packFile writes a build that covered blocks (catalog ids of the layout) in its acceptance +// and its unit tests. +func packFile(t *testing.T, l *layout, acc, unit []int32) string { + t.Helper() + bits := func(ids []int32) bitset { + b := newBitset(len(l.blocks)) + for _, id := range ids { + b.set(id) + } + return b + } + data, _ := encode([]byte(`{"cell":"direct"}`), []*layout{l}, []profile{ + {kind: kindAll, files: []int{0}, bits: bits(acc)}, + {kind: kindUnit, files: []int{0}, bits: bits(unit)}, + }) + p := filepath.Join(t.TempDir(), "coverage.bin.zst") + _, err := writeCompressed(p, data) + require.NoError(t, err) + return p +} + +func TestCompare(t *testing.T) { + repo := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(repo, "m"), 0o755)) + run(t, repo, nil, "git", "init", "-q") + from := commitFile(t, repo, before) + to := commitFile(t, repo, after) + blob, err := exec.Command("git", "-C", repo, "rev-parse", to+":m/a.go").Output() + require.NoError(t, err) + + f := block{file: "m/a.go", sl: 3, sc: 19, el: 5, ec: 2, stmts: 1} + baseLayout := &layout{path: "m/a.go", lines: 5, blocks: []block{f}} + headLayout := &layout{path: "m/a.go", lines: 12, blocks: []block{ + f, + {file: "m/a.go", sl: 7, sc: 19, el: 8, ec: 11, stmts: 1}, + {file: "m/a.go", sl: 8, sc: 11, el: 10, ec: 3, stmts: 1}, + {file: "m/a.go", sl: 11, sc: 2, el: 11, ec: 10, stmts: 1}, + }} + _, err = hex.Decode(headLayout.sha[:], []byte(strings.TrimSpace(string(blob)))) + require.NoError(t, err) + + out := filepath.Join(t.TempDir(), "comment.md") + require.NoError(t, compare(compareOptions{ + base: []string{packFile(t, baseLayout, []int32{0}, []int32{0})}, + head: []string{packFile(t, headLayout, []int32{0, 1, 2}, []int32{0})}, + repo: repo, + from: from, + to: to, + baseDesc: "BASE", + headDesc: "HEAD", + output: out, + })) + data, err := os.ReadFile(out) + require.NoError(t, err) + comment := string(data) + + assert.True(t, strings.HasPrefix(comment, commentMarker+"\n")) + assert.Contains(t, comment, "Linux (`direct`), statements covered by the unit and the acceptance tests. PR: HEAD. Base: BASE.") + // Lines 7-11 are the added lines with code: 7-10 ran in the acceptance tests, 11 in nothing. + assert.Contains(t, comment, "| Combined | 80.0% (4 of 5) | 75.0% | -25.00 pp |") + assert.Contains(t, comment, "| Acceptance | 80.0% (4 of 5) | 75.0% | -25.00 pp |") + assert.Contains(t, comment, "| Unit | 0.0% (0 of 5) | 25.0% | -75.00 pp |") + assert.Contains(t, comment, "- `m/a.go`: 11\n") + assert.Contains(t, comment, "| `m` | 4 (+3) | 75.0% | -25.00 pp | +1 |") + + // Without a base, the totals stand alone. + require.NoError(t, compare(compareOptions{ + head: []string{packFile(t, headLayout, []int32{0, 1, 2}, nil)}, repo: repo, from: from, to: to, output: out, + })) + data, err = os.ReadFile(out) + require.NoError(t, err) + assert.Contains(t, string(data), "| Combined | 80.0% (4 of 5) | 75.0% |\n") + assert.Contains(t, string(data), "No coverage of main was found") + assert.NotContains(t, string(data), "changed packages") + + // A build of another version of the file is left out of the changed lines. + headLayout.sha = [20]byte{} + require.NoError(t, compare(compareOptions{ + head: []string{packFile(t, headLayout, []int32{0}, nil)}, repo: repo, from: from, to: to, output: out, + })) + data, err = os.ReadFile(out) + require.NoError(t, err) + assert.Contains(t, string(data), "| Combined | – | 25.0% |") + assert.Contains(t, string(data), "1 changed files are left out") +} + +func TestAddedLines(t *testing.T) { + // The hunk of x.go adds a line that looks like a header; old.go is deleted, new.go added. + diff := `diff --git a/x.go b/x.go +--- a/x.go ++++ b/x.go +@@ -3 +3,3 @@ func f() { +-a ++b +++++ b/fake.go ++c +@@ -10,2 +12,0 @@ +-gone +-gone too +--- a/old.go ++++ /dev/null +@@ -1 +0,0 @@ +-x +--- /dev/null ++++ b/new.go +@@ -0,0 +1 @@ ++package m +\ No newline at end of file +` + added, touched, err := addedLines(strings.NewReader(diff)) + require.NoError(t, err) + assert.Equal(t, map[string][]int{"x.go": {3, 4, 5}, "new.go": {1}}, added) + assert.Equal(t, map[string]bool{"x.go": true, "old.go": true, "new.go": true}, touched) +} + +func TestDecodeRejectsHostileBlocks(t *testing.T) { + // A block past the end of its file (compare would walk its lines), and a file without + // blocks that claims many bitmaps (they would take no bytes). + for _, l := range []*layout{ + {path: "a.go", lines: 3, blocks: []block{{file: "a.go", sl: 1, sc: 1, el: 1 << 31, ec: 2, stmts: 1}}}, + } { + data, _ := encode([]byte(`{}`), []*layout{l}, []profile{{kind: kindAll, files: []int{0}, bits: bitset{1}}}) + _, err := decode(data) + assert.ErrorIs(t, err, errMalformed) + } + data := []byte(magic) + data = append(data, 0, 1, 1, 'a') // no meta, one layout "a" + data = append(data, make([]byte, 20)...) + data = append(data, 0, 0, 100) // no lines, no blocks, 100 bitmaps + data = append(data, make([]byte, 200)...) + _, err := decode(data) + assert.ErrorIs(t, err, errMalformed) +} + +func TestRangesAndNames(t *testing.T) { + assert.Equal(t, "1-3, 5, 9-10", ranges([]int{1, 2, 3, 5, 9, 10}, 8)) + assert.Equal(t, "1, 3, ...", ranges([]int{1, 3, 5}, 2)) + assert.Equal(t, "a/b_c.go", safeName("a/b_c.go")) + assert.Equal(t, "a?b?.go", safeName("a`b|.go")) + assert.Equal(t, "-1,234", thousands(-1234)) +} diff --git a/tools/coverpack/format.go b/tools/coverpack/format.go index 8c5e1279528..039d821a6a4 100644 --- a/tools/coverpack/format.go +++ b/tools/coverpack/format.go @@ -29,10 +29,14 @@ import ( "errors" "fmt" "io" + "math" ) const magic = "CLICOV1\n" +// maxDecodedBlocks bounds what decode allocates for a file from elsewhere; the CLI has about 130k. +const maxDecodedBlocks = 2_000_000 + const ( kindAll = 0 kindUnit = 1 @@ -166,31 +170,52 @@ func decode(data []byte) (*decoded, error) { str := func() []byte { return c.take(c.count()) } d := &decoded{meta: str()} d.layouts = make([]*layout, c.count()) + blocks := 0 for i := range d.layouts { l := &layout{path: string(str())} copy(l.sha[:], c.take(20)) l.lines = int(c.uleb128()) - l.blocks = make([]block, c.count()) - sl := uint32(0) + n := c.count() + if blocks += n; blocks > maxDecodedBlocks { + return nil, errMalformed + } + l.blocks = make([]block, n) + sl := uint64(0) for j := range l.blocks { - sl += uint32(c.uleb128()) - b := block{file: l.path, sl: sl, sc: uint32(c.uleb128())} - b.el = sl + uint32(c.uleb128()) - b.ec = uint32(c.uleb128()) - b.stmts = uint32(c.uleb128()) - l.blocks[j] = b + sl += c.uleb128() + sc := c.uleb128() + el := sl + c.uleb128() + ec, stmts := c.uleb128(), c.uleb128() + // The lines of a block are within its file, which compare walks (compare.go). + if c.err == nil && (el > math.MaxUint32 || l.lines > 0 && el > uint64(l.lines)) { + return nil, errMalformed + } + l.blocks[j] = block{file: l.path, sl: uint32(sl), sc: uint32(sc), el: uint32(el), ec: uint32(ec), stmts: uint32(stmts)} } d.layouts[i] = l } d.bitmaps = make([][][]byte, len(d.layouts)) for i, l := range d.layouts { - d.bitmaps[i] = make([][]byte, c.count()) + n, size := c.count(), (len(l.blocks)+7)/8 + // count bounds n by the bytes left, but a bitmap of a file without blocks has none. + if size == 0 && n > 1 || size > 0 && n > (len(c.b)-c.off)/size { + return nil, errMalformed + } + d.bitmaps[i] = make([][]byte, n) for j := range d.bitmaps[i] { - d.bitmaps[i][j] = c.take((len(l.blocks) + 7) / 8) + d.bitmaps[i][j] = c.take(size) } } - d.profiles = make([]decodedProfile, c.count()) + // A profile takes at least 3 bytes: its kind, the length of its name and its file count. + n := c.count() + if n > (len(c.b)-c.off)/3 { + return nil, errMalformed + } + d.profiles = make([]decodedProfile, n) for i := range d.profiles { + if c.err != nil { + return nil, c.err + } p := decodedProfile{kind: c.u8(), name: string(str())} n, l := c.count(), 0 for range n { diff --git a/tools/coverpack/main.go b/tools/coverpack/main.go index e4d233d24f8..83f64f44bcb 100644 --- a/tools/coverpack/main.go +++ b/tools/coverpack/main.go @@ -49,10 +49,28 @@ func main() { flag.StringVar(&o.meta, "meta", "{}", "JSON object for the meta (sha, os, cell, ...)") flag.StringVar(&o.output, "o", "", "the .bin.zst to write") dumpPath := flag.String("dump", "", "print `FILE` as Go cover profile text instead") + commitOf := flag.String("commit", "", "print the commit that the build of `FILE` tested instead") + var c compareOptions + compareMode := flag.Bool("compare", false, "write the markdown of a PR comment instead (compare.go)") + flag.Func("base", "with -compare: a `FILE` of the base build (repeated for each cell)", func(s string) error { c.base = append(c.base, s); return nil }) + flag.Func("head", "with -compare: a `FILE` of the PR's build (repeated for each cell)", func(s string) error { c.head = append(c.head, s); return nil }) + flag.StringVar(&c.repo, "repo", ".", "with -compare: the git checkout to read the PR's change from") + flag.StringVar(&c.from, "from", "", "with -compare: the commit the PR's change starts from") + flag.StringVar(&c.to, "to", "", "with -compare: the commit the PR's build tested") + flag.StringVar(&c.baseDesc, "base-desc", "", "with -compare: markdown naming the base build") + flag.StringVar(&c.headDesc, "head-desc", "", "with -compare: markdown naming the PR's build") flag.Parse() var err error switch { + case *compareMode: + c.output = o.output + err = compare(c) + case *commitOf != "": + var commit string + if commit, err = metaCommit(*commitOf); err == nil { + fmt.Println(commit) + } case *dumpPath != "": err = dump(os.Stdout, *dumpPath) case o.output == "": @@ -429,21 +447,7 @@ func writeCompressed(path string, data []byte) (int64, error) { } func dump(out io.Writer, path string) error { - f, err := os.Open(path) - if err != nil { - return err - } - defer f.Close() - r, err := zstd.NewReader(f) - if err != nil { - return err - } - defer r.Close() - data, err := io.ReadAll(r) - if err != nil { - return err - } - d, err := decode(data) + d, err := readPacked(path) if err != nil { return err }