From ecdd1039523449682649934c23d5fa7c59387e77 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 20 Nov 2026 16:08:49 +0000 Subject: [PATCH 1/2] Comment on PRs with their coverage against main A new workflow runs after the build of a PR and, when the build uploaded coverage (Linux, a head commit message with "with-coverage"), posts or updates one comment: the coverage of the PR's changed lines, the totals against the newest main build its change starts from, the changed lines that no test ran, and the packages whose coverage changed. `coverpack -compare` computes it from the packed files of the two builds and git, reading the PR's files as untrusted data. The workflow runs from the default branch, so fork PRs get the comment too. with-coverage Co-authored-by: Isaac --- .github/scripts/coverage-comment.sh | 90 ++++ .github/workflows/coverage-comment.yml | 57 +++ tools/coverpack/compare.go | 590 +++++++++++++++++++++++++ tools/coverpack/compare_test.go | 163 +++++++ tools/coverpack/format.go | 7 + tools/coverpack/main.go | 34 +- 6 files changed, 926 insertions(+), 15 deletions(-) create mode 100755 .github/scripts/coverage-comment.sh create mode 100644 .github/workflows/coverage-comment.yml create mode 100644 tools/coverpack/compare.go create mode 100644 tools/coverpack/compare_test.go diff --git a/.github/scripts/coverage-comment.sh b/.github/scripts/coverage-comment.sh new file mode 100755 index 00000000000..96e5f8cd769 --- /dev/null +++ b/.github/scripts/coverage-comment.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Posts, or updates, the coverage comment of a PR (.github/workflows/coverage-comment.yml): the +# coverage that its build run uploaded, against that of the newest main build its change starts +# from. Exits quietly when the run has no coverage, or its PR has moved on to another commit. +# +# The run's artifacts and their meta come from the PR's author: they are only read as data, by +# `coverpack -compare`, and what they name is checked against git and the GitHub API. +# +# Env: GH_TOKEN, REPO (owner/name), RUN_ID (the PR's build run), HEAD_SHA (the PR commit it +# built), HEAD_OWNER and HEAD_BRANCH (where that commit is). With DRY_RUN=1, prints the comment. +# Run from the root of a full clone of the default branch. +set -euo pipefail + +sha_re='^[0-9a-f]{40}$' +[[ $HEAD_SHA =~ $sha_re ]] || { echo "bad HEAD_SHA" >&2; exit 1; } +[[ $RUN_ID =~ ^[0-9]+$ ]] || { echo "bad RUN_ID" >&2; exit 1; } + +work=$(mktemp -d) +coverpack="$work/coverpack" +go -C tools build -o "$coverpack" ./coverpack + +# The PR whose head is still the commit the run built. +pr=$(gh api -X GET "repos/$REPO/pulls" -f state=open -f head="$HEAD_OWNER:$HEAD_BRANCH" \ + --jq ".[] | select(.head.sha == \"$HEAD_SHA\") | .number" | head -n 1) +if [ -z "$pr" ]; then + echo "No open PR has $HEAD_SHA as its head." + exit 0 +fi + +# download RUN DIR: the coverage files of a run, one per cell, as -head or -base flags. +download() { + gh run download "$1" -R "$REPO" -p 'coverage-linux-*' -D "$2" >/dev/null 2>&1 || true + find "$2" -name coverage.bin.zst 2>/dev/null | sort +} + +mapfile -t head_files < <(download "$RUN_ID" "$work/head") +if [ ${#head_files[@]} -eq 0 ]; then + echo "Run $RUN_ID has no coverage." + exit 0 +fi + +# The PR's build tests GitHub's merge of the PR into main, so its lines are those of that +# commit: the PR's change is that commit against its first parent. +merge=$("$coverpack" -commit "${head_files[0]}") +git fetch -q --no-tags origin "$merge" +read -r _ parent1 parent2 rest < <(git rev-list --parents -n 1 "$merge") +if [ "$parent2" != "$HEAD_SHA" ] || [ -n "${rest:-}" ] || ! git merge-base --is-ancestor "$parent1" origin/main; then + echo "Commit $merge is not a merge of $HEAD_SHA into main." >&2 + exit 1 +fi + +# The base: the newest main build at or before the PR's starting point that has coverage left. +base_args=() +base_desc="" +tries=0 +while read -r run sha; do + [[ $sha =~ $sha_re ]] && git merge-base --is-ancestor "$sha" "$parent1" 2>/dev/null || continue + mapfile -t base_files < <(download "$run" "$work/base-$run") + if [ ${#base_files[@]} -gt 0 ]; then + for f in "${base_files[@]}"; do base_args+=(-base "$f"); done + base_desc="main at [\`${sha:0:9}\`](https://github.com/$REPO/commit/$sha) ([run](https://github.com/$REPO/actions/runs/$run))" + break + fi + tries=$((tries + 1)) + [ "$tries" -lt 10 ] || break +done < <(gh run list -R "$REPO" --workflow push.yml --branch main --event push --limit 200 \ + --json databaseId,headSha,status --jq '.[] | select(.status == "completed") | "\(.databaseId) \(.headSha)"') + +head_args=() +for f in "${head_files[@]}"; do head_args+=(-head "$f"); done +"$coverpack" -compare -repo . -from "$parent1" -to "$merge" "${head_args[@]}" "${base_args[@]}" \ + -head-desc "[\`${HEAD_SHA:0:9}\`](https://github.com/$REPO/commit/$HEAD_SHA) ([run](https://github.com/$REPO/actions/runs/$RUN_ID))" \ + -base-desc "$base_desc" -o "$work/comment.md" + +if [ "${DRY_RUN:-}" = 1 ]; then + echo "PR #$pr:" + cat "$work/comment.md" + exit 0 +fi + +marker=$(head -n 1 "$work/comment.md") +existing=$(gh api --paginate "repos/$REPO/issues/$pr/comments" \ + --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\"))) | .id" | head -n 1) +if [ -n "$existing" ]; then + gh api -X PATCH "repos/$REPO/issues/comments/$existing" -F body=@"$work/comment.md" >/dev/null + echo "Updated the comment on PR #$pr." +else + gh api "repos/$REPO/issues/$pr/comments" -F body=@"$work/comment.md" >/dev/null + echo "Commented on PR #$pr." +fi diff --git a/.github/workflows/coverage-comment.yml b/.github/workflows/coverage-comment.yml new file mode 100644 index 00000000000..17ae8f5afc6 --- /dev/null +++ b/.github/workflows/coverage-comment.yml @@ -0,0 +1,57 @@ +name: Coverage comment + +# Comments on a PR with its coverage against main's, once its build (push.yml) has uploaded it: +# on Linux, for a PR whose head commit message has "with-coverage". A new build updates the same +# comment. This runs from the default branch and reads the build's artifacts only as data +# (.github/scripts/coverage-comment.sh), so it can also comment on fork PRs. +on: + workflow_run: + workflows: [build] + types: [completed] + +# A newer build of the same branch supersedes this one, so an old comment never overwrites a new one. +concurrency: + group: coverage-comment-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: true + +permissions: {} + +jobs: + comment: + if: ${{ github.event.workflow_run.event == 'pull_request' }} + + # The org's IP allow list only lets this runner group call the GitHub API. + runs-on: + group: databricks-deco-testing-runner-group + labels: ubuntu-latest-deco + + permissions: + actions: read + contents: read + id-token: write # JFrog, for the Go modules of coverpack + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + - name: Setup Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: Post or update the comment + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + HEAD_OWNER: ${{ github.event.workflow_run.head_repository.owner.login }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + run: .github/scripts/coverage-comment.sh diff --git a/tools/coverpack/compare.go b/tools/coverpack/compare.go new file mode 100644 index 00000000000..21c98aed2c9 --- /dev/null +++ b/tools/coverpack/compare.go @@ -0,0 +1,590 @@ +package main + +// Compare mode (-compare): the coverage of a PR's build against that of a main build, as the +// markdown of a PR comment (.github/workflows/coverage-comment.yml). The files of a PR come from +// its author, so they are read as untrusted data: bounded in size, and every name that reaches +// the markdown is reduced to a safe set of characters. + +import ( + "bufio" + "bytes" + "cmp" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "os" + "os/exec" + "path" + "regexp" + "slices" + "strconv" + "strings" + + "github.com/klauspost/compress/zstd" +) + +const ( + maxPacked = 64 << 20 // bytes of a file once decompressed; the CLI's are about 5 MB + maxShownFiles = 15 // files listed with their uncovered changed lines + maxShownRanges = 8 // ranges of lines listed per file + maxShownPackage = 25 // packages in the table of changed packages + commentMarker = "" +) + +// The measures of a comment, in the order of its table. Acceptance is the merged profile of +// the acceptance tests (kindAll), unit that of the unit tests, combined either. +const ( + measureCombined = iota + measureAcceptance + measureUnit + measures +) + +var measureNames = [measures]string{"Combined", "Acceptance", "Unit"} + +type compareOptions struct { + base, head []string // the files of the cells of each build + repo string // the git checkout the diff is read from + from, to string // the PR's change: from main to the commit the head build tested + baseDesc string // markdown naming each build (trusted: written by the workflow) + headDesc string + output string +} + +// fileCov is a file's coverage in a build, its cells merged. +type fileCov struct { + sha [20]byte + blocks []block + acc, unit []byte // bitmaps; nil when the file is not part of that build + conflict bool // the cells measured different versions of the file +} + +func (f *fileCov) covered(j int) [measures]bool { + bit := func(bm []byte) bool { return bm != nil && bm[j/8]>>(j%8)&1 != 0 } + acc, unit := bit(f.acc), bit(f.unit) + return [measures]bool{acc || unit, acc, unit} +} + +type buildCov map[string]*fileCov + +func readPacked(p string) (*decoded, error) { + f, err := os.Open(p) + if err != nil { + return nil, err + } + defer f.Close() + r, err := zstd.NewReader(f, zstd.WithDecoderMaxMemory(maxPacked)) + if err != nil { + return nil, err + } + defer r.Close() + data, err := io.ReadAll(io.LimitReader(r, maxPacked+1)) + if err != nil { + return nil, fmt.Errorf("%s: %w", p, err) + } + if len(data) > maxPacked { + return nil, fmt.Errorf("%s: larger than %d bytes", p, maxPacked) + } + d, err := decode(data) + if err != nil { + return nil, fmt.Errorf("%s: %w", p, err) + } + return d, nil +} + +// loadBuild merges the cells of a build: a block is covered if any cell covered it. +func loadBuild(paths []string) (buildCov, []map[string]any, error) { + cov := buildCov{} + var metas []map[string]any + for _, p := range paths { + d, err := readPacked(p) + if err != nil { + return nil, nil, err + } + var meta map[string]any + if err := json.Unmarshal(d.meta, &meta); err != nil { + return nil, nil, fmt.Errorf("%s: meta: %w", p, err) + } + metas = append(metas, meta) + for _, prof := range d.profiles { + if prof.kind != kindAll && prof.kind != kindUnit { + continue + } + for i, l := range prof.files { + lay := d.layouts[l] + f := cov[lay.path] + if f == nil { + f = &fileCov{sha: lay.sha, blocks: lay.blocks} + cov[lay.path] = f + } else if f.sha != lay.sha || len(f.blocks) != len(lay.blocks) { + f.conflict = true + continue + } + dst := &f.acc + if prof.kind == kindUnit { + dst = &f.unit + } + bm := d.bitmaps[l][prof.index[i]] + if *dst == nil { + *dst = make([]byte, len(bm)) + } + for k := range bm { + (*dst)[k] |= bm[k] + } + } + } + } + return cov, metas, nil +} + +type counts struct { + stmts int + covered [measures]int +} + +func (c *counts) add(o counts) { + c.stmts += o.stmts + for m := range c.covered { + c.covered[m] += o.covered[m] + } +} + +func (c *counts) pct(m int) float64 { + if c.stmts == 0 { + return 0 + } + return 100 * float64(c.covered[m]) / float64(c.stmts) +} + +// packages counts the statements of each package (a directory of the module) of a build. +func (b buildCov) packages() map[string]counts { + out := map[string]counts{} + for p, f := range b { + var c counts + for j, blk := range f.blocks { + c.stmts += int(blk.stmts) + for m, ok := range f.covered(j) { + if ok { + c.covered[m] += int(blk.stmts) + } + } + } + pkg := out[path.Dir(p)] + pkg.add(c) + out[path.Dir(p)] = pkg + } + return out +} + +func total(pkgs map[string]counts) counts { + var c counts + for _, p := range pkgs { + c.add(p) + } + return c +} + +// addedLines reads a `git diff -U0`: the added lines of each file, by their number in the new +// version. +func addedLines(diff io.Reader) (map[string][]int, error) { + out := map[string][]int{} + var file string + s := bufio.NewScanner(diff) + s.Buffer(nil, 16<<20) + for s.Scan() { + line := s.Text() + if p, ok := strings.CutPrefix(line, "+++ "); ok { + file, _ = strings.CutPrefix(p, "b/") // "/dev/null" (a deleted file) matches no file + continue + } + rest, ok := strings.CutPrefix(line, "@@ ") + if !ok || file == "" { + continue + } + for field := range strings.FieldsSeq(rest) { + spec, ok := strings.CutPrefix(field, "+") + if !ok { + continue + } + first, n, found := strings.Cut(spec, ",") + if !found { + n = "1" + } + start, err1 := strconv.Atoi(first) + count, err2 := strconv.Atoi(n) + if err := cmp.Or(err1, err2); err != nil { + return nil, fmt.Errorf("hunk %q: %w", line, err) + } + for l := start; l < start+count; l++ { + out[file] = append(out[file], l) + } + break + } + } + return out, s.Err() +} + +// lineStates is, per wanted line that has code, whether any of its code ran, per measure. +func lineStates(f *fileCov, wanted []int) map[int][measures]bool { + want := map[int]bool{} + for _, l := range wanted { + want[l] = true + } + out := map[int][measures]bool{} + for j, blk := range f.blocks { + if blk.stmts == 0 { + continue + } + last := int(blk.el) + if blk.ec <= 1 && blk.el > blk.sl { + last-- // the block ends before the first byte of its last line + } + cov := f.covered(j) + for l := int(blk.sl); l <= last; l++ { + if !want[l] { + continue + } + s := out[l] + for m := range s { + s[m] = s[m] || cov[m] + } + out[l] = s + } + } + return out +} + +type patchFile struct { + path string + uncovered []int // lines with code that nothing ran (combined) +} + +type patch struct { + lines int // changed lines that have code + covered [measures]int + files []patchFile // with uncovered lines, the most first + skipped int // changed files whose coverage is of another version of them +} + +func git(repo string, args ...string) ([]byte, error) { + cmd := exec.Command("git", append([]string{"-C", repo}, args...)...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(stderr.String())) + } + return out, nil +} + +// blobs is the git blob sha of each file of a commit. +func blobs(repo, commit string) (map[string][20]byte, error) { + out, err := git(repo, "ls-tree", "-r", "-z", commit) + if err != nil { + return nil, err + } + result := map[string][20]byte{} + for entry := range bytes.SplitSeq(out, []byte{0}) { + // " blob \t" + meta, p, ok := bytes.Cut(entry, []byte{'\t'}) + fields := bytes.Fields(meta) + if !ok || len(fields) != 3 || string(fields[1]) != "blob" { + continue + } + var sha [20]byte + if _, err := hex.Decode(sha[:], fields[2]); err == nil { + result[string(p)] = sha + } + } + return result, nil +} + +func patchCoverage(head buildCov, o compareOptions) (patch, error) { + var p patch + diff, err := git(o.repo, "diff", "-U0", "--no-color", "--no-ext-diff", "--no-renames", "--src-prefix=a/", "--dst-prefix=b/", o.from, o.to, "--", "*.go") + if err != nil { + return p, err + } + added, err := addedLines(bytes.NewReader(diff)) + if err != nil { + return p, err + } + shas, err := blobs(o.repo, o.to) + if err != nil { + return p, err + } + for _, file := range slices.Sorted(maps.Keys(added)) { + f := head[file] + if f == nil { + continue // not in the build: a test, or not built on this OS + } + if f.conflict || f.sha != shas[file] { + p.skipped++ + continue + } + states := lineStates(f, added[file]) + pf := patchFile{path: file} + for _, l := range added[file] { + s, ok := states[l] + if !ok { + continue + } + p.lines++ + for m, ran := range s { + if ran { + p.covered[m]++ + } + } + if !s[measureCombined] { + pf.uncovered = append(pf.uncovered, l) + } + } + if len(pf.uncovered) > 0 { + p.files = append(p.files, pf) + } + } + slices.SortStableFunc(p.files, func(a, b patchFile) int { return cmp.Compare(len(b.uncovered), len(a.uncovered)) }) + return p, nil +} + +var unsafeName = regexp.MustCompile(`[^A-Za-z0-9_./+=-]`) + +// safeName is a path for inside backticks in markdown. +func safeName(s string) string { + if len(s) > 200 { + s = s[:200] + "..." + } + return unsafeName.ReplaceAllString(s, "?") +} + +func thousands(n int) string { + s := strconv.Itoa(max(n, -n)) + for i := len(s) - 3; i > 0; i -= 3 { + s = s[:i] + "," + s[i:] + } + if n < 0 { + return "-" + s + } + return s +} + +func pct(part, whole int) string { + if whole == 0 { + return "–" + } + return fmt.Sprintf("%.1f%%", 100*float64(part)/float64(whole)) +} + +func delta(pp float64) string { + if pp > -0.005 && pp < 0.005 { + return "0.00 pp" + } + return fmt.Sprintf("%+.2f pp", pp) +} + +// ranges writes lines (ascending) as "3-5, 9". +func ranges(lines []int, limit int) string { + var parts []string + i := 0 + for i < len(lines) { + j := i + for j+1 < len(lines) && lines[j+1] == lines[j]+1 { + j++ + } + if len(parts) == limit { + parts = append(parts, "...") + break + } + if i == j { + parts = append(parts, strconv.Itoa(lines[i])) + } else { + parts = append(parts, fmt.Sprintf("%d-%d", lines[i], lines[j])) + } + i = j + 1 + } + return strings.Join(parts, ", ") +} + +func cells(metas []map[string]any) string { + var names []string + for _, m := range metas { + if c, ok := m["cell"].(string); ok { + names = append(names, safeName(c)) + } + } + slices.Sort(names) + return strings.Join(slices.Compact(names), ", ") +} + +func render(w io.Writer, o compareOptions, base, head map[string]counts, metas []map[string]any, p patch) { + fmt.Fprintln(w, commentMarker) + fmt.Fprintln(w, "### Coverage") + fmt.Fprintln(w) + fmt.Fprintf(w, "Linux (%s), statements covered by the unit and the acceptance tests. PR: %s.", cells(metas), o.headDesc) + if base != nil { + fmt.Fprintf(w, " Base: %s.", o.baseDesc) + } + fmt.Fprintln(w) + fmt.Fprintln(w) + + ht := total(head) + if base != nil { + bt := total(base) + fmt.Fprintln(w, "| | Changed lines | Total | Δ vs base |") + fmt.Fprintln(w, "|---|---|---|---|") + for m, name := range measureNames { + fmt.Fprintf(w, "| %s | %s | %.1f%% | %s |\n", name, patchCell(p, m), ht.pct(m), delta(ht.pct(m)-bt.pct(m))) + } + } else { + fmt.Fprintln(w, "| | Changed lines | Total |") + fmt.Fprintln(w, "|---|---|---|") + for m, name := range measureNames { + fmt.Fprintf(w, "| %s | %s | %.1f%% |\n", name, patchCell(p, m), ht.pct(m)) + } + } + fmt.Fprintln(w) + if base == nil { + fmt.Fprintln(w, "No coverage of main was found at or before this PR's base, so there is nothing to compare the totals with.") + fmt.Fprintln(w) + } + if p.skipped > 0 { + fmt.Fprintf(w, "%d changed files are left out: the build measured another version of them.\n\n", p.skipped) + } + + if len(p.files) > 0 { + missed := 0 + for _, f := range p.files { + missed += len(f.uncovered) + } + fmt.Fprintf(w, "
%s changed lines that no test ran, in %d files\n\n", thousands(missed), len(p.files)) + for i, f := range p.files { + if i == maxShownFiles { + fmt.Fprintf(w, "- and %d more files\n", len(p.files)-maxShownFiles) + break + } + fmt.Fprintf(w, "- `%s`: %s\n", safeName(f.path), ranges(f.uncovered, maxShownRanges)) + } + fmt.Fprintln(w, "\n
") + fmt.Fprintln(w) + } + + if base != nil { + renderPackages(w, base, head) + } +} + +func patchCell(p patch, m int) string { + if p.lines == 0 { + return "–" + } + return fmt.Sprintf("%s (%s of %s)", pct(p.covered[m], p.lines), thousands(p.covered[m]), thousands(p.lines)) +} + +// renderPackages lists the packages whose statements or combined coverage changed, the most +// newly uncovered statements first. +func renderPackages(w io.Writer, base, head map[string]counts) { + type row struct { + name string + base, head counts + uncoveredChanged int + } + var rows []row + for name := range maps.Keys(head) { + rows = append(rows, row{name: name, base: base[name], head: head[name]}) + } + for name := range maps.Keys(base) { + if _, ok := head[name]; !ok { + rows = append(rows, row{name: name, base: base[name]}) + } + } + rows = slices.DeleteFunc(rows, func(r row) bool { + return r.base.stmts == r.head.stmts && r.base.covered[measureCombined] == r.head.covered[measureCombined] + }) + if len(rows) == 0 { + return + } + for i := range rows { + r := &rows[i] + r.uncoveredChanged = (r.head.stmts - r.head.covered[measureCombined]) - (r.base.stmts - r.base.covered[measureCombined]) + } + slices.SortFunc(rows, func(a, b row) int { + return cmp.Or(cmp.Compare(max(b.uncoveredChanged, -b.uncoveredChanged), max(a.uncoveredChanged, -a.uncoveredChanged)), strings.Compare(a.name, b.name)) + }) + fmt.Fprintf(w, "
%d packages changed\n\n", len(rows)) + fmt.Fprintln(w, "| Package | Statements | Combined | Δ | Uncovered statements |") + fmt.Fprintln(w, "|---|---|---|---|---|") + for i, r := range rows { + if i == maxShownPackage { + fmt.Fprintf(w, "\nand %d more packages\n", len(rows)-maxShownPackage) + break + } + name := r.name + if name == "." { + name = "(root)" + } + combined, d := pct(r.head.covered[measureCombined], r.head.stmts), "new" + switch { + case r.head.stmts == 0: + combined, d = "–", "removed" + case r.base.stmts > 0: + d = delta(r.head.pct(measureCombined) - r.base.pct(measureCombined)) + } + fmt.Fprintf(w, "| `%s` | %s (%+d) | %s | %s | %+d |\n", safeName(name), thousands(r.head.stmts), r.head.stmts-r.base.stmts, combined, d, r.uncoveredChanged) + } + fmt.Fprintln(w, "\n
") +} + +var hexSha = regexp.MustCompile(`^[0-9a-f]{40}$`) + +func compare(o compareOptions) error { + if len(o.head) == 0 { + return errors.New("-head is required") + } + if !hexSha.MatchString(o.from) || !hexSha.MatchString(o.to) { + return errors.New("-from and -to must be full commit shas") + } + head, metas, err := loadBuild(o.head) + if err != nil { + return err + } + var base map[string]counts + if len(o.base) > 0 { + b, _, err := loadBuild(o.base) + if err != nil { + return err + } + base = b.packages() + } + p, err := patchCoverage(head, o) + if err != nil { + return err + } + var out bytes.Buffer + render(&out, o, base, head.packages(), metas, p) + if o.output == "" { + _, err = os.Stdout.Write(out.Bytes()) + return err + } + return os.WriteFile(o.output, out.Bytes(), 0o644) +} + +// metaCommit is the commit a packed file says its build tested. +func metaCommit(p string) (string, error) { + d, err := readPacked(p) + if err != nil { + return "", err + } + var meta struct { + Commit string `json:"commit"` + } + if err := json.Unmarshal(d.meta, &meta); err != nil { + return "", err + } + if !hexSha.MatchString(meta.Commit) { + return "", errors.New("the file names no commit") + } + return meta.Commit, nil +} diff --git a/tools/coverpack/compare_test.go b/tools/coverpack/compare_test.go new file mode 100644 index 00000000000..b9fb68c05ed --- /dev/null +++ b/tools/coverpack/compare_test.go @@ -0,0 +1,163 @@ +package main + +import ( + "encoding/hex" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const before = `package m + +func f(x int) int { + return x +} +` + +// The PR adds g: its first branch ran in the acceptance tests, the second ran in nothing. +const after = `package m + +func f(x int) int { + return x +} + +func g(x int) int { + if x > 0 { + return 1 + } + return 2 +} +` + +func commitFile(t *testing.T, repo, content string) string { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(repo, "m", "a.go"), []byte(content), 0o644)) + git := []string{"GIT_CONFIG_GLOBAL=/dev/null", "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t"} + run(t, repo, git, "git", "add", ".") + run(t, repo, git, "git", "commit", "-q", "-m", "change") + out, err := exec.Command("git", "-C", repo, "rev-parse", "HEAD").Output() + require.NoError(t, err) + return strings.TrimSpace(string(out)) +} + +// packFile writes a build that covered blocks (catalog ids of the layout) in its acceptance +// and its unit tests. +func packFile(t *testing.T, l *layout, acc, unit []int32) string { + t.Helper() + bits := func(ids []int32) bitset { + b := newBitset(len(l.blocks)) + for _, id := range ids { + b.set(id) + } + return b + } + data, _ := encode([]byte(`{"cell":"direct"}`), []*layout{l}, []profile{ + {kind: kindAll, files: []int{0}, bits: bits(acc)}, + {kind: kindUnit, files: []int{0}, bits: bits(unit)}, + }) + p := filepath.Join(t.TempDir(), "coverage.bin.zst") + _, err := writeCompressed(p, data) + require.NoError(t, err) + return p +} + +func TestCompare(t *testing.T) { + repo := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(repo, "m"), 0o755)) + run(t, repo, nil, "git", "init", "-q") + from := commitFile(t, repo, before) + to := commitFile(t, repo, after) + blob, err := exec.Command("git", "-C", repo, "rev-parse", to+":m/a.go").Output() + require.NoError(t, err) + + f := block{file: "m/a.go", sl: 3, sc: 19, el: 5, ec: 2, stmts: 1} + baseLayout := &layout{path: "m/a.go", lines: 5, blocks: []block{f}} + headLayout := &layout{path: "m/a.go", lines: 12, blocks: []block{ + f, + {file: "m/a.go", sl: 7, sc: 19, el: 8, ec: 11, stmts: 1}, + {file: "m/a.go", sl: 8, sc: 11, el: 10, ec: 3, stmts: 1}, + {file: "m/a.go", sl: 11, sc: 2, el: 11, ec: 10, stmts: 1}, + }} + _, err = hex.Decode(headLayout.sha[:], []byte(strings.TrimSpace(string(blob)))) + require.NoError(t, err) + + out := filepath.Join(t.TempDir(), "comment.md") + require.NoError(t, compare(compareOptions{ + base: []string{packFile(t, baseLayout, []int32{0}, []int32{0})}, + head: []string{packFile(t, headLayout, []int32{0, 1, 2}, []int32{0})}, + repo: repo, + from: from, + to: to, + baseDesc: "BASE", + headDesc: "HEAD", + output: out, + })) + data, err := os.ReadFile(out) + require.NoError(t, err) + comment := string(data) + + assert.True(t, strings.HasPrefix(comment, commentMarker+"\n")) + assert.Contains(t, comment, "Linux (direct), statements covered by the unit and the acceptance tests. PR: HEAD. Base: BASE.") + // Lines 7-11 are the added lines with code: 7-10 ran in the acceptance tests, 11 in nothing. + assert.Contains(t, comment, "| Combined | 80.0% (4 of 5) | 75.0% | -25.00 pp |") + assert.Contains(t, comment, "| Acceptance | 80.0% (4 of 5) | 75.0% | -25.00 pp |") + assert.Contains(t, comment, "| Unit | 0.0% (0 of 5) | 25.0% | -75.00 pp |") + assert.Contains(t, comment, "- `m/a.go`: 11\n") + assert.Contains(t, comment, "| `m` | 4 (+3) | 75.0% | -25.00 pp | +1 |") + + // Without a base, the totals stand alone. + require.NoError(t, compare(compareOptions{ + head: []string{packFile(t, headLayout, []int32{0, 1, 2}, nil)}, repo: repo, from: from, to: to, output: out, + })) + data, err = os.ReadFile(out) + require.NoError(t, err) + assert.Contains(t, string(data), "| Combined | 80.0% (4 of 5) | 75.0% |\n") + assert.Contains(t, string(data), "No coverage of main was found") + assert.NotContains(t, string(data), "packages changed") + + // A build of another version of the file is left out of the changed lines. + headLayout.sha = [20]byte{} + require.NoError(t, compare(compareOptions{ + head: []string{packFile(t, headLayout, []int32{0}, nil)}, repo: repo, from: from, to: to, output: out, + })) + data, err = os.ReadFile(out) + require.NoError(t, err) + assert.Contains(t, string(data), "| Combined | – | 25.0% |") + assert.Contains(t, string(data), "1 changed files are left out") +} + +func TestAddedLines(t *testing.T) { + diff := `diff --git a/x.go b/x.go +--- a/x.go ++++ b/x.go +@@ -3 +3,2 @@ func f() { +-a ++b ++c +@@ -10,2 +11,0 @@ +-gone +--- a/old.go ++++ /dev/null +@@ -1 +0,0 @@ +-x ++++ b/new.go +@@ -0,0 +1 @@ ++package m +` + got, err := addedLines(strings.NewReader(diff)) + require.NoError(t, err) + assert.Equal(t, map[string][]int{"x.go": {3, 4}, "new.go": {1}}, got) +} + +func TestRangesAndNames(t *testing.T) { + assert.Equal(t, "1-3, 5, 9-10", ranges([]int{1, 2, 3, 5, 9, 10}, 8)) + assert.Equal(t, "1, 3, ...", ranges([]int{1, 3, 5}, 2)) + assert.Equal(t, "a/b_c.go", safeName("a/b_c.go")) + assert.Equal(t, "a?b?.go", safeName("a`b|.go")) + assert.Equal(t, "-1,234", thousands(-1234)) +} diff --git a/tools/coverpack/format.go b/tools/coverpack/format.go index 8c5e1279528..99aba2ffea8 100644 --- a/tools/coverpack/format.go +++ b/tools/coverpack/format.go @@ -33,6 +33,9 @@ import ( const magic = "CLICOV1\n" +// maxDecodedBlocks bounds what decode allocates for a file from elsewhere; the CLI has about 130k. +const maxDecodedBlocks = 2_000_000 + const ( kindAll = 0 kindUnit = 1 @@ -166,11 +169,15 @@ func decode(data []byte) (*decoded, error) { str := func() []byte { return c.take(c.count()) } d := &decoded{meta: str()} d.layouts = make([]*layout, c.count()) + blocks := 0 for i := range d.layouts { l := &layout{path: string(str())} copy(l.sha[:], c.take(20)) l.lines = int(c.uleb128()) l.blocks = make([]block, c.count()) + if blocks += len(l.blocks); blocks > maxDecodedBlocks { + return nil, errMalformed + } sl := uint32(0) for j := range l.blocks { sl += uint32(c.uleb128()) diff --git a/tools/coverpack/main.go b/tools/coverpack/main.go index e4d233d24f8..83f64f44bcb 100644 --- a/tools/coverpack/main.go +++ b/tools/coverpack/main.go @@ -49,10 +49,28 @@ func main() { flag.StringVar(&o.meta, "meta", "{}", "JSON object for the meta (sha, os, cell, ...)") flag.StringVar(&o.output, "o", "", "the .bin.zst to write") dumpPath := flag.String("dump", "", "print `FILE` as Go cover profile text instead") + commitOf := flag.String("commit", "", "print the commit that the build of `FILE` tested instead") + var c compareOptions + compareMode := flag.Bool("compare", false, "write the markdown of a PR comment instead (compare.go)") + flag.Func("base", "with -compare: a `FILE` of the base build (repeated for each cell)", func(s string) error { c.base = append(c.base, s); return nil }) + flag.Func("head", "with -compare: a `FILE` of the PR's build (repeated for each cell)", func(s string) error { c.head = append(c.head, s); return nil }) + flag.StringVar(&c.repo, "repo", ".", "with -compare: the git checkout to read the PR's change from") + flag.StringVar(&c.from, "from", "", "with -compare: the commit the PR's change starts from") + flag.StringVar(&c.to, "to", "", "with -compare: the commit the PR's build tested") + flag.StringVar(&c.baseDesc, "base-desc", "", "with -compare: markdown naming the base build") + flag.StringVar(&c.headDesc, "head-desc", "", "with -compare: markdown naming the PR's build") flag.Parse() var err error switch { + case *compareMode: + c.output = o.output + err = compare(c) + case *commitOf != "": + var commit string + if commit, err = metaCommit(*commitOf); err == nil { + fmt.Println(commit) + } case *dumpPath != "": err = dump(os.Stdout, *dumpPath) case o.output == "": @@ -429,21 +447,7 @@ func writeCompressed(path string, data []byte) (int64, error) { } func dump(out io.Writer, path string) error { - f, err := os.Open(path) - if err != nil { - return err - } - defer f.Close() - r, err := zstd.NewReader(f) - if err != nil { - return err - } - defer r.Close() - data, err := io.ReadAll(r) - if err != nil { - return err - } - d, err := decode(data) + d, err := readPacked(path) if err != nil { return err } From 2b35cb2e0b39885ec332085ef6febee6218ef78c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 20 Nov 2026 16:26:26 +0000 Subject: [PATCH 2/2] coverage comment: bound hostile builds, fail on partial downloads From a review: - decode rejects blocks past the end of their file, bitmaps and profiles that the bytes left cannot hold, and checks the block count before allocating; compare walks only the changed lines of a block, and the job has a 15 minute timeout; - the script lists the artifacts through the API (plain names, under 16 MB, at most 8), fails when a listing or download fails, and takes as the base only a build with the same cells; - cell names are plain words in backticks; the diff is read hunk by hunk, so a changed line that looks like a header is not taken for one; - the package table lists only packages the PR changes, and the comment says how many commits of main the base build is behind. with-coverage Co-authored-by: Isaac --- .github/scripts/coverage-comment.sh | 61 ++++++++--- .github/workflows/coverage-comment.yml | 3 + tools/coverpack/compare.go | 137 +++++++++++++++++-------- tools/coverpack/compare_test.go | 37 +++++-- tools/coverpack/format.go | 42 +++++--- 5 files changed, 206 insertions(+), 74 deletions(-) diff --git a/.github/scripts/coverage-comment.sh b/.github/scripts/coverage-comment.sh index 96e5f8cd769..c90cdba2b98 100755 --- a/.github/scripts/coverage-comment.sh +++ b/.github/scripts/coverage-comment.sh @@ -21,50 +21,85 @@ go -C tools build -o "$coverpack" ./coverpack # The PR whose head is still the commit the run built. pr=$(gh api -X GET "repos/$REPO/pulls" -f state=open -f head="$HEAD_OWNER:$HEAD_BRANCH" \ - --jq ".[] | select(.head.sha == \"$HEAD_SHA\") | .number" | head -n 1) + --jq "[.[] | select(.head.sha == \"$HEAD_SHA\") | .number][0] // empty") if [ -z "$pr" ]; then echo "No open PR has $HEAD_SHA as its head." exit 0 fi -# download RUN DIR: the coverage files of a run, one per cell, as -head or -base flags. +# cells RUN: the cells of a run's coverage artifacts, sorted; a failure to list them fails the +# script. A run of a PR names its artifacts as it likes: only plain names of a bounded size +# are taken, as coverpack bounds what it reads of each. +cells() { + gh api "repos/$REPO/actions/runs/$1/artifacts?per_page=100" --jq '.artifacts[] + | select(.expired | not) | select(.name | test("^coverage-linux-[a-z0-9]{1,32}$")) + | select(.size_in_bytes < 16777216) | .name | ltrimstr("coverage-linux-")' | sort | sed -n 1,8p +} + +# lines VAR TEXT: the non-empty lines of TEXT as the array VAR. (The functions here run in a +# command substitution, not a process one, so that their failure fails the script.) +lines() { + local -n into=$1 + into=() + [ -z "$2" ] || mapfile -t into <<<"$2" +} + +# download RUN DIR CELL...: the coverage file of each cell, or the script fails. download() { - gh run download "$1" -R "$REPO" -p 'coverage-linux-*' -D "$2" >/dev/null 2>&1 || true - find "$2" -name coverage.bin.zst 2>/dev/null | sort + local run=$1 dir=$2 + shift 2 + for cell in "$@"; do + gh run download "$run" -R "$REPO" -n "coverage-linux-$cell" -D "$dir/$cell" >/dev/null + [ -f "$dir/$cell/coverage.bin.zst" ] || { echo "Run $run: no coverage.bin.zst in coverage-linux-$cell" >&2; exit 1; } + echo "$dir/$cell/coverage.bin.zst" + done } -mapfile -t head_files < <(download "$RUN_ID" "$work/head") -if [ ${#head_files[@]} -eq 0 ]; then +out=$(cells "$RUN_ID") +lines head_cells "$out" +if [ ${#head_cells[@]} -eq 0 ]; then echo "Run $RUN_ID has no coverage." exit 0 fi +out=$(download "$RUN_ID" "$work/head" "${head_cells[@]}") +lines head_files "$out" # The PR's build tests GitHub's merge of the PR into main, so its lines are those of that # commit: the PR's change is that commit against its first parent. merge=$("$coverpack" -commit "${head_files[0]}") git fetch -q --no-tags origin "$merge" -read -r _ parent1 parent2 rest < <(git rev-list --parents -n 1 "$merge") +parents=$(git rev-list --parents -n 1 "$merge") +read -r _ parent1 parent2 rest <<<"$parents" if [ "$parent2" != "$HEAD_SHA" ] || [ -n "${rest:-}" ] || ! git merge-base --is-ancestor "$parent1" origin/main; then echo "Commit $merge is not a merge of $HEAD_SHA into main." >&2 exit 1 fi -# The base: the newest main build at or before the PR's starting point that has coverage left. +# The base: the newest main build at or before the PR's starting point that has the coverage of +# the same cells (a build whose cell failed early lacks it). +main_runs=$(gh run list -R "$REPO" --workflow push.yml --branch main --event push --limit 200 \ + --json databaseId,headSha,status --jq '.[] | select(.status == "completed") | "\(.databaseId) \(.headSha)"') base_args=() base_desc="" tries=0 while read -r run sha; do [[ $sha =~ $sha_re ]] && git merge-base --is-ancestor "$sha" "$parent1" 2>/dev/null || continue - mapfile -t base_files < <(download "$run" "$work/base-$run") - if [ ${#base_files[@]} -gt 0 ]; then + out=$(cells "$run") + lines base_cells "$out" + if [ "${base_cells[*]}" = "${head_cells[*]}" ]; then + out=$(download "$run" "$work/base-$run" "${base_cells[@]}") + lines base_files "$out" for f in "${base_files[@]}"; do base_args+=(-base "$f"); done base_desc="main at [\`${sha:0:9}\`](https://github.com/$REPO/commit/$sha) ([run](https://github.com/$REPO/actions/runs/$run))" + behind=$(git rev-list --count --first-parent "$sha..$parent1") + if [ "$behind" -gt 0 ]; then + base_desc+=", $behind commits before the PR's base: the totals include their changes" + fi break fi tries=$((tries + 1)) [ "$tries" -lt 10 ] || break -done < <(gh run list -R "$REPO" --workflow push.yml --branch main --event push --limit 200 \ - --json databaseId,headSha,status --jq '.[] | select(.status == "completed") | "\(.databaseId) \(.headSha)"') +done <<<"$main_runs" head_args=() for f in "${head_files[@]}"; do head_args+=(-head "$f"); done @@ -80,7 +115,7 @@ fi marker=$(head -n 1 "$work/comment.md") existing=$(gh api --paginate "repos/$REPO/issues/$pr/comments" \ - --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\"))) | .id" | head -n 1) + --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"$marker\"))) | .id" | sed -n 1p) if [ -n "$existing" ]; then gh api -X PATCH "repos/$REPO/issues/comments/$existing" -F body=@"$work/comment.md" >/dev/null echo "Updated the comment on PR #$pr." diff --git a/.github/workflows/coverage-comment.yml b/.github/workflows/coverage-comment.yml index 17ae8f5afc6..9089df57fbe 100644 --- a/.github/workflows/coverage-comment.yml +++ b/.github/workflows/coverage-comment.yml @@ -25,6 +25,9 @@ jobs: group: databricks-deco-testing-runner-group labels: ubuntu-latest-deco + # Bounded, as the build's artifacts are the PR author's. + timeout-minutes: 15 + permissions: actions: read contents: read diff --git a/tools/coverpack/compare.go b/tools/coverpack/compare.go index 21c98aed2c9..8441dc9ab15 100644 --- a/tools/coverpack/compare.go +++ b/tools/coverpack/compare.go @@ -32,6 +32,7 @@ const ( maxShownRanges = 8 // ranges of lines listed per file maxShownPackage = 25 // packages in the table of changed packages commentMarker = "" + maxCells = 8 // files of a build ) // The measures of a comment, in the order of its table. Acceptance is the merged profile of @@ -97,6 +98,9 @@ func readPacked(p string) (*decoded, error) { // loadBuild merges the cells of a build: a block is covered if any cell covered it. func loadBuild(paths []string) (buildCov, []map[string]any, error) { + if len(paths) > maxCells { + return nil, nil, fmt.Errorf("%d files: more than %d cells", len(paths), maxCells) + } cov := buildCov{} var metas []map[string]any for _, p := range paths { @@ -188,43 +192,75 @@ func total(pkgs map[string]counts) counts { } // addedLines reads a `git diff -U0`: the added lines of each file, by their number in the new -// version. -func addedLines(diff io.Reader) (map[string][]int, error) { - out := map[string][]int{} +// version, and every file the diff touches. A hunk's body is skipped by its counts, so a line +// of it that looks like a header is not read as one. +func addedLines(diff io.Reader) (map[string][]int, map[string]bool, error) { + added, touched := map[string][]int{}, map[string]bool{} var file string + body := 0 // lines left in the current hunk + afterOld := false s := bufio.NewScanner(diff) s.Buffer(nil, 16<<20) for s.Scan() { line := s.Text() - if p, ok := strings.CutPrefix(line, "+++ "); ok { - file, _ = strings.CutPrefix(p, "b/") // "/dev/null" (a deleted file) matches no file - continue - } - rest, ok := strings.CutPrefix(line, "@@ ") - if !ok || file == "" { + if body > 0 { + if !strings.HasPrefix(line, `\`) { // "\ No newline at end of file" + body-- + } continue } - for field := range strings.FieldsSeq(rest) { - spec, ok := strings.CutPrefix(field, "+") - if !ok { - continue + wasOld := afterOld + afterOld = false + switch { + case strings.HasPrefix(line, "--- "): + afterOld = true + if p, ok := strings.CutPrefix(line[4:], "a/"); ok { + touched[p] = true } - first, n, found := strings.Cut(spec, ",") - if !found { - n = "1" + case wasOld && strings.HasPrefix(line, "+++ "): + file = "" + if p, ok := strings.CutPrefix(line[4:], "b/"); ok { + file = p + touched[p] = true } - start, err1 := strconv.Atoi(first) - count, err2 := strconv.Atoi(n) - if err := cmp.Or(err1, err2); err != nil { - return nil, fmt.Errorf("hunk %q: %w", line, err) + case strings.HasPrefix(line, "@@ "): + oldCount, start, count, err := hunk(line) + if err != nil { + return nil, nil, err } - for l := start; l < start+count; l++ { - out[file] = append(out[file], l) + body = oldCount + count + for l := start; l < start+count && file != ""; l++ { + added[file] = append(added[file], l) } - break } } - return out, s.Err() + return added, touched, s.Err() +} + +// hunk reads "@@ -a,b +c,d @@": b, c and d (a count is 1 when left out). +func hunk(line string) (oldCount, start, count int, err error) { + fields := strings.Fields(line) + if len(fields) < 3 || !strings.HasPrefix(fields[1], "-") || !strings.HasPrefix(fields[2], "+") { + return 0, 0, 0, fmt.Errorf("hunk %q", line) + } + span := func(spec string) (int, int, error) { + first, n, found := strings.Cut(spec, ",") + if !found { + n = "1" + } + a, err1 := strconv.Atoi(first) + b, err2 := strconv.Atoi(n) + if err := cmp.Or(err1, err2); err != nil || b < 0 || b > 1<<24 { + return 0, 0, fmt.Errorf("hunk %q", line) + } + return a, b, nil + } + _, oldCount, err = span(fields[1][1:]) + if err != nil { + return 0, 0, 0, err + } + start, count, err = span(fields[2][1:]) + return oldCount, start, count, err } // lineStates is, per wanted line that has code, whether any of its code ran, per measure. @@ -233,6 +269,7 @@ func lineStates(f *fileCov, wanted []int) map[int][measures]bool { for _, l := range wanted { want[l] = true } + lo, hi := slices.Min(wanted), slices.Max(wanted) out := map[int][measures]bool{} for j, blk := range f.blocks { if blk.stmts == 0 { @@ -243,7 +280,7 @@ func lineStates(f *fileCov, wanted []int) map[int][measures]bool { last-- // the block ends before the first byte of its last line } cov := f.covered(j) - for l := int(blk.sl); l <= last; l++ { + for l := max(int(blk.sl), lo); l <= min(last, hi); l++ { if !want[l] { continue } @@ -263,10 +300,11 @@ type patchFile struct { } type patch struct { - lines int // changed lines that have code - covered [measures]int - files []patchFile // with uncovered lines, the most first - skipped int // changed files whose coverage is of another version of them + lines int // changed lines that have code + covered [measures]int + files []patchFile // with uncovered lines, the most first + skipped int // changed files whose coverage is of another version of them + packages map[string]bool // that the PR changes a Go file of } func git(repo string, args ...string) ([]byte, error) { @@ -304,14 +342,18 @@ func blobs(repo, commit string) (map[string][20]byte, error) { func patchCoverage(head buildCov, o compareOptions) (patch, error) { var p patch - diff, err := git(o.repo, "diff", "-U0", "--no-color", "--no-ext-diff", "--no-renames", "--src-prefix=a/", "--dst-prefix=b/", o.from, o.to, "--", "*.go") + diff, err := git(o.repo, "-c", "core.quotePath=false", "diff", "-U0", "--no-color", "--no-ext-diff", "--no-renames", "--src-prefix=a/", "--dst-prefix=b/", o.from, o.to, "--", "*.go") if err != nil { return p, err } - added, err := addedLines(bytes.NewReader(diff)) + added, touched, err := addedLines(bytes.NewReader(diff)) if err != nil { return p, err } + p.packages = map[string]bool{} + for file := range touched { + p.packages[path.Dir(file)] = true + } shas, err := blobs(o.repo, o.to) if err != nil { return p, err @@ -408,22 +450,27 @@ func ranges(lines []int, limit int) string { return strings.Join(parts, ", ") } -func cells(metas []map[string]any) string { +var cellName = regexp.MustCompile(`^[a-z0-9]{1,32}$`) + +// cells is the names of the cells of a build, sorted: "?" for one that is not a plain name. +func cells(metas []map[string]any) []string { var names []string for _, m := range metas { - if c, ok := m["cell"].(string); ok { - names = append(names, safeName(c)) + c, _ := m["cell"].(string) + if !cellName.MatchString(c) { + c = "?" } + names = append(names, c) } slices.Sort(names) - return strings.Join(slices.Compact(names), ", ") + return names } func render(w io.Writer, o compareOptions, base, head map[string]counts, metas []map[string]any, p patch) { fmt.Fprintln(w, commentMarker) fmt.Fprintln(w, "### Coverage") fmt.Fprintln(w) - fmt.Fprintf(w, "Linux (%s), statements covered by the unit and the acceptance tests. PR: %s.", cells(metas), o.headDesc) + fmt.Fprintf(w, "Linux (`%s`), statements covered by the unit and the acceptance tests. PR: %s.", strings.Join(cells(metas), "`, `"), o.headDesc) if base != nil { fmt.Fprintf(w, " Base: %s.", o.baseDesc) } @@ -472,7 +519,7 @@ func render(w io.Writer, o compareOptions, base, head map[string]counts, metas [ } if base != nil { - renderPackages(w, base, head) + renderPackages(w, base, head, p.packages) } } @@ -483,9 +530,10 @@ func patchCell(p patch, m int) string { return fmt.Sprintf("%s (%s of %s)", pct(p.covered[m], p.lines), thousands(p.covered[m]), thousands(p.lines)) } -// renderPackages lists the packages whose statements or combined coverage changed, the most -// newly uncovered statements first. -func renderPackages(w io.Writer, base, head map[string]counts) { +// renderPackages lists the packages that the PR changes and whose statements or combined +// coverage changed, the most newly uncovered statements first. Other packages can differ too, +// from the commits of main that the base build lacks, but that is not the PR's doing. +func renderPackages(w io.Writer, base, head map[string]counts, changed map[string]bool) { type row struct { name string base, head counts @@ -501,7 +549,7 @@ func renderPackages(w io.Writer, base, head map[string]counts) { } } rows = slices.DeleteFunc(rows, func(r row) bool { - return r.base.stmts == r.head.stmts && r.base.covered[measureCombined] == r.head.covered[measureCombined] + return !changed[r.name] || r.base.stmts == r.head.stmts && r.base.covered[measureCombined] == r.head.covered[measureCombined] }) if len(rows) == 0 { return @@ -513,7 +561,7 @@ func renderPackages(w io.Writer, base, head map[string]counts) { slices.SortFunc(rows, func(a, b row) int { return cmp.Or(cmp.Compare(max(b.uncoveredChanged, -b.uncoveredChanged), max(a.uncoveredChanged, -a.uncoveredChanged)), strings.Compare(a.name, b.name)) }) - fmt.Fprintf(w, "
%d packages changed\n\n", len(rows)) + fmt.Fprintf(w, "
%d changed packages\n\n", len(rows)) fmt.Fprintln(w, "| Package | Statements | Combined | Δ | Uncovered statements |") fmt.Fprintln(w, "|---|---|---|---|---|") for i, r := range rows { @@ -552,10 +600,13 @@ func compare(o compareOptions) error { } var base map[string]counts if len(o.base) > 0 { - b, _, err := loadBuild(o.base) + b, baseMetas, err := loadBuild(o.base) if err != nil { return err } + if !slices.Equal(cells(baseMetas), cells(metas)) { + return fmt.Errorf("the base build has cells %v, the PR's %v", cells(baseMetas), cells(metas)) + } base = b.packages() } p, err := patchCoverage(head, o) diff --git a/tools/coverpack/compare_test.go b/tools/coverpack/compare_test.go index b9fb68c05ed..d96fbff87a0 100644 --- a/tools/coverpack/compare_test.go +++ b/tools/coverpack/compare_test.go @@ -102,7 +102,7 @@ func TestCompare(t *testing.T) { comment := string(data) assert.True(t, strings.HasPrefix(comment, commentMarker+"\n")) - assert.Contains(t, comment, "Linux (direct), statements covered by the unit and the acceptance tests. PR: HEAD. Base: BASE.") + assert.Contains(t, comment, "Linux (`direct`), statements covered by the unit and the acceptance tests. PR: HEAD. Base: BASE.") // Lines 7-11 are the added lines with code: 7-10 ran in the acceptance tests, 11 in nothing. assert.Contains(t, comment, "| Combined | 80.0% (4 of 5) | 75.0% | -25.00 pp |") assert.Contains(t, comment, "| Acceptance | 80.0% (4 of 5) | 75.0% | -25.00 pp |") @@ -118,7 +118,7 @@ func TestCompare(t *testing.T) { require.NoError(t, err) assert.Contains(t, string(data), "| Combined | 80.0% (4 of 5) | 75.0% |\n") assert.Contains(t, string(data), "No coverage of main was found") - assert.NotContains(t, string(data), "packages changed") + assert.NotContains(t, string(data), "changed packages") // A build of another version of the file is left out of the changed lines. headLayout.sha = [20]byte{} @@ -132,26 +132,51 @@ func TestCompare(t *testing.T) { } func TestAddedLines(t *testing.T) { + // The hunk of x.go adds a line that looks like a header; old.go is deleted, new.go added. diff := `diff --git a/x.go b/x.go --- a/x.go +++ b/x.go -@@ -3 +3,2 @@ func f() { +@@ -3 +3,3 @@ func f() { -a +b +++++ b/fake.go +c -@@ -10,2 +11,0 @@ +@@ -10,2 +12,0 @@ -gone +-gone too --- a/old.go +++ /dev/null @@ -1 +0,0 @@ -x +--- /dev/null +++ b/new.go @@ -0,0 +1 @@ +package m +\ No newline at end of file ` - got, err := addedLines(strings.NewReader(diff)) + added, touched, err := addedLines(strings.NewReader(diff)) require.NoError(t, err) - assert.Equal(t, map[string][]int{"x.go": {3, 4}, "new.go": {1}}, got) + assert.Equal(t, map[string][]int{"x.go": {3, 4, 5}, "new.go": {1}}, added) + assert.Equal(t, map[string]bool{"x.go": true, "old.go": true, "new.go": true}, touched) +} + +func TestDecodeRejectsHostileBlocks(t *testing.T) { + // A block past the end of its file (compare would walk its lines), and a file without + // blocks that claims many bitmaps (they would take no bytes). + for _, l := range []*layout{ + {path: "a.go", lines: 3, blocks: []block{{file: "a.go", sl: 1, sc: 1, el: 1 << 31, ec: 2, stmts: 1}}}, + } { + data, _ := encode([]byte(`{}`), []*layout{l}, []profile{{kind: kindAll, files: []int{0}, bits: bitset{1}}}) + _, err := decode(data) + assert.ErrorIs(t, err, errMalformed) + } + data := []byte(magic) + data = append(data, 0, 1, 1, 'a') // no meta, one layout "a" + data = append(data, make([]byte, 20)...) + data = append(data, 0, 0, 100) // no lines, no blocks, 100 bitmaps + data = append(data, make([]byte, 200)...) + _, err := decode(data) + assert.ErrorIs(t, err, errMalformed) } func TestRangesAndNames(t *testing.T) { diff --git a/tools/coverpack/format.go b/tools/coverpack/format.go index 99aba2ffea8..039d821a6a4 100644 --- a/tools/coverpack/format.go +++ b/tools/coverpack/format.go @@ -29,6 +29,7 @@ import ( "errors" "fmt" "io" + "math" ) const magic = "CLICOV1\n" @@ -174,30 +175,47 @@ func decode(data []byte) (*decoded, error) { l := &layout{path: string(str())} copy(l.sha[:], c.take(20)) l.lines = int(c.uleb128()) - l.blocks = make([]block, c.count()) - if blocks += len(l.blocks); blocks > maxDecodedBlocks { + n := c.count() + if blocks += n; blocks > maxDecodedBlocks { return nil, errMalformed } - sl := uint32(0) + l.blocks = make([]block, n) + sl := uint64(0) for j := range l.blocks { - sl += uint32(c.uleb128()) - b := block{file: l.path, sl: sl, sc: uint32(c.uleb128())} - b.el = sl + uint32(c.uleb128()) - b.ec = uint32(c.uleb128()) - b.stmts = uint32(c.uleb128()) - l.blocks[j] = b + sl += c.uleb128() + sc := c.uleb128() + el := sl + c.uleb128() + ec, stmts := c.uleb128(), c.uleb128() + // The lines of a block are within its file, which compare walks (compare.go). + if c.err == nil && (el > math.MaxUint32 || l.lines > 0 && el > uint64(l.lines)) { + return nil, errMalformed + } + l.blocks[j] = block{file: l.path, sl: uint32(sl), sc: uint32(sc), el: uint32(el), ec: uint32(ec), stmts: uint32(stmts)} } d.layouts[i] = l } d.bitmaps = make([][][]byte, len(d.layouts)) for i, l := range d.layouts { - d.bitmaps[i] = make([][]byte, c.count()) + n, size := c.count(), (len(l.blocks)+7)/8 + // count bounds n by the bytes left, but a bitmap of a file without blocks has none. + if size == 0 && n > 1 || size > 0 && n > (len(c.b)-c.off)/size { + return nil, errMalformed + } + d.bitmaps[i] = make([][]byte, n) for j := range d.bitmaps[i] { - d.bitmaps[i][j] = c.take((len(l.blocks) + 7) / 8) + d.bitmaps[i][j] = c.take(size) } } - d.profiles = make([]decodedProfile, c.count()) + // A profile takes at least 3 bytes: its kind, the length of its name and its file count. + n := c.count() + if n > (len(c.b)-c.off)/3 { + return nil, errMalformed + } + d.profiles = make([]decodedProfile, n) for i := range d.profiles { + if c.err != nil { + return nil, c.err + } p := decodedProfile{kind: c.u8(), name: string(str())} n, l := c.count(), 0 for range n {