Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions packages/benchmark/src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ Compare options:
--output <file> Output file (default: stdout)
--format <type> Output format: "console" or "markdown" (default: console)
--detailed Show per-rule/per-emitter-step breakdown
--changes-only Only show metrics with notable changes

Generate-history options:
--dir <dir> Read results from a directory instead of the benchmark-data git branch
Expand Down Expand Up @@ -122,7 +121,6 @@ async function compareCommand(args: Record<string, string>): Promise<void> {

const threshold = args["threshold"] ? parseFloat(args["threshold"]) : undefined;
const format = args["format"] ?? "console";
const changesOnly = args["changes-only"] === "true";
const outputFile = args["output"];

const baseline = await loadJson<BenchmarkResult>(baselineFile);
Expand All @@ -133,7 +131,6 @@ async function compareCommand(args: Record<string, string>): Promise<void> {
if (format === "markdown") {
output = formatPrComment(comparisons, baseline.commit, current.commit, {
threshold,
changesOnly,
});
} else {
output = formatConsoleSummary(comparisons, threshold);
Expand Down
44 changes: 28 additions & 16 deletions packages/benchmark/src/format-comment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -145,8 +145,6 @@ const LEGEND =
export interface FormatOptions {
/** Change threshold for highlighting (default: 5%). */
threshold?: number;
/** Only show metrics with notable changes. */
changesOnly?: boolean;
}

/** Format comparison results as a GitHub PR comment markdown. */
Expand All @@ -157,30 +155,26 @@ export function formatPrComment(
options: FormatOptions = {},
): string {
const threshold = options.threshold ?? DEFAULT_THRESHOLD;
const changesOnly = options.changesOnly ?? false;

const lines: string[] = [];
lines.push("## ⚡ Benchmark Results\n");
lines.push(
`Comparing [\`${currentCommit.slice(0, 7)}\`] against baseline [\`${baselineCommit.slice(0, 7)}\`]\n`,
);

// Average metrics across all specs
const averaged = averageComparisonMetrics(comparisons);
const regressions = averaged.filter((m) => m.percentChange >= threshold);

let metrics = averaged;
if (changesOnly) {
metrics = metrics.filter((m) => Math.abs(m.percentChange) >= threshold);
}

if (metrics.length === 0) {
lines.push("_No notable changes._\n");
// Top-level summary: show regressions prominently, otherwise a simple ok message
if (regressions.length === 0) {
lines.push("✅ No performance regressions detected.\n");
} else {
lines.push(
`⚠️ **${regressions.length} metric(s) regressed** above the +${threshold}% threshold:\n`,
);
lines.push("| Metric | Baseline | Current | Change |");
lines.push("|--------|----------|---------|--------|");
for (const m of metrics) {
const indicator = changeIndicator(m.percentChange, threshold);
const changeStr = `${formatPercent(m.percentChange)} ${indicator}`.trim();
for (const m of regressions) {
const changeStr =
`${formatPercent(m.percentChange)} ${changeIndicator(m.percentChange, threshold)}`.trim();
const th = thresholdsFor(m.label);
lines.push(
`| ${displayLabel(m.label)} | ${formatMsColored(m.baseline, th)} | ${formatMsColored(m.current, th)} | ${changeStr} |`,
Expand All @@ -189,9 +183,27 @@ export function formatPrComment(
lines.push("");
}

// Full details collapsed
const specNames = comparisons.map((c) => c.specName).join(", ");
lines.push("<details>");
lines.push(
`<summary>Full details – comparing <code>${currentCommit.slice(0, 7)}</code> vs baseline <code>${baselineCommit.slice(0, 7)}</code></summary>\n`,
);
lines.push("| Metric | Baseline | Current | Change |");
lines.push("|--------|----------|---------|--------|");
for (const m of averaged) {
const changeStr =
`${formatPercent(m.percentChange)} ${changeIndicator(m.percentChange, threshold)}`.trim();
const th = thresholdsFor(m.label);
lines.push(
`| ${displayLabel(m.label)} | ${formatMsColored(m.baseline, th)} | ${formatMsColored(m.current, th)} | ${changeStr} |`,
);
}
lines.push("");
lines.push(`> Averaged across ${comparisons.length} specs (${specNames}).`);
lines.push(`> Threshold: changes > ±${threshold}% are highlighted.`);
lines.push(LEGEND);
lines.push("</details>");

return lines.join("\n");
}
Expand Down
Loading