From e382f99d5fa397a31b9039505049efc4b7415a84 Mon Sep 17 00:00:00 2001 From: venkat1701 Date: Sun, 4 Oct 2026 12:21:12 +0530 Subject: [PATCH 1/2] feat(bench): report the median of several runs run.sh now runs each store RUNS times (default 3), alternating them, and the report shows the median per workload with the range next to it. Results only count as agreeing when every run of both stores matches. --- benchmarks/run.sh | 13 ++-- .../main/java/io/hstore/bench/Comparison.java | 2 +- .../src/main/java/io/hstore/bench/Report.java | 62 ++++++++++++++----- 3 files changed, 54 insertions(+), 23 deletions(-) diff --git a/benchmarks/run.sh b/benchmarks/run.sh index 4c9b758..fd8ef6f 100755 --- a/benchmarks/run.sh +++ b/benchmarks/run.sh @@ -7,6 +7,7 @@ durability="${DURABILITY:-async}" threads="${THREADS:-$(getconf _NPROCESSORS_ONLN)}" heap="${HEAP:-4g}" history="${HISTORY:-64}" +runs="${RUNS:-3}" results="results/scale-$scale-$durability-history-$history" ../mvnw -q -f ../pom.xml -Pbenchmarks -DskipTests install @@ -17,9 +18,11 @@ jvm=(-Xms"$heap" -Xmx"$heap" -XX:+UseParallelGC --add-opens java.base/java.util=ALL-UNNAMED -cp "$classpath") -mkdir -p "$results" -for store in hstore hypergraphdb; do - java "${jvm[@]}" io.hstore.bench.Comparison run --store "$store" --scale "$scale" --durability "$durability" \ - --threads "$threads" --history "$history" --out "$results/$store.json" +rm -rf "$results" && mkdir -p "$results" +for run in $(seq 1 "$runs"); do + for store in hstore hypergraphdb; do + java "${jvm[@]}" io.hstore.bench.Comparison run --store "$store" --scale "$scale" --durability "$durability" \ + --threads "$threads" --history "$history" --out "$results/$store-$run.json" + done done -java "${jvm[@]}" io.hstore.bench.Comparison report "$results/hstore.json" "$results/hypergraphdb.json" | tee "$results/report.md" +java "${jvm[@]}" io.hstore.bench.Comparison report "$results"/hstore-*.json "$results"/hypergraphdb-*.json | tee "$results/report.md" diff --git a/benchmarks/src/main/java/io/hstore/bench/Comparison.java b/benchmarks/src/main/java/io/hstore/bench/Comparison.java index dd256b9..b74ec3a 100644 --- a/benchmarks/src/main/java/io/hstore/bench/Comparison.java +++ b/benchmarks/src/main/java/io/hstore/bench/Comparison.java @@ -55,7 +55,7 @@ public static void main(String[] args) throws Exception { default -> { IO.println(""" usage: Comparison run --store hstore|hypergraphdb [--scale N] [--durability async|sync] [--threads N] [--history N] --out FILE - Comparison report FILE..."""); + Comparison report FILE... (runs of two stores; cells show the median and range)"""); System.exit(2); } } diff --git a/benchmarks/src/main/java/io/hstore/bench/Report.java b/benchmarks/src/main/java/io/hstore/bench/Report.java index c270bc1..6a3e675 100644 --- a/benchmarks/src/main/java/io/hstore/bench/Report.java +++ b/benchmarks/src/main/java/io/hstore/bench/Report.java @@ -11,7 +11,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Objects; +import java.util.stream.Stream; final class Report { @@ -22,12 +22,17 @@ private Report() { } static String markdown(String[] args) { - List runs = Arrays.stream(args).skip(1).map(Path::of).map(Report::load).toList(); - if (runs.size() < 2) { - throw new IllegalArgumentException("report needs at least two result files"); + Map> byStore = new LinkedHashMap<>(); + Arrays.stream(args).skip(1).map(Path::of).map(Report::load) + .forEach(run -> byStore.computeIfAbsent(run.store(), _ -> new ArrayList<>()).add(run)); + if (byStore.size() != 2) { + throw new IllegalArgumentException("report needs runs of exactly two stores, got " + byStore.keySet()); } - Run baseline = runs.getLast(); - Run subject = runs.getFirst(); + List> stores = List.copyOf(byStore.values()); + List subjects = stores.getFirst(); + List baselines = stores.getLast(); + Run subject = subjects.getFirst(); + Run baseline = baselines.getFirst(); StringBuilder out = new StringBuilder(); Map header = subject.header(); out.append("Dataset: %s nodes, %s hyperedges, %s incidences; durability %s; %s threads.%n" @@ -37,30 +42,53 @@ static String markdown(String[] args) { text(header.get("java")), number(header.get("maxHeap")) / (1 << 20))); out.append("Caches: %s uses a %s; %s uses a %s.%n%n".formatted(subject.store(), text(subject.header().get("cache")), baseline.store(), text(baseline.header().get("cache")))); - out.append("| Workload | %s | %s | Ratio | Results agree |%n".formatted(subject.store(), baseline.store())); + out.append("Each cell is the median of %d runs of %s and %d runs of %s, with the range in brackets.%n%n" + .formatted(subjects.size(), subject.store(), baselines.size(), baseline.store())); + out.append("| Workload | %s | %s | Ratio of medians | Results agree |%n".formatted(subject.store(), baseline.store())); out.append("|---|---:|---:|---:|:---:|%n".formatted()); for (Map.Entry> entry : subject.results().entrySet()) { String workload = entry.getKey(); - Map mine = entry.getValue(); - Map theirs = baseline.results().get(workload); - if (theirs == null) { + if (baselines.stream().anyMatch(run -> !run.results().containsKey(workload))) { continue; } boolean disk = workload.equals("disk"); - boolean single = number(mine.get("operations")) == 1; - double a = disk ? number(mine.get("operations")) : single ? number(mine.get("millis")) : rate(mine); - double b = disk ? number(theirs.get("operations")) : single ? number(theirs.get("millis")) : rate(theirs); - double ratio = disk || single ? b / a : a / b; + boolean single = number(entry.getValue().get("operations")) == 1; + double[] mine = values(subjects, workload, disk, single); + double[] theirs = values(baselines, workload, disk, single); + double ratio = disk || single ? median(theirs) / median(mine) : median(mine) / median(theirs); boolean agree = disk || workload.startsWith("ingest") || workload.equals("reopen") - || Objects.equals(number(mine.get("checksum")), number(theirs.get("checksum"))); - out.append("| `%s` — %s | %s | %s | **%.2f×** | %s |%n".formatted(workload, text(mine.get("description")), - format(a, disk, single), format(b, disk, single), ratio, agree ? "yes" : "**no**")); + || Stream.concat(subjects.stream(), baselines.stream()) + .map(run -> number(run.results().get(workload).get("checksum"))).distinct().count() == 1; + out.append("| `%s`: %s | %s | %s | **%.2f×** | %s |%n".formatted(workload, text(entry.getValue().get("description")), + cell(mine, disk, single), cell(theirs, disk, single), ratio, agree ? "yes" : "**no**")); } out.append("%nRatios above 1 favour %s: throughput ratios divide %s by %s; latency and size ratios divide %s by %s.%n" .formatted(subject.store(), subject.store(), baseline.store(), baseline.store(), subject.store())); return out.toString(); } + private static double[] values(List runs, String workload, boolean disk, boolean single) { + return runs.stream().mapToDouble(run -> { + Map result = run.results().get(workload); + return disk ? number(result.get("operations")) : single ? number(result.get("millis")) : rate(result); + }).sorted().toArray(); + } + + private static double median(double[] sorted) { + int middle = sorted.length / 2; + return sorted.length % 2 == 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; + } + + private static String cell(double[] sorted, boolean disk, boolean single) { + String median = format(median(sorted), disk, single); + return sorted.length == 1 ? median + : median + " (" + bare(sorted[0], disk) + " to " + bare(sorted[sorted.length - 1], disk) + ")"; + } + + private static String bare(double value, boolean disk) { + return disk ? "%.1f".formatted(value / (1 << 20)) : "%,.0f".formatted(value); + } + private static String format(double value, boolean disk, boolean single) { if (disk) { return "%.1f MiB".formatted(value / (1 << 20)); From f02c2b6d1b9c16124a66b55bf6bbbb2ba9dd82dd Mon Sep 17 00:00:00 2001 From: venkat1701 Date: Sun, 4 Oct 2026 12:21:12 +0530 Subject: [PATCH 2/2] docs(bench): describe repeated runs and medians --- docs/benchmarks.md | 9 ++++++--- docs/development.md | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index db4ddc9..88c4be4 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -176,7 +176,10 @@ SCALE=4 DURABILITY=async benchmarks/run.sh SCALE=4 DURABILITY=async HISTORY=1 benchmarks/run.sh ``` -`run.sh` builds the `benchmarks` module (`-Pbenchmarks`) and runs each store in its own JVM. It writes -`benchmarks/results/scale---history-/{hstore,hypergraphdb}.json` and the Markdown report -`report.md` in the same directory. `THREADS` and `HEAP` override the thread count and heap size. Close other +`run.sh` builds the `benchmarks` module (`-Pbenchmarks`) and runs each store in its own JVM, `RUNS` times +(default 3), alternating the two stores so slow periods hit both. It writes +`benchmarks/results/scale---history-/{hstore,hypergraphdb}-.json`, plus a Markdown report +`report.md` in the same directory. Each cell in the report is the median over the runs, with the fastest and slowest +run in brackets, and "Results agree" requires every run of both stores to return the same checksum. `THREADS` +and `HEAP` override the thread count and heap size. Close other workloads first: the numbers are only meaningful on an otherwise idle machine. diff --git a/docs/development.md b/docs/development.md index 168b7c2..972751f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -211,7 +211,7 @@ SCALE=1 DURABILITY=sync benchmarks/run.sh `core/src/java` and `storage/bdb-je/src/java` with `javac --release 11` against Berkeley DB JE 5.0.73, and installs `org.hypergraphdb:hgdb` and `org.hypergraphdb:hgbdbje` version `1.4-99485a1`. -`run.sh` builds with `-Pbenchmarks`, runs each store in its own JVM on the same deterministic dataset (seeded +`run.sh` builds with `-Pbenchmarks`, runs each store `RUNS` times (default 3) in its own JVM on the same deterministic dataset (seeded generator, `SCALE` × 50,000 nodes and × 100,000 hyperedges), writes `benchmarks/results/scale-N-durability/*.json` and prints a Markdown report whose *Results agree* column cross-checks every query checksum between the two engines. Methodology and published numbers: [benchmarks.md](benchmarks.md).