diff --git a/benchmarks/run.sh b/benchmarks/run.sh index 4c9b758..fd8ef6f 100755 --- a/benchmarks/run.sh +++ b/benchmarks/run.sh @@ -7,6 +7,7 @@ durability="${DURABILITY:-async}" threads="${THREADS:-$(getconf _NPROCESSORS_ONLN)}" heap="${HEAP:-4g}" history="${HISTORY:-64}" +runs="${RUNS:-3}" results="results/scale-$scale-$durability-history-$history" ../mvnw -q -f ../pom.xml -Pbenchmarks -DskipTests install @@ -17,9 +18,11 @@ jvm=(-Xms"$heap" -Xmx"$heap" -XX:+UseParallelGC --add-opens java.base/java.util=ALL-UNNAMED -cp "$classpath") -mkdir -p "$results" -for store in hstore hypergraphdb; do - java "${jvm[@]}" io.hstore.bench.Comparison run --store "$store" --scale "$scale" --durability "$durability" \ - --threads "$threads" --history "$history" --out "$results/$store.json" +rm -rf "$results" && mkdir -p "$results" +for run in $(seq 1 "$runs"); do + for store in hstore hypergraphdb; do + java "${jvm[@]}" io.hstore.bench.Comparison run --store "$store" --scale "$scale" --durability "$durability" \ + --threads "$threads" --history "$history" --out "$results/$store-$run.json" + done done -java "${jvm[@]}" io.hstore.bench.Comparison report "$results/hstore.json" "$results/hypergraphdb.json" | tee "$results/report.md" +java "${jvm[@]}" io.hstore.bench.Comparison report "$results"/hstore-*.json "$results"/hypergraphdb-*.json | tee "$results/report.md" diff --git a/benchmarks/src/main/java/io/hstore/bench/Comparison.java b/benchmarks/src/main/java/io/hstore/bench/Comparison.java index dd256b9..b74ec3a 100644 --- a/benchmarks/src/main/java/io/hstore/bench/Comparison.java +++ b/benchmarks/src/main/java/io/hstore/bench/Comparison.java @@ -55,7 +55,7 @@ public static void main(String[] args) throws Exception { default -> { IO.println(""" usage: Comparison run --store hstore|hypergraphdb [--scale N] [--durability async|sync] [--threads N] [--history N] --out FILE - Comparison report FILE..."""); + Comparison report FILE... (runs of two stores; cells show the median and range)"""); System.exit(2); } } diff --git a/benchmarks/src/main/java/io/hstore/bench/Report.java b/benchmarks/src/main/java/io/hstore/bench/Report.java index c270bc1..6a3e675 100644 --- a/benchmarks/src/main/java/io/hstore/bench/Report.java +++ b/benchmarks/src/main/java/io/hstore/bench/Report.java @@ -11,7 +11,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Objects; +import java.util.stream.Stream; final class Report { @@ -22,12 +22,17 @@ private Report() { } static String markdown(String[] args) { - List runs = Arrays.stream(args).skip(1).map(Path::of).map(Report::load).toList(); - if (runs.size() < 2) { - throw new IllegalArgumentException("report needs at least two result files"); + Map> byStore = new LinkedHashMap<>(); + Arrays.stream(args).skip(1).map(Path::of).map(Report::load) + .forEach(run -> byStore.computeIfAbsent(run.store(), _ -> new ArrayList<>()).add(run)); + if (byStore.size() != 2) { + throw new IllegalArgumentException("report needs runs of exactly two stores, got " + byStore.keySet()); } - Run baseline = runs.getLast(); - Run subject = runs.getFirst(); + List> stores = List.copyOf(byStore.values()); + List subjects = stores.getFirst(); + List baselines = stores.getLast(); + Run subject = subjects.getFirst(); + Run baseline = baselines.getFirst(); StringBuilder out = new StringBuilder(); Map header = subject.header(); out.append("Dataset: %s nodes, %s hyperedges, %s incidences; durability %s; %s threads.%n" @@ -37,30 +42,53 @@ static String markdown(String[] args) { text(header.get("java")), number(header.get("maxHeap")) / (1 << 20))); out.append("Caches: %s uses a %s; %s uses a %s.%n%n".formatted(subject.store(), text(subject.header().get("cache")), baseline.store(), text(baseline.header().get("cache")))); - out.append("| Workload | %s | %s | Ratio | Results agree |%n".formatted(subject.store(), baseline.store())); + out.append("Each cell is the median of %d runs of %s and %d runs of %s, with the range in brackets.%n%n" + .formatted(subjects.size(), subject.store(), baselines.size(), baseline.store())); + out.append("| Workload | %s | %s | Ratio of medians | Results agree |%n".formatted(subject.store(), baseline.store())); out.append("|---|---:|---:|---:|:---:|%n".formatted()); for (Map.Entry> entry : subject.results().entrySet()) { String workload = entry.getKey(); - Map mine = entry.getValue(); - Map theirs = baseline.results().get(workload); - if (theirs == null) { + if (baselines.stream().anyMatch(run -> !run.results().containsKey(workload))) { continue; } boolean disk = workload.equals("disk"); - boolean single = number(mine.get("operations")) == 1; - double a = disk ? number(mine.get("operations")) : single ? number(mine.get("millis")) : rate(mine); - double b = disk ? number(theirs.get("operations")) : single ? number(theirs.get("millis")) : rate(theirs); - double ratio = disk || single ? b / a : a / b; + boolean single = number(entry.getValue().get("operations")) == 1; + double[] mine = values(subjects, workload, disk, single); + double[] theirs = values(baselines, workload, disk, single); + double ratio = disk || single ? median(theirs) / median(mine) : median(mine) / median(theirs); boolean agree = disk || workload.startsWith("ingest") || workload.equals("reopen") - || Objects.equals(number(mine.get("checksum")), number(theirs.get("checksum"))); - out.append("| `%s` — %s | %s | %s | **%.2f×** | %s |%n".formatted(workload, text(mine.get("description")), - format(a, disk, single), format(b, disk, single), ratio, agree ? "yes" : "**no**")); + || Stream.concat(subjects.stream(), baselines.stream()) + .map(run -> number(run.results().get(workload).get("checksum"))).distinct().count() == 1; + out.append("| `%s`: %s | %s | %s | **%.2f×** | %s |%n".formatted(workload, text(entry.getValue().get("description")), + cell(mine, disk, single), cell(theirs, disk, single), ratio, agree ? "yes" : "**no**")); } out.append("%nRatios above 1 favour %s: throughput ratios divide %s by %s; latency and size ratios divide %s by %s.%n" .formatted(subject.store(), subject.store(), baseline.store(), baseline.store(), subject.store())); return out.toString(); } + private static double[] values(List runs, String workload, boolean disk, boolean single) { + return runs.stream().mapToDouble(run -> { + Map result = run.results().get(workload); + return disk ? number(result.get("operations")) : single ? number(result.get("millis")) : rate(result); + }).sorted().toArray(); + } + + private static double median(double[] sorted) { + int middle = sorted.length / 2; + return sorted.length % 2 == 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2; + } + + private static String cell(double[] sorted, boolean disk, boolean single) { + String median = format(median(sorted), disk, single); + return sorted.length == 1 ? median + : median + " (" + bare(sorted[0], disk) + " to " + bare(sorted[sorted.length - 1], disk) + ")"; + } + + private static String bare(double value, boolean disk) { + return disk ? "%.1f".formatted(value / (1 << 20)) : "%,.0f".formatted(value); + } + private static String format(double value, boolean disk, boolean single) { if (disk) { return "%.1f MiB".formatted(value / (1 << 20)); diff --git a/docs/benchmarks.md b/docs/benchmarks.md index db4ddc9..88c4be4 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -176,7 +176,10 @@ SCALE=4 DURABILITY=async benchmarks/run.sh SCALE=4 DURABILITY=async HISTORY=1 benchmarks/run.sh ``` -`run.sh` builds the `benchmarks` module (`-Pbenchmarks`) and runs each store in its own JVM. It writes -`benchmarks/results/scale---history-/{hstore,hypergraphdb}.json` and the Markdown report -`report.md` in the same directory. `THREADS` and `HEAP` override the thread count and heap size. Close other +`run.sh` builds the `benchmarks` module (`-Pbenchmarks`) and runs each store in its own JVM, `RUNS` times +(default 3), alternating the two stores so slow periods hit both. It writes +`benchmarks/results/scale---history-/{hstore,hypergraphdb}-.json`, plus a Markdown report +`report.md` in the same directory. Each cell in the report is the median over the runs, with the fastest and slowest +run in brackets, and "Results agree" requires every run of both stores to return the same checksum. `THREADS` +and `HEAP` override the thread count and heap size. Close other workloads first: the numbers are only meaningful on an otherwise idle machine. diff --git a/docs/development.md b/docs/development.md index 168b7c2..972751f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -211,7 +211,7 @@ SCALE=1 DURABILITY=sync benchmarks/run.sh `core/src/java` and `storage/bdb-je/src/java` with `javac --release 11` against Berkeley DB JE 5.0.73, and installs `org.hypergraphdb:hgdb` and `org.hypergraphdb:hgbdbje` version `1.4-99485a1`. -`run.sh` builds with `-Pbenchmarks`, runs each store in its own JVM on the same deterministic dataset (seeded +`run.sh` builds with `-Pbenchmarks`, runs each store `RUNS` times (default 3) in its own JVM on the same deterministic dataset (seeded generator, `SCALE` × 50,000 nodes and × 100,000 hyperedges), writes `benchmarks/results/scale-N-durability/*.json` and prints a Markdown report whose *Results agree* column cross-checks every query checksum between the two engines. Methodology and published numbers: [benchmarks.md](benchmarks.md).