From 5a82d5aa25cebce5cc14e134f9da9a94462675d5 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 14 Nov 2026 08:26:30 +0900 Subject: [PATCH 1/5] 0.8.3: each call measured by itself, as a loop (bench::laps) --- CHANGELOG.md | 11 ++++++ README.md | 2 +- README.zh-CN.md | 2 +- adapters/gtest/mcpp.toml | 2 +- adapters/gtest/src/protocol.cpp | 2 +- capi/mcpp.toml | 2 +- capi/src/mtest.cpp | 2 +- conformance/mcpp.toml | 2 +- engine/mcpp.toml | 2 +- engine/src/commands.cpp | 2 +- engine/src/main.cpp | 4 +-- engine/src/service.cpp | 2 +- lib/docs/writing-tests.md | 6 ++-- lib/mcpp.toml | 2 +- lib/src/bench.cpp | 39 ++++++++++++++++++++ lib/src/bench.cppm | 64 ++++++++++++++++++--------------- lib/src/mtest.cpp | 2 +- lib/tests/companions.cpp | 25 +++++++++++++ manifest/mcpp.toml | 2 +- 19 files changed, 130 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4706a87..b3b6c46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## [0.8.3] - 2026-11-14 + +- **Each call measured by itself, as a loop** (`bench::laps`). `bench::each` takes the call as a + function; a body that keeps its state in the function it runs in writes the same measurement as + a loop: `bench::laps run{"draw"}; while (run.next()) { ... }`. The time from one `next` to the + following one is a call's, `run` is the call's `lap` (`pause`, `resume`, `items`, `part`, `time`, + `value`, `say`, `last`), and when `next` returns false the metrics have been reported as `each` + reports them; `result()` is where the calls lay. `each` is that loop around its function: one + way of measuring, in two forms. +- Additive: `each` measures and reports as it did. + ## [0.8.2] - 2026-11-14 - 0.8.1 as released: its tag was made and its release was not (a test of `bench::each` held that diff --git a/README.md b/README.md index c8df32a..c2e4da7 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ int main() { ```toml [dev-dependencies] -mcpplibs.mtest = "0.8.2" +mcpplibs.mtest = "0.8.3" ``` A failure names its place and both sides (`expected 2, actual 3`). Under a runner the same program diff --git a/README.zh-CN.md b/README.zh-CN.md index 870381c..35f970d 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -37,7 +37,7 @@ int main() { ```toml [dev-dependencies] -mcpplibs.mtest = "0.8.2" +mcpplibs.mtest = "0.8.3" ``` 失败时给出位置与两侧的值(`expected 2, actual 3`)。在运行器之下,同一程序按协议报告;单独运行时每个用例输出一行。 diff --git a/adapters/gtest/mcpp.toml b/adapters/gtest/mcpp.toml index d4b9e0e..84e55ac 100644 --- a/adapters/gtest/mcpp.toml +++ b/adapters/gtest/mcpp.toml @@ -13,7 +13,7 @@ [package] namespace = "mcpplibs.mtest" name = "gtest" -version = "0.8.2" +version = "0.8.3" description = "GoogleTest speaking the mtest protocol: a test main that lists, selects and reports GoogleTest's cases" license = "Apache-2.0" diff --git a/adapters/gtest/src/protocol.cpp b/adapters/gtest/src/protocol.cpp index 7ce7423..9000d43 100644 --- a/adapters/gtest/src/protocol.cpp +++ b/adapters/gtest/src/protocol.cpp @@ -128,7 +128,7 @@ int mcpplibs_mtest_gtest_main(int argc, char** argv) { {"protocol", 1}, {"level", 1}, {"framework", "googletest"}, - {"version", "0.8.2"}, + {"version", "0.8.3"}, {"isolation", {"process", "case"}}, {"guide", "https://github.com/mcpp-community/mtest/blob/main/adapters/gtest/docs/" "writing-tests.md"}}); diff --git a/capi/mcpp.toml b/capi/mcpp.toml index f4209da..ed7d635 100644 --- a/capi/mcpp.toml +++ b/capi/mcpp.toml @@ -4,7 +4,7 @@ [package] namespace = "mcpplibs.mtest" name = "capi" -version = "0.8.2" +version = "0.8.3" description = "mtest's C ABI: the protocol's events for any language with a C FFI" license = "Apache-2.0" diff --git a/capi/src/mtest.cpp b/capi/src/mtest.cpp index 66e4883..3dc5b4f 100644 --- a/capi/src/mtest.cpp +++ b/capi/src/mtest.cpp @@ -120,7 +120,7 @@ void mtest_init(const char* framework, int level) { {"protocol", 1}, {"level", std::clamp(level, 1, 3)}, {"framework", text(framework)}, - {"version", "0.8.2"}, + {"version", "0.8.3"}, {"isolation", {"process", "case"}}}); } diff --git a/conformance/mcpp.toml b/conformance/mcpp.toml index 86e302a..f70b417 100644 --- a/conformance/mcpp.toml +++ b/conformance/mcpp.toml @@ -8,7 +8,7 @@ [package] namespace = "mcpplibs.mtest" name = "conformance" -version = "0.8.2" +version = "0.8.3" description = "The conformance suite of the mtest protocol, for every implementation of its program side" license = "Apache-2.0" diff --git a/engine/mcpp.toml b/engine/mcpp.toml index d4805c6..2e91352 100644 --- a/engine/mcpp.toml +++ b/engine/mcpp.toml @@ -10,7 +10,7 @@ [package] namespace = "mtest" name = "mtest" -version = "0.8.2" +version = "0.8.3" description = "The mtest engine: run, schedule, store, explain and analyse tests of any language" license = "Apache-2.0" authors = ["mcpp-community"] diff --git a/engine/src/commands.cpp b/engine/src/commands.cpp index ba6f81b..13e4434 100644 --- a/engine/src/commands.cpp +++ b/engine/src/commands.cpp @@ -1958,7 +1958,7 @@ json catalog() { plain({{"out", {{"type", "string"}}}, {"argv", {{"type", "array"}, {"items", {{"type", "string"}}}}}}), true); - return {{"schema", "mtest.catalog/1"}, {"version", "0.8.2"}, {"tools", tools}}; + return {{"schema", "mtest.catalog/1"}, {"version", "0.8.3"}, {"tools", tools}}; } } // namespace mcpplibs::mtest::engine diff --git a/engine/src/main.cpp b/engine/src/main.cpp index d2f0ffe..f24b77e 100644 --- a/engine/src/main.cpp +++ b/engine/src/main.cpp @@ -98,7 +98,7 @@ const json* tool_named(const json& tools, std::string_view name) { cmdline::App application(const json& tools) { auto app = cmdline::App("mtest") - .version("0.8.2") + .version("0.8.3") .description("run tests of any language, and ask about their runs") .option(cmdline::Option("json").global().help("print the answer's data")); for (const auto& t : tools) @@ -253,7 +253,7 @@ int main(int argc, char** argv) { const std::string name(parsed->subcommand_name()); const auto& sub = parsed->subcommand()->get(); if (name == "version") { - std::println("mtest 0.8.2"); + std::println("mtest 0.8.3"); return 0; } const auto cwd = std::filesystem::current_path(); diff --git a/engine/src/service.cpp b/engine/src/service.cpp index a7ba5ef..abd6678 100644 --- a/engine/src/service.cpp +++ b/engine/src/service.cpp @@ -56,7 +56,7 @@ int serve(const std::filesystem::path& cwd) { {"result", {{"protocolVersion", params.value("protocolVersion", std::string("2025-06-18"))}, {"capabilities", {{"tools", {{"listChanged", false}}}}}, - {"serverInfo", {{"name", "mtest"}, {"version", "0.8.2"}}}, + {"serverInfo", {{"name", "mtest"}, {"version", "0.8.3"}}}, {"instructions", "Run tests and ask about runs: run, plan, explain, history, analyze, graph, " "gate, ci. A test session (test/plan, test/run, test/add, test/cancel, " diff --git a/lib/docs/writing-tests.md b/lib/docs/writing-tests.md index 11a852f..842e89f 100644 --- a/lib/docs/writing-tests.md +++ b/lib/docs/writing-tests.md @@ -53,7 +53,7 @@ E2E cli{"cli", resource{"serial:home"}, timeout{std::chrono::minutes{2}}, [] { / The entry enters the child before any case; the copy runs no case and reports nothing. The files of one directory are then one program (mcpp `[test] programs = "directory"`) however many of them start copies. -- The manifest names the library once: `[dev-dependencies] mcpplibs.mtest = "0.8.2"`. +- The manifest names the library once: `[dev-dependencies] mcpplibs.mtest = "0.8.3"`. - Identifiers are stable: a selection, a history and a shard name a case by them. ## Expectations @@ -158,7 +158,9 @@ How the tests of a project are laid out, from the directory to the line, is in `operator new` and calls `bench::allocated(n)` from it gets each call's allocations too: `draw.allocations`, `draw.allocations.max`, `draw.allocated` (bytes). With `{.limit = 8333us}`, `draw.over` counts the calls that took more (and they are the ones stated one by one); - `l.last()` ends the run at this call. + `l.last()` ends the run at this call. The same as a loop, for a body that keeps its state where it + runs: `bench::laps run{"draw"}; while (run.next()) { ... }` -- `run` is the call's lap, and + `run.result()` where the calls lay. ## Programs under test diff --git a/lib/mcpp.toml b/lib/mcpp.toml index 7ecf76f..9c0a5a4 100644 --- a/lib/mcpp.toml +++ b/lib/mcpp.toml @@ -17,7 +17,7 @@ [package] namespace = "mcpplibs" name = "mtest" -version = "0.8.2" +version = "0.8.3" description = "The C++ test library of mtest: cases, expectations, fixtures, bench, conformance and end-to-end helpers, with no macro and no header, speaking the mtest protocol" license = "Apache-2.0" authors = ["mcpp-community"] diff --git a/lib/src/bench.cpp b/lib/src/bench.cpp index e42bf43..a72f49e 100644 --- a/lib/src/bench.cpp +++ b/lib/src/bench.cpp @@ -213,6 +213,45 @@ distribution distribute(std::vector samples, const lap& l, std::size_t s return d; } +laps::laps(std::string_view name, each_options o) : name_(name), o_(o) { + samples_.reserve(std::min(o_.most, 4096)); +} + +bool laps::next() { + const auto now = clock::now(); + if (running_) { + // The call that was running ends here: what `next` does after is no call's. + const auto spent = std::max(clock::duration{}, now - began_ - lap_access::end(*this)); + if (warmed_ < o_.warmup) { + ++warmed_; + lap_access::drop(*this); + } else { + total_ += spent; + samples_.push_back(std::chrono::duration(spent).count()); + lap_access::keep(*this); + } + running_ = false; + } + const bool warming = warmed_ < o_.warmup; + const bool more = + !lap_access::last(*this) && (warming || (samples_.size() < o_.most && + (samples_.size() < o_.least || total_ < o_.time))); + if (!more) { + if (!done_) { + done_ = true; + result_ = distribute(std::move(samples_), *this, o_.slowest, + std::chrono::duration(o_.limit).count()); + record(name_, result_); + } + return false; + } + lap_access::warmup(*this, warming); + lap_access::begin(*this); + running_ = true; + began_ = clock::now(); + return true; +} + // The calls differ by their nature, so the samples are not stated with the metrics: the verdict on // samples that spread (P6.2) is for a measurement meant to be one number. void record(std::string_view name, const distribution& d) { diff --git a/lib/src/bench.cppm b/lib/src/bench.cppm index 911059e..462745c 100644 --- a/lib/src/bench.cppm +++ b/lib/src/bench.cppm @@ -265,41 +265,49 @@ distribution distribute(std::vector samples, const lap& l, std::size_t s // `.max`; the slowest calls a line each in the case's output. void record(std::string_view name, const distribution& d); +// THE SAME, AS A LOOP: for a body that keeps its state in the function it runs in. +// bench::laps run{"draw", {.limit = std::chrono::microseconds{8333}}}; +// while (run.next()) { +// run.pause(); scene.advance(); run.resume(); +// scene.draw(); +// run.value("nodes", scene.nodes()); +// } +// The time from one `next` that returned true to the following `next` is a call's; `run` is the +// call's `lap`. `next` returns false once what the options ask has been measured (or a call said +// `last`), having reported the metrics as `each` does; `result()` is then where the calls lay. +// `each` is this loop around its function: one way of measuring, in two forms. +class laps : public lap { +public: + explicit laps(std::string_view name, each_options o = {}); + laps(const laps&) = delete; + laps& operator=(const laps&) = delete; + [[nodiscard]] bool next(); + // Where the calls lay: after `next` has returned false. + [[nodiscard]] const distribution& result() const noexcept { + return result_; + } +private: + std::string name_; + each_options o_; + std::vector samples_; + clock::duration total_{}; + clock::time_point began_{}; + std::size_t warmed_ = 0; + bool running_ = false, done_ = false; + distribution result_; +}; + template requires std::invocable || std::invocable distribution each(std::string_view name, F&& f, each_options o = {}) { - using clock = lap::clock; - lap l; - const auto call = [&] { - lap_access::begin(l); - const auto t0 = clock::now(); + laps run{name, o}; + while (run.next()) { if constexpr (std::invocable) - std::invoke(f, l); + std::invoke(f, static_cast(run)); else std::invoke(f); - const auto t1 = clock::now(); - return std::max(clock::duration{}, t1 - t0 - lap_access::end(l)); - }; - lap_access::warmup(l, true); - for (std::size_t i = 0; i < o.warmup && !lap_access::last(l); ++i) { - call(); - lap_access::drop(l); - } - lap_access::warmup(l, false); - std::vector samples; - samples.reserve(std::min(o.most, 4096)); - clock::duration total{}; - while (!lap_access::last(l) && samples.size() < o.most && - (samples.size() < o.least || total < o.time)) { - const auto spent = call(); - total += spent; - samples.push_back(std::chrono::duration(spent).count()); - lap_access::keep(l); } - auto d = distribute(std::move(samples), l, o.slowest, - std::chrono::duration(o.limit).count()); - record(name, d); - return d; + return run.result(); } } // namespace mcpplibs::mtest::bench diff --git a/lib/src/mtest.cpp b/lib/src/mtest.cpp index 1bc2d31..661614d 100644 --- a/lib/src/mtest.cpp +++ b/lib/src/mtest.cpp @@ -324,7 +324,7 @@ void write_program() { {"protocol", 1}, {"level", 3}, {"framework", "mcpplibs.mtest"}, - {"version", "0.8.2"}, + {"version", "0.8.3"}, {"isolation", {"process", "case"}}, {"guide", "https://github.com/mcpp-community/mtest/blob/main/lib/docs/writing-tests.md"}}); } diff --git a/lib/tests/companions.cpp b/lib/tests/companions.cpp index 16f7af8..1c06ebf 100644 --- a/lib/tests/companions.cpp +++ b/lib/tests/companions.cpp @@ -136,6 +136,31 @@ int main(int argc, char** argv) { expect(std::ranges::contains(d.slowest, late, &bench::slow_call::index)) << late; }; + "the same as a loop: the calls between one next and the following one"_test[kind::bench] = [] { + std::size_t calls = 0, warm = 0; + std::uint64_t state = 1; // kept in the function the loop runs in + bench::laps run{"loop", {.least = 30, .most = 30, .warmup = 2, .slowest = 1}}; + while (run.next()) { + run.warmup() ? ++warm : ++calls; + run.pause(); + state = state * 6364136223846793005ULL + 1442695040888963407ULL; // not the call's + run.resume(); + bench::keep(state); + run.value("turn", double(run.index())); + if (run.index() == 2 + 19) + run.last(); // the 20th measured call is the last + } + expect(!run.next()) << "and it stays ended"; + const bench::distribution& d = run.result(); + expect(eq(warm, std::size_t(2)) && eq(calls, std::size_t(20)) && + eq(d.samples.size(), std::size_t(20))); + expect(le(d.min, d.p50) && le(d.p50, d.p99) && le(d.p99, d.max)); + expect(eq(d.values.size(), std::size_t(1)) && eq(d.values[0].min, 2.0) && + eq(d.values[0].max, 21.0)) + << "a warm-up call's is not kept"; + expect(eq(d.slowest.size(), std::size_t(1)) && + eq(d.slowest[0].values[0].second, double(d.slowest[0].index + 2))); + }; "calls that say nothing keep nothing"_test[kind::bench] = [] { auto d = bench::each("quiet", [](bench::lap&) {}, {.least = 5, .most = 5, .warmup = 1, .slowest = 1}); diff --git a/manifest/mcpp.toml b/manifest/mcpp.toml index 552edd4..ff26ccf 100644 --- a/manifest/mcpp.toml +++ b/manifest/mcpp.toml @@ -8,7 +8,7 @@ [package] namespace = "mcpplibs.mtest" name = "manifest" -version = "0.8.2" +version = "0.8.3" description = "The static case manifest of mtest: the cases a source file declares, read without building it" license = "Apache-2.0" authors = ["mcpp-community"] From 1577cdad1556fe13428364eab0268d546131b3e0 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 14 Nov 2026 08:32:00 +0900 Subject: [PATCH 2/5] run alone, a program shows what it measured: a measurement's line, its parts and quantities --- CHANGELOG.md | 4 ++++ lib/src/bench.cpp | 39 +++++++++++++++++++++++++++++---------- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3b6c46..f1fbcf2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ `value`, `say`, `last`), and when `next` returns false the metrics have been reported as `each` reports them; `result()` is where the calls lay. `each` is that loop around its function: one way of measuring, in two forms. +- **Run alone, a program shows what it measured.** A measurement's line in the program's output -- + `draw: 2.100 ms (p95 5.300 ms, p99 7.700 ms, max 8.400 ms; mean 2.900 ms; 139 calls; 0 over + 8.333 ms)`, then a line for each part and quantity -- where no runner keeps the run's record. + Under a runner nothing is added to the case's output: the record has the metrics. - Additive: `each` measures and reports as it did. ## [0.8.2] - 2026-11-14 diff --git a/lib/src/bench.cpp b/lib/src/bench.cpp index a72f49e..e808d01 100644 --- a/lib/src/bench.cpp +++ b/lib/src/bench.cpp @@ -12,6 +12,17 @@ double median_of(std::vector v) { const auto n = v.size(); return n % 2 ? v[n / 2] : (v[n / 2 - 1] + v[n / 2]) / 2; } + +// A time for a reader: 9.100 ms, 412.0 us, 1.250 s. +std::string readable(double seconds) { + if (seconds >= 1) + return std::format("{:.3f} s", seconds); + if (seconds >= 1e-3) + return std::format("{:.3f} ms", seconds * 1e3); + if (seconds >= 1e-6) + return std::format("{:.1f} us", seconds * 1e6); + return std::format("{:.0f} ns", seconds * 1e9); +} } // namespace summary summarize(std::vector samples, std::size_t iterations) { @@ -32,6 +43,10 @@ summary summarize(std::vector samples, std::size_t iterations) { void record(std::string_view name, const summary& s) { metric(name, s.median, "s", s.samples, "lower"); + // Run alone, a program shows what it measured; under a runner the run's record has it. + if (!under_runner()) + std::println("{}: {} (from {} to {}; {} samples of {} calls)", name, readable(s.median), + readable(s.min), readable(s.max), s.samples.size(), s.iterations); } double percentile(std::span sorted, double q) { @@ -110,16 +125,6 @@ quantity over(std::string name, const Rows& rows, std::size_t at, Pick pick) { return q; } -// A time for a reader: 9.100 ms, 412.0 us, 1.250 s. -std::string readable(double seconds) { - if (seconds >= 1) - return std::format("{:.3f} s", seconds); - if (seconds >= 1e-3) - return std::format("{:.3f} ms", seconds * 1e3); - if (seconds >= 1e-6) - return std::format("{:.1f} us", seconds * 1e6); - return std::format("{:.0f} ns", seconds * 1e9); -} } // namespace namespace { @@ -280,6 +285,20 @@ void record(std::string_view name, const distribution& d) { metric(base + ".over.share", d.samples.empty() ? 0 : double(d.over) / double(d.samples.size()), ""); } + if (!under_runner()) { + std::string line = std::format("{}: {} (p95 {}, p99 {}, max {}; mean {}; {} calls", name, + readable(d.p50), readable(d.p95), readable(d.p99), + readable(d.max), readable(d.mean), d.samples.size()); + if (d.limit > 0) + line += std::format("; {} over {}", d.over, readable(d.limit)); + line += ")"; + for (const quantity& q : d.parts) + line += std::format("\n {}: {} (p99 {}, max {})", q.name, readable(q.p50), + readable(q.p99), readable(q.max)); + for (const quantity& q : d.values) + line += std::format("\n {}: {:g} (max {:g})", q.name, q.mean, q.max); + std::println("{}", line); + } for (const slow_call& c : d.slowest) { std::string line = std::format("{}: call {} took {}", name, c.index, readable(c.seconds)); const char* between = ": "; From ca1aa1a48848b7f3075b76ed58cee81baa6eee14 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 14 Nov 2026 08:36:40 +0900 Subject: [PATCH 3/5] what the lap allocates to keep what a call says is not the call's; a test program that counts its allocations --- CHANGELOG.md | 3 ++ lib/src/bench.cpp | 10 ++++++ lib/tests/allocations.cpp | 72 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 lib/tests/allocations.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index f1fbcf2..c0dc580 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ `draw: 2.100 ms (p95 5.300 ms, p99 7.700 ms, max 8.400 ms; mean 2.900 ms; 139 calls; 0 over 8.333 ms)`, then a line for each part and quantity -- where no runner keeps the run's record. Under a runner nothing is added to the case's output: the record has the metrics. +- **What the lap keeps is not the call's.** The allocations `part`, `time` and `value` make to keep + what a call says are left out of the call's count: a frame that allocates nothing reports none + while it states its parts (`lib/tests/allocations.cpp`, a program that replaces `operator new`). - Additive: `each` measures and reports as it did. ## [0.8.2] - 2026-11-14 diff --git a/lib/src/bench.cpp b/lib/src/bench.cpp index e808d01..939ffbc 100644 --- a/lib/src/bench.cpp +++ b/lib/src/bench.cpp @@ -143,14 +143,24 @@ allocation_count allocations() noexcept { return {allocation_calls, allocation_bytes}; } +// What the lap allocates to keep what a call says is not the call's: a frame that allocates nothing +// still reports none while it states its parts. void lap::part(std::string_view name, clock::duration spent) { + const allocation_count before = allocations(); double& at = cell(now_.parts, column(part_names_, name)); const double seconds = std::chrono::duration(spent).count(); at = std::isnan(at) ? seconds : at + seconds; + const allocation_count after = allocations(); + out_count_ += after.count - before.count; + out_bytes_ += after.bytes - before.bytes; } void lap::value(std::string_view name, double v) { + const allocation_count before = allocations(); cell(now_.values, column(value_names_, name)) = v; + const allocation_count after = allocations(); + out_count_ += after.count - before.count; + out_bytes_ += after.bytes - before.bytes; } void lap_access::keep(lap& l) { diff --git a/lib/tests/allocations.cpp b/lib/tests/allocations.cpp new file mode 100644 index 0000000..95224ab --- /dev/null +++ b/lib/tests/allocations.cpp @@ -0,0 +1,72 @@ +// A program that counts its allocations (bench::allocated from its own operator new): a call's are +// the body's -- what the lap allocates to keep what the call said, and what a call paused for, are +// not. A program of its own, since it replaces the global operator new. +import std; +import mcpplibs.mtest; +import mcpplibs.mtest.bench; + +void* operator new(std::size_t size) { + mcpplibs::mtest::bench::allocated(size); + if (void* memory = std::malloc(size ? size : 1)) + return memory; + std::abort(); +} +void* operator new[](std::size_t size) { + return ::operator new(size); +} +void operator delete(void* memory) noexcept { + std::free(memory); +} +void operator delete[](void* memory) noexcept { + std::free(memory); +} +void operator delete(void* memory, std::size_t) noexcept { + std::free(memory); +} +void operator delete[](void* memory, std::size_t) noexcept { + std::free(memory); +} + +using namespace mcpplibs::mtest; + +int main() { + "a call that allocates nothing reports none, though it states parts and quantities"_test + [kind::bench] = [] { + std::uint64_t turn = 0; + bench::laps run{"frame", {.least = 40, .most = 40, .warmup = 2}}; + while (run.next()) { + bench::keep(++turn); + run.part("draw", std::chrono::microseconds{3}); + run.value("nodes", double(turn)); + run.value(turn % 2 ? "odd" : "even", + 1); // a name first said late: a column made then + } + const bench::distribution& d = run.result(); + const auto counted = + std::ranges::find(d.values, std::string("allocations"), &bench::quantity::name); + expect(counted != d.values.end()) << "the program counts, so each call's are stated"; + if (counted != d.values.end()) + expect(eq(counted->max, 0.0)) << "what the lap keeps is not the call's"; + }; + "a call's own allocations are counted, and what it paused for is not"_test[kind::bench] = [] { + bench::laps run{"work", {.least = 10, .most = 10, .warmup = 1}}; + while (run.next()) { + run.pause(); + auto setup = std::make_unique(1); // not the call's + run.resume(); + auto made = std::make_unique>(); + bench::keep(made); + bench::keep(setup); + } + const bench::distribution& d = run.result(); + const auto counted = + std::ranges::find(d.values, std::string("allocations"), &bench::quantity::name); + const auto bytes = + std::ranges::find(d.values, std::string("allocated"), &bench::quantity::name); + expect(counted != d.values.end() && bytes != d.values.end()); + if (counted != d.values.end() && bytes != d.values.end()) + expect(eq(counted->min, 1.0) && eq(counted->max, 1.0) && + eq(bytes->max, double(sizeof(std::array)))); + }; + return report(); +} From 50d6f95a5df596ec17f9ccdd4d21f5275c47ca54 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 14 Nov 2026 08:45:54 +0900 Subject: [PATCH 4/5] a loop that ran out by its options stays ended, however often it is asked --- lib/src/bench.cpp | 2 ++ lib/tests/companions.cpp | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/lib/src/bench.cpp b/lib/src/bench.cpp index 939ffbc..927afbe 100644 --- a/lib/src/bench.cpp +++ b/lib/src/bench.cpp @@ -234,6 +234,8 @@ laps::laps(std::string_view name, each_options o) : name_(name), o_(o) { bool laps::next() { const auto now = clock::now(); + if (done_) + return false; // ended: it stays so, however often it is asked if (running_) { // The call that was running ends here: what `next` does after is no call's. const auto spent = std::max(clock::duration{}, now - began_ - lap_access::end(*this)); diff --git a/lib/tests/companions.cpp b/lib/tests/companions.cpp index 1c06ebf..09ff5be 100644 --- a/lib/tests/companions.cpp +++ b/lib/tests/companions.cpp @@ -161,6 +161,14 @@ int main(int argc, char** argv) { expect(eq(d.slowest.size(), std::size_t(1)) && eq(d.slowest[0].values[0].second, double(d.slowest[0].index + 2))); }; + "a loop that ran out by its options stays ended"_test[kind::bench] = [] { + std::size_t calls = 0; + bench::laps run{"counted", {.least = 4, .most = 4, .warmup = 1}}; + while (run.next()) + ++calls; + expect(!run.next() && !run.next()) << "asked again"; + expect(eq(calls, std::size_t(5)) && eq(run.result().samples.size(), std::size_t(4))); + }; "calls that say nothing keep nothing"_test[kind::bench] = [] { auto d = bench::each("quiet", [](bench::lap&) {}, {.least = 5, .most = 5, .warmup = 1, .slowest = 1}); From 19611ed47d304a04983fdc484b8a8441a08ad5a0 Mon Sep 17 00:00:00 2001 From: Sunrisepeak Date: Sat, 14 Nov 2026 09:00:38 +0900 Subject: [PATCH 5/5] a program run alone selects by identifier, by the name cases lie under, or by a pattern Its interface said arguments select by prefix; the program took whole identifiers, and a name cases lie under was run and then reported as no case. Now: an identifier, a name (suite/group), or a pattern with * (as mcpp test --case takes one), matched as the cases declare themselves; a selection no case answers fails the run. lib/tests/alone.cpp. --- CHANGELOG.md | 5 +++ lib/docs/writing-tests.md | 1 + lib/src/mtest.cpp | 91 ++++++++++++++++++++++++++++++++++++--- lib/src/mtest.cppm | 7 ++- lib/tests/alone.cpp | 78 +++++++++++++++++++++++++++++++++ 5 files changed, 174 insertions(+), 8 deletions(-) create mode 100644 lib/tests/alone.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index c0dc580..e55b987 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ - **What the lap keeps is not the call's.** The allocations `part`, `time` and `value` make to keep what a call says are left out of the call's count: a frame that allocates nothing reports none while it states its parts (`lib/tests/allocations.cpp`, a program that replaces `operator new`). +- **A program run alone selects as the runner does.** Its arguments name cases by identifier, by + the name they lie under (`suite/group`, which was run and then reported as no case), or by a + pattern with `*` (the cases it matches, as `mcpp test --case` takes one); a selection no + case answers fails the run. The interface said "by prefix" and the program took whole + identifiers: the statement and the behaviour are now one (`lib/tests/alone.cpp`). - Additive: `each` measures and reports as it did. ## [0.8.2] - 2026-11-14 diff --git a/lib/docs/writing-tests.md b/lib/docs/writing-tests.md index 842e89f..62e3399 100644 --- a/lib/docs/writing-tests.md +++ b/lib/docs/writing-tests.md @@ -274,6 +274,7 @@ its precision, so a bound is written as for any metric: `at_most{"mem.peak", 64. | `mcpp test --cases` | each program's cases, running none | | `mcpp test --message-format json` | a record per program with each case's result, place and both sides | | `mcpp test --print-launch` | how a test starts, for a debugger | +| ` [--list] [identifier \| name \| 'pattern*' ...]` | the program alone: the cases named — one by its identifier, those under a name (`suite/group`), those a pattern matches; a measurement's numbers are printed | | `mtest run --isolate --jobs N --shard I/N` | the cases scheduled by their resources and history, a part of them | A failure reads `case fields/a field holds its text failed at tests/fields.cpp:9: ...`; the JSON diff --git a/lib/src/mtest.cpp b/lib/src/mtest.cpp index 661614d..f381af5 100644 --- a/lib/src/mtest.cpp +++ b/lib/src/mtest.cpp @@ -60,6 +60,9 @@ struct state { bool protocol = false; bool listing = false; std::optional> selection; + std::vector patterns; // a person's, with `*` + std::set patterns_met; + bool alone = false; // the selection is a person's, given as the program's arguments std::set seen; std::ofstream events; std::filesystem::path tmp_root; @@ -356,6 +359,55 @@ attributes inherited() { return s.stack.empty() ? attributes{} : s.stack.back().attrs; } +// Whether an identifier is what a pattern says: `*` is any run of characters, the rest itself. +// `begins`: whether an identifier that begins so could be. +bool matches(std::string_view pattern, std::string_view id, bool begins = false) { + std::size_t at = 0, in = 0, star = std::string_view::npos, resumed = 0; + while (in < id.size()) { + if (at < pattern.size() && pattern[at] == '*') { + star = at++; + resumed = in; + } else if (at < pattern.size() && pattern[at] == id[in]) { + ++at; + ++in; + } else if (star != std::string_view::npos) { + at = star + 1; + in = ++resumed; + } else { + return false; + } + } + if (begins) + return true; + while (at < pattern.size() && pattern[at] == '*') + ++at; + return at == pattern.size(); +} + +// The pattern a case answers, itself or by a case it is declared in; met from then on. +bool answers_pattern(const std::string& id) { + auto& s = the(); + bool any = false; + for (const auto& pattern : s.patterns) { + bool is = matches(pattern, id); + for (auto slash = id.find('/'); !is && slash != std::string::npos; + slash = id.find('/', slash + 1)) + is = matches(pattern, std::string_view(id).substr(0, slash)); + if (is) { + s.patterns_met.insert(pattern); + any = true; + } + } + return any; +} + +// Whether a case declared inside this one could answer a pattern. +bool may_declare_pattern(const std::string& id) { + return std::ranges::any_of(the().patterns, [&](const std::string& pattern) { + return matches(pattern, id + "/", true); + }); +} + // Whether a case is reported: it is selected, or declared inside a selected case (P4.1). A case // that runs only because it declares a selected one is the scope of that case (P3.4). bool reported(const std::string& id) { @@ -366,7 +418,7 @@ bool reported(const std::string& id) { if (want == id || (id.starts_with(want) && id.size() > want.size() && id[want.size()] == '/')) return true; - return false; + return answers_pattern(id); } // A case runs when it is selected, or is an ancestor or a descendant of a selected case (P4). @@ -382,7 +434,7 @@ bool selected(const std::string& id) { if (id.starts_with(want) && id.size() > want.size() && id[want.size()] == '/') return true; } - return false; + return answers_pattern(id) || may_declare_pattern(id); } std::string sanitized(std::string_view id) { @@ -794,6 +846,13 @@ void test::operator=(body b) { skipped.push_back(want); s.seen.insert(want); } + // What a pattern would have found in it is not declared: the scope stands for it. + for (const auto& pattern : s.patterns) + if (matches(pattern, id + "/", true)) { + s.patterns_met.insert(pattern); + if (!std::ranges::contains(skipped, id)) + skipped.push_back(id); + } } for (const auto& one : skipped) { write_event({{"e", "begin"}, {"id", one}, {"t", since_start()}}); @@ -1011,11 +1070,23 @@ int report() { for (const auto& want : *s.selection) { if (s.seen.contains(want)) continue; + // A person may name a group of cases by the part of their names before a `/`. + if (s.alone && std::ranges::any_of(s.seen, [&](const std::string& id) { + return id.size() > want.size() && id.starts_with(want) && + id[want.size()] == '/'; + })) + continue; missing = true; write_event({{"e", "missing"}, {"id", want}}); say(std::format(" FAIL {} -- no such case", want)); } } + if (!s.listing) + for (const auto& pattern : s.patterns) + if (!s.patterns_met.contains(pattern)) { + missing = true; + say(std::format(" FAIL {} -- no such case", pattern)); + } const bool failed = s.any_failed || missing; write_event({{"e", "done"}, {"status", failed ? "fail" : "pass"}, {"counts", s.counts}}); if (!s.listing) { @@ -1046,14 +1117,14 @@ int run(int argc, char** argv) { s.done = true; return it->second(std::vector(argv + 3, argv + argc)); } - std::vector prefixes; + std::vector wanted; bool list = false; for (int i = 1; i < argc; ++i) { std::string_view a = argv[i]; if (a == "--list") list = true; else if (!a.starts_with("--")) - prefixes.emplace_back(a); + wanted.emplace_back(a); } if (list && !s.protocol) { // The cases, by identifier, for a person: run the suites in list mode with no events. @@ -1064,9 +1135,17 @@ int run(int argc, char** argv) { s.done = true; return 0; } - if (!prefixes.empty() && !s.protocol) { - std::set ids(prefixes.begin(), prefixes.end()); + if (!wanted.empty() && !s.protocol) { + // A pattern is met as the cases declare themselves (a case inside a case is not listed); + // the rest are identifiers and names. + std::set ids; + for (auto& want : wanted) + if (want.contains('*')) + s.patterns.push_back(std::move(want)); + else + ids.insert(std::move(want)); s.selection = std::move(ids); + s.alone = true; } return report(); } diff --git a/lib/src/mtest.cppm b/lib/src/mtest.cppm index bc9112d..d31e4ed 100644 --- a/lib/src/mtest.cppm +++ b/lib/src/mtest.cppm @@ -613,8 +613,11 @@ using suite = Suite; // Runs every suite not yet run, writes the summary (the `done` event under a // runner), and returns the program's exit status (P5.5). int report(); -// The same, for a `main` that receives its arguments: `--list` prints the cases' -// identifiers, and the remaining arguments select cases by identifier prefix. +// The same, for a `main` that receives its arguments, which a person gives a program run alone: +// `--list` prints the cases' identifiers; any other argument selects — a case by its identifier, +// the cases under a name (`suite`, `suite/group`: what lies before a `/`), or the cases a pattern +// with +// `*` (any run of characters) matches. A selection no case answers fails the run. int run(int argc, char** argv); // ---- fixtures --------------------------------------------------------------- diff --git a/lib/tests/alone.cpp b/lib/tests/alone.cpp new file mode 100644 index 0000000..6b3fad2 --- /dev/null +++ b/lib/tests/alone.cpp @@ -0,0 +1,78 @@ +// The program run alone, by a person: its arguments select what runs. +import std; +import mcpplibs.mtest; +import mcpplibs.mtest.e2e; + +using namespace mcpplibs::mtest; + +namespace { + +e2e::outcome alone(std::vector arguments) { + auto argv = self_command(); + argv.insert(argv.end(), arguments.begin(), arguments.end()); + return e2e::run({.argv = argv}); +} + +bool ran(const e2e::outcome& o, std::string_view id) { + return o.out.find(std::format("ok {}\n", id)) != std::string::npos; +} + +} // namespace + +UTest alone_subject{"alone.subject", [] { + "near/one"_test = [] {}; + "near/two"_test = [] {}; + "far/one"_test = [] {}; + }}; + +UTest + alone_cases{"alone", [] { + "--list prints every case's identifier and runs none"_test = [] { + const auto o = alone({"--list"}); + expect(eq(o.status, 0)); + expect(o.out.find("alone.subject\n") != std::string::npos) << o.out; + expect(o.out.find("ok ") == std::string::npos) << o.out; + }; + "an identifier selects that case"_test = [] { + const auto o = alone({"alone.subject/near/one"}); + expect(eq(o.status, 0)) << o.out; + expect(ran(o, "alone.subject/near/one")) << o.out; + expect(!ran(o, "alone.subject/near/two")) << o.out; + }; + "a name selects the cases under it"_test = [] { + const auto o = alone({"alone.subject/near"}); + expect(eq(o.status, 0)) << o.out; + expect(ran(o, "alone.subject/near/one") && ran(o, "alone.subject/near/two")) + << o.out; + expect(!ran(o, "alone.subject/far/one")) << o.out; + expect(o.out.find("no such case") == std::string::npos) << o.out; + }; + "a name is whole: part of one selects nothing and fails"_test = [] { + const auto o = alone({"alone.subject/ne"}); + expect(eq(o.status, 1)); + expect(o.out.find("FAIL alone.subject/ne -- no such case") != + std::string::npos) + << o.out; + }; + "a pattern selects the cases it matches"_test = [] { + const auto o = alone({"alone.subject/*/one"}); + expect(eq(o.status, 0)) << o.out; + expect(ran(o, "alone.subject/near/one") && ran(o, "alone.subject/far/one")) + << o.out; + expect(!ran(o, "alone.subject/near/two")) << o.out; + }; + "a pattern beside an identifier: both"_test = [] { + const auto o = alone({"alone.subject/f*", "alone.subject/near/two"}); + expect(eq(o.status, 0)) << o.out; + expect(ran(o, "alone.subject/far/one") && ran(o, "alone.subject/near/two")) + << o.out; + expect(!ran(o, "alone.subject/near/one")) << o.out; + }; + "a pattern nothing matches fails"_test = [] { + const auto o = alone({"alone.subject/*/three"}); + expect(eq(o.status, 1)); + expect(o.out.find("FAIL alone.subject/*/three -- no such case") != + std::string::npos) + << o.out; + }; + }};