27#include <fmt/format.h>
29#include <seastar/core/future.hh>
30#include <seastar/core/loop.hh>
31#include <seastar/testing/linux_perf_event.hh>
40using clock_type = std::chrono::steady_clock;
44 uint64_t allocations = 0;
45 uint64_t tasks_executed = 0;
46 uint64_t instructions_retired = 0;
47 uint64_t cpu_cycles_retired = 0;
50 static uint64_t perf_mallocs();
51 static uint64_t perf_tasks_processed();
55 perf_stats(uint64_t allocations_, uint64_t tasks_executed_, uint64_t instructions_retired_ = 0, uint64_t cpu_cycles_retired_ = 0)
56 : allocations(allocations_)
57 , tasks_executed(tasks_executed_)
58 , instructions_retired(instructions_retired_)
59 , cpu_cycles_retired(cpu_cycles_retired_)
62 : allocations(std::exchange(o.allocations, 0))
63 , tasks_executed(std::exchange(o.tasks_executed, 0))
64 , instructions_retired(std::exchange(o.instructions_retired, 0))
65 , cpu_cycles_retired(std::exchange(o.cpu_cycles_retired, 0))
81 a.allocations += b.allocations;
82 a.tasks_executed += b.tasks_executed;
83 a.instructions_retired += b.instructions_retired;
84 a.cpu_cycles_retired += b.cpu_cycles_retired;
90operator-(perf_stats a, perf_stats b) {
91 a.allocations -= b.allocations;
92 a.tasks_executed -= b.tasks_executed;
93 a.instructions_retired -= b.instructions_retired;
94 a.cpu_cycles_retired -= b.cpu_cycles_retired;
98inline perf_stats& perf_stats::operator+=(perf_stats b) {
99 allocations += b.allocations;
100 tasks_executed += b.tasks_executed;
101 instructions_retired += b.instructions_retired;
102 cpu_cycles_retired += b.cpu_cycles_retired;
106inline perf_stats& perf_stats::operator-=(perf_stats b) {
107 allocations -= b.allocations;
108 tasks_executed -= b.tasks_executed;
109 instructions_retired -= b.instructions_retired;
110 cpu_cycles_retired -= b.cpu_cycles_retired;
115 std::string _test_case;
116 std::string _test_group;
118 uint64_t _single_run_iterations = 0;
119 std::atomic<uint64_t> _max_single_run_iterations;
121 linux_perf_event _instructions_retired_counter = linux_perf_event::user_instructions_retired();
122 linux_perf_event _cpu_cycles_retired_counter = linux_perf_event::user_cpu_cycles_retired();
124 void do_run(
const config&);
127 clock_type::duration duration;
131 [[gnu::always_inline]] [[gnu::hot]]
133 return _single_run_iterations >= _max_single_run_iterations.load(std::memory_order_relaxed);
136 [[gnu::always_inline]] [[gnu::hot]]
137 void next_iteration(
size_t n) {
138 _single_run_iterations += n;
141 virtual void set_up() = 0;
142 virtual void tear_down() noexcept = 0;
143 virtual
future<run_result> do_single_run() = 0;
145 performance_test(const
std::
string& test_case, const
std::
string& test_group)
146 : _test_case(test_case)
147 , _test_group(test_group)
150 virtual ~performance_test() =
default;
152 const std::string& test_case()
const {
return _test_case; }
153 const std::string& test_group()
const {
return _test_group; }
154 std::string name()
const {
return fmt::format(
"{}.{}", test_group(), test_case()); }
156 void run(
const config&);
158 static void register_test(std::unique_ptr<performance_test>);
167 clock_type::time_point _run_start_time;
168 clock_type::time_point _start_time;
169 clock_type::duration _total_time;
178 [[gnu::always_inline]] [[gnu::hot]]
180 _instructions_retired_counter = instructions_retired_counter;
181 _cpu_cycles_retired_counter = cpu_cycles_retired_counter;
187 _start_stats = perf_stats::snapshot(_instructions_retired_counter, _cpu_cycles_retired_counter);
190 [[gnu::always_inline]] [[gnu::hot]]
194 if (_start_time == _run_start_time) {
195 ret.duration = t - _start_time;
196 auto stats = perf_stats::snapshot(_instructions_retired_counter, _cpu_cycles_retired_counter);
197 ret.stats = stats - _start_stats;
199 ret.duration = _total_time;
200 ret.stats = _total_stats;
202 _instructions_retired_counter =
nullptr;
203 _cpu_cycles_retired_counter =
nullptr;
207 [[gnu::always_inline]] [[gnu::hot]]
208 void start_iteration() {
210 _start_stats = perf_stats::snapshot(_instructions_retired_counter, _cpu_cycles_retired_counter);
213 [[gnu::always_inline]] [[gnu::hot]]
216 _total_time += t - _start_time;
218 stats = perf_stats::snapshot(_instructions_retired_counter, _cpu_cycles_retired_counter);
219 _total_stats += stats - _start_stats;
227template<
bool Condition,
typename TrueFn,
typename FalseFn>
228struct do_if_constexpr_ : FalseFn {
229 do_if_constexpr_(TrueFn, FalseFn false_fn) : FalseFn(
std::move(false_fn)) { }
230 decltype(
auto)
operator()()
const {
232 return FalseFn::operator()(0);
235template<
typename TrueFn,
typename FalseFn>
236struct do_if_constexpr_<true, TrueFn, FalseFn> : TrueFn {
237 do_if_constexpr_(TrueFn true_fn, FalseFn) : TrueFn(
std::move(true_fn)) { }
238 decltype(
auto)
operator()()
const {
return TrueFn::operator()(0); }
241template<
bool Condition,
typename TrueFn,
typename FalseFn>
242do_if_constexpr_<Condition, TrueFn, FalseFn> if_constexpr_(TrueFn&& true_fn, FalseFn&& false_fn)
244 return do_if_constexpr_<Condition, TrueFn, FalseFn>(std::forward<TrueFn>(true_fn),
245 std::forward<FalseFn>(false_fn));
250template<
typename Test>
252 std::optional<Test> _test;
254 template<
typename... Args>
255 auto run_test(Args&&...) {
260 virtual void set_up()
override {
264 virtual void tear_down()
noexcept override {
265 _test = std::nullopt;
271 _instructions_retired_counter.enable();
272 _cpu_cycles_retired_counter.enable();
273 return if_constexpr_<
is_future<
decltype(_test->run())>::value>([&] (
auto&&...) {
274 measure_time.start_run(&_instructions_retired_counter);
276 return if_constexpr_<std::is_same_v<
decltype(_test->run()),
future<>>>([&] (
auto&&...) {
277 this->next_iteration(1);
279 }, [&] (
auto&&... dependency) {
282 return run_test(dependency...).then([&] (
size_t n) {
283 this->next_iteration(n);
287 return measure_time.stop_run();
289 _instructions_retired_counter.disable();
290 _cpu_cycles_retired_counter.disable();
293 measure_time.start_run(&_instructions_retired_counter, &_cpu_cycles_retired_counter);
295 if_constexpr_<std::is_void_v<
decltype(_test->run())>>([&] (
auto&&...) {
297 this->next_iteration(1);
298 }, [&] (
auto&&... dependency) {
301 this->next_iteration(run_test(dependency...));
304 auto ret = measure_time.stop_run();
305 _instructions_retired_counter.disable();
306 _cpu_cycles_retired_counter.disable();
307 return make_ready_future<run_result>(std::move(ret));
311 using performance_test::performance_test;
314void register_test(std::unique_ptr<performance_test>);
316template<
typename Test>
318 test_registrar(
const std::string& test_group,
const std::string& test_case) {
319 auto test = std::make_unique<concrete_performance_test<Test>>(test_case, test_group);
320 performance_test::register_test(std::move(test));
326[[gnu::always_inline]]
327inline void start_measuring_time()
329 internal::measure_time.start_iteration();
332[[gnu::always_inline]]
333inline void stop_measuring_time()
335 internal::measure_time.stop_iteration();
340void do_not_optimize(
const T& v)
342 asm volatile(
"" : :
"r,m" (v));
359#define PERF_TEST_F(test_group, test_case) \
360 struct test_##test_group##_##test_case : test_group { \
361 [[gnu::always_inline]] inline auto run(); \
363 static ::perf_tests::internal::test_registrar<test_##test_group##_##test_case> \
364 test_##test_group##_##test_case##_registrar(#test_group, #test_case); \
365 [[gnu::always_inline]] auto test_##test_group##_##test_case::run()
367#define PERF_TEST(test_group, test_case) \
368 struct test_##test_group##_##test_case { \
369 [[gnu::always_inline]] inline auto run(); \
371 static ::perf_tests::internal::test_registrar<test_##test_group##_##test_case> \
372 test_##test_group##_##test_case##_registrar(#test_group, #test_case); \
373 [[gnu::always_inline]] auto test_##test_group##_##test_case::run()
376#define PERF_TEST_C(test_group, test_case) \
377 struct test_##test_group##_##test_case : test_group { \
378 inline future<> run(); \
380 static ::perf_tests::internal::test_registrar<test_##test_group##_##test_case> \
381 test_##test_group##_##test_case##_registrar(#test_group, #test_case); \
382 future<> test_##test_group##_##test_case::run()
384#define PERF_TEST_CN(test_group, test_case) \
385 struct test_##test_group##_##test_case : test_group { \
386 inline future<size_t> run(); \
388 static ::perf_tests::internal::test_registrar<test_##test_group##_##test_case> \
389 test_##test_group##_##test_case##_registrar(#test_group, #test_case); \
390 future<size_t> test_##test_group##_##test_case::run()
Definition: linux_perf_event.hh:36
Definition: perf_tests.hh:42
Definition: perf_tests.hh:166
Type-safe boolean.
Definition: bool_class.hh:58
A representation of a possibly not-yet-computed value.
Definition: future.hh:1240
future do_until(StopCondition stop_cond, AsyncAction action) noexcept
Definition: loop.hh:339
future now()
Returns a ready future.
Definition: later.hh:35
Seastar API namespace.
Definition: abort_on_ebadf.hh:26
Definition: perf_tests.hh:317
Check whether a type is a future.
Definition: future.hh:1032