aboutsummaryrefslogtreecommitdiffstats
path: root/lib/stdlib/test/stdlib_bench_SUITE.erl
diff options
context:
space:
mode:
authorRaimo Niskanen <[email protected]>2018-01-16 15:40:50 +0100
committerRaimo Niskanen <[email protected]>2018-01-16 16:34:44 +0100
commit0f051cfe47532e8a008fec47e25715eeb8479a0c (patch)
treede82f7b21b33f91f43d024cdac620ae88c1f3968 /lib/stdlib/test/stdlib_bench_SUITE.erl
parent9de9c8f4e16dc22125617cf69e95a5ab9efbad9b (diff)
downloadotp-0f051cfe47532e8a008fec47e25715eeb8479a0c.tar.gz
otp-0f051cfe47532e8a008fec47e25715eeb8479a0c.tar.bz2
otp-0f051cfe47532e8a008fec47e25715eeb8479a0c.zip
Introduce gen_statem vs gen_fsm benchmark
Conflicts: lib/stdlib/test/stdlib_bench_SUITE.erl
Diffstat (limited to 'lib/stdlib/test/stdlib_bench_SUITE.erl')
-rw-r--r--lib/stdlib/test/stdlib_bench_SUITE.erl230
1 files changed, 161 insertions, 69 deletions
diff --git a/lib/stdlib/test/stdlib_bench_SUITE.erl b/lib/stdlib/test/stdlib_bench_SUITE.erl
index 9ad4bae2f5..2ce089dbb7 100644
--- a/lib/stdlib/test/stdlib_bench_SUITE.erl
+++ b/lib/stdlib/test/stdlib_bench_SUITE.erl
@@ -30,7 +30,8 @@ suite() -> [{ct_hooks,[{ts_install_cth,[{nodenames,2}]}]}].
all() ->
[{group,unicode},{group,base64},
- {group,gen_server},{group,gen_server_comparison}].
+ {group,gen_server},{group,gen_statem},
+ {group,gen_server_comparison},{group,gen_statem_comparison}].
groups() ->
[{unicode,[{repeat,5}],
@@ -44,20 +45,39 @@ groups() ->
encode_list, encode_list_to_string,
mime_binary_decode, mime_binary_decode_to_string,
mime_list_decode, mime_list_decode_to_string]},
- {gen_server, [{repeat,5}],
- [simple, simple_timer, simple_mon, simple_timer_mon,
- generic, generic_timer]},
+ {gen_server, [{repeat,5}], cases(gen_server)},
+ {gen_statem, [{repeat,3}], cases(gen_statem)},
{gen_server_comparison, [],
[single_small, single_medium, single_big,
sched_small, sched_medium, sched_big,
- multi_small, multi_medium, multi_big]}].
+ multi_small, multi_medium, multi_big]},
+ {gen_statem_comparison, [],
+ [single_small, single_big,
+ sched_small, sched_big,
+ multi_small, multi_big]}].
-init_per_group(GroupName, Config) when GroupName =:= gen_server;
- GroupName =:= gen_server_comparison ->
- DataDir = ?config(data_dir, Config),
- Files = filelib:wildcard(filename:join(DataDir, "{simple,generic}*.erl")),
- _ = [{ok, _} = compile:file(File) || File <- Files],
- Config;
+cases(gen_server) ->
+ [simple, simple_timer, simple_mon, simple_timer_mon,
+ generic, generic_timer];
+cases(gen_statem) ->
+ [generic, generic_fsm, generic_fsm_transit,
+ generic_statem, generic_statem_transit,
+ generic_statem_complex].
+
+init_per_group(gen_server, Config) ->
+ compile_servers(Config),
+ [{benchmark_suite,"stdlib_gen_server"}|Config];
+init_per_group(gen_statem, Config) ->
+ compile_servers(Config),
+ [{benchmark_suite,"stdlib_gen_statem"}|Config];
+init_per_group(gen_server_comparison, Config) ->
+ compile_servers(Config),
+ [{cases,cases(gen_server)},
+ {benchmark_suite,"stdlib_gen_server"}|Config];
+init_per_group(gen_statem_comparison, Config) ->
+ compile_servers(Config),
+ [{cases,cases(gen_statem)},
+ {benchmark_suite,"stdlib_gen_statem"}|Config];
init_per_group(_GroupName, Config) ->
Config.
@@ -77,23 +97,33 @@ end_per_testcase(_Func, _Conf) ->
ok.
+compile_servers(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Files = filelib:wildcard(filename:join(DataDir, "{simple,generic}*.erl")),
+ _ = [{ok, _} = compile:file(File) || File <- Files],
+ ok.
+
+comment(Value) ->
+ C = lists:flatten(io_lib:format("~p", [Value])),
+ {comment, C}.
+
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-define(REPEAT_NORM, 5).
norm_nfc_list(Config) ->
Bin = norm_data(Config),
{_N, Mean, _Stddev, Res} = unicode_util_SUITE:time_count(nfc, list, Bin, ?REPEAT_NORM),
- report(1000.0*Res / Mean).
+ comment(report(1000.0*Res / Mean)).
norm_nfc_deep_l(Config) ->
Bin = norm_data(Config),
{_N, Mean, _Stddev, Res} = unicode_util_SUITE:time_count(nfc, deep_l, Bin, ?REPEAT_NORM),
- report(1000.0*Res / Mean).
+ comment(report(1000.0*Res / Mean)).
norm_nfc_binary(Config) ->
Bin = norm_data(Config),
{_N, Mean, _Stddev, Res} = unicode_util_SUITE:time_count(nfc, binary, Bin, ?REPEAT_NORM),
- report(1000.0*Res / Mean).
+ comment(report(1000.0*Res / Mean)).
string_lexemes_list(Config) ->
@@ -102,7 +132,7 @@ string_lexemes_list(Config) ->
Bin = norm_data(Config),
Fun = fun(Str) -> string:nth_lexeme(Str, 200000, [$;,$\n,$\r]), 200000 end,
{_N, Mean, _Stddev, Res} = string_SUITE:time_func(Fun, list, Bin, 15),
- report(1000.0*Res / Mean).
+ comment(report(1000.0*Res / Mean)).
string_lexemes_binary(Config) ->
%% Use nth_lexeme instead of lexemes to avoid building a result of
@@ -110,7 +140,7 @@ string_lexemes_binary(Config) ->
Bin = norm_data(Config),
Fun = fun(Str) -> string:nth_lexeme(Str, 200000, [$;,$\n,$\r]), 200000 end,
{_N, Mean, _Stddev, Res} = string_SUITE:time_func(Fun, binary, Bin, ?REPEAT_NORM),
- report(1000.0*Res / Mean).
+ comment(report(1000.0*Res / Mean)).
%%%
report(Tps) ->
@@ -128,40 +158,40 @@ norm_data(Config) ->
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
decode_binary(_Config) ->
- test(decode, encoded_binary()).
+ comment(test(decode, encoded_binary())).
decode_binary_to_string(_Config) ->
- test(decode_to_string, encoded_binary()).
+ comment(test(decode_to_string, encoded_binary())).
decode_list(_Config) ->
- test(decode, encoded_list()).
+ comment(test(decode, encoded_list())).
decode_list_to_string(_Config) ->
- test(decode_to_string, encoded_list()).
+ comment(test(decode_to_string, encoded_list())).
encode_binary(_Config) ->
- test(encode, binary()).
+ comment(test(encode, binary())).
encode_binary_to_string(_Config) ->
- test(encode_to_string, binary()).
+ comment(test(encode_to_string, binary())).
encode_list(_Config) ->
- test(encode, list()).
+ comment(test(encode, list())).
encode_list_to_string(_Config) ->
- test(encode_to_string, list()).
+ comment(test(encode_to_string, list())).
mime_binary_decode(_Config) ->
- test(mime_decode, encoded_binary()).
+ comment(test(mime_decode, encoded_binary())).
mime_binary_decode_to_string(_Config) ->
- test(mime_decode_to_string, encoded_binary()).
+ comment(test(mime_decode_to_string, encoded_binary())).
mime_list_decode(_Config) ->
- test(mime_decode, encoded_list()).
+ comment(test(mime_decode, encoded_list())).
mime_list_decode_to_string(_Config) ->
- test(mime_decode_to_string, encoded_list()).
+ comment(test(mime_decode_to_string, encoded_list())).
-define(SIZE, 10000).
-define(N, 1000).
@@ -223,76 +253,81 @@ mbb(N, Acc) ->
lists:reverse(Acc, B).
simple(Config) when is_list(Config) ->
- do_tests(simple, single_small).
+ comment(do_tests(simple, single_small, Config)).
simple_timer(Config) when is_list(Config) ->
- do_tests(simple_timer, single_small).
+ comment(do_tests(simple_timer, single_small, Config)).
simple_mon(Config) when is_list(Config) ->
- do_tests(simple_mon, single_small).
+ comment(do_tests(simple_mon, single_small, Config)).
simple_timer_mon(Config) when is_list(Config) ->
- do_tests(simple_timer_mon, single_small).
+ comment(do_tests(simple_timer_mon, single_small, Config)).
generic(Config) when is_list(Config) ->
- do_tests(generic, single_small).
+ comment(do_tests(generic, single_small, Config)).
generic_timer(Config) when is_list(Config) ->
- do_tests(generic_timer, single_small).
+ comment(do_tests(generic_timer, single_small, Config)).
+
+generic_statem(Config) when is_list(Config) ->
+ comment(do_tests(generic_statem, single_small, Config)).
+
+generic_statem_transit(Config) when is_list(Config) ->
+ comment(do_tests(generic_statem_transit, single_small, Config)).
+
+generic_statem_complex(Config) when is_list(Config) ->
+ comment(do_tests(generic_statem_complex, single_small, Config)).
+
+generic_fsm(Config) when is_list(Config) ->
+ comment(do_tests(generic_fsm, single_small, Config)).
+
+generic_fsm_transit(Config) when is_list(Config) ->
+ comment(do_tests(generic_fsm_transit, single_small, Config)).
single_small(Config) when is_list(Config) ->
- comparison(single_small).
+ comparison(?config(cases, Config), single_small, Config).
single_medium(Config) when is_list(Config) ->
- comparison(single_medium).
+ comparison(?config(cases, Config), single_medium, Config).
single_big(Config) when is_list(Config) ->
- comparison(single_big).
+ comparison(?config(cases, Config), single_big, Config).
sched_small(Config) when is_list(Config) ->
- comparison(sched_small).
+ comparison(?config(cases, Config), sched_small, Config).
sched_medium(Config) when is_list(Config) ->
- comparison(sched_medium).
+ comparison(?config(cases, Config), sched_medium, Config).
sched_big(Config) when is_list(Config) ->
- comparison(sched_big).
+ comparison(?config(cases, Config), sched_big, Config).
multi_small(Config) when is_list(Config) ->
- comparison(multi_small).
+ comparison(?config(cases, Config), multi_small, Config).
multi_medium(Config) when is_list(Config) ->
- comparison(multi_medium).
+ comparison(?config(cases, Config), multi_medium, Config).
multi_big(Config) when is_list(Config) ->
- comparison(multi_big).
-
-comparison(Kind) ->
- Simple0 = do_tests(simple, Kind),
- SimpleTimer0 = do_tests(simple_timer, Kind),
- SimpleMon0 = do_tests(simple_mon, Kind),
- SimpleTimerMon0 = do_tests(simple_timer_mon, Kind),
- Generic0 = do_tests(generic, Kind),
- GenericTimer0 = do_tests(generic_timer, Kind),
- %% Normalize
- Simple = norm(Simple0, Simple0),
- SimpleTimer = norm(SimpleTimer0, Simple0),
- SimpleMon = norm(SimpleMon0, Simple0),
- SimpleTimerMon = norm(SimpleTimerMon0, Simple0),
- Generic = norm(Generic0, Simple0),
- GenericTimer = norm(GenericTimer0, Simple0),
+ comparison(?config(cases, Config), multi_big, Config).
+
+comparison(Cases, Kind, Config) ->
+ Cases = ?config(cases, Config),
+ [RefResult|_] = Results =
+ [do_tests(Case, Kind, Config) || Case <- Cases],
+ Normalized = [norm(Result, RefResult) || Result <- Results],
{Parallelism, Message} = bench_params(Kind),
Wordsize = erlang:system_info(wordsize),
MSize = Wordsize * erts_debug:flat_size(Message),
What = io_lib:format("#parallel gen_server instances: ~.4w, "
"message flat size: ~.5w bytes",
[Parallelism, MSize]),
- C = io_lib:format("~s: "
- "Simple: ~s Simple+Timer: ~s "
- "Simple+Monitor: ~s Simple+Timer+Monitor: ~s "
- "Generic: ~s Generic+Timer: ~s",
- [What, Simple, SimpleTimer, SimpleMon, SimpleTimerMon,
- Generic, GenericTimer]),
+ Format =
+ lists:flatten(
+ ["~s: "] ++
+ [[atom_to_list(Case),": ~s "] || Case <- Cases]),
+ C = lists:flatten(io_lib:format(Format, [What] ++ Normalized)),
{comment, C}.
norm(T, Ref) ->
@@ -302,15 +337,17 @@ norm(T, Ref) ->
-define(MAX_TIME, 1000 * ?MAX_TIME_SECS). % ms
-define(CALLS_PER_LOOP, 5).
-do_tests(Test, ParamSet) ->
+do_tests(Test, ParamSet, Config) ->
+ BenchmarkSuite = ?config(benchmark_suite, Config),
{Client, ServerMod} = bench(Test),
{Parallelism, Message} = bench_params(ParamSet),
Fun = create_clients(Message, ServerMod, Client, Parallelism),
{TotalLoops, AllPidTime} = run_test(Fun),
PerSecond = ?CALLS_PER_LOOP * round((1000 * TotalLoops) / AllPidTime),
- ct_event:notify(#event{name = benchmark_data,
- data = [{suite,"stdlib_gen_server"},
- {value,PerSecond}]}),
+ ct_event:notify(
+ #event{
+ name = benchmark_data,
+ data = [{suite,BenchmarkSuite},{value,PerSecond}]}),
PerSecond.
-define(COUNTER, n).
@@ -369,6 +406,51 @@ generic_timer_client(N, M, P) ->
_ = generic_server_timer:reply(P, M),
generic_timer_client(N+1, M, P).
+generic_statem_client(N, M, P) ->
+ put(?COUNTER, N),
+ _ = generic_statem:reply(P, M),
+ _ = generic_statem:reply(P, M),
+ _ = generic_statem:reply(P, M),
+ _ = generic_statem:reply(P, M),
+ _ = generic_statem:reply(P, M),
+ generic_statem_client(N+1, M, P).
+
+generic_statem_transit_client(N, M, P) ->
+ put(?COUNTER, N),
+ _ = generic_statem:transit(P, M),
+ _ = generic_statem:transit(P, M),
+ _ = generic_statem:transit(P, M),
+ _ = generic_statem:transit(P, M),
+ _ = generic_statem:transit(P, M),
+ generic_statem_transit_client(N+1, M, P).
+
+generic_statem_complex_client(N, M, P) ->
+ put(?COUNTER, N),
+ _ = generic_statem:reply(P, M),
+ _ = generic_statem:reply(P, M),
+ _ = generic_statem:reply(P, M),
+ _ = generic_statem:reply(P, M),
+ _ = generic_statem:reply(P, M),
+ generic_statem_complex_client(N+1, M, P).
+
+generic_fsm_client(N, M, P) ->
+ put(?COUNTER, N),
+ _ = generic_fsm:reply(P, M),
+ _ = generic_fsm:reply(P, M),
+ _ = generic_fsm:reply(P, M),
+ _ = generic_fsm:reply(P, M),
+ _ = generic_fsm:reply(P, M),
+ generic_fsm_client(N+1, M, P).
+
+generic_fsm_transit_client(N, M, P) ->
+ put(?COUNTER, N),
+ _ = generic_fsm:transit(P, M),
+ _ = generic_fsm:transit(P, M),
+ _ = generic_fsm:transit(P, M),
+ _ = generic_fsm:transit(P, M),
+ _ = generic_fsm:transit(P, M),
+ generic_fsm_transit_client(N+1, M, P).
+
bench(simple) ->
{fun simple_client/3, simple_server};
bench(simple_timer) ->
@@ -380,7 +462,17 @@ bench(simple_timer_mon) ->
bench(generic) ->
{fun generic_client/3, generic_server};
bench(generic_timer) ->
- {fun generic_timer_client/3, generic_server_timer}.
+ {fun generic_timer_client/3, generic_server_timer};
+bench(generic_statem) ->
+ {fun generic_statem_client/3, generic_statem};
+bench(generic_statem_transit) ->
+ {fun generic_statem_transit_client/3, generic_statem};
+bench(generic_statem_complex) ->
+ {fun generic_statem_complex_client/3, generic_statem_complex};
+bench(generic_fsm) ->
+ {fun generic_fsm_client/3, generic_fsm};
+bench(generic_fsm_transit) ->
+ {fun generic_fsm_transit_client/3, generic_fsm}.
%% -> {Parallelism, MessageTerm}
bench_params(single_small) -> {1, small()};