Introduce gen_statem vs gen_fsm benchmark

Conflicts:
	lib/stdlib/test/stdlib_bench_SUITE.erl

1 files changed, 161 insertions, 69 deletions

diff --git a/lib/stdlib/test/stdlib_bench_SUITE.erl b/lib/stdlib/test/stdlib_bench_SUITE.erl
index 9ad4bae2f5..2ce089dbb7 100644
--- a/lib/stdlib/test/stdlib_bench_SUITE.erl
+++ b/lib/stdlib/test/stdlib_bench_SUITE.erl
@@ -30,7 +30,8 @@ suite() -> [{ct_hooks,[{ts_install_cth,[{nodenames,2}]}]}].
 
 all() ->
     [{group,unicode},{group,base64},
-     {group,gen_server},{group,gen_server_comparison}].
+     {group,gen_server},{group,gen_statem},
+     {group,gen_server_comparison},{group,gen_statem_comparison}].
 
 groups() ->
     [{unicode,[{repeat,5}],
@@ -44,20 +45,39 @@ groups() ->
        encode_list, encode_list_to_string,
        mime_binary_decode, mime_binary_decode_to_string,
        mime_list_decode, mime_list_decode_to_string]},
-     {gen_server, [{repeat,5}],
-      [simple, simple_timer, simple_mon, simple_timer_mon,
-       generic, generic_timer]},
+     {gen_server, [{repeat,5}], cases(gen_server)},
+     {gen_statem, [{repeat,3}], cases(gen_statem)},
      {gen_server_comparison, [],
       [single_small, single_medium, single_big,
        sched_small, sched_medium, sched_big,
-       multi_small, multi_medium, multi_big]}].
+       multi_small, multi_medium, multi_big]},
+     {gen_statem_comparison, [],
+      [single_small, single_big,
+       sched_small, sched_big,
+       multi_small, multi_big]}].
 
-init_per_group(GroupName, Config) when GroupName =:= gen_server;
-                                       GroupName =:= gen_server_comparison ->
-    DataDir = ?config(data_dir, Config),
-    Files = filelib:wildcard(filename:join(DataDir, "{simple,generic}*.erl")),
-    _ = [{ok, _} = compile:file(File) || File <- Files],
-    Config;
+cases(gen_server) ->
+      [simple, simple_timer, simple_mon, simple_timer_mon,
+       generic, generic_timer];
+cases(gen_statem) ->
+    [generic, generic_fsm, generic_fsm_transit,
+     generic_statem, generic_statem_transit,
+     generic_statem_complex].
+
+init_per_group(gen_server, Config) ->
+    compile_servers(Config),
+    [{benchmark_suite,"stdlib_gen_server"}|Config];
+init_per_group(gen_statem, Config) ->
+    compile_servers(Config),
+    [{benchmark_suite,"stdlib_gen_statem"}|Config];
+init_per_group(gen_server_comparison, Config) ->
+    compile_servers(Config),
+    [{cases,cases(gen_server)},
+     {benchmark_suite,"stdlib_gen_server"}|Config];
+init_per_group(gen_statem_comparison, Config) ->
+    compile_servers(Config),
+    [{cases,cases(gen_statem)},
+     {benchmark_suite,"stdlib_gen_statem"}|Config];
 init_per_group(_GroupName, Config) ->
     Config.
 
@@ -77,23 +97,33 @@ end_per_testcase(_Func, _Conf) ->
     ok.
 
 
+compile_servers(Config) ->
+    DataDir = ?config(data_dir, Config),
+    Files = filelib:wildcard(filename:join(DataDir, "{simple,generic}*.erl")),
+    _ = [{ok, _} = compile:file(File) || File <- Files],
+    ok.
+
+comment(Value) ->
+    C = lists:flatten(io_lib:format("~p", [Value])),
+    {comment, C}.
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 -define(REPEAT_NORM, 5).
 
 norm_nfc_list(Config) ->
     Bin = norm_data(Config),
     {_N, Mean, _Stddev, Res} = unicode_util_SUITE:time_count(nfc, list, Bin, ?REPEAT_NORM),
-    report(1000.0*Res / Mean).
+    comment(report(1000.0*Res / Mean)).
 
 norm_nfc_deep_l(Config) ->
     Bin = norm_data(Config),
     {_N, Mean, _Stddev, Res} = unicode_util_SUITE:time_count(nfc, deep_l, Bin, ?REPEAT_NORM),
-    report(1000.0*Res / Mean).
+    comment(report(1000.0*Res / Mean)).
 
 norm_nfc_binary(Config) ->
     Bin = norm_data(Config),
     {_N, Mean, _Stddev, Res} = unicode_util_SUITE:time_count(nfc, binary, Bin, ?REPEAT_NORM),
-    report(1000.0*Res / Mean).
+    comment(report(1000.0*Res / Mean)).
 
 
 string_lexemes_list(Config) ->
@@ -102,7 +132,7 @@ string_lexemes_list(Config) ->
     Bin = norm_data(Config),
     Fun = fun(Str) -> string:nth_lexeme(Str, 200000, [$;,$\n,$\r]), 200000 end,
     {_N, Mean, _Stddev, Res} = string_SUITE:time_func(Fun, list, Bin, 15),
-    report(1000.0*Res / Mean).
+    comment(report(1000.0*Res / Mean)).
 
 string_lexemes_binary(Config) ->
     %% Use nth_lexeme instead of lexemes to avoid building a result of
@@ -110,7 +140,7 @@ string_lexemes_binary(Config) ->
     Bin = norm_data(Config),
     Fun = fun(Str) -> string:nth_lexeme(Str, 200000, [$;,$\n,$\r]), 200000 end,
     {_N, Mean, _Stddev, Res} = string_SUITE:time_func(Fun, binary, Bin, ?REPEAT_NORM),
-    report(1000.0*Res / Mean).
+    comment(report(1000.0*Res / Mean)).
 
 %%%
 report(Tps) ->
@@ -128,40 +158,40 @@ norm_data(Config) ->
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 decode_binary(_Config) ->
-    test(decode, encoded_binary()).
+    comment(test(decode, encoded_binary())).
 
 decode_binary_to_string(_Config) ->
-    test(decode_to_string, encoded_binary()).
+    comment(test(decode_to_string, encoded_binary())).
 
 decode_list(_Config) ->
-    test(decode, encoded_list()).
+    comment(test(decode, encoded_list())).
 
 decode_list_to_string(_Config) ->
-    test(decode_to_string, encoded_list()).
+    comment(test(decode_to_string, encoded_list())).
 
 encode_binary(_Config) ->
-    test(encode, binary()).
+    comment(test(encode, binary())).
 
 encode_binary_to_string(_Config) ->
-    test(encode_to_string, binary()).
+    comment(test(encode_to_string, binary())).
 
 encode_list(_Config) ->
-    test(encode, list()).
+    comment(test(encode, list())).
 
 encode_list_to_string(_Config) ->
-    test(encode_to_string, list()).
+    comment(test(encode_to_string, list())).
 
 mime_binary_decode(_Config) ->
-    test(mime_decode, encoded_binary()).
+    comment(test(mime_decode, encoded_binary())).
 
 mime_binary_decode_to_string(_Config) ->
-    test(mime_decode_to_string, encoded_binary()).
+    comment(test(mime_decode_to_string, encoded_binary())).
 
 mime_list_decode(_Config) ->
-    test(mime_decode, encoded_list()).
+    comment(test(mime_decode, encoded_list())).
 
 mime_list_decode_to_string(_Config) ->
-    test(mime_decode_to_string, encoded_list()).
+    comment(test(mime_decode_to_string, encoded_list())).
 
 -define(SIZE, 10000).
 -define(N, 1000).
@@ -223,76 +253,81 @@ mbb(N, Acc) ->
     lists:reverse(Acc, B).
 
 simple(Config) when is_list(Config) ->
-    do_tests(simple, single_small).
+    comment(do_tests(simple, single_small, Config)).
 
 simple_timer(Config) when is_list(Config) ->
-    do_tests(simple_timer, single_small).
+    comment(do_tests(simple_timer, single_small, Config)).
 
 simple_mon(Config) when is_list(Config) ->
-    do_tests(simple_mon, single_small).
+    comment(do_tests(simple_mon, single_small, Config)).
 
 simple_timer_mon(Config) when is_list(Config) ->
-    do_tests(simple_timer_mon, single_small).
+    comment(do_tests(simple_timer_mon, single_small, Config)).
 
 generic(Config) when is_list(Config) ->
-    do_tests(generic, single_small).
+    comment(do_tests(generic, single_small, Config)).
 
 generic_timer(Config) when is_list(Config) ->
-    do_tests(generic_timer, single_small).
+    comment(do_tests(generic_timer, single_small, Config)).
+
+generic_statem(Config) when is_list(Config) ->
+    comment(do_tests(generic_statem, single_small, Config)).
+
+generic_statem_transit(Config) when is_list(Config) ->
+    comment(do_tests(generic_statem_transit, single_small, Config)).
+
+generic_statem_complex(Config) when is_list(Config) ->
+    comment(do_tests(generic_statem_complex, single_small, Config)).
+
+generic_fsm(Config) when is_list(Config) ->
+    comment(do_tests(generic_fsm, single_small, Config)).
+
+generic_fsm_transit(Config) when is_list(Config) ->
+    comment(do_tests(generic_fsm_transit, single_small, Config)).
 
 single_small(Config) when is_list(Config) ->
-    comparison(single_small).
+    comparison(?config(cases, Config), single_small, Config).
 
 single_medium(Config) when is_list(Config) ->
-    comparison(single_medium).
+    comparison(?config(cases, Config), single_medium, Config).
 
 single_big(Config) when is_list(Config) ->
-    comparison(single_big).
+    comparison(?config(cases, Config), single_big, Config).
 
 sched_small(Config) when is_list(Config) ->
-    comparison(sched_small).
+    comparison(?config(cases, Config), sched_small, Config).
 
 sched_medium(Config) when is_list(Config) ->
-    comparison(sched_medium).
+    comparison(?config(cases, Config), sched_medium, Config).
 
 sched_big(Config) when is_list(Config) ->
-    comparison(sched_big).
+    comparison(?config(cases, Config), sched_big, Config).
 
 multi_small(Config) when is_list(Config) ->
-    comparison(multi_small).
+    comparison(?config(cases, Config), multi_small, Config).
 
 multi_medium(Config) when is_list(Config) ->
-    comparison(multi_medium).
+    comparison(?config(cases, Config), multi_medium, Config).
 
 multi_big(Config) when is_list(Config) ->
-    comparison(multi_big).
-
-comparison(Kind) ->
-    Simple0 = do_tests(simple, Kind),
-    SimpleTimer0 = do_tests(simple_timer, Kind),
-    SimpleMon0 = do_tests(simple_mon, Kind),
-    SimpleTimerMon0 = do_tests(simple_timer_mon, Kind),
-    Generic0 = do_tests(generic, Kind),
-    GenericTimer0 = do_tests(generic_timer, Kind),
-    %% Normalize
-    Simple = norm(Simple0, Simple0),
-    SimpleTimer = norm(SimpleTimer0, Simple0),
-    SimpleMon = norm(SimpleMon0, Simple0),
-    SimpleTimerMon = norm(SimpleTimerMon0, Simple0),
-    Generic = norm(Generic0, Simple0),
-    GenericTimer = norm(GenericTimer0, Simple0),
+    comparison(?config(cases, Config), multi_big, Config).
+
+comparison(Cases, Kind, Config) ->
+    Cases = ?config(cases, Config),
+    [RefResult|_] = Results =
+        [do_tests(Case, Kind, Config) || Case <- Cases],
+    Normalized = [norm(Result, RefResult) || Result <- Results],
     {Parallelism, Message} = bench_params(Kind),
     Wordsize = erlang:system_info(wordsize),
     MSize = Wordsize * erts_debug:flat_size(Message),
     What = io_lib:format("#parallel gen_server instances: ~.4w, "
                          "message flat size: ~.5w bytes",
                          [Parallelism, MSize]),
-    C = io_lib:format("~s: "
-                      "Simple: ~s Simple+Timer: ~s "
-                      "Simple+Monitor: ~s Simple+Timer+Monitor: ~s "
-                      "Generic: ~s Generic+Timer: ~s",
-                     [What, Simple, SimpleTimer, SimpleMon, SimpleTimerMon,
-                     Generic, GenericTimer]),
+    Format =
+        lists:flatten(
+          ["~s: "] ++
+              [[atom_to_list(Case),": ~s "] || Case <- Cases]),
+    C = lists:flatten(io_lib:format(Format, [What] ++ Normalized)),
     {comment, C}.
 
 norm(T, Ref) ->
@@ -302,15 +337,17 @@ norm(T, Ref) ->
 -define(MAX_TIME, 1000 * ?MAX_TIME_SECS). % ms
 -define(CALLS_PER_LOOP, 5).
 
-do_tests(Test, ParamSet) ->
+do_tests(Test, ParamSet, Config) ->
+    BenchmarkSuite = ?config(benchmark_suite, Config),
     {Client, ServerMod} = bench(Test),
     {Parallelism, Message} = bench_params(ParamSet),
     Fun = create_clients(Message, ServerMod, Client, Parallelism),
     {TotalLoops, AllPidTime} = run_test(Fun),
     PerSecond = ?CALLS_PER_LOOP * round((1000 * TotalLoops) / AllPidTime),
-    ct_event:notify(#event{name = benchmark_data,
-                           data = [{suite,"stdlib_gen_server"},
-                                   {value,PerSecond}]}),
+    ct_event:notify(
+      #event{
+         name = benchmark_data,
+         data = [{suite,BenchmarkSuite},{value,PerSecond}]}),
     PerSecond.
 
 -define(COUNTER, n).
@@ -369,6 +406,51 @@ generic_timer_client(N, M, P) ->
     _ = generic_server_timer:reply(P, M),
     generic_timer_client(N+1, M, P).
 
+generic_statem_client(N, M, P) ->
+    put(?COUNTER, N),
+    _ = generic_statem:reply(P, M),
+    _ = generic_statem:reply(P, M),
+    _ = generic_statem:reply(P, M),
+    _ = generic_statem:reply(P, M),
+    _ = generic_statem:reply(P, M),
+    generic_statem_client(N+1, M, P).
+
+generic_statem_transit_client(N, M, P) ->
+    put(?COUNTER, N),
+    _ = generic_statem:transit(P, M),
+    _ = generic_statem:transit(P, M),
+    _ = generic_statem:transit(P, M),
+    _ = generic_statem:transit(P, M),
+    _ = generic_statem:transit(P, M),
+    generic_statem_transit_client(N+1, M, P).
+
+generic_statem_complex_client(N, M, P) ->
+    put(?COUNTER, N),
+    _ = generic_statem:reply(P, M),
+    _ = generic_statem:reply(P, M),
+    _ = generic_statem:reply(P, M),
+    _ = generic_statem:reply(P, M),
+    _ = generic_statem:reply(P, M),
+    generic_statem_complex_client(N+1, M, P).
+
+generic_fsm_client(N, M, P) ->
+    put(?COUNTER, N),
+    _ = generic_fsm:reply(P, M),
+    _ = generic_fsm:reply(P, M),
+    _ = generic_fsm:reply(P, M),
+    _ = generic_fsm:reply(P, M),
+    _ = generic_fsm:reply(P, M),
+    generic_fsm_client(N+1, M, P).
+
+generic_fsm_transit_client(N, M, P) ->
+    put(?COUNTER, N),
+    _ = generic_fsm:transit(P, M),
+    _ = generic_fsm:transit(P, M),
+    _ = generic_fsm:transit(P, M),
+    _ = generic_fsm:transit(P, M),
+    _ = generic_fsm:transit(P, M),
+    generic_fsm_transit_client(N+1, M, P).
+
 bench(simple) ->
     {fun simple_client/3, simple_server};
 bench(simple_timer) ->
@@ -380,7 +462,17 @@ bench(simple_timer_mon) ->
 bench(generic) ->
     {fun generic_client/3, generic_server};
 bench(generic_timer) ->
-    {fun generic_timer_client/3, generic_server_timer}.
+    {fun generic_timer_client/3, generic_server_timer};
+bench(generic_statem) ->
+    {fun generic_statem_client/3, generic_statem};
+bench(generic_statem_transit) ->
+    {fun generic_statem_transit_client/3, generic_statem};
+bench(generic_statem_complex) ->
+    {fun generic_statem_complex_client/3, generic_statem_complex};
+bench(generic_fsm) ->
+    {fun generic_fsm_client/3, generic_fsm};
+bench(generic_fsm_transit) ->
+    {fun generic_fsm_transit_client/3, generic_fsm}.
 
 %% -> {Parallelism, MessageTerm}
 bench_params(single_small) -> {1, small()};