Merge branch 'lukas/benchmarks/OTP-10156'

* lukas/benchmarks/OTP-10156:
  Add mnesia tpc benchmark
  Add estone benchmarks to ts benchmark framework
  Add framework to ts to run benchmarks

8 files changed, 263 insertions, 16 deletions

diff --git a/lib/mnesia/examples/mnesia_tpcb.erl b/lib/mnesia/examples/mnesia_tpcb.erl
index 903c53a21c..f36b43a495 100644
--- a/lib/mnesia/examples/mnesia_tpcb.erl
+++ b/lib/mnesia/examples/mnesia_tpcb.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 1996-2009. All Rights Reserved.
+%% Copyright Ericsson AB 1996-2010. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -99,9 +99,13 @@
 	 replica_test/1,
 	 sticky_replica_test/1,
 	 remote_test/1,
-	 remote_frag2_test/1
+	 remote_frag2_test/1,
+
+	 conflict_benchmark/1
 	]).
 
+-include_lib("common_test/include/ct_event.hrl").
+
 -define(SECOND, 1000000).
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -191,8 +195,10 @@
 	  driver_nodes = [node()],
 	  n_drivers_per_node = 1,
 	  use_running_mnesia = false,
+	  seed,
 	  stop_after = timer:minutes(15), % Minimum 15 min
 	  report_interval = timer:minutes(1),
+	  send_bench_report = false,
 	  use_sticky_locks = false,
 	  spawn_near_branch = false,
 	  activity_type = transaction,
@@ -397,8 +403,30 @@ config(remote_frag2_test, ReplicaType) ->
      {stop_after, timer:minutes(1)},
      {report_interval, timer:seconds(10)},
      {reuse_history_id, true}
+    ];
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% Ten drivers per node, tables replicated to all nodes, single branch
+
+config(conflict_benchmark, ReplicaType) ->
+    Remote = nodes(),
+    Local = node(),
+    Nodes = [Local | Remote], 
+    [{seed, {1326,448637,337711}},
+     {db_nodes, Nodes},
+     {driver_nodes, Nodes},
+     {replica_nodes, Nodes},
+     {n_drivers_per_node, 10},
+     {n_branches, 1},
+     {n_accounts_per_branch, 10},
+     {replica_type, ReplicaType},
+     {stop_after, timer:minutes(1)},
+     {report_interval, timer:seconds(10)},
+     {send_bench_report, true},
+     {reuse_history_id, true}
     ].
 
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 start(What, ReplicaType) ->
@@ -422,6 +450,9 @@ remote_test(ReplicaType) ->
 remote_frag2_test(ReplicaType) ->
     start(remote_frag2_test, ReplicaType).
 
+conflict_benchmark(ReplicaType) ->
+    start(config(conflict_benchmark, ReplicaType)).
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 %% Args is a list of {Key, Val} tuples where Key is a field name
 %% in either the record tab_config or run_config. Unknown keys are ignored.
@@ -866,6 +897,7 @@ add_time(Acc, New) ->
 show_report(State) ->
     Now    = now_to_micros(erlang:now()),
     Iters  = State#reporter_state.n_iters,
+    Cfg    = State#reporter_state.run_config,
     Time   = State#reporter_state.curr,
     Max    = Time#time.max_time,
     N      = Time#time.n_trans,
@@ -888,7 +920,17 @@ show_report(State) ->
 		      "duration of longest transaction was ~p milliseconds~n",
 		      [Tps, BruttoTps, Max div 1000])
     end,
-    State#reporter_state{prev_tps = Tps, prev_micros = Now}.
+    case Cfg#run_config.send_bench_report of
+	true ->
+	    ct_event:notify(
+	      #event{name = benchmark_data, 
+		     data = [{suite,"mnesia_tpcb"},
+			     {value,Tps}]});
+	_ ->
+	    ok
+    end,
+
+    State#reporter_state{prev_tps = Tps, prev_micros = Now}.	    
 
 signed_diff(Iters, Curr, Prev) ->
     case Iters > 1 of
@@ -955,7 +997,13 @@ alloc_local_branches([], Specs, OrphanBranches) ->
     {Specs, OrphanBranches}.
     
 driver_init(DS, AllBranches) ->
-    Seed = erlang:now(),
+    case (DS#driver_state.run_config)#run_config.seed of
+	undefined ->
+	    Seed = erlang:now();
+	Seed ->
+	    Seed
+    end,
+    
     DS2 =
 	if
 	    DS#driver_state.n_local_branches =:= 0 ->
diff --git a/lib/mnesia/test/Makefile b/lib/mnesia/test/Makefile
index 0451da697d..32453bc74e 100644
--- a/lib/mnesia/test/Makefile
+++ b/lib/mnesia/test/Makefile
@@ -26,6 +26,7 @@ include $(ERL_TOP)/make/$(TARGET)/otp.mk
 MODULES= \
 	mt \
 	mnesia_SUITE \
+	mnesia_bench_SUITE \
 	mnesia_test_lib \
 	mnesia_install_test \
 	mnesia_registry_test \
@@ -117,7 +118,7 @@ release_spec: opt
 
 release_tests_spec: opt
 	$(INSTALL_DIR) "$(RELSYSDIR)"
-	$(INSTALL_DATA) mnesia.spec mnesia.cover $(ERL_FILES) $(HRL_FILES) "$(RELSYSDIR)"
+	$(INSTALL_DATA) mnesia.spec mnesia_bench.spec mnesia.cover $(ERL_FILES) $(HRL_FILES) "$(RELSYSDIR)"
 	$(INSTALL_SCRIPT) mt $(INSTALL_PROGS) "$(RELSYSDIR)"
 #	chmod -R u+w "$(RELSYSDIR)"
 #	@tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -)
diff --git a/lib/mnesia/test/mnesia_bench.spec b/lib/mnesia/test/mnesia_bench.spec
new file mode 100644
index 0000000000..7b17cb5c4e
--- /dev/null
+++ b/lib/mnesia/test/mnesia_bench.spec
@@ -0,0 +1 @@
+{suites,"../mnesia_test",[mnesia_bench_SUITE]}.
\ No newline at end of file
diff --git a/lib/mnesia/test/mnesia_bench_SUITE.erl b/lib/mnesia/test/mnesia_bench_SUITE.erl
new file mode 100644
index 0000000000..7cbf77f046
--- /dev/null
+++ b/lib/mnesia/test/mnesia_bench_SUITE.erl
@@ -0,0 +1,69 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%
+-module(mnesia_bench_SUITE).
+-author('[email protected]').
+-compile(export_all).
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+suite() -> [{ct_hooks,[{ts_install_cth,[{nodenames,2}]}]}].
+
+
+all() -> 
+    [{group,tpcb}].
+
+groups() -> 
+    [{tpcb,[{repeat,2}],[tpcb_conflict_ramcopies,
+			 tpcb_conflict_disk_only_copies]}].
+
+init_per_group(_GroupName, Config) ->
+	Config.
+
+end_per_group(_GroupName, Config) ->
+	Config.
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(Config) ->
+    Config.
+
+init_per_testcase(_Func, Conf) ->
+    Conf.
+
+end_per_testcase(_Func, _Conf) ->
+    ok.
+
+    
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+tpcb_conflict_ramcopies(_Config) ->
+    mnesia_tpcb:conflict_benchmark(ram_copies).
+
+tpcb_conflict_disk_only_copies(_Config) ->
+    mnesia_tpcb:conflict_benchmark(disc_only_copies).
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+
+    
+
+
diff --git a/lib/test_server/src/Makefile b/lib/test_server/src/Makefile
index a75855eaab..b47cf9ff64 100644
--- a/lib/test_server/src/Makefile
+++ b/lib/test_server/src/Makefile
@@ -57,7 +57,8 @@ TS_MODULES= \
 	ts_autoconf_win32 \
         ts_autoconf_vxworks \
 	ts_install \
-	ts_install_cth
+	ts_install_cth \
+	ts_benchmark
 
 TARGET_MODULES= $(MODULES:%=$(EBIN)/%)
 TS_TARGET_MODULES= $(TS_MODULES:%=$(EBIN)/%)
diff --git a/lib/test_server/src/ts.erl b/lib/test_server/src/ts.erl
index 4899f38d2b..75631216b0 100644
--- a/lib/test_server/src/ts.erl
+++ b/lib/test_server/src/ts.erl
@@ -28,6 +28,7 @@
 	 clean/0, clean/1,
 	 tests/0, tests/1,
 	 install/0, install/1, index/0,
+	 bench/0, bench/1, bench/2, benchmarks/0,
 	 estone/0, estone/1,
 	 cross_cover_analyse/1,
 	 compile_testcases/0, compile_testcases/1,
@@ -48,12 +49,12 @@
 %%%       |                +------  ts_make
 %%%       |                |
 %%%       +-- ts_run  -----+
-%%%                        |	    			     ts_filelib
-%%%                        +------  ts_make_erl
-%%%                        |
-%%%                        +------  ts_reports (indirectly)
-%%%       
-%%%       
+%%%       |                |	    			     ts_filelib
+%%%       |                +------  ts_make_erl
+%%%       |                |
+%%%       |                +------  ts_reports (indirectly)
+%%%       |
+%%%       +-- ts_benchmark
 %%%
 %%% The modules ts_lib and ts_filelib contains utilities used by
 %%% the other modules.
@@ -81,6 +82,7 @@
 %%%			 of the tests run.
 %%% ts_lib		 Miscellanous utility functions, each used by several
 %%%			 other modules.
+%%% ts_benchmark         Supervises otp benchmarks and collects results.
 %%%----------------------------------------------------------------------
 
 -include_lib("kernel/include/file.hrl").
@@ -128,7 +130,7 @@ help(installed) ->
 	 "  ts:run(Spec, Mod) - Run a single test suite.\n",
 	 "  ts:run(Spec, Mod, Case)\n",
 	 "                    - Run a single test case.\n",
-	 "  All above run functions can have the additional Options argument\n",
+	 "  All above run functions can have an additional Options argument\n",
 	 "  which is a list of options.\n",
 	 "\n",
 	 "Run options supported:\n",
@@ -158,7 +160,7 @@ help(installed) ->
 	 "  {ctp | ctpl, Mod, Func}\n",
 	 "  {ctp | ctpl, Mod, Func, Arity}\n",
 	 "\n",
-	 "Support functions\n",
+	 "Support functions:\n",
 	 "  ts:tests()        - Shows all available families of tests.\n",
 	 "  ts:tests(Spec)    - Shows all available test modules in Spec,\n",
 	 "                      i.e. ../Spec_test/*_SUITE.erl\n",
@@ -179,6 +181,13 @@ help(installed) ->
 	 "                    - Compile all testcases for usage in a cross ~n"
 	 "                      compile environment."
 	 " \n"
+	 "Benchmark functions:\n"
+	 "  ts:benchmarks()   - Get all available families of benchmarks\n"
+	 "  ts:bench()        - Runs all benchmarks\n"
+	 "  ts:bench(Spec)    - Runs all benchmarks in the given spec file.\n"
+	 "                      The spec file is actually ../*_test/Spec_bench.spec\n\n"
+	 "                      ts:bench can take the same Options argument as ts:run.\n"
+	 "\n"
 	 "Installation (already done):\n"
 	],
     show_help([H,?install_help]).
@@ -491,6 +500,25 @@ tests(Spec) ->
     {ok, Cwd} = file:get_cwd(),
     ts_lib:suites(Cwd, atom_to_list(Spec)).
 
+%% Benchmark related functions
+
+bench() ->
+    bench([]).
+
+bench(Opts) when is_list(Opts) ->
+    bench(benchmarks(),Opts);
+bench(Spec) ->
+    bench([Spec],[]).
+
+bench(Spec, Opts) when is_atom(Spec) ->
+    bench([Spec],Opts);
+bench(Specs, Opts) ->
+    check_and_run(fun(Vars) -> ts_benchmark:run(Specs, Opts, Vars) end).
+
+benchmarks() ->
+    ts_benchmark:benchmarks().
+
+
 
 %% 
 %% estone/0, estone/1
diff --git a/lib/test_server/src/ts_benchmark.erl b/lib/test_server/src/ts_benchmark.erl
new file mode 100644
index 0000000000..516d22fd2d
--- /dev/null
+++ b/lib/test_server/src/ts_benchmark.erl
@@ -0,0 +1,91 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2012-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(ts_benchmark).
+
+-include_lib("common_test/include/ct_event.hrl").
+-include_lib("kernel/include/file.hrl").
+-include("ts.hrl").
+
+-export([benchmarks/0,
+	 run/3]).
+
+%% gen_event callbacks
+-export([init/1, handle_event/2]).
+
+benchmarks() ->
+    {ok, Cwd} = file:get_cwd(),
+    Benches = filelib:wildcard(
+		filename:join([Cwd,"..","*_test","*_bench.spec"])),
+    [begin
+	 Base = filename:basename(N),
+	 list_to_atom(string:substr(Base,1,string:rstr(Base,"_")-1))
+     end || N <- Benches].
+
+run(Specs, Opts, Vars) ->
+    {ok, Cwd} = file:get_cwd(),
+    {{YY,MM,DD},{HH,Mi,SS}} = calendar:local_time(),
+    BName = lists:concat([YY,"_",MM,"_",DD,"T",HH,"_",Mi,"_",SS]),
+    BDir = filename:join([Cwd,BName]),
+    file:make_dir(BDir),
+    [ts_run:run(atom_to_list(Spec),
+		[{spec, [atom_to_list(Spec)++"_bench.spec"]}],
+		[{event_handler, {ts_benchmark, [Spec,BDir]}}|Opts],Vars) 
+     || Spec <- Specs],
+    file:delete(filename:join(Cwd,"latest_benchmark")),
+    {ok,D} = file:open(filename:join(Cwd,"latest_benchmark"),[write]),
+    io:format(D,BDir,[]),
+    file:close(D).
+    
+
+%%%===================================================================
+%%% gen_event callbacks
+%%%===================================================================
+
+-record(state, { spec, suite, tc, stats_dir}).
+
+init([Spec,Dir]) ->
+    {ok, #state{ spec = Spec, stats_dir = Dir }}.
+
+handle_event(#event{name = tc_start, data = {Suite,Tc}}, State) ->
+    {ok,State#state{ suite = Suite, tc = Tc}};
+handle_event(#event{name = benchmark_data, data = Data}, State) ->
+    Spec = proplists:get_value(application, Data, State#state.spec),
+    Suite = proplists:get_value(suite, Data, State#state.suite),
+    Tc = proplists:get_value(name, Data, State#state.tc),
+    Value = proplists:get_value(value, Data),
+    {ok, D} = file:open(filename:join(
+			  [State#state.stats_dir,
+			   lists:concat([e(Spec),"-",e(Suite),"-",
+					 e(Tc),".ebench"])]),
+			[append]),
+    io:format(D, "~p~n",[Value]),
+    file:close(D),
+    {ok, State};
+handle_event(_Event, State) ->
+    {ok, State}.
+
+
+e(Atom) when is_atom(Atom) ->
+    Atom;
+e(Str) when is_list(Str) ->
+    lists:map(fun($/) ->
+		      $\\;
+		 (C) ->
+		      C
+	      end,Str).
diff --git a/lib/test_server/src/ts_lib.erl b/lib/test_server/src/ts_lib.erl
index d521d2beda..93d5d78289 100644
--- a/lib/test_server/src/ts_lib.erl
+++ b/lib/test_server/src/ts_lib.erl
@@ -108,8 +108,16 @@ interesting_logs(Dir) ->
 
 specs(Dir) ->
     Specs = filelib:wildcard(filename:join([filename:dirname(Dir),
-					    "*_test", "*.{dyn,}spec"])), 
-    sort_tests([filename_to_atom(Name) || Name <- Specs]).
+					    "*_test", "*.{dyn,}spec"])),
+    % Filter away all spec which end with _bench.spec
+    NoBench = fun(SpecName) ->
+		      case lists:reverse(SpecName) of
+			  "ceps.hcneb_"++_ -> false;
+			  _ -> true
+		      end
+	      end,
+
+    sort_tests([filename_to_atom(Name) || Name <- Specs, NoBench(Name)]).
 
 suites(Dir, Spec) ->
     Glob=filename:join([filename:dirname(Dir), Spec++"_test",