Merge branch 'peppe/common_test_r14_dev_2' into dev

* peppe/common_test_r14_dev_2:
  Misc documentation updates
  Add documentation for run_test program
  Step vsn for test_server to 3.4
  Make {repeat*,N} property in group execute the group N times exactly
  Fix so that ct_run converts relative diretories in the code path to absolute
  Fix bug in handling framework:end_tc timeouts
  Fix bug that prevents the interactive shell mode to start properly
  Fix failing multiply timetrap test case
  Minor fixes in code and test suites
  Add support for executing pre-loaded suites (e.g. modules loaded as binaries)
  Add test suite for remote loading of binary suites
  Fix error with {repeat,0} property in groups causing double iterations
  Add support for config info functions (e.g. init_per_suite/0)
  Add support for dynamic timetrap handling
  Have end_per_testcase run even after timetrap_timeout and abort_testcase
  Flush old DOWN messages in demonitor
  Add groups in test specifications
  Add new tests for test case groups and test specifications
  Improve and fix various test suites
  Add event_handler_init start flag that can pass init arguments to event handlers
  ...

OTP-8703 peppe/common_test_r14_dev_2

1 files changed, 102 insertions, 18 deletions

diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl
index 6148e3280e..6a75c8f789 100644
--- a/lib/common_test/test/ct_test_support.erl
+++ b/lib/common_test/test/ct_test_support.erl
@@ -1,19 +1,19 @@
 %%
 %% %CopyrightBegin%
-%% 
-%% Copyright Ericsson AB 2008-2009. All Rights Reserved.
-%% 
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
 %% compliance with the License. You should have received a copy of the
 %% Erlang Public License along with this software. If not, it can be
 %% retrieved online at http://www.erlang.org/.
-%% 
+%%
 %% Software distributed under the License is distributed on an "AS IS"
 %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 %% the License for the specific language governing rights and limitations
 %% under the License.
-%% 
+%%
 %% %CopyrightEnd%
 %%
 
@@ -27,8 +27,9 @@
 -include_lib("common_test/include/ct_event.hrl").
 
 -export([init_per_suite/1, init_per_suite/2, end_per_suite/1,
-	 init_per_testcase/2, end_per_testcase/2, write_testspec/3, 
-	 run/4, get_opts/1, wait_for_ct_stop/1]).
+	 init_per_testcase/2, end_per_testcase/2,
+	 write_testspec/2, write_testspec/3,
+	 run/2, run/4, get_opts/1, wait_for_ct_stop/1]).
 
 -export([handle_event/2, start_event_receiver/1, get_events/2,
 	 verify_events/3, reformat/2, log_events/3]).
@@ -55,17 +56,28 @@ init_per_suite(Config, Level) ->
 	    test_server:fail(Reason);
 	{ok,CTNode} ->
 	    test_server:format(0, "Node ~p started~n", [CTNode]),
+	    IsCover = test_server:is_cover(),
+	    if IsCover ->
+		cover:start(CTNode);
+	    true->
+		ok
+	    end,
 	    DataDir = ?config(data_dir, Config),
 	    PrivDir = ?config(priv_dir, Config),
 
 	    %% PrivDir as well as directory of Test Server suites
 	    %% have to be in code path on Common Test node.
-	    true = rpc:call(CTNode, code, add_patha, [PrivDir]),
 	    [_ | Parts] = lists:reverse(filename:split(DataDir)),
 	    TSDir = filename:join(lists:reverse(Parts)),
-	    true = rpc:call(CTNode, code, add_patha, [TSDir]),
-	    test_server:format(Level, "Dirs added to code path (on ~w):~n"
-			       "~s~n~s~n", [CTNode,TSDir,PrivDir]),
+	    AddPathDirs = case ?config(path_dirs, Config) of
+			      undefined -> [];
+			      Ds -> Ds
+			  end,
+	    PathDirs = [PrivDir,TSDir | AddPathDirs],
+	    [true = rpc:call(CTNode, code, add_patha, [D]) || D <- PathDirs],
+	    test_server:format(Level, "Dirs added to code path (on ~w):~n",
+			       [CTNode]),
+	    [io:format("~s~n", [D]) || D <- PathDirs],
 
 	    TraceFile = filename:join(DataDir, "ct.trace"),
 	    case file:read_file_info(TraceFile) of
@@ -87,6 +99,7 @@ end_per_suite(Config) ->
     CTNode = ?config(ct_node, Config),
     PrivDir = ?config(priv_dir, Config),
     true = rpc:call(CTNode, code, del_path, [filename:join(PrivDir,"")]),
+    cover:stop(CTNode),
     slave:stop(CTNode),
     ok.
 
@@ -95,9 +108,16 @@ end_per_suite(Config) ->
 
 init_per_testcase(_TestCase, Config) ->
     {_,{_,LogDir}} = lists:keysearch(logdir, 1, get_opts(Config)),
-    test_server:format("See Common Test logs here:\n"
+    case lists:keysearch(master, 1, Config) of
+	false->
+	    test_server:format("See Common Test logs here:\n\n"
 		       "<a href=\"file://~s/all_runs.html\">~s/all_runs.html</a>",
-		       [LogDir,LogDir]),
+		       [LogDir,LogDir]);
+	{value, _}->
+	    test_server:format("See CT Master Test logs here:\n\n"
+		       "<a href=\"file://~s/master_runs.html\">~s/master_runs.html</a>",
+		       [LogDir,LogDir])
+    end,
     Config.
 
 %%%-----------------------------------------------------------------
@@ -111,9 +131,10 @@ end_per_testcase(_TestCase, Config) ->
 
 %%%-----------------------------------------------------------------
 %%% 
-
 write_testspec(TestSpec, Dir, Name) ->
-    TSFile = filename:join(Dir, Name),
+    write_testspec(TestSpec, filename:join(Dir, Name)).
+
+write_testspec(TestSpec, TSFile) ->
     {ok,Dev} = file:open(TSFile, [write]),
     [io:format(Dev, "~p.~n", [Entry]) || Entry <- TestSpec],
     file:close(Dev),
@@ -158,10 +179,32 @@ get_opts(Config) ->
 
 %%%-----------------------------------------------------------------
 %%% 
+run(Opts, Config) ->
+    CTNode = ?config(ct_node, Config),
+    Level = ?config(trace_level, Config),
+    %% use ct interface
+    test_server:format(Level, "~n[RUN #1] Calling ct:run_test(~p) on ~p~n",
+		       [Opts, CTNode]),
+    Result1 = rpc:call(CTNode, ct, run_test, [Opts]),
+
+    %% use run_test interface (simulated)
+    test_server:format(Level, "Saving start opts on ~p: ~p~n", [CTNode,Opts]),
+    rpc:call(CTNode, application, set_env, [common_test, run_test_start_opts, Opts]),
+    test_server:format(Level, "[RUN #2] Calling ct_run:script_start() on ~p~n", [CTNode]),
+    Result2 = rpc:call(CTNode, ct_run, script_start, []),
+    case {Result1,Result2} of
+	{ok,ok} ->
+	    ok;
+	{E,_} when E =/= ok ->
+	    E;
+	{_,E} when E =/= ok ->
+	    E
+    end.
+
 run(M, F, A, Config) ->
     CTNode = ?config(ct_node, Config),
     Level = ?config(trace_level, Config),
-    test_server:format(Level, "Calling ~w:~w(~p) on ~p~n", 
+    test_server:format(Level, "~nCalling ~w:~w(~p) on ~p~n",
 		       [M, F, A, CTNode]),
     rpc:call(CTNode, M, F, A).
 
@@ -231,8 +274,12 @@ verify_events(TEvs, Evs, Config) ->
 	    ok
     end.
 
+verify_events1([TestEv|_], [{TEH,#event{name=stop_logging,node=Node,data=_}}|_], Node, _)
+  when element(1,TestEv) == TEH, element(2,TestEv) =/= stop_logging ->
+    test_server:format("Failed to find ~p in the list of events!~n", [TestEv]),
+    exit({event_not_found,TestEv});
+
 verify_events1(TEvs = [TestEv | TestEvs], Evs = [_|Events], Node, Config) ->
-%%    test_server:format("Next expected event: ~p~n", [TestEv]),
     case catch locate(TestEv, Node, Evs, Config) of
 	nomatch ->
 	    verify_events1(TEvs, Events, Node, Config);
@@ -332,6 +379,10 @@ locate({parallel,TEvs}, Node, Evs, Config) ->
 					     EH == TEH, EvNode == Node, 
 					     Mod == M, Func == F ->
 					   false;
+				      ({EH,#event{name=stop_logging,
+						  node=EvNode,data=_}}) when
+					     EH == TEH, EvNode == Node ->
+					   exit({tc_start_not_found,TEv});
 				      (_) ->
 					   true
 				   end, Evs1),
@@ -345,6 +396,10 @@ locate({parallel,TEvs}, Node, Evs, Config) ->
 					  EH == TEH, EvNode == Node, 
 					  Mod == M, Func == F ->
 					false;
+				   ({EH,#event{name=stop_logging,
+					       node=EvNode,data=_}}) when
+					  EH == TEH, EvNode == Node ->
+					exit({tc_done_not_found,TEv});
 				   (_) ->
 					true
 				end, Evs2),
@@ -388,6 +443,10 @@ locate({parallel,TEvs}, Node, Evs, Config) ->
 					  EH == TEH, EvNode == Node, Mod == M,
 					  EvGName == GroupName, EvProps == Props ->
 					false;
+				   ({EH,#event{name=stop_logging,
+					       node=EvNode,data=_}}) when
+					  EH == TEH, EvNode == Node ->
+					exit({tc_start_not_found,TEv});
 				   (_) ->
 					true
 				end, RemEvs),
@@ -410,6 +469,10 @@ locate({parallel,TEvs}, Node, Evs, Config) ->
 					  EH == TEH, EvNode == Node, Mod == M,
 					  EvGName == GroupName, EvProps == Props, Res == R ->
 					false;
+				   ({EH,#event{name=stop_logging,
+					       node=EvNode,data=_}}) when
+					  EH == TEH, EvNode == Node ->
+					exit({tc_done_not_found,TEv});
 				   (_) ->
 					true
 				end, RemEvs),
@@ -429,6 +492,10 @@ locate({parallel,TEvs}, Node, Evs, Config) ->
 					       data={Mod,end_per_group,Reason}}}) when
 				   EH == TEH, EvNode == Node, Mod == M, Reason == R ->
 					false;
+				   ({EH,#event{name=stop_logging,
+					       node=EvNode,data=_}}) when
+					  EH == TEH, EvNode == Node ->
+					exit({tc_auto_skip_not_found,TEv});
 				   (_) ->
 					true
 				end, RemEvs),
@@ -533,6 +600,10 @@ locate({shuffle,TEvs}, Node, Evs, Config) ->
 					     EH == TEH, EvNode == Node, 
 					     Mod == M, Func == F ->
 					   false;
+				      ({EH,#event{name=stop_logging,
+						  node=EvNode,data=_}}) when
+					     EH == TEH, EvNode == Node ->
+					   exit({tc_start_not_found,TEv});
 				      (_) ->
 					   true
 				   end, Evs1),
@@ -576,6 +647,10 @@ locate({shuffle,TEvs}, Node, Evs, Config) ->
 					  EH == TEH, EvNode == Node, Mod == M,
 					  EvGName == GroupName ->
 					false;
+				   ({EH,#event{name=stop_logging,
+					       node=EvNode,data=_}}) when
+					  EH == TEH, EvNode == Node ->
+					exit({tc_start_not_found,TEv});
 				   (_) ->
 					true
 				end, RemEvs),
@@ -611,6 +686,10 @@ locate({shuffle,TEvs}, Node, Evs, Config) ->
 					  EH == TEH, EvNode == Node, Mod == M,
 					  EvGName == GroupName, Res == R ->
 					false;
+				   ({EH,#event{name=stop_logging,
+					       node=EvNode,data=_}}) when
+					  EH == TEH, EvNode == Node ->
+					exit({tc_done_not_found,TEv});
 				   (_) ->
 					true
 				end, RemEvs),
@@ -643,6 +722,10 @@ locate({shuffle,TEvs}, Node, Evs, Config) ->
 					       data={Mod,end_per_group,Reason}}}) when
 				   EH == TEH, EvNode == Node, Mod == M, Reason == R ->
 					false;
+				   ({EH,#event{name=stop_logging,
+					       node=EvNode,data=_}}) when
+					  EH == TEH, EvNode == Node ->
+					exit({tc_auto_skip_not_found,TEv});
 				   (_) ->
 					true
 				end, RemEvs),
@@ -785,7 +868,8 @@ log_events(TC, Events, PrivDir) ->
     io:format(Dev, "[~n", []),
     log_events1(Events, Dev, " "),
     file:close(Dev),
-    io:format("Events written to logfile: ~p~n", [LogFile]),
+    io:format("Events written to logfile: <a href=\"file://~s\">~s</a>~n",
+	      [LogFile,LogFile]),
     io:format(user, "Events written to logfile: ~p~n", [LogFile]).
 
 log_events1(Evs, Dev, "") ->