Merge branch 'peppe/common_test_r14_dev_2' into dev

* peppe/common_test_r14_dev_2:
  Misc documentation updates
  Add documentation for run_test program
  Step vsn for test_server to 3.4
  Make {repeat*,N} property in group execute the group N times exactly
  Fix so that ct_run converts relative diretories in the code path to absolute
  Fix bug in handling framework:end_tc timeouts
  Fix bug that prevents the interactive shell mode to start properly
  Fix failing multiply timetrap test case
  Minor fixes in code and test suites
  Add support for executing pre-loaded suites (e.g. modules loaded as binaries)
  Add test suite for remote loading of binary suites
  Fix error with {repeat,0} property in groups causing double iterations
  Add support for config info functions (e.g. init_per_suite/0)
  Add support for dynamic timetrap handling
  Have end_per_testcase run even after timetrap_timeout and abort_testcase
  Flush old DOWN messages in demonitor
  Add groups in test specifications
  Add new tests for test case groups and test specifications
  Improve and fix various test suites
  Add event_handler_init start flag that can pass init arguments to event handlers
  ...

OTP-8703 peppe/common_test_r14_dev_2

6 files changed, 465 insertions, 0 deletions

diff --git a/lib/common_test/test/ct_config_SUITE_data/config/config.txt b/lib/common_test/test/ct_config_SUITE_data/config/config.txt
new file mode 100644
index 0000000000..fcbffcd7f3
--- /dev/null
+++ b/lib/common_test/test/ct_config_SUITE_data/config/config.txt
@@ -0,0 +1,31 @@
+{x, suite}.
+{gen_cfg,
+	[
+		{a,a_value},
+		{b,b_value}
+	]}.
+{gen_cfg2,
+	[
+		{c, "Hello, world!"},
+		{d, atom_value},
+		{e, {tuple,1,"third value",[]}},
+		{f, []},
+		{g, [1,atom,"string",13.6,{1,2,3}]}
+	]}.
+{gen_cfg3,
+	[
+		{h,
+		[
+			{i, third1},
+			{j, "Third2"},
+			{k, 'THIRD3'}
+		]},
+		{l,
+		[
+			{m,
+			[
+				{n, "N"},
+				{o, 'O'}
+			]}
+		]}
+	]}.
diff --git a/lib/common_test/test/ct_config_SUITE_data/config/config.xml b/lib/common_test/test/ct_config_SUITE_data/config/config.xml
new file mode 100644
index 0000000000..0a3e5f2e31
--- /dev/null
+++ b/lib/common_test/test/ct_config_SUITE_data/config/config.xml
@@ -0,0 +1,27 @@
+<config>
+	<x>suite</x>
+	<gen_cfg>
+		<a>a_value</a>
+		<b>b_value</b>
+	</gen_cfg>
+	<gen_cfg2>
+		<c>"Hello, world!"</c>
+		<d>atom_value</d>
+		<e>{tuple,1,"third value",[]}</e>
+		<f>[]</f>
+		<g>[1,atom,"string",13.6,{1,2,3}]</g>
+	</gen_cfg2>
+	<gen_cfg3>
+		<h>
+			<i>third1</i>
+			<j>"Third2"</j>
+			<k>'THIRD3'</k>
+		</h>
+		<l>
+			<m>
+				<n>"N"</n>
+				<o>'O'</o>
+			</m>
+		</l>
+	</gen_cfg3>
+</config>
diff --git a/lib/common_test/test/ct_config_SUITE_data/config/test/config_driver.erl b/lib/common_test/test/ct_config_SUITE_data/config/test/config_driver.erl
new file mode 100644
index 0000000000..d93faf6ec6
--- /dev/null
+++ b/lib/common_test/test/ct_config_SUITE_data/config/test/config_driver.erl
@@ -0,0 +1,46 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_config_SUITE
+%%%
+%%% Description:
+%%% Config driver used in the CT's tests (config_2_SUITE)
+%%%-------------------------------------------------------------------
+-module(config_driver).
+-export([read_config/1, check_parameter/1]).
+
+read_config(ServerName)->
+    ServerModule = list_to_atom(ServerName),
+    ServerModule:start(),
+    ServerModule:get_config().
+
+check_parameter(ServerName)->
+    ServerModule = list_to_atom(ServerName),
+    case code:is_loaded(ServerModule) of
+	{file, _}->
+	    {ok, {config, ServerName}};
+	false->
+	    case code:load_file(ServerModule) of
+		{module, ServerModule}->
+		    {ok, {config, ServerName}};
+		{error, nofile}->
+		    {error, {wrong_config, "File not found: " ++ ServerName ++ ".beam"}}
+	    end
+    end.
diff --git a/lib/common_test/test/ct_config_SUITE_data/config/test/config_dynamic_SUITE.erl b/lib/common_test/test/ct_config_SUITE_data/config/test/config_dynamic_SUITE.erl
new file mode 100644
index 0000000000..8ee12a2e4d
--- /dev/null
+++ b/lib/common_test/test/ct_config_SUITE_data/config/test/config_dynamic_SUITE.erl
@@ -0,0 +1,145 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: config_dynamic_SUITE
+%%%
+%%% Description:
+%%% Test suite for common_test which tests the userconfig functionality
+%%%-------------------------------------------------------------------
+-module(config_dynamic_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%% This suite will be run with dynamic userconfig
+%% test_driver.erl is compliant to ct_config_* callback
+%% and test_server is simple server for getting runtime-changing data
+%% which will return the list with the following variables:
+%% localtime = the erlang:localtime() result in list [{date, Date}, {time, Time}]
+%% node = erlang:node() - can be compared in the testcase
+%% now = erlang:now() - easier to compare than localtime()
+%% config_server_pid - pid of the config server, should NOT change!
+%% config_server_vsn - .19
+%% config_server_iteration - a number of iteration config_server's loop done
+%% disappearable_variable - hereAmI - will be absent on even iterations
+
+suite() ->
+    [
+     {timetrap, {seconds,10}}
+    ].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_) ->
+    ok.
+
+all() -> [test_get_known_variable, test_localtime_update,
+	  test_server_pid, test_disappearable_variable,
+	  test_disappearable_variable_alias].
+
+init_per_testcase(_, Config) ->
+    Config.
+
+end_per_testcase(_, _) ->
+    ok.
+
+% test that usual config works
+test_get_known_variable(_)->
+    Node = erlang:node(),
+    0.19 = ct:get_config(config_server_vsn),
+    Node = ct:get_config(node),
+    ok.
+
+% localtime will be updated in 5 seconds, check that
+test_localtime_update(_)->
+    Seconds = 5,
+    LT1 = ct:get_config(localtime),
+    timer:sleep(Seconds*1000),
+    LT2 = ct:reload_config(localtime),
+    case is_diff_ok(LT1, LT2, Seconds) of
+	{false, Actual, Exp}->
+	    ct:fail(io_lib:format("Time difference ~p is not ok, expected ~p", [Actual, Exp]));
+	true->
+	    ok
+    end.
+
+% server pid should not change
+test_server_pid()->
+    [{require, cfvsn, config_server_vsn}].
+test_server_pid(_)->
+    Pid = ct:get_config(config_server_pid),
+    Pid = ct:reload_config(config_server_pid),
+    Vsn = ct:get_config(config_server_vsn),
+    % aliases remain after config reloading
+    Vsn = ct:get_config(cfvsn),
+    ok.
+
+% test that variables may disappear from the config_2_SUITE
+test_disappearable_variable(_)->
+    % ask CT for config_server_iteration variable
+    Iter = ct:reload_config(config_server_iteration),
+    % here we should reload this variable in case it's odd
+    if Iter rem 2 == 1->
+	Iter2 = ct:reload_config(config_server_iteration),
+	Iter2 = Iter+1;
+	true->ok
+    end,
+    % now disappearable_variable should be in place
+    hereAmI = ct:get_config(disappearable_variable),
+    % and now it should disappear
+    undefined = ct:reload_config(disappearable_variable).
+
+% alias of disappearable_variable should disappear too
+test_disappearable_variable_alias(_)->
+    % the same rules apply for this testcase as for previous one
+    Iter = ct:reload_config(config_server_iteration),
+    Iter2 = if
+	Iter rem 2 == 1 ->
+	    NewIter = ct:reload_config(config_server_iteration),
+	    NewIter = Iter+1;
+	true->
+	    Iter
+    end,
+    ct:require(diav, disappearable_variable),
+    hereAmI = ct:get_config(disappearable_variable),
+    hereAmI = ct:get_config(diav),
+    ct:reload_config(disappearable_variable),
+    undefined = ct:get_config(disappearable_variable),
+    % after reloading, it's even again
+    Iter3=ct:get_config(config_server_iteration),
+    Iter3 = Iter2+1,
+    % and alias does not exist
+    undefined = ct:get_config(diav).
+
+my_dt_to_datetime([{date, D},{time, T}])->
+    {D, T}.
+
+is_diff_ok(DT1, DT2, Seconds)->
+    GS1 = calendar:datetime_to_gregorian_seconds(my_dt_to_datetime(DT1)),
+    GS2 = calendar:datetime_to_gregorian_seconds(my_dt_to_datetime(DT2)),
+    if
+	GS2-GS1 > Seconds+Seconds/2;
+	GS2-GS1 < Seconds-Seconds/2->
+	    {false, GS2-GS1, Seconds};
+	true->
+	    true
+    end.
diff --git a/lib/common_test/test/ct_config_SUITE_data/config/test/config_server.erl b/lib/common_test/test/ct_config_SUITE_data/config/test/config_server.erl
new file mode 100644
index 0000000000..8463fea645
--- /dev/null
+++ b/lib/common_test/test/ct_config_SUITE_data/config/test/config_server.erl
@@ -0,0 +1,93 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_config_SUITE
+%%%
+%%% Description:
+%%% Config server used in the CT's tests (config_2_SUITE)
+%%%-------------------------------------------------------------------
+-module(config_server).
+-export([start/0, stop/0, loop/1, init/1, get_config/0]).
+
+-define(REGISTERED_NAME, ct_test_config_server).
+-define(vsn, 0.19).
+
+start()->
+    case whereis(?REGISTERED_NAME) of
+        undefined->
+	    spawn(?MODULE, init, [?REGISTERED_NAME]),
+	    wait();
+	_Pid->
+	    ok
+    end,
+    ?REGISTERED_NAME.
+
+init(Name)->
+    register(Name, self()),
+    loop(0).
+
+get_config()->
+    call(self(), get_config).
+
+stop()->
+    call(self(), stop).
+
+call(Client, Request)->
+    case whereis(?REGISTERED_NAME) of
+	undefined->
+	    {error, not_started, Request};
+	Pid->
+	    Pid ! {Client, Request},
+	    receive
+	        Reply->
+		    {ok, Reply}
+	    after 4000->
+		{error, timeout, Request}
+	    end
+    end.
+
+loop(Iteration)->
+    receive
+	{Pid, stop}->
+	    Pid ! ok;
+	{Pid, get_config}->
+	    {D,T} = erlang:localtime(),
+	    Config =
+		[{localtime, [{date, D}, {time, T}]},
+		 {node, erlang:node()},
+		 {config_server_iteration, Iteration},
+		 {now, erlang:now()},
+		 {config_server_pid, self()},
+		 {config_server_vsn, ?vsn}],
+	    Config2 = if Iteration rem 2 == 0->
+		Config ++ [{disappearable_variable, hereAmI}];
+		true-> Config
+	    end,
+	    Pid ! Config2,
+	    ?MODULE:loop(Iteration+1)
+    end.
+
+wait()->
+    case whereis(?REGISTERED_NAME) of
+	undefined->
+	    wait();
+	_Pid->
+	    ok
+    end.
diff --git a/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl b/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl
new file mode 100644
index 0000000000..8751a2e8f3
--- /dev/null
+++ b/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl
@@ -0,0 +1,123 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: config_static_SUITE
+%%%
+%%% Description:
+%%% Test suite for common_test which tests the get_config and require
+%%% functionality
+%%%-------------------------------------------------------------------
+-module(config_static_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+% The config contains variables:
+% x - atom
+% gen_cfg - list with two key-values tagged with a and b
+% gen_cfg2 - list of five key-values tagged with c, d, e, f and g
+% gen_cfg3 - list of two complex key-values taggen with:
+%	h: three elements inside - i, j and k
+%	l: m inside, contains n and o
+
+suite() ->
+    [
+     {timetrap, {seconds,10}},
+     %% x1 doesn't exist in cfg-file!
+     {require, x1, x},
+     {require, gen_cfg3},
+     {require, alias, gen_cfg},
+     %% x1 default value
+     {x1, {x,suite}}
+    ].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_) ->
+    ok.
+
+all() -> [test_get_config_simple, test_get_config_nested, test_default_suitewide,
+	  test_config_name_already_in_use1, test_default_tclocal,
+	  test_config_name_already_in_use2, test_alias_tclocal,
+	  test_get_config_undefined].
+
+init_per_testcase(_, Config) ->
+    Config.
+
+end_per_testcase(_, _) ->
+    ok.
+
+%% test getting a simple value
+test_get_config_simple(_)->
+    suite = ct:get_config(x),
+    ok.
+
+%% test getting a nested value
+test_get_config_nested(_)->
+    a_value = ct:get_config({gen_cfg, a}),
+    ok.
+
+%% test suite-wide default value
+test_default_suitewide(_)->
+    suite = ct:get_config(x1),
+    ok.
+
+%% should get skipped
+test_config_name_already_in_use1() ->
+    [{timetrap, {seconds,2}},
+     {require, x1, x},
+     {x1, {x,test2}}].
+test_config_name_already_in_use1(_) ->
+    ct:fail("Test should've been skipped, you shouldn't see this!"),
+    ok.
+
+%% test defaults in a testcase
+test_default_tclocal() ->
+    [{timetrap, {seconds,3}},
+     {require, y1, y},
+     {y1, {y,test3}}].
+test_default_tclocal(_) ->
+    test3 = ct:get_config(y1),
+    ok.
+
+%% should get skipped
+test_config_name_already_in_use2() ->
+    [{require,alias,something},
+     {alias,{something,else}},
+     {require, x1, x},
+     {x1, {x,test4}}].
+test_config_name_already_in_use2(_) ->
+    ct:fail("Test should've been skipped, you shouldn't see this!"),
+    ok.
+
+%% test aliases
+test_alias_tclocal() ->
+    [{require,newalias,gen_cfg}].
+test_alias_tclocal(_) ->
+    A = [{a,a_value},{b,b_value}] = ct:get_config(newalias),
+    A = ct:get_config(gen_cfg),
+    ok.
+
+%% test for getting undefined variables
+test_get_config_undefined(_) ->
+    undefined = ct:get_config(y1),
+    ok.