aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/test
diff options
context:
space:
mode:
Diffstat (limited to 'lib/common_test/test')
-rw-r--r--lib/common_test/test/Makefile10
-rw-r--r--lib/common_test/test/common_test.cover10
-rw-r--r--lib/common_test/test/ct_config_info_SUITE.erl14
-rw-r--r--lib/common_test/test/ct_cover_SUITE.erl312
-rw-r--r--lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl156
-rw-r--r--lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore0
-rw-r--r--lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl4
-rw-r--r--lib/common_test/test/ct_error_SUITE.erl509
-rw-r--r--lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl175
-rw-r--r--lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl12
-rw-r--r--lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl258
-rw-r--r--lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl239
-rw-r--r--lib/common_test/test/ct_group_leader_SUITE.erl181
-rw-r--r--lib/common_test/test/ct_group_leader_SUITE_data/group_leader_SUITE.erl252
-rw-r--r--lib/common_test/test/ct_hooks_SUITE.erl2
-rw-r--r--lib/common_test/test/ct_master_SUITE.erl74
-rw-r--r--lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl11
-rw-r--r--lib/common_test/test/ct_netconfc_SUITE.erl5
-rw-r--r--lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl7
-rw-r--r--lib/common_test/test/ct_netconfc_SUITE_data/ns.erl21
-rw-r--r--lib/common_test/test/ct_snmp_SUITE.erl141
-rw-r--r--lib/common_test/test/ct_snmp_SUITE_data/snmp.cfg44
-rw-r--r--lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE.erl395
-rw-r--r--lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/community.conf1
-rw-r--r--lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/context.conf1
-rw-r--r--lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/notify.conf1
-rw-r--r--lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/standard.conf7
-rw-r--r--lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/target_addr.conf2
-rw-r--r--lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/target_params.conf1
-rw-r--r--lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/usm.conf1
-rw-r--r--lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/vacm.conf6
-rw-r--r--lib/common_test/test/ct_surefire_SUITE.erl374
-rw-r--r--lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl92
-rw-r--r--lib/common_test/test/ct_system_error_SUITE.erl132
-rw-r--r--lib/common_test/test/ct_system_error_SUITE_data/a_SUITE.erl122
-rw-r--r--lib/common_test/test/ct_test_support.erl102
-rw-r--r--lib/common_test/test/ct_testspec_1_SUITE.erl108
-rw-r--r--lib/common_test/test/ct_testspec_2_SUITE.erl41
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE.erl745
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg111
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg121
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg131
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/config2/cfg211
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/flat_spec14
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join12
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join_sep15
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep12
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep_join12
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both12
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both_join16
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_join11
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_sep13
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/flat_spec25
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join25
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join_sep25
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep25
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep_join25
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both24
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both_join29
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_join25
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_sep25
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl175
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl175
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl174
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl177
-rw-r--r--lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl158
66 files changed, 5314 insertions, 192 deletions
diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index 374fd8a824..760cc20410 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -40,6 +40,7 @@ MODULES= \
ct_repeat_1_SUITE \
ct_testspec_1_SUITE \
ct_testspec_2_SUITE \
+ ct_testspec_3_SUITE \
ct_skip_SUITE \
ct_error_SUITE \
ct_test_server_if_1_SUITE \
@@ -52,7 +53,12 @@ MODULES= \
ct_auto_compile_SUITE \
ct_verbosity_SUITE \
ct_shell_SUITE \
- ct_groups_search_SUITE
+ ct_system_error_SUITE \
+ ct_snmp_SUITE \
+ ct_group_leader_SUITE \
+ ct_cover_SUITE \
+ ct_groups_search_SUITE \
+ ct_surefire_SUITE
ERL_FILES= $(MODULES:%=%.erl)
@@ -106,7 +112,7 @@ release_spec: opt
release_tests_spec:
$(INSTALL_DIR) "$(RELSYSDIR)"
$(INSTALL_DATA) $(ERL_FILES) $(COVERFILE) "$(RELSYSDIR)"
- $(INSTALL_DATA) common_test.spec "$(RELSYSDIR)"
+ $(INSTALL_DATA) common_test.spec common_test.cover "$(RELSYSDIR)"
chmod -R u+w "$(RELSYSDIR)"
@tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -)
diff --git a/lib/common_test/test/common_test.cover b/lib/common_test/test/common_test.cover
new file mode 100644
index 0000000000..3aa49623e7
--- /dev/null
+++ b/lib/common_test/test/common_test.cover
@@ -0,0 +1,10 @@
+%% -*- erlang -*-
+{incl_app,common_test,details}.
+{cross,common_test,[{test_server,[erl2html2,
+ test_server,
+ test_server_ctrl,
+ test_server_gl,
+ test_server_h,
+ test_server_io,
+ test_server_node,
+ test_server_sup]}]}.
diff --git a/lib/common_test/test/ct_config_info_SUITE.erl b/lib/common_test/test/ct_config_info_SUITE.erl
index 40da377ee5..10fe8286dd 100644
--- a/lib/common_test/test/ct_config_info_SUITE.erl
+++ b/lib/common_test/test/ct_config_info_SUITE.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2009-2011. All Rights Reserved.
+%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -123,8 +123,7 @@ test_events(config_info) ->
{?eh,tc_done,{config_info_1_SUITE,init_per_suite,ok}},
[{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g1,[]}}},
- {?eh,tc_done,{config_info_1_SUITE,
- {init_per_group,unknown,[]},
+ {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g1,[]},
{failed,{timetrap_timeout,350}}}},
{?eh,tc_auto_skip,{config_info_1_SUITE,t11,
{failed,{config_info_1_SUITE,init_per_group,{timetrap_timeout,350}}}}},
@@ -136,14 +135,12 @@ test_events(config_info) ->
{?eh,tc_done,{config_info_1_SUITE,{init_per_group,g2,[]},ok}},
{?eh,tc_done,{config_info_1_SUITE,t21,ok}},
{?eh,tc_start,{config_info_1_SUITE,{end_per_group,g2,[]}}},
- {?eh,tc_done,{config_info_1_SUITE,
- {end_per_group,unknown,[]},
+ {?eh,tc_done,{config_info_1_SUITE,{end_per_group,g2,[]},
{failed,{timetrap_timeout,450}}}}],
[{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g3,[]}}},
{?eh,tc_done,{config_info_1_SUITE,{init_per_group,g3,[]},ok}},
[{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g4,[]}}},
- {?eh,tc_done,{config_info_1_SUITE,
- {init_per_group,unknown,[]},
+ {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g4,[]},
{failed,{timetrap_timeout,400}}}},
{?eh,tc_auto_skip,{config_info_1_SUITE,t41,
{failed,{config_info_1_SUITE,init_per_group,
@@ -164,8 +161,7 @@ test_events(config_info) ->
{?eh,tc_done,{config_info_1_SUITE,{init_per_group,g5,[]},ok}},
{?eh,tc_done,{config_info_1_SUITE,t51,ok}},
{?eh,tc_start,{config_info_1_SUITE,{end_per_group,g5,[]}}},
- {?eh,tc_done,{config_info_1_SUITE,
- {end_per_group,unknown,[]},
+ {?eh,tc_done,{config_info_1_SUITE,{end_per_group,g5,[]},
{failed,{timetrap_timeout,400}}}}],
{?eh,tc_start,{config_info_1_SUITE,{end_per_group,g3,[]}}},
{?eh,tc_done,{config_info_1_SUITE,{end_per_group,g3,[]},ok}}],
diff --git a/lib/common_test/test/ct_cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE.erl
new file mode 100644
index 0000000000..cb49dc423f
--- /dev/null
+++ b/lib/common_test/test/ct_cover_SUITE.erl
@@ -0,0 +1,312 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_cover_SUITE
+%%%
+%%% Description:
+%%% Test code cover analysis support
+%%%
+%%%-------------------------------------------------------------------
+-module(ct_cover_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+-define(suite, cover_SUITE).
+-define(mod, cover_test_mod).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ case test_server:is_cover() of
+ true ->
+ {skip,"Test server is running cover already - skipping"};
+ false ->
+ ct_test_support:init_per_suite(Config)
+ end.
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ Node = fullname(existing_node),
+ case lists:member(Node,nodes()) of
+ true -> rpc:call(Node,erlang,halt,[]);
+ false -> ok
+ end,
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [
+ default,
+ cover_stop_true,
+ cover_stop_false,
+ slave,
+ slave_start_slave,
+ cover_node_option,
+ ct_cover_add_remove_nodes,
+ otp_9956,
+ cross
+ ].
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%% Check that cover is collected from test node
+%% Also check that cover is by default stopped after test is completed
+default(Config) ->
+ {ok,Events} = run_test(default,Config),
+ false = check_cover(Config),
+ check_calls(Events,1),
+ ok.
+
+%% Check that cover is stopped when cover_stop option is set to true
+cover_stop_true(Config) ->
+ {ok,_Events} = run_test(cover_stop_true,[{cover_stop,true}],Config),
+ false = check_cover(Config).
+
+%% Check that cover is not stopped when cover_stop option is set to false
+cover_stop_false(Config) ->
+ {ok,_Events} = run_test(cover_stop_false,[{cover_stop,false}],Config),
+ {true,[],[?mod]} = check_cover(Config),
+ CTNode = proplists:get_value(ct_node, Config),
+ ok = rpc:call(CTNode,cover,stop,[]),
+ false = check_cover(Config),
+ ok.
+
+%% Let test node start a slave node - check that cover is collected
+%% from both nodes
+slave(Config) ->
+ {ok,Events} = run_test(slave,slave,[],Config),
+ check_calls(Events,2),
+ ok.
+
+%% Let test node start a slave node which in turn starts another slave
+%% node - check that cover is collected from all three nodes
+slave_start_slave(Config) ->
+ {ok,Events} = run_test(slave_start_slave,slave_start_slave,[],Config),
+ check_calls(Events,3),
+ ok.
+
+%% Start a slave node before test starts - the node is listed in cover
+%% spec file.
+%% Check that cover is collected from test node and slave node.
+cover_node_option(Config) ->
+ {ok, HostStr}=inet:gethostname(),
+ Host = list_to_atom(HostStr),
+ DataDir = ?config(data_dir,Config),
+ {ok,Node} = ct_slave:start(Host,existing_node,
+ [{erl_flags,"-pa " ++ DataDir}]),
+ false = check_cover(Node),
+ CoverSpec = default_cover_file_content() ++ [{nodes,[Node]}],
+ CoverFile = create_cover_file(cover_node_option,CoverSpec,Config),
+ {ok,Events} = run_test(cover_node_option,cover_node_option,
+ [{cover,CoverFile}],Config),
+ check_calls(Events,2),
+ {ok,Node} = ct_slave:stop(existing_node),
+ ok.
+
+%% Test ct_cover:add_nodes/1 and ct_cover:remove_nodes/1
+%% Check that cover is collected from added node
+ct_cover_add_remove_nodes(Config) ->
+ {ok, HostStr}=inet:gethostname(),
+ Host = list_to_atom(HostStr),
+ DataDir = ?config(data_dir,Config),
+ {ok,Node} = ct_slave:start(Host,existing_node,
+ [{erl_flags,"-pa " ++ DataDir}]),
+ false = check_cover(Node),
+ {ok,Events} = run_test(ct_cover_add_remove_nodes,ct_cover_add_remove_nodes,
+ [],Config),
+ check_calls(Events,2),
+ {ok,Node} = ct_slave:stop(existing_node),
+ ok.
+
+%% Test that the test suite itself can be cover compiled and that
+%% data_dir is set correctly (OTP-9956)
+otp_9956(Config) ->
+ CoverFile = create_cover_file(otp_9956,[{incl_mods,[?suite]}],Config),
+ {ok,Events} = run_test(otp_9956,otp_9956,[{cover,CoverFile}],Config),
+ check_calls(Events,{?suite,otp_9956,1},1),
+ ok.
+
+%% Test cross cover mechanism
+cross(Config) ->
+ {ok,Events1} = run_test(cross1,Config),
+ check_calls(Events1,1),
+
+ CoverFile2 = create_cover_file(cross1,[{cross,[{cross1,[?mod]}]}],Config),
+ {ok,Events2} = run_test(cross2,[{cover,CoverFile2}],Config),
+ check_calls(Events2,1),
+
+ %% Get the log dirs for each test and run cross cover analyse
+ [D11,D12] = lists:sort(get_run_dirs(Events1)),
+ [D21,D22] = lists:sort(get_run_dirs(Events2)),
+
+ ct_cover:cross_cover_analyse(details,[{cross1,D11},{cross2,D21}]),
+ ct_cover:cross_cover_analyse(details,[{cross1,D12},{cross2,D22}]),
+
+ %% Get the cross cover logs and read for each test
+ [C11,C12,C21,C22] =
+ [filename:join(D,"cross_cover.html") || D <- [D11,D12,D21,D22]],
+
+ {ok,CrossData} = file:read_file(C11),
+ {ok,CrossData} = file:read_file(C12),
+
+ {ok,Def} = file:read_file(C21),
+ {ok,Def} = file:read_file(C22),
+
+ %% A simple test: just check that the test module exists in the
+ %% log from cross1 test, and that it does not exist in the log
+ %% from cross2 test.
+ TestMod = list_to_binary(atom_to_list(?mod)),
+ {_,_} = binary:match(CrossData,TestMod),
+ nomatch = binary:match(Def,TestMod),
+ {_,_} = binary:match(Def,
+ <<"No cross cover modules exist for this application">>),
+
+ ok.
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+run_test(Label,Config) ->
+ run_test(Label,[],Config).
+run_test(Label,ExtraOpts,Config) ->
+ run_test(Label,default,ExtraOpts,Config).
+run_test(Label,Testcase,ExtraOpts,Config) ->
+ DataDir = ?config(data_dir, Config),
+ Suite = filename:join(DataDir, ?suite),
+ CoverFile =
+ case proplists:get_value(cover,ExtraOpts) of
+ undefined ->
+ create_default_cover_file(Label,Config);
+ CF ->
+ CF
+ end,
+ RestOpts = lists:keydelete(cover,1,ExtraOpts),
+ {Opts,ERPid} = setup([{suite,Suite},{testcase,Testcase},
+ {cover,CoverFile},{label,Label}] ++ RestOpts, Config),
+ execute(Label, Testcase, Opts, ERPid, Config).
+
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+execute(Name, Testcase, Opts, ERPid, Config) ->
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(Name,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+ TestEvents = events_to_check(Testcase),
+ R = ct_test_support:verify_events(TestEvents, Events, Config),
+ {R,Events}.
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+
+events_to_check(Testcase) ->
+ OneTest =
+ [{?eh,start_logging,{'DEF','RUNDIR'}}] ++
+ [{?eh,tc_done,{?suite,Testcase,ok}}] ++
+ [{?eh,stop_logging,[]}],
+
+ %% 2 tests (ct:run_test + script_start) is default
+ OneTest ++ OneTest.
+
+check_cover(Config) when is_list(Config) ->
+ CTNode = proplists:get_value(ct_node, Config),
+ check_cover(CTNode);
+check_cover(Node) when is_atom(Node) ->
+ case rpc:call(Node,test_server,is_cover,[]) of
+ true ->
+ {true,
+ rpc:call(Node,cover,which_nodes,[]),
+ rpc:call(Node,cover,modules,[])};
+ false ->
+ false
+ end.
+
+%% Get the log dir "run.<timestamp>" for all (both!) tests
+get_run_dirs(Events) ->
+ [filename:dirname(TCLog) ||
+ {ct_test_support_eh,
+ {event,tc_logfile,_Node,
+ {{?suite,init_per_suite},TCLog}}} <- Events].
+
+%% Check that each coverlog includes N calls to ?mod:foo/0
+check_calls(Events,N) ->
+ check_calls(Events,{?mod,foo,0},N).
+check_calls(Events,MFA,N) ->
+ CoverLogs = [filename:join(D,"all.coverdata") || D <- get_run_dirs(Events)],
+ do_check_logs(CoverLogs,MFA,N).
+
+do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) ->
+ {ok,_} = cover:start(),
+ ok = cover:import(CoverLog),
+ {ok,Calls} = cover:analyse(Mod,calls,function),
+ ok = cover:stop(),
+ {MFA,N} = lists:keyfind(MFA,1,Calls),
+ do_check_logs(CoverLogs,MFA,N);
+do_check_logs([],_,_) ->
+ ok.
+
+fullname(Name) ->
+ {ok,Host} = inet:gethostname(),
+ list_to_atom(atom_to_list(Name) ++ "@" ++ Host).
+
+default_cover_file_content() ->
+ [{incl_mods,[?mod]}].
+
+create_default_cover_file(Filename,Config) ->
+ create_cover_file(Filename,default_cover_file_content(),Config).
+
+create_cover_file(Filename,Terms,Config) ->
+ PrivDir = ?config(priv_dir,Config),
+ File = filename:join(PrivDir,Filename) ++ ".cover",
+ {ok,Fd} = file:open(File,[write]),
+ lists:foreach(fun(Term) ->
+ file:write(Fd,io_lib:format("~p.~n",[Term]))
+ end,Terms),
+ ok = file:close(Fd),
+ File.
diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl
new file mode 100644
index 0000000000..fdc3323f0a
--- /dev/null
+++ b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl
@@ -0,0 +1,156 @@
+%%--------------------------------------------------------------------
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%----------------------------------------------------------------------
+%% File: cover_SUITE.erl
+%%
+%% Description:
+%% This file contains the test cases for the code coverage support
+%%
+%% @author Support
+%% @doc Test of code coverage support in common_test
+%% @end
+%%----------------------------------------------------------------------
+%%----------------------------------------------------------------------
+-module(cover_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+%% Default timetrap timeout (set in init_per_testcase).
+-define(default_timeout, ?t:minutes(1)).
+
+suite() ->
+ [].
+
+all() ->
+ [].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ Config.
+
+init_per_testcase(_Case, Config) ->
+ Dog = test_server:timetrap(?default_timeout),
+ [{watchdog, Dog}|Config].
+
+end_per_testcase(Case, Config) ->
+ %% try apply(?MODULE,Case,[cleanup,Config])
+ %% catch error:undef -> ok
+ %% end,
+
+ kill_slaves(Case,nodes()),
+ Dog=?config(watchdog, Config),
+ test_server:timetrap_cancel(Dog),
+ ok.
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+break(_Config) ->
+ test_server:break(""),
+ ok.
+
+default(Config) ->
+ cover_compiled = code:which(cover_test_mod),
+ cover_test_mod:foo(),
+ ok.
+
+slave(Config) ->
+ cover_compiled = code:which(cover_test_mod),
+ cover_test_mod:foo(),
+ N1 = nodename(slave,1),
+ {ok,Node} = ct_slave:start(N1),
+ cover_compiled = rpc:call(Node,code,which,[cover_test_mod]),
+ rpc:call(Node,cover_test_mod,foo,[]),
+ {ok,Node} = ct_slave:stop(N1),
+ ok.
+
+slave_start_slave(Config) ->
+ cover_compiled = code:which(cover_test_mod),
+ cover_test_mod:foo(),
+ N1 = nodename(slave_start_slave,1),
+ N2 = nodename(slave_start_slave,2),
+ {ok,Node} = ct_slave:start(N1),
+ cover_compiled = rpc:call(Node,code,which,[cover_test_mod]),
+ rpc:call(Node,cover_test_mod,foo,[]),
+ {ok,Node2} = rpc:call(Node,ct_slave,start,[N2]),
+ rpc:call(Node2,cover_test_mod,foo,[]),
+ {ok,Node2} = rpc:call(Node,ct_slave,stop,[N2]),
+ {ok,Node} = ct_slave:stop(N1),
+ ok.
+
+cover_node_option(Config) ->
+ cover_compiled = code:which(cover_test_mod),
+ cover_test_mod:foo(),
+ Node = fullname(existing_node),
+ cover_compiled = rpc:call(Node,code,which,[cover_test_mod]),
+ rpc:call(Node,cover_test_mod,foo,[]),
+ ok.
+
+ct_cover_add_remove_nodes(Config) ->
+ cover_compiled = code:which(cover_test_mod),
+ cover_test_mod:foo(),
+ Node = fullname(existing_node),
+ Beam = rpc:call(Node,code,which,[cover_test_mod]),
+ false = (Beam == cover_compiled),
+
+ rpc:call(Node,cover_test_mod,foo,[]), % should not be collected
+ {ok,[Node]} = ct_cover:add_nodes([Node]),
+ cover_compiled = rpc:call(Node,code,which,[cover_test_mod]),
+ rpc:call(Node,cover_test_mod,foo,[]), % should be collected
+ ok = ct_cover:remove_nodes([Node]),
+ rpc:call(Node,cover_test_mod,foo,[]), % should not be collected
+
+ Beam = rpc:call(Node,code,which,[cover_test_mod]),
+
+ ok.
+
+otp_9956(Config) ->
+ cover_compiled = code:which(?MODULE),
+ DataDir = ?config(data_dir,Config),
+ absolute = filename:pathtype(DataDir),
+ true = filelib:is_dir(DataDir),
+ ok.
+
+
+%%%-----------------------------------------------------------------
+%%% Internal
+nodename(Case,N) ->
+ list_to_atom(nodeprefix(Case) ++ integer_to_list(N)).
+
+nodeprefix(Case) ->
+ atom_to_list(?MODULE) ++ "_" ++ atom_to_list(Case) ++ "_node".
+
+
+fullname(Name) ->
+ {ok,Host} = inet:gethostname(),
+ list_to_atom(atom_to_list(Name) ++ "@" ++ Host).
+
+kill_slaves(Case, [Node|Nodes]) ->
+ Prefix = nodeprefix(Case),
+ case lists:prefix(Prefix,atom_to_list(Node)) of
+ true ->
+ rpc:call(Node,erlang,halt,[]);
+ _ ->
+ ok
+ end,
+ kill_slaves(Case,Nodes);
+kill_slaves(_,[]) ->
+ ok.
diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore
diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl b/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl
new file mode 100644
index 0000000000..d4f69452c3
--- /dev/null
+++ b/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl
@@ -0,0 +1,4 @@
+-module(cover_test_mod).
+-compile(export_all).
+foo() ->
+ ok.
diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl
index 338e76264e..86ec71c98e 100644
--- a/lib/common_test/test/ct_error_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE.erl
@@ -44,8 +44,12 @@
%% there will be clashes with logging processes etc).
%%--------------------------------------------------------------------
init_per_suite(Config) ->
- Config1 = ct_test_support:init_per_suite(Config),
- Config1.
+ DataDir = ?config(data_dir, Config),
+ TestDir = filename:join(DataDir, "error/test/"),
+ CTH = filename:join(TestDir, "verify_config.erl"),
+ ct:pal("Compiling ~p: ~p",
+ [CTH,compile:file(CTH,[{outdir,TestDir},debug_info])]),
+ ct_test_support:init_per_suite([{path_dirs,[TestDir]} | Config]).
end_per_suite(Config) ->
ct_test_support:end_per_suite(Config).
@@ -61,7 +65,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}].
all() ->
[cfg_error, lib_error, no_compile, timetrap_end_conf,
timetrap_normal, timetrap_extended, timetrap_parallel,
- timetrap_fun, misc_errors].
+ timetrap_fun, timetrap_fun_group, misc_errors,
+ config_restored].
groups() ->
[].
@@ -251,6 +256,24 @@ timetrap_fun(Config) when is_list(Config) ->
%%%-----------------------------------------------------------------
%%%
+timetrap_fun_group(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Join = fun(D, S) -> filename:join(D, "error/test/"++S) end,
+ Suites = [Join(DataDir, "timetrap_8_SUITE")],
+ {Opts,ERPid} = setup([{suite,Suites}], Config),
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(timetrap_fun_group,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(timetrap_fun_group),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+%%%-----------------------------------------------------------------
+%%%
misc_errors(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
Join = fun(D, S) -> filename:join(D, "error/test/"++S) end,
@@ -267,6 +290,24 @@ misc_errors(Config) when is_list(Config) ->
TestEvents = events_to_check(misc_errors),
ok = ct_test_support:verify_events(TestEvents, Events, Config).
+%%%-----------------------------------------------------------------
+%%%
+config_restored(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Suite = filename:join(DataDir, "error/test/config_restored_SUITE"),
+ {Opts,ERPid} = setup([{suite,Suite},
+ {ct_hooks,[verify_config]}],
+ Config),
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(config_restored,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(config_restored),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
@@ -429,8 +470,7 @@ test_events(cfg_error) ->
{'EXIT',{init_per_group_fails,g1}}}}}}],
[{?eh,tc_start,{cfg_error_8_SUITE,{init_per_group,g2,[]}}},
- {?eh,tc_done,{cfg_error_8_SUITE,
- {init_per_group,unknown,[]},
+ {?eh,tc_done,{cfg_error_8_SUITE,{init_per_group,g2,[]},
{failed,{timetrap_timeout,2000}}}},
{?eh,tc_auto_skip,{cfg_error_8_SUITE,tc1,
{failed,{cfg_error_8_SUITE,init_per_group,
@@ -500,7 +540,7 @@ test_events(cfg_error) ->
{?eh,tc_done,{cfg_error_8_SUITE,tc1,ok}},
{?eh,test_stats,{9,0,{0,14}}},
{?eh,tc_start,{cfg_error_8_SUITE,{end_per_group,g12,[]}}},
- {?eh,tc_done,{cfg_error_8_SUITE,{end_per_group,unknown,[]},
+ {?eh,tc_done,{cfg_error_8_SUITE,{end_per_group,g12,[]},
{failed,{timetrap_timeout,2000}}}}],
{?eh,tc_start,{cfg_error_8_SUITE,end_per_suite}},
@@ -971,11 +1011,423 @@ test_events(timetrap_fun) ->
{?eh,stop_logging,[]}
];
+test_events(timetrap_fun_group) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,58}},
+ {?eh,tc_start,{timetrap_8_SUITE,init_per_suite}},
+ {?eh,tc_done,{timetrap_8_SUITE,init_per_suite,ok}},
+
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g0,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g0,[]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc0}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc0,
+ {failed,{timetrap_timeout,{'$approx',1000}}}}},
+ {?eh,test_stats,{0,1,{0,0}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc2}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_timeout,{'$approx',500}}}}},
+ {?eh,test_stats,{0,2,{0,0}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g0,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g0,[]},ok}}],
+
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g1,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g1,[]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc0}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc0,
+ {failed,{timetrap_timeout,{'$approx',1000}}}}},
+ {?eh,test_stats,{0,3,{0,0}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc2}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_timeout,{'$approx',500}}}}},
+ {?eh,test_stats,{0,4,{0,0}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g1,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g1,[]},ok}}],
+
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g2,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g2,[]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc1}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc1,
+ {failed,{timetrap_timeout,{'$approx',1000}}}}},
+ {?eh,test_stats,{0,5,{0,0}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc2}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_timeout,{'$approx',500}}}}},
+ {?eh,test_stats,{0,6,{0,0}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g2,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g2,[]},ok}}],
+
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g3,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g3,[]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc4}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc4,
+ {failed,{timetrap_timeout,{'$approx',2000}}}}},
+ {?eh,test_stats,{0,7,{0,0}}},
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g1,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g1,[]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc0}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc0,
+ {failed,{timetrap_timeout,{'$approx',1000}}}}},
+ {?eh,test_stats,{0,8,{0,0}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc2}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_timeout,{'$approx',500}}}}},
+ {?eh,test_stats,{0,9,{0,0}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g1,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g1,[]},ok}}],
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g2,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g2,[]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc1}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc1,
+ {failed,{timetrap_timeout,{'$approx',1000}}}}},
+ {?eh,test_stats,{0,10,{0,0}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc2}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_timeout,{'$approx',500}}}}},
+ {?eh,test_stats,{0,11,{0,0}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g2,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g2,[]},ok}}],
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g3,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g3,[]},ok}}],
+
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g4,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g4,[]},
+ {user_timetrap_error,{kaboom,'_'}}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {user_timetrap_error,{kaboom,'_'}}}}}},
+ {?eh,test_stats,{0,11,{0,1}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {user_timetrap_error,{kaboom,'_'}}}}}},
+ {?eh,test_stats,{0,11,{0,2}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {user_timetrap_error,{kaboom,'_'}}}}}}],
+
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g5,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g5,[]},
+ {user_timetrap_error,{kaboom,'_'}}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {user_timetrap_error,{kaboom,'_'}}}}}},
+ {?eh,test_stats,{0,11,{0,3}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {user_timetrap_error,{kaboom,'_'}}}}}},
+ {?eh,test_stats,{0,11,{0,4}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {user_timetrap_error,{kaboom,'_'}}}}}}],
+
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g6,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g6,[]},
+ {failed,{timetrap_timeout,{'$approx',500}}}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {timetrap_timeout,'_'}}}}},
+ {?eh,test_stats,{0,11,{0,5}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {timetrap_timeout,'_'}}}}},
+ {?eh,test_stats,{0,11,{0,6}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {timetrap_timeout,'_'}}}}}],
+
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g7,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g7,[]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc5}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}},
+ {?eh,test_stats,{1,11,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g7,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g7,[]},
+ {user_timetrap_error,{kaboom,'_'}}}}],
+
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g8,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g8,[]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc5}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}},
+ {?eh,test_stats,{2,11,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g8,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g8,[]},
+ {failed,{timetrap_timeout,{'$approx',500}}}}}],
+
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g9,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g9,[]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc5}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}},
+ {?eh,test_stats,{3,11,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc0}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc0,
+ {user_timetrap_error,{kaboom,'_'}}}},
+ {?eh,test_stats,{3,12,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g9,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g9,[]},ok}}],
+
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g10,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g10,[]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc0}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc0,
+ {user_timetrap_error,{kaboom,'_'}}}},
+ {?eh,test_stats,{3,13,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc5}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}},
+ {?eh,test_stats,{4,13,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g10,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g10,[]},ok}}],
+
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g11,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g11,[]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc3}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc3,
+ {failed,{timetrap_timeout,{'$approx',4000}}}}},
+ {?eh,test_stats,{4,14,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc2}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_timeout,{'$approx',500}}}}},
+ {?eh,test_stats,{4,15,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g11,[]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g11,[]},ok}}],
+
+ {parallel,
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg0,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg0,[parallel]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc0}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc0,
+ {failed,{timetrap_timeout,{'$approx',1000}}}}},
+ {?eh,test_stats,{4,16,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc2}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_timeout,{'$approx',500}}}}},
+ {?eh,test_stats,{4,17,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg0,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg0,[parallel]},ok}}]},
+
+ {parallel,
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc0}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc0,
+ {failed,{timetrap_timeout,{'$approx',1000}}}}},
+ {?eh,test_stats,{4,18,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc2}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_timeout,{'$approx',500}}}}},
+ {?eh,test_stats,{4,19,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]},ok}}]},
+
+ {parallel,
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc1}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc1,
+ {failed,{timetrap_timeout,{'$approx',1000}}}}},
+ {?eh,test_stats,{4,20,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc2}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_timeout,{'$approx',500}}}}},
+ {?eh,test_stats,{4,21,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]},ok}}]},
+
+ {parallel,
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg3,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg3,[parallel]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc4}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc4,
+ {failed,{timetrap_timeout,{'$approx',2000}}}}},
+ {?eh,test_stats,{4,22,{0,6}}},
+ {parallel,
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]},
+ ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc0}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc0,
+ {failed,{timetrap_timeout,{'$approx',1000}}}}},
+ {?eh,test_stats,{4,23,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc2}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_timeout,{'$approx',500}}}}},
+ {?eh,test_stats,{4,24,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]},
+ ok}}]},
+ {parallel,
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]},
+ ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc1}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc1,
+ {failed,{timetrap_timeout,{'$approx',1000}}}}},
+ {?eh,test_stats,{4,25,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc2}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_timeout,{'$approx',500}}}}},
+ {?eh,test_stats,{4,26,{0,6}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]},
+ ok}}]},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg3,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg3,[parallel]},ok}}]},
+
+ {parallel,
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg4,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg4,[parallel]},
+ {user_timetrap_error,{kaboom,'_'}}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {user_timetrap_error,{kaboom,'_'}}}}}},
+ {?eh,test_stats,{4,26,{0,7}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {user_timetrap_error,{kaboom,'_'}}}}}},
+ {?eh,test_stats,{4,26,{0,8}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {user_timetrap_error,{kaboom,'_'}}}}}}]},
+
+ {parallel,
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg5,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg5,[parallel]},
+ {user_timetrap_error,{kaboom,'_'}}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {user_timetrap_error,{kaboom,'_'}}}}}},
+ {?eh,test_stats,{4,26,{0,9}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {user_timetrap_error,{kaboom,'_'}}}}}},
+ {?eh,test_stats,{4,26,{0,10}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {user_timetrap_error,{kaboom,'_'}}}}}}]},
+
+ {parallel,
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg6,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg6,[parallel]},
+ {failed,{timetrap_timeout,{'$approx',500}}}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {timetrap_timeout,'_'}}}}},
+ {?eh,test_stats,{4,26,{0,11}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {timetrap_timeout,'_'}}}}},
+ {?eh,test_stats,{4,26,{0,12}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group,
+ {failed,{timetrap_8_SUITE,init_per_group,
+ {timetrap_timeout,'_'}}}}}]},
+
+ {parallel,
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg7,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg7,[parallel]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc5}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}},
+ {?eh,test_stats,{5,26,{0,12}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg7,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg7,[parallel]},
+ {user_timetrap_error,{kaboom,'_'}}}}]},
+
+ {parallel,
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg8,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg8,[parallel]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc5}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}},
+ {?eh,test_stats,{6,26,{0,12}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg8,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg8,[parallel]},
+ {failed,{timetrap_timeout,{'$approx',500}}}}}]},
+
+ {parallel,
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg9,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg9,[parallel]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc5}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc0}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc0,
+ {user_timetrap_error,{kaboom,'_'}}}},
+ %% Due to parallelism only checking final test stat in group
+ {?eh,test_stats,{7,27,{0,12}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg9,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg9,[parallel]},ok}}]},
+
+ {parallel,
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg10,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg10,[parallel]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc0}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc0,
+ {user_timetrap_error,{kaboom,'_'}}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc5}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}},
+ %% Due to parallelism only checking final test stat in group
+ {?eh,test_stats,{8,28,{0,12}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg10,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg10,[parallel]},ok}}]},
+
+ {parallel,
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg11,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg11,[parallel]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc3}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc3,
+ {failed,{timetrap_timeout,{'$approx',4000}}}}},
+ {?eh,test_stats,{8,29,{0,12}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc2}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_timeout,{'$approx',500}}}}},
+ {?eh,test_stats,{8,30,{0,12}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg11,[parallel]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg11,[parallel]},ok}}]},
+
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,sg1,[sequence]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,sg1,[sequence]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc5}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}},
+ {?eh,test_stats,{9,30,{0,12}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc0}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc0,
+ {user_timetrap_error,{kaboom,'_'}}}},
+ {?eh,test_stats,{9,31,{0,12}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc1,
+ {failed,{timetrap_8_SUITE,tc0}}}},
+ {?eh,test_stats,{9,31,{0,13}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_8_SUITE,tc0}}}},
+ {?eh,test_stats,{9,31,{0,14}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,sg1,[sequence]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,sg1,[sequence]},ok}}],
+
+ [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,sg2,[sequence]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,sg2,[sequence]},ok}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc5}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}},
+ {?eh,test_stats,{10,31,{0,14}}},
+ {?eh,tc_start,{timetrap_8_SUITE,tc0}},
+ {?eh,tc_done,{timetrap_8_SUITE,tc0,
+ {failed,{timetrap_timeout,{'$approx',1000}}}}},
+ {?eh,test_stats,{10,32,{0,14}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc1,
+ {failed,{timetrap_8_SUITE,tc0}}}},
+ {?eh,test_stats,{10,32,{0,15}}},
+ {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2,
+ {failed,{timetrap_8_SUITE,tc0}}}},
+ {?eh,test_stats,{10,32,{0,16}}},
+ {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,sg2,[sequence]}}},
+ {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,sg2,[sequence]},ok}}],
+
+ {?eh,tc_start,{timetrap_8_SUITE,end_per_suite}},
+ {?eh,tc_done,{timetrap_8_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ];
+
test_events(misc_errors) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
- {?eh,start_info,{1,1,7}},
+ {?eh,start_info,{1,1,9}},
{?eh,tc_start,{misc_error_1_SUITE,ct_fail_1}},
{?eh,tc_done,{misc_error_1_SUITE,ct_fail_1,
{failed,{error,{test_case_failed,{error,this_is_expected}}}}}},
@@ -1002,7 +1454,48 @@ test_events(misc_errors) ->
{?eh,tc_start,{misc_error_1_SUITE,killed_by_signal_2}},
{?eh,tc_done,{misc_error_1_SUITE,killed_by_signal_2,
{failed,testcase_aborted_or_killed}}},
- {?eh,test_stats,{0,7,{0,0}}},
+ {parallel,
+ [{?eh,tc_start,{misc_error_1_SUITE,p1}},
+ {?eh,tc_done,{misc_error_1_SUITE,p1,ok}},
+ {?eh,tc_start,{misc_error_1_SUITE,p2}},
+ {?eh,tc_done,{misc_error_1_SUITE,p2,ok}}]},
+ {?eh,test_stats,{2,7,{0,0}}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(config_restored) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,4}},
+ {?eh,tc_start,{config_restored_SUITE,init_per_suite}},
+ {?eh,tc_done,{config_restored_SUITE,init_per_suite,ok}},
+ {?eh,tc_start,{config_restored_SUITE,to_tc}},
+ {?eh,cth,{verify_config,post_end_per_testcase,{to_tc,diff_ok}}},
+ {?eh,tc_done,
+ {config_restored_SUITE,to_tc,{failed,{timetrap_timeout,1000}}}},
+ {?eh,test_stats,{0,1,{0,0}}},
+ {?eh,tc_start,{config_restored_SUITE,exit_tc}},
+ {?eh,cth,{verify_config,post_end_per_testcase,{exit_tc,diff_ok}}},
+ {?eh,tc_done,{config_restored_SUITE,exit_tc,
+ {failed,{error,{test_case_failed,"Goodbye!"}}}}},
+ {?eh,test_stats,{0,2,{0,0}}},
+ [{?eh,tc_start,{config_restored_SUITE,{init_per_group,g1,[]}}},
+ {?eh,tc_start,{config_restored_SUITE,to_tc}},
+ {?eh,cth,{verify_config,post_end_per_testcase,{to_tc,diff_ok}}},
+ {?eh,tc_done,
+ {config_restored_SUITE,to_tc,{failed,{timetrap_timeout,1000}}}},
+ {?eh,test_stats,{0,3,{0,0}}},
+ {?eh,tc_start,{config_restored_SUITE,exit_tc}},
+ {?eh,cth,{verify_config,post_end_per_testcase,{exit_tc,diff_ok}}},
+ {?eh,tc_done,{config_restored_SUITE,exit_tc,
+ {failed,{error,{test_case_failed,"Goodbye!"}}}}},
+ {?eh,test_stats,{0,4,{0,0}}},
+ {?eh,tc_start,{config_restored_SUITE,{end_per_group,g1,[]}}},
+ {?eh,tc_done,{config_restored_SUITE,{end_per_group,g1,[]},ok}}],
+ {?eh,tc_start,{config_restored_SUITE,end_per_suite}},
+ {?eh,tc_done,{config_restored_SUITE,end_per_suite,ok}},
{?eh,test_done,{'DEF','STOP_TIME'}},
{?eh,stop_logging,[]}
].
diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl
new file mode 100644
index 0000000000..fb47f2bcb9
--- /dev/null
+++ b/lib/common_test/test/ct_error_SUITE_data/error/test/config_restored_SUITE.erl
@@ -0,0 +1,175 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(config_restored_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% Function: suite() -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,1000}].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config1 = [{init_per_suite,?MODULE} | Config],
+ TabPid = spawn(fun() ->
+ ets:new(?MODULE, [named_table, set, public]),
+ receive _ -> ok end
+ end),
+ [{tab,TabPid} | Config1].
+
+%%--------------------------------------------------------------------
+%% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_suite(Config) ->
+ ets:delete(?MODULE),
+ exit(?config(tab, Config), kill),
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_group(GroupName, Config) ->
+ [{init_per_group,GroupName} | Config].
+
+%%--------------------------------------------------------------------
+%% Function: end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_testcase(TC, Config) ->
+ Config1 = [{init_per_testcase,TC} | Config],
+ ets:insert(?MODULE, {config,Config}),
+ %% ct:pal("Config after init_per_testcase(~w) = ~p", [TC,Config1]),
+ Config1.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_testcase(TC, Config) ->
+ ct:pal("Config in end_per_testcase(~w) = ~p", [TC,Config]),
+ [{_,MemConfig}] = ets:lookup(?MODULE, config),
+ diff_config(Config, MemConfig, [tc_status]),
+ ?MODULE = proplists:get_value(init_per_suite, Config),
+ TC = proplists:get_value(init_per_testcase, Config),
+ case ?config(tc_group_properties, Config) of
+ undefined ->
+ ok;
+ Props ->
+ GName = proplists:get_value(name, Props),
+ GName = proplists:get_value(init_per_group, Config)
+ end,
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%%--------------------------------------------------------------------
+groups() ->
+ [{g1,[],[to_tc, exit_tc]}].
+
+%%--------------------------------------------------------------------
+%% Function: all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%%--------------------------------------------------------------------
+all() ->
+ [to_tc, exit_tc,
+ {group,g1}].
+
+
+to_tc(Config) ->
+ %% ct:pal("Config for to_tc = ~p", [Config]),
+ [{_,MemConfig}] = ets:lookup(?MODULE, config),
+ diff_config(Config, MemConfig, []),
+ ct:sleep(2000).
+
+exit_tc(Config) ->
+ %% ct:pal("Config for exit_tc = ~p", [Config]),
+ [{_,MemConfig}] = ets:lookup(?MODULE, config),
+ diff_config(Config, MemConfig, []),
+ ct:fail("Goodbye!").
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+
+diff_config(Cfg1, Cfg2, DiffKeys) ->
+ diff_config(Cfg1, Cfg2, DiffKeys, []).
+
+diff_config([{K,V} | Cfg1], Cfg2, DiffKeys, RemKeys) ->
+ case proplists:get_value(K, Cfg2) of
+ undefined ->
+ diff_config(Cfg1, Cfg2, proplists:delete(K, DiffKeys), RemKeys);
+ V ->
+ diff_config(Cfg1, proplists:delete(K, Cfg2), DiffKeys, RemKeys);
+ _Other ->
+ case proplists:is_defined(K, DiffKeys) of
+ true ->
+ diff_config(Cfg1, Cfg2, proplists:delete(K, DiffKeys), RemKeys);
+ false ->
+ diff_config(Cfg1, Cfg2, DiffKeys, [K | RemKeys])
+ end
+ end;
+diff_config([], [], [], []) ->
+ ct:pal("Diff ok!", []),
+ ok;
+diff_config([], Cfg2, DiffKeys, RemKeys) ->
+ Result = {diff_failed, {cfg2,Cfg2}, {diffkeys,DiffKeys}, {remkeys,RemKeys}},
+ ct:pal("Diff failed! Result = ~p", [Result]),
+ exit(Result).
+
diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl
index 99c3ed05ec..61f3fa7e59 100644
--- a/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl
@@ -96,7 +96,7 @@ end_per_testcase(_TestCase, _Config) ->
%% N = integer() | forever
%%--------------------------------------------------------------------
groups() ->
- [].
+ [{p,[parallel],[p1,p2]}].
%%--------------------------------------------------------------------
%% Function: all() -> GroupsAndTestCases | {skip,Reason}
@@ -107,7 +107,8 @@ groups() ->
%%--------------------------------------------------------------------
all() ->
[ct_fail_1, ct_fail_2, ct_fail_3, ts_fail_1, ts_fail_2,
- killed_by_signal_1, killed_by_signal_2].
+ killed_by_signal_1, killed_by_signal_2,
+ {group,p}].
ct_fail_1(_) ->
ct:fail({error,this_is_expected}),
@@ -152,3 +153,10 @@ killed_by_signal_2(_) ->
end),
ct:sleep(1000),
exit(this_should_not_be_seen).
+
+p1(_) ->
+ {error,parallel_group} = ct:abort_current_testcase(aborted),
+ ok.
+
+p2(_) ->
+ receive after 1000 -> ok end.
diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl
new file mode 100644
index 0000000000..ff138f38b5
--- /dev/null
+++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl
@@ -0,0 +1,258 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(timetrap_8_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+-define(TO, 4).
+
+%%--------------------------------------------------------------------
+%% Function: suite() -> Info
+%% Info = [tuple()]
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,{timetrap_utils,timetrap_val,[{seconds,?TO}]}}].
+
+%%--------------------------------------------------------------------
+%% Function: init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_group(G6, Config) when G6==g6; G6==pg6 ->
+ ct:sleep({seconds,1}),
+ Config;
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_group(G7or8, _Config) when G7or8==g7; G7or8==pg7; G7or8==g8; G7or8==pg8 ->
+ ct:sleep({seconds,5}),
+ ok;
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%%--------------------------------------------------------------------
+init_per_testcase(_, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% Function: end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%%--------------------------------------------------------------------
+end_per_testcase(_, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%%--------------------------------------------------------------------
+groups() ->
+ [
+ {g0,[],[tc0,tc2]}, % group override suite and tc overrides group
+ {g1,[],[tc0,tc2]}, % group override suite and tc overrides group
+ {g2,[],[tc1,tc2]}, % tc override group
+ {g3,[],[tc4,{group,g1},{group,g2}]}, % subgroup override group
+ {g4,[],[tc0,tc2]}, % exit during init_per_group
+ {g5,[],[tc0,tc2]}, % exit during init_per_group
+ {g6,[],[tc0,tc2]}, % timeout during init_per_group
+ {g7,[],[tc5]}, % exit during end_per_group
+ {g8,[],[tc5]}, % timeout during end_per_group
+ {g9,[],[tc5,tc0]}, % exit during testcase
+ {g10,[],[tc0,tc5]}, % exit during testcase
+ {g11,[],[tc3,tc2]}, % suite is valid if nothing else is specified
+ {pg0,[parallel],[tc0,tc2]}, % group override suite and tc overrides group
+ {pg1,[parallel],[tc0,tc2]}, % group override suite and tc overrides group
+ {pg2,[parallel],[tc1,tc2]}, % tc override group
+ {pg3,[parallel],[tc4,{group,pg1},{group,pg2}]}, % subgroup override group
+ {pg4,[parallel],[tc0,tc2]}, % exit during init_per_group
+ {pg5,[parallel],[tc0,tc2]}, % exit during init_per_group
+ {pg6,[parallel],[tc0,tc2]}, % timeout during init_per_group
+ {pg7,[parallel],[tc5]}, % exit during end_per_group
+ {pg8,[parallel],[tc5]}, % timeout during end_per_group
+ {pg9,[parallel],[tc5,tc0]}, % exit during testcase
+ {pg10,[parallel],[tc0,tc5]},% exit during testcase
+ {pg11,[parallel],[tc3,tc2]},% suite is valid if nothing else is specified
+ {sg1,[sequence],[tc5,tc0,tc1,tc2]}, % exit during sequencial testcase
+ {sg2,[sequence],[tc5,tc0,tc1,tc2]}].% timeout during sequencial testcase
+
+group(g0) ->
+ [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}];
+group(g1) ->
+ [{timetrap,fun() -> timetrap_utils:timetrap_val(1000) end}];
+group(g2) ->
+ [{timetrap,fun() -> timetrap_utils:timetrap_val(3000) end}];
+group(g3) ->
+ [{timetrap,fun() -> timetrap_utils:timetrap_val(2000) end}];
+group(g4) ->
+ [{timetrap,{timetrap_utils,timetrap_exit,[kaboom]}}];
+group(g5) ->
+ [{timetrap,fun() -> exit(kaboom) end}];
+group(g6) ->
+ [{timetrap,{timetrap_utils,timetrap_val,[500]}}];
+group(g7) ->
+ [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}];
+group(g8) ->
+ [{timetrap,{timetrap_utils,timetrap_val,[500]}}];
+group(g9) ->
+ [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}];
+group(g10) ->
+ [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}];
+group(g11) ->
+ [];
+group(pg0) ->
+ [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}];
+group(pg1) ->
+ [{timetrap,fun() -> timetrap_utils:timetrap_val(1000) end}];
+group(pg2) ->
+ [{timetrap,fun() -> timetrap_utils:timetrap_val(3000) end}];
+group(pg3) ->
+ [{timetrap,fun() -> timetrap_utils:timetrap_val(2000) end}];
+group(pg4) ->
+ [{timetrap,{timetrap_utils,timetrap_exit,[kaboom]}}];
+group(pg5) ->
+ [{timetrap,fun() -> exit(kaboom) end}];
+group(pg6) ->
+ [{timetrap,{timetrap_utils,timetrap_val,[500]}}];
+group(pg7) ->
+ [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}];
+group(pg8) ->
+ [{timetrap,{timetrap_utils,timetrap_val,[500]}}];
+group(pg9) ->
+ [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}];
+group(pg10) ->
+ [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}];
+group(pg11) ->
+ [];
+group(sg1) ->
+ [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}];
+group(sg2) ->
+ [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}].
+
+
+%%--------------------------------------------------------------------
+%% Function: all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%%--------------------------------------------------------------------
+all() ->
+ [
+ {group,g0},
+ {group,g1},
+ {group,g2},
+ {group,g3},
+ {group,g4},
+ {group,g5},
+ {group,g6},
+ {group,g7},
+ {group,g8},
+ {group,g9},
+ {group,g10},
+ {group,g11},
+ {group,pg0},
+ {group,pg1},
+ {group,pg2},
+ {group,pg3},
+ {group,pg4},
+ {group,pg5},
+ {group,pg6},
+ {group,pg7},
+ {group,pg8},
+ {group,pg9},
+ {group,pg10},
+ {group,pg11},
+ {group,sg1},
+ {group,sg2}].
+
+
+
+tc0(_) ->
+ ct:comment("TO set by group"),
+ ct:sleep({seconds,5}),
+ ok.
+
+tc1() ->
+ [{timetrap,{timetrap_utils,timetrap_val,[1000]}}].
+tc1(_) ->
+ ct:comment("TO after 1 sec"),
+ ct:sleep({seconds,2}),
+ ok.
+
+tc2() ->
+ [{timetrap,fun() -> timetrap_utils:timetrap_val(500) end}].
+tc2(_) ->
+ ct:comment("TO after 0.5 sec"),
+ ct:sleep({seconds,2}),
+ ok.
+
+tc3(_) ->
+ ct:comment(io_lib:format("TO after ~w sec", [?TO])),
+ ct:sleep({seconds,5}),
+ ok.
+
+tc4(_) ->
+ ct:comment("TO set by group"),
+ ct:sleep({seconds,5}),
+ ok.
+
+tc5(_) ->
+ ct:comment("No TO in this testcase, maybe later"),
+ ok.
diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl
new file mode 100644
index 0000000000..40a54b9f99
--- /dev/null
+++ b/lib/common_test/test/ct_error_SUITE_data/error/test/verify_config.erl
@@ -0,0 +1,239 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%% @doc Common Test Example Suite Callback module.
+%%%
+%%% <p>This module gives an example of a common test CTH (Common Test Hook).
+%%% There are many ways to add a CTH to a test run, you can do it either in
+%%% the command line using -ct_hook, in a test spec using
+%%% {ct_hook,M} or in the suite it self by returning ct_hook
+%%% from either suite/0, init_per_suite/1, init_per_group/2 and
+%%% init_per_testcase/2. The scope of the CTH is determined by where is it
+%%% started. If it is started in the command line or test spec then it will
+%%% be stopped at the end of all tests. If it is started in init_per_suite,
+%%% it will be stopped after end_per_suite and so on. See terminate
+%%% documentation for a table describing the scoping machanics.
+%%%
+%%% All of callbacks except init/1 in a CTH are optional.</p>
+
+-module(verify_config).
+
+%% CT Hooks
+-export([id/1]).
+-export([init/2]).
+
+-export([pre_init_per_suite/3]).
+-export([post_init_per_suite/4]).
+-export([pre_end_per_suite/3]).
+-export([post_end_per_suite/4]).
+
+-export([pre_init_per_group/3]).
+-export([post_init_per_group/4]).
+-export([pre_end_per_group/3]).
+-export([post_end_per_group/4]).
+
+-export([pre_init_per_testcase/3]).
+-export([post_end_per_testcase/4]).
+
+-export([on_tc_fail/3]).
+-export([on_tc_skip/3]).
+
+-export([terminate/1]).
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-type config() :: proplists:proplist().
+-type reason() :: term().
+-type skip_or_fail() :: {skip, reason()} |
+ {auto_skip, reason()} |
+ {fail, reason()} |
+ {'EXIT',reason()}.
+
+-record(state, { id = ?MODULE :: term()}).
+
+%% @doc Always called before any other callback function. Use this to initiate
+%% any common state. It should return an state for this CTH.
+-spec init(Id :: term(), Opts :: proplists:proplist()) ->
+ {ok, State :: #state{}}.
+init(Id, Opts) ->
+ {ok,Opts}.
+
+%% @doc The ID is used to uniquly identify an CTH instance, if two CTH's
+%% return the same ID the seconds CTH is ignored. This function should NOT
+%% have any side effects as it might be called multiple times by common test.
+-spec id(Opts :: proplists:proplist()) ->
+ Id :: term().
+id(Opts) ->
+ now().
+
+%% @doc Called before init_per_suite is called. Note that this callback is
+%% only called if the CTH is added before init_per_suite is run (eg. in a test
+%% specification, suite/0 function etc).
+%% You can change the config in the this function.
+-spec pre_init_per_suite(Suite :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_suite(Suite,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after init_per_suite.
+%% you can change the return value in this function.
+-spec post_init_per_suite(Suite :: atom(),
+ Config :: config(),
+ Return :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+post_init_per_suite(Suite,Config,Return,State) ->
+ {Return, State}.
+
+%% @doc Called before end_per_suite. The config/state can be changed here,
+%% though it will only affect the *end_per_suite function.
+-spec pre_end_per_suite(Suite :: atom(),
+ Config :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+pre_end_per_suite(Suite,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after end_per_suite. Note that the config cannot be
+%% changed here, only the status of the suite.
+-spec post_end_per_suite(Suite :: atom(),
+ Config :: config(),
+ Return :: term(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_suite(Suite,Config,Return,State) ->
+ {Return, State}.
+
+%% @doc Called before each init_per_group.
+%% You can change the config in this function.
+-spec pre_init_per_group(Group :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_group(Group,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after each init_per_group.
+%% You can change the return value in this function.
+-spec post_init_per_group(Group :: atom(),
+ Config :: config(),
+ Return :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+post_init_per_group(Group,Config,Return,State) ->
+ {Return, State}.
+
+%% @doc Called after each end_per_group. The config/state can be changed here,
+%% though it will only affect the *end_per_group functions.
+-spec pre_end_per_group(Group :: atom(),
+ Config :: config() | skip_or_fail(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+pre_end_per_group(Group,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after each end_per_group. Note that the config cannot be
+%% changed here, only the status of the group.
+-spec post_end_per_group(Group :: atom(),
+ Config :: config(),
+ Return :: term(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_group(Group,Config,Return,State) ->
+ {Return, State}.
+
+%% @doc Called before each test case.
+%% You can change the config in this function.
+-spec pre_init_per_testcase(TC :: atom(),
+ Config :: config(),
+ State :: #state{}) ->
+ {config() | skip_or_fail(), NewState :: #state{}}.
+pre_init_per_testcase(TC,Config,State) ->
+ {Config, State}.
+
+%% @doc Called after each test case. Note that the config cannot be
+%% changed here, only the status of the test case.
+-spec post_end_per_testcase(TC :: atom(),
+ Config :: config(),
+ Return :: term(),
+ State :: #state{}) ->
+ {ok | skip_or_fail(), NewState :: #state{}}.
+post_end_per_testcase(TC,Config,Return,State) ->
+ %% check that config has been restored
+ ct:pal("Config in verify_config:post_end_per_testcase(~w) = ~p",
+ [TC,Config]),
+ [{_,MemConfig}] = ets:lookup(config_restored_SUITE, config),
+ try config_restored_SUITE:diff_config(Config, MemConfig, [tc_status]) of
+ ok ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_end_per_testcase,
+ {TC,diff_ok}}})
+ catch
+ _:_ ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_end_per_testcase,
+ {TC,diff_failed}}})
+ end,
+ {Return, State}.
+
+%% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,
+%% post_end_per_group and post_end_per_tc if the suite, group or test case failed.
+%% This function should be used for extra cleanup which might be needed.
+%% It is not possible to modify the config or the status of the test run.
+-spec on_tc_fail(TC :: init_per_suite | end_per_suite |
+ init_per_group | end_per_group | atom(),
+ Reason :: term(), State :: #state{}) ->
+ NewState :: #state{}.
+on_tc_fail(TC, Reason, State) ->
+ State.
+
+%% @doc Called when a test case is skipped by either user action
+%% or due to an init function failing. Test case can be
+%% end_per_suite, init_per_group, end_per_group and the actual test cases.
+-spec on_tc_skip(TC :: end_per_suite |
+ init_per_group | end_per_group | atom(),
+ {tc_auto_skip, {failed, {Mod :: atom(), Function :: atom(), Reason :: term()}}} |
+ {tc_user_skip, {skipped, Reason :: term()}},
+ State :: #state{}) ->
+ NewState :: #state{}.
+on_tc_skip(TC, Reason, State) ->
+ State.
+
+%% @doc Called when the scope of the CTH is done, this depends on
+%% when the CTH was specified. This translation table describes when this
+%% function is called.
+%%
+%% | Started in | terminate called |
+%% |---------------------|-------------------------|
+%% | command_line | after all tests are run |
+%% | test spec | after all tests are run |
+%% | suite/0 | after SUITE is done |
+%% | init_per_suite/1 | after SUITE is done |
+%% | init_per_group/2 | after group is done |
+%% |-----------------------------------------------|
+%%
+-spec terminate(State :: #state{}) ->
+ term().
+terminate(State) ->
+ ok.
diff --git a/lib/common_test/test/ct_group_leader_SUITE.erl b/lib/common_test/test/ct_group_leader_SUITE.erl
new file mode 100644
index 0000000000..cde3061d6a
--- /dev/null
+++ b/lib/common_test/test/ct_group_leader_SUITE.erl
@@ -0,0 +1,181 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_system_error_SUITE
+%%%
+%%% Description:
+%%%
+%%% Test the group leader functionality in the test_server application.
+%%%-------------------------------------------------------------------
+-module(ct_group_leader_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config1 = ct_test_support:init_per_suite(Config),
+ Config1.
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [
+ basic
+ ].
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+basic(Config) ->
+ TC = basic,
+ DataDir = ?config(data_dir, Config),
+ Suite = filename:join(DataDir, "group_leader_SUITE"),
+ {Opts,ERPid} = setup([{suite,Suite},{label,TC}], Config),
+ SuiteLog = execute(TC, Opts, ERPid, Config),
+ {ok,Data} = file:read_file(SuiteLog),
+ Lines = binary:split(Data, <<"\n">>, [global]),
+ {ok,RE} = re:compile("(\\S+):(\\S+)$"),
+ Cases0 = [begin
+ {match,[M,F]} = re:run(Case, RE, [{capture,all_but_first,list}]),
+ {list_to_atom(M),list_to_atom(F)}
+ end || <<"=case ",Case/binary>> <- Lines],
+ Cases = [MF || {_,F}=MF <- Cases0,
+ F =/= init_per_suite,
+ F =/= end_per_suite,
+ F =/= init_per_group,
+ F =/= end_per_group],
+ io:format("~p\n", [Cases]),
+ [] = verify_cases(events_to_check(TC), Cases, false),
+ ok.
+
+verify_cases([{parallel,P}|Ts], Cases0, Par) ->
+ Cases = verify_cases(P, Cases0, true),
+ verify_cases(Ts, Cases, Par);
+verify_cases([{?eh,tc_done,{M,F,_}}|Ts], Cases0, false) ->
+ [{M,F}|Cases] = Cases0,
+ verify_cases(Ts, Cases, false);
+verify_cases([{?eh,tc_done,{M,F,_}}|Ts], Cases0, true) ->
+ case lists:member({M,F}, Cases0) of
+ true ->
+ Cases = Cases0 -- [{M,F}],
+ verify_cases(Ts, Cases, true);
+ false ->
+ io:format("~p not found\n", [{M,F}]),
+ ?t:fail()
+ end;
+verify_cases([{?eh,_,_}|Ts], Cases, Par) ->
+ verify_cases(Ts, Cases, Par);
+verify_cases([], Cases, _) ->
+ Cases;
+verify_cases([List|Ts], Cases0, Par) when is_list(List) ->
+ Cases = verify_cases(List, Cases0, false),
+ verify_cases(Ts, Cases, Par).
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+execute(Name, Opts, ERPid, Config) ->
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(Name,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(Name),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config),
+ {event,tc_logfile,_,{_,File}} =
+ lists:keyfind(tc_logfile, 2, [Ev || {?eh,Ev} <- Events]),
+ LogDir = filename:dirname(File),
+ filename:join(LogDir, "suite.log").
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+
+events_to_check(_Test) ->
+ [{?eh,tc_done,{group_leader_SUITE,tc1,ok}},
+ {parallel,[{?eh,tc_start,{group_leader_SUITE,p1}},
+ {?eh,tc_done,{group_leader_SUITE,p1,ok}},
+ {?eh,tc_start,{group_leader_SUITE,p2}},
+ {?eh,tc_done,{group_leader_SUITE,p2,ok}}]},
+ {?eh,tc_done,{group_leader_SUITE,p_restart_my_io_server,ok}},
+ {?eh,tc_done,{group_leader_SUITE,p3,ok}},
+ {parallel,[
+ {?eh,tc_start,{group_leader_SUITE,p10}},
+ {?eh,tc_start,{group_leader_SUITE,p11}},
+ {?eh,tc_done,{group_leader_SUITE,p10,ok}},
+ {?eh,tc_done,{group_leader_SUITE,p11,ok}},
+ [{?eh,tc_done,{group_leader_SUITE,s1,ok}},
+ {?eh,tc_done,{group_leader_SUITE,s2,ok}},
+ {?eh,tc_done,{group_leader_SUITE,s3,ok}}],
+ {?eh,tc_start,{group_leader_SUITE,p12}},
+ {?eh,tc_done,{group_leader_SUITE,p12,ok}},
+ [{?eh,tc_done,{group_leader_SUITE,s4,ok}},
+ {?eh,tc_done,{group_leader_SUITE,s5,ok}}],
+ {?eh,tc_start,{group_leader_SUITE,p13}},
+ {?eh,tc_done,{group_leader_SUITE,p13,ok}} ]},
+ {?eh,tc_done,{group_leader_SUITE,cap1,ok}},
+ {?eh,tc_done,{group_leader_SUITE,cap2,ok}},
+ {parallel,[{?eh,tc_start,{group_leader_SUITE,cap1}},
+ {?eh,tc_done,{group_leader_SUITE,cap1,ok}},
+ {?eh,tc_start,{group_leader_SUITE,cap2}},
+ {?eh,tc_done,{group_leader_SUITE,cap2,ok}}]},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ].
diff --git a/lib/common_test/test/ct_group_leader_SUITE_data/group_leader_SUITE.erl b/lib/common_test/test/ct_group_leader_SUITE_data/group_leader_SUITE.erl
new file mode 100644
index 0000000000..3f1844b4ae
--- /dev/null
+++ b/lib/common_test/test/ct_group_leader_SUITE_data/group_leader_SUITE.erl
@@ -0,0 +1,252 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(group_leader_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,{seconds,10}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ start_my_io_server(),
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ my_io_server ! die,
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [{p,[parallel],[p1,p2]},
+ {p_restart,[parallel],[p_restart_my_io_server]},
+ {seq,[],[s1,s2,s3]},
+ {seq2,[],[s4,s5]},
+ {seq_in_par,[parallel],[p10,p11,{group,seq},p12,{group,seq2},p13]},
+ {capture_io,[parallel],[cap1,cap2]}].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [tc1,{group,p},{group,p_restart},p3,
+ {group,seq_in_par},
+ cap1,cap2,
+ {group,capture_io}].
+
+tc1(_C) ->
+ ok.
+
+p1(_) ->
+ %% OTP-10101:
+ %%
+ %% External apps/processes started by init_per_suite (common operation),
+ %% will inherit the group leader of the init_per_suite process, i.e. the
+ %% test_server test case control process (executing run_test_case_msgloop/7).
+ %% If, later, a parallel test case triggers the external app to print with
+ %% e.g. io:format() (also common operation), the calling process will hang!
+ %% The reason for this is that a parallel test case has a dedicated IO
+ %% server process, other than the central test case control process. The
+ %% latter process is not executing run_test_case_msgloop/7 and will not
+ %% respond to IO messages. The process is still group leader for the
+ %% external app, however, which is wrong. It's the IO process for the
+ %% parallel test case that should be group leader - but only for the
+ %% particular invokation, since other parallel test cases could be
+ %% invoking the external app too.
+ print("hej\n").
+
+p2(_) ->
+ print("hopp\n").
+
+p_restart_my_io_server(_) ->
+ %% Restart the IO server and change its group leader. This used
+ %% to set to the group leader to a process that would soon die.
+ Ref = erlang:monitor(process, my_io_server),
+ my_io_server ! die,
+ receive
+ {'DOWN',Ref,_,_,_} ->
+ start_my_io_server()
+ end.
+
+p3(_) ->
+ %% OTP-10125. This would crash since the group leader process
+ %% for the my_io_server had died.
+ print("hoppsan\n").
+
+print(String) ->
+ my_io_server ! {print,self(),String},
+ receive
+ {printed,String} ->
+ ok
+ end.
+
+start_my_io_server() ->
+ Parent = self(),
+ Pid = spawn(fun() -> my_io_server(Parent) end),
+ receive
+ {Pid,started} ->
+ io:format("~p\n", [process_info(Pid)]),
+ ok
+ end.
+
+my_io_server(Parent) ->
+ register(my_io_server, self()),
+ Parent ! {self(),started},
+ my_io_server_loop().
+
+my_io_server_loop() ->
+ receive
+ {print,From,String} ->
+ io:put_chars(String),
+ From ! {printed,String},
+ my_io_server_loop();
+ die ->
+ ok
+ end.
+
+p10(_) ->
+ receive after 1 -> ok end.
+
+p11(_) ->
+ ok.
+
+p12(_) ->
+ ok.
+
+p13(_) ->
+ ok.
+
+s1(_) ->
+ ok.
+
+s2(_) ->
+ ok.
+
+s3(_) ->
+ ok.
+
+s4(_) ->
+ ok.
+
+s5(_) ->
+ ok.
+
+cap1(_) ->
+ ct:capture_start(),
+ IO = gen_io(cap1, 10, []),
+ ct:capture_stop(),
+ IO = ct:capture_get(),
+ ok.
+
+cap2(_) ->
+ ct:capture_start(),
+ {Pid,Ref} = spawn_monitor(fun() ->
+ exit(gen_io(cap2, 42, []))
+ end),
+ receive
+ {'DOWN',Ref,process,Pid,IO} ->
+ ct:capture_stop(),
+ IO = ct:capture_get(),
+ ok
+ end.
+
+gen_io(_, 0, Acc) ->
+ lists:reverse(Acc);
+gen_io(Label, N, Acc) ->
+ S = lists:flatten(io_lib:format("~s: ~p\n", [Label,N])),
+ io:put_chars(S),
+ gen_io(Label, N-1, [S|Acc]).
diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 405df1e978..796a0832d7 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -64,7 +64,7 @@ end_per_testcase(TestCase, Config) ->
suite() ->
- [{timetrap,{seconds,20}}].
+ [{timetrap,{minutes,1}}].
all() ->
all(suite).
diff --git a/lib/common_test/test/ct_master_SUITE.erl b/lib/common_test/test/ct_master_SUITE.erl
index 27243a0067..0f336d2d79 100644
--- a/lib/common_test/test/ct_master_SUITE.erl
+++ b/lib/common_test/test/ct_master_SUITE.erl
@@ -109,7 +109,7 @@ ct_master_test(Config) when is_list(Config) ->
ERPid = ct_test_support:start_event_receiver(Config),
- [{TSFile,ok}] = run_test(ct_master_test, FileName, Config),
+ [{[TSFile],ok}] = run_test(ct_master_test, FileName, Config),
Events = ct_test_support:get_events(ERPid, Config),
@@ -117,14 +117,8 @@ ct_master_test(Config) when is_list(Config) ->
reformat(Events, ?eh),
PrivDir, []),
- find_events(NodeNames, [{tc_start,{master_SUITE,init_per_suite}},
- {tc_start,{master_SUITE,first_testcase}},
- {tc_start,{master_SUITE,second_testcase}},
- {tc_start,{master_SUITE,third_testcase}},
- {tc_start,{master_SUITE,end_per_suite}}],
- Events),
-
- ok.
+ TestEvents = events_to_check(ct_master_test),
+ ok = find_events(NodeNames, TestEvents, Events, Config).
%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
@@ -153,13 +147,18 @@ make_spec(DataDir, FileName, NodeNames, Suites, Config) ->
CM = [{config,master,filename:join(DataDir,"master/config.txt")}],
+ Env = [{"THIS_MUST_BE_SET","yes"},
+ {"SO_MUST_THIS","value"}],
NS = lists:map(
fun(NodeName) ->
{init,NodeName,[
{node_start,[{startup_functions,[]},
- {monitor_master,true}]},
- {eval,{erlang,nodes,[]}}
- ]
+ {monitor_master,true},
+ {boot_timeout,10},
+ {init_timeout,10},
+ {startup_timeout,10},
+ {env,Env}]},
+ {eval,{erlang,nodes,[]}}]
}
end,
NodeNames),
@@ -196,13 +195,12 @@ get_log_dir(_,PrivDir,NodeName) ->
run_test(_Name, FileName, Config) ->
%% run the test twice, using different html versions
- [{FileName,ok}] = ct_test_support:run({ct_master,run,[FileName]},
- [{ct_master,basic_html,[true]}],
- Config),
- timer:sleep(5000),
- [{FileName,ok}] = ct_test_support:run({ct_master,run,[FileName]},
- [{ct_master,basic_html,[false]}],
- Config).
+ [{[FileName],ok}] = ct_test_support:run({ct_master,run,[FileName]},
+ [{ct_master,basic_html,[true]}],
+ Config),
+ [{[FileName],ok}] = ct_test_support:run({ct_master,run,[FileName]},
+ [{ct_master,basic_html,[false]}],
+ Config).
reformat(Events, EH) ->
ct_test_support:reformat(Events, EH).
@@ -210,28 +208,26 @@ reformat(Events, EH) ->
%%%-----------------------------------------------------------------
%%% TEST EVENTS
%%%-----------------------------------------------------------------
-find_events([], _CheckEvents, _) ->
- ok;
-find_events([NodeName|NodeNames],CheckEvents,AllEvents) ->
- find_events(NodeNames, CheckEvents,
- remove_events(add_host(NodeName),CheckEvents, AllEvents, [])).
-
-remove_events(Node,[{Name,Data} | RestChecks],
- [{?eh,#event{ name = Name, node = Node, data = Data }}|RestEvs],
- Acc) ->
- remove_events(Node, RestChecks, RestEvs, Acc);
-remove_events(Node, Checks, [Event|RestEvs], Acc) ->
- remove_events(Node, Checks, RestEvs, [Event | Acc]);
-remove_events(_Node, [], [], Acc) ->
- lists:reverse(Acc);
-remove_events(Node, Events, [], Acc) ->
- test_server:format("Could not find events: ~p in ~p for node ~p",
- [Events, lists:reverse(Acc), Node]),
- exit(event_not_found).
+
+find_events(NodeNames, TestEvents, Events, Config) ->
+ [begin
+ Node = add_host(Node0),
+ io:format("Searching for events for node: ~s", [Node]),
+ ok = ct_test_support:verify_events(TestEvents, Events, Node, Config),
+ io:nl()
+ end || Node0 <- NodeNames],
+ ok.
add_host(NodeName) ->
{ok, HostName} = inet:gethostname(),
list_to_atom(atom_to_list(NodeName)++"@"++HostName).
-expected_events(_) ->
- [].
+events_to_check(_) ->
+ [{?eh,tc_start,{master_SUITE,first_testcase}},
+ {?eh,tc_done,{master_SUITE,first_testcase,ok}},
+ {?eh,tc_start,{master_SUITE,second_testcase}},
+ {?eh,tc_done,{master_SUITE,second_testcase,ok}},
+ {?eh,tc_start,{master_SUITE,third_testcase}},
+ {?eh,tc_done,{master_SUITE,third_testcase,ok}},
+ {?eh,tc_start,{master_SUITE,env_vars}},
+ {?eh,tc_done,{master_SUITE,env_vars,ok}}].
diff --git a/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl b/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl
index 032d69ad9f..df54c4419c 100644
--- a/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl
+++ b/lib/common_test/test/ct_master_SUITE_data/master/master_SUITE.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2010. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2013. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -39,7 +39,8 @@ init_per_suite(Config) ->
end_per_suite(_) ->
ok.
-all() -> [first_testcase, second_testcase, third_testcase].
+all() -> [first_testcase, second_testcase, third_testcase,
+ env_vars].
init_per_testcase(_, Config) ->
Config.
@@ -56,3 +57,9 @@ second_testcase(_)->
third_testcase(_)->
A = 4,
A = 2*2.
+
+env_vars(_) ->
+ io:format("~p\n", [os:getenv()]),
+ "yes" = os:getenv("THIS_MUST_BE_SET"),
+ "value" = os:getenv("SO_MUST_THIS"),
+ ok.
diff --git a/lib/common_test/test/ct_netconfc_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE.erl
index 3042a924fe..c89a4cdabe 100644
--- a/lib/common_test/test/ct_netconfc_SUITE.erl
+++ b/lib/common_test/test/ct_netconfc_SUITE.erl
@@ -43,12 +43,11 @@
%% there will be clashes with logging processes etc).
%%--------------------------------------------------------------------
init_per_suite(Config) ->
- Config1 = ct_test_support:init_per_suite(Config),
case application:load(crypto) of
- {error,Reason} ->
+ {error,Reason} when Reason=/={already_loaded,crypto} ->
{skip, Reason};
_ ->
- Config1
+ ct_test_support:init_per_suite(Config)
end.
end_per_suite(Config) ->
diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl
index d337158bce..54526e8e83 100644
--- a/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl
+++ b/lib/common_test/test/ct_netconfc_SUITE_data/netconfc1_SUITE.erl
@@ -1044,12 +1044,9 @@ gen_dsa(LSize,NSize) when is_integer(LSize), is_integer(NSize) ->
Key = gen_dsa2(LSize, NSize),
{Key, encode_key(Key)}.
-encode_key(Key = #'RSAPrivateKey'{}) ->
- {ok, Der} = 'OTP-PUB-KEY':encode('RSAPrivateKey', Key),
- {'RSAPrivateKey', list_to_binary(Der), not_encrypted};
encode_key(Key = #'DSAPrivateKey'{}) ->
- {ok, Der} = 'OTP-PUB-KEY':encode('DSAPrivateKey', Key),
- {'DSAPrivateKey', list_to_binary(Der), not_encrypted}.
+ Der = public_key:der_encode('DSAPrivateKey', Key),
+ {'DSAPrivateKey', Der, not_encrypted}.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
diff --git a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl
index 2427f37f52..09217f60a3 100644
--- a/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl
+++ b/lib/common_test/test/ct_netconfc_SUITE_data/ns.erl
@@ -1,7 +1,7 @@
%%--------------------------------------------------------------------
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2012. All Rights Reserved.
+%% Copyright Ericsson AB 2012-2013. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -426,7 +426,7 @@ reply(ConnRef,Reply) ->
send(ConnRef, make_msg(Reply)).
from_simple(Simple) ->
- list_to_binary(xmerl:export_simple_element(Simple,xmerl_xml)).
+ unicode_c2b(xmerl:export_simple_element(Simple,xmerl_xml)).
xml(Content) ->
<<"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n",
@@ -435,30 +435,30 @@ xml(Content) ->
rpc_reply(Content) when is_binary(Content) ->
MsgId = case erase(msg_id) of
undefined -> <<>>;
- Id -> list_to_binary([" message-id=\"",Id,"\""])
+ Id -> unicode_c2b([" message-id=\"",Id,"\""])
end,
<<"<rpc-reply xmlns=\"",?NETCONF_NAMESPACE,"\"",MsgId/binary,">\n",
Content/binary,"\n</rpc-reply>">>;
rpc_reply(Content) ->
- rpc_reply(list_to_binary(Content)).
+ rpc_reply(unicode_c2b(Content)).
session_id(no_session_id) ->
<<>>;
session_id(SessionId0) ->
- SessionId = list_to_binary(integer_to_list(SessionId0)),
+ SessionId = unicode_c2b(integer_to_list(SessionId0)),
<<"<session-id>",SessionId/binary,"</session-id>\n">>.
capabilities(undefined) ->
- CapsXml = list_to_binary([["<capability>",C,"</capability>\n"]
+ CapsXml = unicode_c2b([["<capability>",C,"</capability>\n"]
|| C <- ?CAPABILITIES]),
<<"<capabilities>\n",CapsXml/binary,"</capabilities>\n">>;
capabilities({base,Vsn}) ->
- CapsXml = list_to_binary([["<capability>",C,"</capability>\n"]
+ CapsXml = unicode_c2b([["<capability>",C,"</capability>\n"]
|| C <- ?CAPABILITIES_VSN(Vsn)]),
<<"<capabilities>\n",CapsXml/binary,"</capabilities>\n">>;
capabilities(no_base) ->
[_|Caps] = ?CAPABILITIES,
- CapsXml = list_to_binary([["<capability>",C,"</capability>\n"] || C <- Caps]),
+ CapsXml = unicode_c2b([["<capability>",C,"</capability>\n"] || C <- Caps]),
<<"<capabilities>\n",CapsXml/binary,"</capabilities>\n">>;
capabilities(no_caps) ->
<<>>.
@@ -553,3 +553,8 @@ make_msg(Xml) when is_binary(Xml) ->
xml(Xml);
make_msg(Simple) when is_tuple(Simple) ->
xml(from_simple(Simple)).
+
+%%%-----------------------------------------------------------------
+%%% Convert to unicode binary, since we use UTF-8 encoding in XML
+unicode_c2b(Characters) ->
+ unicode:characters_to_binary(Characters).
diff --git a/lib/common_test/test/ct_snmp_SUITE.erl b/lib/common_test/test/ct_snmp_SUITE.erl
new file mode 100644
index 0000000000..f8b4543770
--- /dev/null
+++ b/lib/common_test/test/ct_snmp_SUITE.erl
@@ -0,0 +1,141 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_snmp_SUITE
+%%%
+%%% Description:
+%%% Test ct_snmp module
+%%%
+%%%-------------------------------------------------------------------
+-module(ct_snmp_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config1 = ct_test_support:init_per_suite(Config),
+ Config1.
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [
+ default
+ ].
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+default(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Suite = filename:join(DataDir, "snmp_SUITE"),
+ CfgFile = filename:join(DataDir, "snmp.cfg"),
+ {Opts,ERPid} = setup([{suite,Suite},{config,CfgFile},
+ {label,default}], Config),
+
+ ok = execute(default, Opts, ERPid, Config).
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+execute(Name, Opts, ERPid, Config) ->
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(Name,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(Name,Config),
+ ct_test_support:verify_events(TestEvents, Events, Config).
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+events_to_check(_TestName,Config) ->
+ {module,_} = code:load_abs(filename:join(?config(data_dir,Config),
+ snmp_SUITE)),
+ TCs = get_tcs(),
+ code:purge(snmp_SUITE),
+ code:delete(snmp_SUITE),
+
+ OneTest =
+ [{?eh,start_logging,{'DEF','RUNDIR'}}] ++
+ [{?eh,tc_done,{snmp_SUITE,TC,ok}} || TC <- TCs] ++
+ [{?eh,stop_logging,[]}],
+
+ %% 2 tests (ct:run_test + script_start) is default
+ OneTest ++ OneTest.
+
+
+get_tcs() ->
+ All = snmp_SUITE:all(),
+ Groups =
+ try snmp_SUITE:groups()
+ catch error:undef -> []
+ end,
+ flatten_tcs(All,Groups).
+
+flatten_tcs([H|T],Groups) when is_atom(H) ->
+ [H|flatten_tcs(T,Groups)];
+flatten_tcs([{group,Group}|T],Groups) ->
+ TCs = proplists:get_value(Group,Groups),
+ flatten_tcs(TCs ++ T,Groups);
+flatten_tcs([],_) ->
+ [].
diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp.cfg b/lib/common_test/test/ct_snmp_SUITE_data/snmp.cfg
new file mode 100644
index 0000000000..895e097de6
--- /dev/null
+++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp.cfg
@@ -0,0 +1,44 @@
+%% -*- erlang -*-
+{snmp1, [{start_agent,true},
+ {users,[{user_name,[snmpm_user_default,[]]}]},
+ {managed_agents,[{agent_name, [user_name, {127,0,0,1}, 4000,
+ [{engine_id,"ct_snmp-test-engine"},
+ {version,v2}]]}]},
+ {engine_id,"ct_snmp-test-engine"},
+ {agent_vsns,[v2]}
+ ]}.
+{snmp2, [{start_agent,true},
+ {engine_id,"ct_snmp-test-engine"}
+ ]}.
+{snmp3, [{start_agent,true},
+ {engine_id,"ct_snmp-test-engine"},
+ {agent_vsns,[v1,v2,v3]},
+ {agent_contexts,{data_dir_file,"context.conf"}},
+ {agent_usm,{data_dir_file,"usm.conf"}},
+ {agent_community,{data_dir_file,"community.conf"}},
+ {agent_notify_def,{data_dir_file,"notify.conf"}},
+ {agent_sysinfo,{data_dir_file,"standard.conf"}},
+ {agent_target_address_def,{data_dir_file,"target_addr.conf"}},
+ {agent_target_param_def,{data_dir_file,"target_params.conf"}},
+ {agent_vacm,{data_dir_file,"vacm.conf"}}]}.
+{snmp_app1,[{manager, [{config, [{verbosity, silence}]},
+ {server,[{verbosity,silence}]},
+ {net_if,[{verbosity,silence}]},
+ {versions,[v2]}
+ ]},
+ {agent, [{config, [{verbosity, silence}]},
+ {net_if,[{verbosity,silence}]},
+ {mib_server,[{verbosity,silence}]},
+ {local_db,[{verbosity,silence}]},
+ {agent_verbosity,silence}
+ ]}]}.
+{snmp_app2,[{manager, [{config, [{verbosity, silence}]},
+ {server,[{verbosity,silence}]},
+ {net_if,[{verbosity,silence}]}
+ ]},
+ {agent, [{config, [{verbosity, silence}]},
+ {net_if,[{verbosity,silence}]},
+ {mib_server,[{verbosity,silence}]},
+ {local_db,[{verbosity,silence}]},
+ {agent_verbosity,silence}
+ ]}]}.
diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE.erl b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE.erl
new file mode 100644
index 0000000000..16b2b5690c
--- /dev/null
+++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE.erl
@@ -0,0 +1,395 @@
+%%--------------------------------------------------------------------
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%----------------------------------------------------------------------
+%% File: ct_snmp_SUITE.erl
+%%
+%% Description:
+%% This file contains the test cases for the ct_snmp API.
+%%
+%% @author Support
+%% @doc Test of SNMP support in common_test
+%% @end
+%%----------------------------------------------------------------------
+%%----------------------------------------------------------------------
+-module(snmp_SUITE).
+-include_lib("common_test/include/ct.hrl").
+-include_lib("snmp/include/STANDARD-MIB.hrl").
+-include_lib("snmp/include/SNMP-USER-BASED-SM-MIB.hrl").
+-include_lib("snmp/include/snmp_types.hrl").
+
+-compile(export_all).
+
+%% Default timetrap timeout (set in init_per_testcase).
+-define(default_timeout, ?t:minutes(1)).
+
+-define(AGENT_UDP, 4000).
+
+suite() ->
+ [
+ {require, snmp1, snmp1},
+ {require, snmp_app1, snmp_app1},
+ {require, snmp2, snmp2},
+ {require, snmp_app2, snmp_app2},
+ {require, snmp3, snmp3}
+ ].
+
+all() ->
+ [start_stop,
+ {group,get_set},
+ {group,register},
+ {group,override},
+ set_info].
+
+
+groups() ->
+ [{get_set,[get_values,
+ get_next_values,
+ set_values,
+ load_mibs]},
+ {register,[register_users,
+ register_users_fail,
+ register_agents,
+ register_agents_fail,
+ register_usm_users,
+ register_usm_users_fail]},
+ {override,[override_usm,
+ override_standard,
+ override_context,
+ override_community,
+ override_notify,
+ override_target_addr,
+ override_target_params,
+ override_vacm]}].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ Config.
+
+init_per_group(get_set, Config) ->
+ ok = ct_snmp:start(Config,snmp1,snmp_app1),
+ Config;
+init_per_group(register, Config) ->
+ ok = ct_snmp:start(Config,snmp2,snmp_app2),
+ Config;
+init_per_group(_, Config) ->
+ ok = ct_snmp:start(Config,snmp3,snmp_app2),
+ Config.
+
+end_per_group(_Group, Config) ->
+ catch ct_snmp:stop(Config),
+ Config.
+
+init_per_testcase(_Case, Config) ->
+ Dog = test_server:timetrap(?default_timeout),
+ [{watchdog, Dog}|Config].
+
+end_per_testcase(Case, Config) ->
+ try apply(?MODULE,Case,[cleanup,Config])
+ catch error:undef -> ok
+ end,
+ Dog=?config(watchdog, Config),
+ test_server:timetrap_cancel(Dog),
+ ok.
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+break(_Config) ->
+ test_server:break(""),
+ ok.
+
+start_stop(Config) ->
+ ok = ct_snmp:start(Config,snmp1,snmp_app1),
+ timer:sleep(1000),
+ {snmp,_,_} = lists:keyfind(snmp,1,application:which_applications()),
+ [_|_] = filelib:wildcard("*/*.conf",?config(priv_dir,Config)),
+
+ ok = ct_snmp:stop(Config),
+ timer:sleep(1000),
+ false = lists:keyfind(snmp,1,application:which_applications()),
+ [] = filelib:wildcard("*/*.conf",?config(priv_dir,Config)),
+ ok.
+
+get_values(_Config) ->
+ Oids1 = [?sysDescr_instance, ?sysName_instance],
+ {noError,_,V1} = ct_snmp:get_values(agent_name,Oids1,snmp1),
+ [#varbind{oid=?sysDescr_instance,value="Erlang SNMP agent"},
+ #varbind{oid=?sysName_instance,value="ct_test"}] = V1,
+ ok.
+
+get_next_values(_Config) ->
+ Oids2 = [?system],
+ {noError,_,V2} = ct_snmp:get_next_values(agent_name,Oids2,snmp1),
+ [#varbind{oid=?sysDescr_instance,value="Erlang SNMP agent"}] = V2,
+ ok.
+
+set_values(Config) ->
+ Oid3 = ?sysName_instance,
+ NewName = "ct_test changed by " ++ atom_to_list(?MODULE),
+ VarsAndVals = [{Oid3,s,NewName}],
+ {noError,_,_} =
+ ct_snmp:set_values(agent_name,VarsAndVals,snmp1,Config),
+
+ Oids4 = [?sysName_instance],
+ {noError,_,V4} = ct_snmp:get_values(agent_name,Oids4,snmp1),
+ [#varbind{oid=?sysName_instance,value=NewName}] = V4,
+
+ ok.
+
+load_mibs(_Config) ->
+ [{'SNMPv2-MIB',_}=SnmpV2Mib] = snmpa:which_mibs(),
+ Mib = filename:join([code:priv_dir(snmp),"mibs","SNMP-USER-BASED-SM-MIB"]),
+ ok = ct_snmp:load_mibs([Mib]),
+ TwoMibs = [_,_] = snmpa:which_mibs(),
+ [{'SNMP-USER-BASED-SM-MIB',_}] = lists:delete(SnmpV2Mib,TwoMibs),
+ ok = ct_snmp:unload_mibs([Mib]),
+ [{'SNMPv2-MIB',_}] = snmpa:which_mibs(),
+ ok.
+
+
+register_users(_Config) ->
+ [] = snmpm:which_users(),
+ ok = ct_snmp:register_users(snmp2,[{reg_user1,[snmpm_user_default,[]]}]),
+ [_] = snmpm:which_users(),
+ [_] = ct:get_config({snmp2,users}),
+ ok = ct_snmp:register_users(snmp2,[{reg_user2,[snmpm_user_default,[]]}]),
+ [_,_] = snmpm:which_users(),
+ [_,_] = ct:get_config({snmp2,users}),
+ ok = ct_snmp:register_users(snmp2,[{reg_user3,[snmpm_user_default,[]]}]),
+ [_,_,_] = snmpm:which_users(),
+ [_,_,_] = ct:get_config({snmp2,users}),
+ ok = ct_snmp:unregister_users(snmp2,[reg_user3]),
+ [_,_] = snmpm:which_users(),
+ [_,_] = ct:get_config({snmp2,users}),
+ ok = ct_snmp:unregister_users(snmp2),
+ [] = snmpm:which_users(),
+ [] = ct:get_config({snmp2,users}),
+ ok.
+register_users(cleanup,_Config) ->
+ ct_snmp:unregister_users(snmp2).
+
+register_users_fail(_Config) ->
+ [] = snmpm:which_users(),
+ {error,_} = ct_snmp:register_users(snmp2,[{reg_user3,[unknown_module,[]]}]),
+ [] = snmpm:which_users(),
+ ok.
+register_users_fail(cleanup,_Config) ->
+ ct_snmp:unregister_users(snmp2).
+
+register_agents(_Config) ->
+ {ok, HostName} = inet:gethostname(),
+ {ok, Addr} = inet:getaddr(HostName, inet),
+
+ [] = snmpm:which_agents(),
+ ok = ct_snmp:register_users(snmp2,[{reg_user1,[snmpm_user_default,[]]}]),
+ ok = ct_snmp:register_agents(snmp2,[{reg_agent1,
+ [reg_user1,Addr,?AGENT_UDP,[]]}]),
+ [_] = snmpm:which_agents(),
+ [_] = ct:get_config({snmp2,managed_agents}),
+ ok = ct_snmp:register_agents(snmp2,[{reg_agent2,
+ [reg_user1,Addr,?AGENT_UDP,[]]}]),
+ [_,_] = snmpm:which_agents(),
+ [_,_] = ct:get_config({snmp2,managed_agents}),
+
+ ok = ct_snmp:register_agents(snmp2,[{reg_agent3,
+ [reg_user1,Addr,?AGENT_UDP,[]]}]),
+ [_,_,_] = snmpm:which_agents(),
+ [_,_,_] = ct:get_config({snmp2,managed_agents}),
+
+ ok = ct_snmp:unregister_agents(snmp2,[reg_agent3]),
+ [_,_] = snmpm:which_agents(),
+ [_,_] = ct:get_config({snmp2,managed_agents}),
+
+ ok = ct_snmp:unregister_agents(snmp2),
+ ok = ct_snmp:unregister_users(snmp2),
+ [] = snmpm:which_agents(),
+ [] = ct:get_config({snmp2,managed_agents}),
+ ok.
+register_agents(cleanup,_Config) ->
+ ct_snmp:unregister_agents(snmp2),
+ ct_snmp:unregister_users(snmp2).
+
+register_agents_fail(_Config) ->
+ {ok, HostName} = inet:gethostname(),
+ {ok, Addr} = inet:getaddr(HostName, inet),
+
+ [] = snmpm:which_agents(),
+ {error,_}
+ = ct_snmp:register_agents(snmp2,[{reg_agent3,
+ [unknown_user,Addr,?AGENT_UDP,[]]}]),
+ [] = snmpm:which_agents(),
+ ok.
+register_agents_fail(cleanup,_Config) ->
+ ct_snmp:unregister_agents(snmp2).
+
+register_usm_users(_Config) ->
+ [] = snmpm:which_usm_users(),
+ ok = ct_snmp:register_usm_users(snmp2,[{"reg_usm_user1",[]}]),
+ [_] = snmpm:which_usm_users(),
+ [_] = ct:get_config({snmp2,usm_users}),
+ ok = ct_snmp:register_usm_users(snmp2,[{"reg_usm_user2",[]}]),
+ [_,_] = snmpm:which_usm_users(),
+ [_,_] = ct:get_config({snmp2,usm_users}),
+ ok = ct_snmp:register_usm_users(snmp2,[{"reg_usm_user3",[]}]),
+ [_,_,_] = snmpm:which_usm_users(),
+ [_,_,_] = ct:get_config({snmp2,usm_users}),
+ ok = ct_snmp:unregister_usm_users(snmp2,["reg_usm_user3"]),
+ [_,_] = snmpm:which_usm_users(),
+ [_,_] = ct:get_config({snmp2,usm_users}),
+ ok = ct_snmp:unregister_usm_users(snmp2),
+ [] = snmpm:which_usm_users(),
+ [] = ct:get_config({snmp2,usm_users}),
+ ok.
+register_usm_users(cleanup,_Config) ->
+ ct_snmp:unregister_usm_users(snmp2).
+
+register_usm_users_fail(_Config) ->
+ [] = snmpm:which_usm_users(),
+ {error,_}
+ = ct_snmp:register_usm_users(snmp2,[{"reg_usm_user3",
+ [{sec_name,invalid_data_type}]}]),
+ [] = snmpm:which_usm_users(),
+ ok.
+register_usm_users_fail(cleanup,_Config) ->
+ ct_snmp:unregister_usm_users(snmp2).
+
+%% Test that functionality for overriding default configuration file
+%% works - i.e. that the files are written and that the configuration
+%% is actually used.
+%%
+%% Note that the config files used in this test case do not
+%% necessarily make up a reasonable configuration for the snmp
+%% application...
+override_usm(Config) ->
+ DataDir = ?config(data_dir,Config),
+ PrivDir = ?config(priv_dir,Config),
+ ConfDir = filename:join(PrivDir,"conf"),
+
+ Mib = filename:join([code:priv_dir(snmp),"mibs","SNMP-USER-BASED-SM-MIB"]),
+ ok = ct_snmp:load_mibs([Mib]),
+
+ %% Check that usm.conf is overwritten
+ {ok,MyUsm} = snmpa_conf:read_usm_config(DataDir),
+ {ok,UsedUsm} = snmpa_conf:read_usm_config(ConfDir),
+ true = (MyUsm == UsedUsm),
+
+ %% Check that the usm user is actually configured...
+ [{Index,"secname"}] =
+ snmp_user_based_sm_mib:usmUserTable(get_next,?usmUserEntry,[3]),
+ true = lists:suffix("usm_user_name",Index),
+ ok.
+
+override_standard(Config) ->
+ DataDir = ?config(data_dir,Config),
+ PrivDir = ?config(priv_dir,Config),
+ ConfDir = filename:join(PrivDir,"conf"),
+
+ %% Check that standard.conf is overwritten
+ {ok,MyStandard} = snmpa_conf:read_standard_config(DataDir),
+ {ok,UsedStandard} = snmpa_conf:read_standard_config(ConfDir),
+ true = (MyStandard == UsedStandard),
+
+ %% Check that the values from standard.conf is actually configured...
+ {value,"name for override test"} = snmp_standard_mib:sysName(get),
+ {value,"agent for ct_snmp override test"} = snmp_standard_mib:sysDescr(get),
+ ok.
+
+override_context(Config) ->
+ DataDir = ?config(data_dir,Config),
+ PrivDir = ?config(priv_dir,Config),
+ ConfDir = filename:join(PrivDir,"conf"),
+
+ %% Check that context.conf is overwritten
+ {ok,MyContext} = snmpa_conf:read_context_config(DataDir),
+ {ok,UsedContext} = snmpa_conf:read_context_config(ConfDir),
+ true = (MyContext == UsedContext),
+ ok.
+
+override_community(Config) ->
+ DataDir = ?config(data_dir,Config),
+ PrivDir = ?config(priv_dir,Config),
+ ConfDir = filename:join(PrivDir,"conf"),
+
+ %% Check that community.conf is overwritten
+ {ok,MyCommunity} = snmpa_conf:read_community_config(DataDir),
+ {ok,UsedCommunity} = snmpa_conf:read_community_config(ConfDir),
+ true = (MyCommunity == UsedCommunity),
+ ok.
+
+override_notify(Config) ->
+ DataDir = ?config(data_dir,Config),
+ PrivDir = ?config(priv_dir,Config),
+ ConfDir = filename:join(PrivDir,"conf"),
+
+ %% Check that notify.conf is overwritten
+ {ok,MyNotify} = snmpa_conf:read_notify_config(DataDir),
+ {ok,UsedNotify} = snmpa_conf:read_notify_config(ConfDir),
+ true = (MyNotify == UsedNotify),
+ ok.
+
+override_target_addr(Config) ->
+ DataDir = ?config(data_dir,Config),
+ PrivDir = ?config(priv_dir,Config),
+ ConfDir = filename:join(PrivDir,"conf"),
+
+ %% Check that target_addr.conf is overwritten
+ {ok,MyTargetAddr} = snmpa_conf:read_target_addr_config(DataDir),
+ {ok,UsedTargetAddr} = snmpa_conf:read_target_addr_config(ConfDir),
+ true = (MyTargetAddr == UsedTargetAddr),
+ ok.
+
+override_target_params(Config) ->
+ DataDir = ?config(data_dir,Config),
+ PrivDir = ?config(priv_dir,Config),
+ ConfDir = filename:join(PrivDir,"conf"),
+
+ %% Check that target_params.conf is overwritten
+ {ok,MyTargetParams} = snmpa_conf:read_target_params_config(DataDir),
+ {ok,UsedTargetParams} = snmpa_conf:read_target_params_config(ConfDir),
+ true = (MyTargetParams == UsedTargetParams),
+ ok.
+
+override_vacm(Config) ->
+ DataDir = ?config(data_dir,Config),
+ PrivDir = ?config(priv_dir,Config),
+ ConfDir = filename:join(PrivDir,"conf"),
+
+ %% Check that vacm.conf is overwritten
+ {ok,MyVacm} = snmpa_conf:read_vacm_config(DataDir),
+ {ok,UsedVacm} = snmpa_conf:read_vacm_config(ConfDir),
+ true = (MyVacm == UsedVacm),
+ ok.
+
+
+
+
+%% NOTE!! This test must always be executed last in the suite, and
+%% should match all set requests performed in the suite. I.e. if you
+%% add a set request, you must add an entry in the return value of
+%% ct_snmp:set_info/1 below.
+set_info(Config) ->
+ %% From test case set_values/1:
+ Oid1 = ?sysName_instance,
+ NewValue1 = "ct_test changed by " ++ atom_to_list(?MODULE),
+
+ %% The test...
+ [{_AgentName,_,[{Oid1,_,NewValue1}]}]
+ = ct_snmp:set_info(Config),
+ ok.
diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/community.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/community.conf
new file mode 100644
index 0000000000..5a64df6605
--- /dev/null
+++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/community.conf
@@ -0,0 +1 @@
+{"public", "public", "initial", "", ""}.
diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/context.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/context.conf
new file mode 100644
index 0000000000..feed5e1d11
--- /dev/null
+++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/context.conf
@@ -0,0 +1 @@
+"testcontext".
diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/notify.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/notify.conf
new file mode 100644
index 0000000000..367ba3aa4b
--- /dev/null
+++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/notify.conf
@@ -0,0 +1 @@
+{"standard inform", "std_inform", inform}.
diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/standard.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/standard.conf
new file mode 100644
index 0000000000..79908fb355
--- /dev/null
+++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/standard.conf
@@ -0,0 +1,7 @@
+{sysDescr, "agent for ct_snmp override test"}.
+{sysObjectID, [1,2,3]}.
+{sysContact, "[email protected]"}.
+{sysLocation, "erlang"}.
+{sysServices, 72}.
+{snmpEnableAuthenTraps, enabled}.
+{sysName, "name for override test"}.
diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/target_addr.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/target_addr.conf
new file mode 100644
index 0000000000..d02672a074
--- /dev/null
+++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/target_addr.conf
@@ -0,0 +1,2 @@
+{"target1", snmpUDPDomain, [147,214,122,73], 5000, 1500, 3, "std_trap", "target_v3", "", [], 2048}.
+{"target2", snmpUDPDomain, [147,214,122,73], 5000, 1500, 3, "std_inform", "target_v3", "", [], 2048}.
diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/target_params.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/target_params.conf
new file mode 100644
index 0000000000..5a9a619422
--- /dev/null
+++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/target_params.conf
@@ -0,0 +1 @@
+{"target_v3", v3, usm, "initial", noAuthNoPriv}.
diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/usm.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/usm.conf
new file mode 100644
index 0000000000..d6e245914e
--- /dev/null
+++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/usm.conf
@@ -0,0 +1 @@
+{"ct_snmp-test-engine","usm_user_name","secname",zeroDotZero,usmNoAuthProtocol,"","",usmNoPrivProtocol,"","","","",""}.
diff --git a/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/vacm.conf b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/vacm.conf
new file mode 100644
index 0000000000..158fe02e3b
--- /dev/null
+++ b/lib/common_test/test/ct_snmp_SUITE_data/snmp_SUITE_data/vacm.conf
@@ -0,0 +1,6 @@
+{vacmSecurityToGroup, usm, "initial", "initial"}.
+{vacmAccess, "initial", "", any, noAuthNoPriv, exact, "restricted", "", "restricted"}.
+{vacmAccess, "initial", "", usm, authNoPriv, exact, "internet", "internet", "internet"}.
+{vacmAccess, "initial", "", usm, authPriv, exact, "internet", "internet", "internet"}.
+{vacmViewTreeFamily, "restricted", [1,3,6,1], included, null}.
+{vacmViewTreeFamily, "internet", [1,3,6,1], included, null}.
diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl
new file mode 100644
index 0000000000..b86b47f0a2
--- /dev/null
+++ b/lib/common_test/test/ct_surefire_SUITE.erl
@@ -0,0 +1,374 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2012-2013. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_surefire_SUITE
+%%%
+%%% Description:
+%%% Test cth_surefire hook
+%%%
+%%%-------------------------------------------------------------------
+-module(ct_surefire_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-include_lib("xmerl/include/xmerl.hrl").
+-include_lib("kernel/include/file.hrl").
+
+-define(eh, ct_test_support_eh).
+
+-define(url_base,"http://my.host.com/").
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config1 = ct_test_support:init_per_suite(Config),
+ Config1.
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [
+ default,
+ absolute_path,
+ relative_path,
+ url,
+ logdir
+ ].
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+default(Config) when is_list(Config) ->
+ run(default,[cth_surefire],"junit_report.xml",Config).
+
+absolute_path(Config) when is_list(Config) ->
+ PrivDir = ?config(priv_dir,Config),
+ Path = filename:join(PrivDir,"abspath.xml"),
+ run(absolute_path,[{cth_surefire,[{path,Path}]}],Path,Config).
+
+relative_path(Config) when is_list(Config) ->
+ Path = "relpath.xml",
+ run(relative_path,[{cth_surefire,[{path,Path}]}],Path,Config).
+
+url(Config) when is_list(Config) ->
+ Path = "url.xml",
+ run(url,[{cth_surefire,[{url_base,?url_base},{path,Path}]}],
+ Path,Config).
+
+logdir(Config) when is_list(Config) ->
+ Opts = ct_test_support:get_opts(Config),
+ LogDir =
+ case lists:keyfind(logdir,1,Opts) of
+ {logdir,LD} -> LD;
+ false -> ?config(priv_dir,Config)
+ end,
+ MyLogDir = filename:join(LogDir,"specific_logdir"),
+ ensure_exists_empty(MyLogDir),
+ Path = "logdir.xml",
+ run(logdir,[{cth_surefire,[{path,Path}]}],Path,Config,[{logdir,MyLogDir}]).
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+run(Case,CTHs,Report,Config) ->
+ run(Case,CTHs,Report,Config,[]).
+run(Case,CTHs,Report,Config,ExtraOpts) ->
+ DataDir = ?config(data_dir, Config),
+ Suite = filename:join(DataDir, "surefire_SUITE"),
+ {Opts,ERPid} = setup([{suite,Suite},{ct_hooks,CTHs},{label,Case}|ExtraOpts],
+ Config),
+ ok = execute(Case, Opts, ERPid, Config),
+ LogDir =
+ case lists:keyfind(logdir,1,Opts) of
+ {logdir,LD} -> LD;
+ false -> ?config(priv_dir,Config)
+ end,
+ Re = filename:join([LogDir,"*",Report]),
+ check_xml(Case,Re).
+
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Opts1 =
+ case lists:keymember(logdir,1,Test) of
+ true -> lists:keydelete(logdir,1,Opts0);
+ false -> Opts0
+ end,
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts1 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+execute(Name, Opts, ERPid, Config) ->
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(Name,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(Name),
+ ct_test_support:verify_events(TestEvents, Events, Config).
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+events_to_check(Test) ->
+ %% 2 tests (ct:run_test + script_start) is default
+ events_to_check(Test, 2).
+
+events_to_check(_, 0) ->
+ [];
+events_to_check(Test, N) ->
+ test_events(Test) ++ events_to_check(Test, N-1).
+
+test_events(_) ->
+ [{?eh,start_logging,'_'},
+ {?eh,start_info,{1,1,9}},
+ {?eh,tc_start,{surefire_SUITE,init_per_suite}},
+ {?eh,tc_done,{surefire_SUITE,init_per_suite,ok}},
+ {?eh,tc_start,{surefire_SUITE,tc_ok}},
+ {?eh,tc_done,{surefire_SUITE,tc_ok,ok}},
+ {?eh,test_stats,{1,0,{0,0}}},
+ {?eh,tc_start,{surefire_SUITE,tc_fail}},
+ {?eh,tc_done,{surefire_SUITE,tc_fail,
+ {failed,{error,{test_case_failed,"this test should fail"}}}}},
+ {?eh,test_stats,{1,1,{0,0}}},
+ {?eh,tc_start,{surefire_SUITE,tc_skip}},
+ {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}},
+ {?eh,test_stats,{1,1,{1,0}}},
+ {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}},
+ {?eh,tc_done,{surefire_SUITE,tc_autoskip_require,
+ {skipped,{require_failed,'_'}}}},
+ {?eh,test_stats,{1,1,{1,1}}},
+ [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}},
+ {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}},
+ {?eh,tc_start,{surefire_SUITE,tc_ok}},
+ {?eh,tc_done,{surefire_SUITE,tc_ok,ok}},
+ {?eh,test_stats,{2,1,{1,1}}},
+ {?eh,tc_start,{surefire_SUITE,tc_fail}},
+ {?eh,tc_done,{surefire_SUITE,tc_fail,
+ {failed,{error,{test_case_failed,"this test should fail"}}}}},
+ {?eh,test_stats,{2,2,{1,1}}},
+ {?eh,tc_start,{surefire_SUITE,tc_skip}},
+ {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}},
+ {?eh,test_stats,{2,2,{2,1}}},
+ {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}},
+ {?eh,tc_done,{surefire_SUITE,tc_autoskip_require,
+ {skipped,{require_failed,'_'}}}},
+ {?eh,test_stats,{2,2,{2,2}}},
+ {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}},
+ {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}],
+ [{?eh,tc_start,{surefire_SUITE,{init_per_group,g_fail,[]}}},
+ {?eh,tc_done,{surefire_SUITE,{init_per_group,g_fail,[]},
+ {failed,{error,all_cases_should_be_skipped}}}},
+ {?eh,tc_auto_skip,{surefire_SUITE,tc_ok,
+ {failed,
+ {surefire_SUITE,init_per_group,
+ {'EXIT',all_cases_should_be_skipped}}}}},
+ {?eh,test_stats,{2,2,{2,3}}},
+ {?eh,tc_auto_skip,{surefire_SUITE,end_per_group,
+ {failed,
+ {surefire_SUITE,init_per_group,
+ {'EXIT',all_cases_should_be_skipped}}}}}],
+ {?eh,tc_start,{surefire_SUITE,end_per_suite}},
+ {?eh,tc_done,{surefire_SUITE,end_per_suite,ok}},
+ {?eh,stop_logging,[]}].
+
+
+%%%-----------------------------------------------------------------
+%%% Check generated xml log files
+check_xml(Case,XmlRe) ->
+ case filelib:wildcard(XmlRe) of
+ [] ->
+ ct:fail("No xml files found with regexp ~p~n", [XmlRe]);
+ [_] = Xmls when Case==absolute_path ->
+ do_check_xml(Case,Xmls);
+ [_,_] = Xmls ->
+ do_check_xml(Case,Xmls)
+ end.
+
+%% Allowed structure:
+%% <testsuites>
+%% <testsuite>
+%% <properties>
+%% <property/>
+%% ...
+%% </properties>
+%% <testcase>
+%% [<failure/> | <error/> | <skipped/> ]
+%% </testcase>
+%% ...
+%% </testsuite>
+%% ...
+%% </testsuites>
+do_check_xml(Case,[Xml|Xmls]) ->
+ ct:log("Checking <a href=~p>~s</a>~n",[Xml,Xml]),
+ {E,_} = xmerl_scan:file(Xml),
+ Expected = events_to_result(lists:flatten(test_events(Case))),
+ ParseResult = testsuites(Case,E),
+ ct:log("Expecting: ~p~n",[[Expected]]),
+ ct:log("Actual : ~p~n",[ParseResult]),
+ [Expected] = ParseResult,
+ do_check_xml(Case,Xmls);
+do_check_xml(_,[]) ->
+ ok.
+
+%% Scanning the XML to get the same type of result as events_to_result/1
+testsuites(Case,#xmlElement{name=testsuites,content=TS}) ->
+ %% OTP-10589 - move properties element to <testsuite>
+ false = lists:keytake(properties,#xmlElement.name,TS),
+ testsuite(Case,TS).
+
+testsuite(Case,[#xmlElement{name=testsuite,content=TC,attributes=A}|TS]) ->
+ {ET,EF,ES} = events_to_numbers(lists:flatten(test_events(Case))),
+ {T,E,F,S} = get_numbers_from_attrs(A,false,false,false,false),
+ ct:log("Expecting total:~p, error:~p, failure:~p, skipped:~p~n",[ET,0,EF,ES]),
+ ct:log("Actual total:~p, error:~p, failure:~p, skipped:~p~n",[T,E,F,S]),
+ {ET,0,EF,ES} = {T,E,F,S},
+
+ %% properties should only be there if given a options to hook
+ false = lists:keytake(properties,#xmlElement.name,TC),
+ %% system-out and system-err is not used by common_test
+ false = lists:keytake('system-out',#xmlElement.name,TC),
+ false = lists:keytake('system-err',#xmlElement.name,TC),
+ R=testcase(Case,TC),
+ [R|testsuite(Case,TS)];
+testsuite(_Case,[]) ->
+ [].
+
+testcase(url=Case,[#xmlElement{name=testcase,attributes=A,content=C}|TC]) ->
+ R = failed_or_skipped(C),
+ case R of
+ [s] ->
+ case lists:keyfind(url,#xmlAttribute.name,A) of
+ false -> ok;
+ #xmlAttribute{value=UrlAttr} ->
+ lists:keyfind(url,#xmlAttribute.name,A),
+ true = lists:prefix(?url_base,UrlAttr)
+ end;
+ _ ->
+ #xmlAttribute{value=UrlAttr} =
+ lists:keyfind(url,#xmlAttribute.name,A),
+ true = lists:prefix(?url_base,UrlAttr)
+ end,
+ [R|testcase(Case,TC)];
+testcase(Case,[#xmlElement{name=testcase,attributes=A,content=C}|TC]) ->
+ false = lists:keyfind(url,#xmlAttribute.name,A),
+ R = failed_or_skipped(C),
+ [R|testcase(Case,TC)];
+testcase(_Case,[]) ->
+ [].
+
+failed_or_skipped([#xmlElement{name=failure}|E]) ->
+ [f|failed_or_skipped(E)];
+failed_or_skipped([#xmlElement{name=error}|E]) ->
+ [e|failed_or_skipped(E)];
+failed_or_skipped([#xmlElement{name=skipped}|E]) ->
+ [s|failed_or_skipped(E)];
+failed_or_skipped([]) ->
+ [].
+
+%% Using the expected events to produce the expected result of the XML scanning.
+%% The result is a list of test suites:
+%% Testsuites = [Testsuite]
+%% Testsuite = [Testcase]
+%% Testcase = [] | [f] | [s], indicating ok, failed and skipped respectively
+events_to_result([{?eh,tc_done,{_Suite,_Case,R}}|E]) ->
+ [result(R)|events_to_result(E)];
+events_to_result([{?eh,tc_auto_skip,_}|E]) ->
+ [[s]|events_to_result(E)];
+events_to_result([_|E]) ->
+ events_to_result(E);
+events_to_result([]) ->
+ [].
+
+result(ok) ->[];
+result({skipped,_}) -> [s];
+result({failed,_}) -> [f].
+
+%% Using the expected events' last test_stats element to produce the
+%% expected number of totla, errors, failed and skipped testcases.
+events_to_numbers(E) ->
+ RevE = lists:reverse(E),
+ {?eh,test_stats,{Ok,F,{US,AS}}} = lists:keyfind(test_stats,2,RevE),
+ {Ok+F+US+AS,F,US+AS}.
+
+get_numbers_from_attrs([#xmlAttribute{name=tests,value=X}|A],false,E,F,S) ->
+ get_numbers_from_attrs(A,list_to_integer(X),E,F,S);
+get_numbers_from_attrs([#xmlAttribute{name=errors,value=X}|A],T,false,F,S) ->
+ get_numbers_from_attrs(A,T,list_to_integer(X),F,S);
+get_numbers_from_attrs([#xmlAttribute{name=failures,value=X}|A],T,E,false,S) ->
+ get_numbers_from_attrs(A,T,E,list_to_integer(X),S);
+get_numbers_from_attrs([#xmlAttribute{name=skipped,value=X}|A],T,E,F,false) ->
+ get_numbers_from_attrs(A,T,E,F,list_to_integer(X));
+get_numbers_from_attrs([_|A],T,E,F,S) ->
+ get_numbers_from_attrs(A,T,E,F,S);
+get_numbers_from_attrs([],T,E,F,S) ->
+ {T,E,F,S}.
+
+ensure_exists_empty(Dir) ->
+ case file:list_dir(Dir) of
+ {error,enoent} ->
+ file:make_dir(Dir);
+ {ok,Files} ->
+ del_files(Dir,Files)
+ end.
+
+del_files(Dir,[F0|Fs] ) ->
+ F = filename:join(Dir,F0),
+ case file:read_file_info(F) of
+ {ok,#file_info{type=directory}} ->
+ {ok,Files} = file:list_dir(F),
+ del_files(F,Files),
+ file:del_dir(F),
+ del_files(Dir,Fs);
+ _ ->
+ file:delete(F),
+ del_files(Dir,Fs)
+ end;
+del_files(_,[]) ->
+ ok.
diff --git a/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl
new file mode 100644
index 0000000000..677aee46c5
--- /dev/null
+++ b/lib/common_test/test/ct_surefire_SUITE_data/surefire_SUITE.erl
@@ -0,0 +1,92 @@
+%%--------------------------------------------------------------------
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%----------------------------------------------------------------------
+%% File: surefire_SUITE.erl
+%%
+%% Description:
+%% This file contains the test cases for cth_surefire.
+%%
+%% @author Support
+%% @doc Test of surefire support in common_test
+%% @end
+%%----------------------------------------------------------------------
+%%----------------------------------------------------------------------
+-module(surefire_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+%% Default timetrap timeout (set in init_per_testcase).
+-define(default_timeout, ?t:minutes(1)).
+
+all() ->
+ testcases() ++ [{group,g},{group,g_fail}].
+
+groups() ->
+ [{g,testcases()},
+ {g_fail,[tc_ok]}].
+
+testcases() ->
+ [tc_ok,
+ tc_fail,
+ tc_skip,
+ tc_autoskip_require].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ Config.
+
+init_per_group(g_fail, _Config) ->
+ exit(all_cases_should_be_skipped);
+init_per_group(_, Config) ->
+ Config.
+
+end_per_group(_Group, Config) ->
+ Config.
+
+init_per_testcase(_Case, Config) ->
+ Dog = test_server:timetrap(?default_timeout),
+ [{watchdog, Dog}|Config].
+
+end_per_testcase(_Case, Config) ->
+ Dog=?config(watchdog, Config),
+ test_server:timetrap_cancel(Dog),
+ ok.
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+break(_Config) ->
+ test_server:break(""),
+ ok.
+
+tc_ok(_Config) ->
+ ok.
+
+tc_fail(_Config) ->
+ ct:fail("this test should fail").
+
+tc_skip(_Config) ->
+ {skip,"this test is skipped"}.
+
+tc_autoskip_require() ->
+ [{require,whatever}].
+tc_autoskip_require(Config) ->
+ ct:fail("this test should never be executed - it should be autoskipped").
diff --git a/lib/common_test/test/ct_system_error_SUITE.erl b/lib/common_test/test/ct_system_error_SUITE.erl
new file mode 100644
index 0000000000..f2d6ef4b1b
--- /dev/null
+++ b/lib/common_test/test/ct_system_error_SUITE.erl
@@ -0,0 +1,132 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_system_error_SUITE
+%%%
+%%% Description:
+%%%
+%%% Test that severe system errors (such as failure to write logs) are
+%%% noticed and handled.
+%%%-------------------------------------------------------------------
+-module(ct_system_error_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config1 = ct_test_support:init_per_suite(Config),
+ Config1.
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [
+ test_server_failing_logs
+ ].
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+test_server_failing_logs(Config) ->
+ TC = test_server_failing_logs,
+ DataDir = ?config(data_dir, Config),
+ Suite = filename:join(DataDir, "a_SUITE"),
+ {Opts,ERPid} = setup([{suite,Suite},{label,TC}], Config),
+ crash_test_server(Config),
+ {error,{cannot_create_log_dir,__}} = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+ ct_test_support:log_events(TC,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(TC),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+crash_test_server(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Root = proplists:get_value(logdir, ct_test_support:get_opts(Config)),
+ [$@|Host] = lists:dropwhile(fun(C) ->
+ C =/= $@
+ end, atom_to_list(node())),
+ Format = filename:join(Root,
+ "ct_run.ct@" ++ Host ++
+ ".~4..0w-~2..0w-~2..0w_"
+ "~2..0w.~2..0w.~2..0w"),
+ [C2,C1|_] = lists:reverse(filename:split(DataDir)),
+ LogDir = C1 ++ "." ++ C2 ++ ".a_SUITE.logs",
+ T = calendar:datetime_to_gregorian_seconds(calendar:local_time()),
+ [begin
+ {{Y,Mon,D},{H,Min,S}} =
+ calendar:gregorian_seconds_to_datetime(T+Offset),
+ Dir0 = io_lib:format(Format, [Y,Mon,D,H,Min,S]),
+ Dir = lists:flatten(Dir0),
+ file:make_dir(Dir),
+ File = filename:join(Dir, LogDir),
+ file:write_file(File, "anything goes\n")
+ end || Offset <- lists:seq(0, 20)],
+ ok.
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+
+events_to_check(_Test) ->
+ [{?eh,severe_error,{cannot_create_log_dir,{'_','_'}}}].
diff --git a/lib/common_test/test/ct_system_error_SUITE_data/a_SUITE.erl b/lib/common_test/test/ct_system_error_SUITE_data/a_SUITE.erl
new file mode 100644
index 0000000000..c6e3ddfd5d
--- /dev/null
+++ b/lib/common_test/test/ct_system_error_SUITE_data/a_SUITE.erl
@@ -0,0 +1,122 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(a_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,{seconds,10}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [tc1].
+
+tc1(_C) ->
+ ok.
diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl
index 80cca4a1cc..7c33fd404d 100644
--- a/lib/common_test/test/ct_test_support.erl
+++ b/lib/common_test/test/ct_test_support.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2008-2012. All Rights Reserved.
+%% Copyright Ericsson AB 2008-2013. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -32,7 +32,7 @@
run/2, run/3, run/4, get_opts/1, wait_for_ct_stop/1]).
-export([handle_event/2, start_event_receiver/1, get_events/2,
- verify_events/3, reformat/2, log_events/4,
+ verify_events/3, verify_events/4, reformat/2, log_events/4,
join_abs_dirs/2]).
-export([ct_test_halt/1]).
@@ -117,8 +117,7 @@ end_per_suite(Config) ->
CTNode = proplists:get_value(ct_node, Config),
PrivDir = proplists:get_value(priv_dir, Config),
true = rpc:call(CTNode, code, del_path, [filename:join(PrivDir,"")]),
- cover:stop(CTNode),
- slave:stop(CTNode),
+ slave_stop(CTNode),
ok.
%%%-----------------------------------------------------------------
@@ -149,8 +148,7 @@ end_per_testcase(_TestCase, Config) ->
case wait_for_ct_stop(CTNode) of
%% Common test was not stopped to we restart node.
false ->
- cover:stop(CTNode),
- slave:stop(CTNode),
+ slave_stop(CTNode),
start_slave(Config,proplists:get_value(trace_level,Config)),
{fail, "Could not stop common_test"};
true ->
@@ -364,6 +362,14 @@ verify_events(TEvs, Evs, Config) ->
ok
end.
+verify_events(TEvs, Evs, Node, Config) ->
+ case catch verify_events1(TEvs, Evs, Node, Config) of
+ {'EXIT',Reason} ->
+ Reason;
+ _ ->
+ ok
+ end.
+
verify_events1([TestEv|_], [{TEH,#event{name=stop_logging,node=Node,data=_}}|_], Node, _)
when element(1,TestEv) == TEH, element(2,TestEv) =/= stop_logging ->
test_server:format("Failed to find ~p in the list of events!~n", [TestEv]),
@@ -612,8 +618,11 @@ locate({parallel,TEvs}, Node, Evs, Config) ->
fun({EH,#event{name=tc_auto_skip,
node=EvNode,
data={Mod,end_per_group,Reason}}}) when
- EH == TEH, EvNode == Node, Mod == M, Reason == R ->
- false;
+ EH == TEH, EvNode == Node, Mod == M ->
+ case match_data(R, Reason) of
+ match -> false;
+ _ -> true
+ end;
({EH,#event{name=stop_logging,
node=EvNode,data=_}}) when
EH == TEH, EvNode == Node ->
@@ -627,23 +636,12 @@ locate({parallel,TEvs}, Node, Evs, Config) ->
[_AutoSkip | RemEvs2] ->
{Done,RemEvs2,length(RemEvs2)}
end;
- %% match other event than test case
- (TEv={TEH,N,D}, Acc) when D == '_' ->
- case [E || E={EH,#event{name=Name,
- node=EvNode,
- data=_}} <- Evs1,
- EH == TEH, EvNode == Node, Name == N] of
- [] ->
- exit({unmatched,TEv});
- _ ->
- test_server:format("Found ~p!", [TEv]),
- Acc
- end;
(TEv={TEH,N,D}, Acc) ->
case [E || E={EH,#event{name=Name,
node=EvNode,
data=Data}} <- Evs1,
- EH == TEH, EvNode == Node, Name == N, Data == D] of
+ EH == TEH, EvNode == Node, Name == N,
+ match == match_data(D,Data)] of
[] ->
exit({unmatched,TEv});
_ ->
@@ -1002,33 +1000,39 @@ locate({TEH,Name,Data}, Node, [{TEH,#event{name=Name,
data = EvData,
node = Node}}|Evs],
Config) ->
- try match_data(Data, EvData) of
+ case match_data(Data, EvData) of
match ->
- {Config,Evs}
- catch _:_ ->
+ {Config,Evs};
+ _ ->
nomatch
end;
locate({_TEH,_Name,_Data}, _Node, [_|_Evs], _Config) ->
nomatch.
-match_data(D,D) ->
+match_data(Data, EvData) ->
+ try do_match_data(Data, EvData)
+ catch _:_ ->
+ nomatch
+ end.
+
+do_match_data(D,D) ->
match;
-match_data('_',_) ->
+do_match_data('_',_) ->
match;
-match_data(Fun,Data) when is_function(Fun) ->
+do_match_data(Fun,Data) when is_function(Fun) ->
Fun(Data);
-match_data('$proplist',Proplist) ->
- match_data(
+do_match_data('$proplist',Proplist) ->
+ do_match_data(
fun(List) ->
lists:foreach(fun({_,_}) -> ok end,List)
end,Proplist);
-match_data([H1|MatchT],[H2|ValT]) ->
- match_data(H1,H2),
- match_data(MatchT,ValT);
-match_data(Tuple1,Tuple2) when is_tuple(Tuple1),is_tuple(Tuple2) ->
- match_data(tuple_to_list(Tuple1),tuple_to_list(Tuple2));
-match_data([],[]) ->
+do_match_data([H1|MatchT],[H2|ValT]) ->
+ do_match_data(H1,H2),
+ do_match_data(MatchT,ValT);
+do_match_data(Tuple1,Tuple2) when is_tuple(Tuple1),is_tuple(Tuple2) ->
+ do_match_data(tuple_to_list(Tuple1),tuple_to_list(Tuple2));
+do_match_data([],[]) ->
match.
result_match({SkipOrFail,{ErrorInd,{Why,'_'}}},
@@ -1039,10 +1043,13 @@ result_match({SkipOrFail,{ErrorInd,{EMod,EFunc,{Why,'_'}}}},
true;
result_match({failed,{timetrap_timeout,{'$approx',Num}}},
{failed,{timetrap_timeout,Value}}) ->
- if Value >= trunc(Num-0.02*Num),
- Value =< trunc(Num+0.02*Num) -> true;
+ if Value >= trunc(Num-0.05*Num),
+ Value =< trunc(Num+0.05*Num) -> true;
true -> false
end;
+result_match({user_timetrap_error,{Why,'_'}},
+ {user_timetrap_error,{Why,_Stack}}) ->
+ true;
result_match(Result, Result) ->
true;
result_match(_, _) ->
@@ -1125,6 +1132,8 @@ reformat([{_EH,#event{name=test_start,data=_}} | Events], EH) ->
[{EH,test_start,{'DEF',{'START_TIME','LOGDIR'}}} | reformat(Events, EH)];
reformat([{_EH,#event{name=test_done,data=_}} | Events], EH) ->
[{EH,test_done,{'DEF','STOP_TIME'}} | reformat(Events, EH)];
+reformat([{_EH,#event{name=tc_logfile,data=_}} | Events], EH) ->
+ reformat(Events, EH);
reformat([{_EH,#event{name=test_stats,data=Data}} | Events], EH) ->
[{EH,test_stats,Data} | reformat(Events, EH)];
%% use this to only print the last test_stats event:
@@ -1259,3 +1268,22 @@ rm_files([F | Fs]) ->
rm_files([]) ->
ok.
+%%%-----------------------------------------------------------------
+%%%
+slave_stop(Node) ->
+ Cover = test_server:is_cover(),
+ if Cover-> cover:flush(Node);
+ true -> ok
+ end,
+ erlang:monitor_node(Node, true),
+ slave:stop(Node),
+ receive
+ {nodedown, Node} ->
+ if Cover -> cover:stop(Node);
+ true -> ok
+ end
+ after 5000 ->
+ erlang:monitor_node(Node, false),
+ receive {nodedown, Node} -> ok after 0 -> ok end %flush
+ end,
+ ok.
diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl
index b7e19f25dd..6a4a4acd80 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE.erl
@@ -58,7 +58,7 @@ end_per_testcase(TestCase, Config) ->
suite() -> [{ct_hooks,[ts_install_cth]}].
-all() ->
+all() ->
[all_suites, skip_all_suites, suite, skip_suite,
all_testcases, skip_all_testcases, testcase,
skip_testcase, all_groups, skip_all_groups, group,
@@ -67,23 +67,23 @@ all() ->
skip_group_testcase, topgroup, subgroup, skip_subgroup,
subgroup_all_testcases, skip_subgroup_all_testcases,
subgroup_testcase, skip_subgroup_testcase,
- sub_skipped_by_top, testcase_in_multiple_groups,
- order_of_tests_in_multiple_dirs_no_merge_tests,
- order_of_tests_in_multiple_suites_no_merge_tests,
- order_of_suites_in_multiple_dirs_no_merge_tests,
- order_of_groups_in_multiple_dirs_no_merge_tests,
- order_of_groups_in_multiple_suites_no_merge_tests,
- order_of_tests_in_multiple_dirs,
- order_of_tests_in_multiple_suites,
- order_of_suites_in_multiple_dirs,
- order_of_groups_in_multiple_dirs,
- order_of_groups_in_multiple_suites,
- order_of_tests_in_multiple_suites_with_skip_no_merge_tests,
- order_of_tests_in_multiple_suites_with_skip,
+ sub_skipped_by_top, testcase_many_groups,
+ order_of_tests_many_dirs_no_merge_tests,
+ order_of_tests_many_suites_no_merge_tests,
+ order_of_suites_many_dirs_no_merge_tests,
+ order_of_groups_many_dirs_no_merge_tests,
+ order_of_groups_many_suites_no_merge_tests,
+ order_of_tests_many_dirs,
+ order_of_tests_many_suites,
+ order_of_suites_many_dirs,
+ order_of_groups_many_dirs,
+ order_of_groups_many_suites,
+ order_of_tests_many_suites_with_skip_no_merge_tests,
+ order_of_tests_many_suites_with_skip,
all_plus_one_tc_no_merge_tests,
all_plus_one_tc].
-groups() ->
+groups() ->
[].
init_per_group(_GroupName, Config) ->
@@ -373,19 +373,19 @@ sub_skipped_by_top(Config) when is_list(Config) ->
%%%-----------------------------------------------------------------
%%%
-testcase_in_multiple_groups(Config) when is_list(Config) ->
+testcase_many_groups(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
TestDir = filename:join(DataDir, "groups_1"),
TestSpec = [{cases,TestDir,groups_12_SUITE,[testcase_1a,testcase_1b]},
{skip_cases,TestDir,groups_12_SUITE,[testcase_1b],"SKIPPED!"}],
- setup_and_execute(testcase_in_multiple_groups, TestSpec, Config).
+ setup_and_execute(testcase_many_groups, TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
-order_of_tests_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) ->
+order_of_tests_many_dirs_no_merge_tests(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
TestDir1 = filename:join(DataDir, "groups_1"),
@@ -395,13 +395,13 @@ order_of_tests_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) ->
{cases,TestDir2,groups_22_SUITE,[testcase_1]},
{cases,TestDir1,groups_12_SUITE,[testcase_1b]}],
- setup_and_execute(order_of_tests_in_multiple_dirs_no_merge_tests,
+ setup_and_execute(order_of_tests_many_dirs_no_merge_tests,
TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
-order_of_tests_in_multiple_suites_no_merge_tests(Config) when is_list(Config) ->
+order_of_tests_many_suites_no_merge_tests(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
TestDir1 = filename:join(DataDir, "groups_1"),
@@ -410,13 +410,13 @@ order_of_tests_in_multiple_suites_no_merge_tests(Config) when is_list(Config) ->
{cases,TestDir1,groups_11_SUITE,[testcase_1]},
{cases,TestDir1,groups_12_SUITE,[testcase_1b]}],
- setup_and_execute(order_of_tests_in_multiple_suites_no_merge_tests,
+ setup_and_execute(order_of_tests_many_suites_no_merge_tests,
TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
-order_of_suites_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) ->
+order_of_suites_many_dirs_no_merge_tests(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
TestDir1 = filename:join(DataDir, "groups_1"),
@@ -426,13 +426,13 @@ order_of_suites_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) ->
{suites,TestDir2,groups_22_SUITE},
{suites,TestDir1,groups_11_SUITE}],
- setup_and_execute(order_of_suites_in_multiple_dirs_no_merge_tests,
+ setup_and_execute(order_of_suites_many_dirs_no_merge_tests,
TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
-order_of_groups_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) ->
+order_of_groups_many_dirs_no_merge_tests(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
TestDir1 = filename:join(DataDir, "groups_1"),
@@ -442,13 +442,13 @@ order_of_groups_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) ->
{groups,TestDir2,groups_22_SUITE,test_group_1a},
{groups,TestDir1,groups_12_SUITE,test_group_1b}],
- setup_and_execute(order_of_groups_in_multiple_dirs_no_merge_tests,
+ setup_and_execute(order_of_groups_many_dirs_no_merge_tests,
TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
-order_of_groups_in_multiple_suites_no_merge_tests(Config)
+order_of_groups_many_suites_no_merge_tests(Config)
when is_list(Config) ->
DataDir = ?config(data_dir, Config),
@@ -458,13 +458,13 @@ order_of_groups_in_multiple_suites_no_merge_tests(Config)
{groups,TestDir1,groups_11_SUITE,test_group_1a},
{groups,TestDir1,groups_12_SUITE,test_group_1b}],
- setup_and_execute(order_of_groups_in_multiple_suites_no_merge_tests,
+ setup_and_execute(order_of_groups_many_suites_no_merge_tests,
TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
-order_of_tests_in_multiple_suites_with_skip_no_merge_tests(Config)
+order_of_tests_many_suites_with_skip_no_merge_tests(Config)
when is_list(Config) ->
DataDir = ?config(data_dir, Config),
@@ -477,14 +477,14 @@ order_of_tests_in_multiple_suites_with_skip_no_merge_tests(Config)
{skip_cases,TestDir1,groups_12_SUITE,[testcase_1b],"Skip it"}],
setup_and_execute(
- order_of_tests_in_multiple_suites_with_skip_no_merge_tests,
+ order_of_tests_many_suites_with_skip_no_merge_tests,
TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
-order_of_tests_in_multiple_dirs(Config) when is_list(Config) ->
+order_of_tests_many_dirs(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
TestDir1 = filename:join(DataDir, "groups_1"),
@@ -493,13 +493,13 @@ order_of_tests_in_multiple_dirs(Config) when is_list(Config) ->
{cases,TestDir2,groups_22_SUITE,[testcase_1]},
{cases,TestDir1,groups_12_SUITE,[testcase_1b]}],
- setup_and_execute(order_of_tests_in_multiple_dirs,
+ setup_and_execute(order_of_tests_many_dirs,
TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
-order_of_tests_in_multiple_suites(Config) when is_list(Config) ->
+order_of_tests_many_suites(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
TestDir1 = filename:join(DataDir, "groups_1"),
@@ -507,13 +507,13 @@ order_of_tests_in_multiple_suites(Config) when is_list(Config) ->
{cases,TestDir1,groups_11_SUITE,[testcase_1]},
{cases,TestDir1,groups_12_SUITE,[testcase_1b]}],
- setup_and_execute(order_of_tests_in_multiple_suites,
+ setup_and_execute(order_of_tests_many_suites,
TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
-order_of_suites_in_multiple_dirs(Config) when is_list(Config) ->
+order_of_suites_many_dirs(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
TestDir1 = filename:join(DataDir, "groups_1"),
@@ -522,13 +522,13 @@ order_of_suites_in_multiple_dirs(Config) when is_list(Config) ->
{suites,TestDir2,groups_22_SUITE},
{suites,TestDir1,groups_11_SUITE}],
- setup_and_execute(order_of_suites_in_multiple_dirs,
+ setup_and_execute(order_of_suites_many_dirs,
TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
-order_of_groups_in_multiple_dirs(Config) when is_list(Config) ->
+order_of_groups_many_dirs(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
TestDir1 = filename:join(DataDir, "groups_1"),
@@ -537,13 +537,13 @@ order_of_groups_in_multiple_dirs(Config) when is_list(Config) ->
{groups,TestDir2,groups_22_SUITE,test_group_1a},
{groups,TestDir1,groups_12_SUITE,test_group_1b}],
- setup_and_execute(order_of_groups_in_multiple_dirs,
+ setup_and_execute(order_of_groups_many_dirs,
TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
-order_of_groups_in_multiple_suites(Config) when is_list(Config) ->
+order_of_groups_many_suites(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
TestDir1 = filename:join(DataDir, "groups_1"),
@@ -551,13 +551,13 @@ order_of_groups_in_multiple_suites(Config) when is_list(Config) ->
{groups,TestDir1,groups_11_SUITE,test_group_1a},
{groups,TestDir1,groups_12_SUITE,test_group_1b}],
- setup_and_execute(order_of_groups_in_multiple_suites,
+ setup_and_execute(order_of_groups_many_suites,
TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
-order_of_tests_in_multiple_suites_with_skip(Config) when is_list(Config) ->
+order_of_tests_many_suites_with_skip(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
TestDir1 = filename:join(DataDir, "groups_1"),
@@ -567,7 +567,7 @@ order_of_tests_in_multiple_suites_with_skip(Config) when is_list(Config) ->
{cases,TestDir1,groups_11_SUITE,[testcase_2]},
{skip_cases,TestDir1,groups_12_SUITE,[testcase_1b],"Skip it!"}],
- setup_and_execute(order_of_tests_in_multiple_suites_with_skip,
+ setup_and_execute(order_of_tests_many_suites_with_skip,
TestSpec, Config).
%%%-----------------------------------------------------------------
@@ -1204,10 +1204,10 @@ test_events(sub_skipped_by_top) ->
{negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
];
-test_events(testcase_in_multiple_groups) ->
+test_events(testcase_many_groups) ->
[];
-test_events(order_of_tests_in_multiple_dirs_no_merge_tests) ->
+test_events(order_of_tests_many_dirs_no_merge_tests) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,tc_start,{groups_12_SUITE,testcase_1a}},
{?eh,tc_done, {groups_12_SUITE,testcase_1a,
@@ -1219,7 +1219,7 @@ test_events(order_of_tests_in_multiple_dirs_no_merge_tests) ->
{failed,{error,{test_case_failed,no_group_data}}}}},
{?eh,stop_logging,[]}
];
-test_events(order_of_tests_in_multiple_suites_no_merge_tests) ->
+test_events(order_of_tests_many_suites_no_merge_tests) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,tc_start,{groups_12_SUITE,testcase_1a}},
{?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}},
@@ -1229,7 +1229,7 @@ test_events(order_of_tests_in_multiple_suites_no_merge_tests) ->
{?eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}},
{?eh,stop_logging,[]}
];
-test_events(order_of_suites_in_multiple_dirs_no_merge_tests) ->
+test_events(order_of_suites_many_dirs_no_merge_tests) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,tc_start,{groups_12_SUITE,init_per_suite}},
{?eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}},
@@ -1244,7 +1244,7 @@ test_events(order_of_suites_in_multiple_dirs_no_merge_tests) ->
{?eh,tc_start,{groups_11_SUITE,end_per_suite}},
{?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
{?eh,stop_logging,[]}];
-test_events(order_of_groups_in_multiple_dirs_no_merge_tests) ->
+test_events(order_of_groups_many_dirs_no_merge_tests) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}},
@@ -1257,7 +1257,7 @@ test_events(order_of_groups_in_multiple_dirs_no_merge_tests) ->
{?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}},
{?eh,stop_logging,[]}];
-test_events(order_of_groups_in_multiple_suites_no_merge_tests) ->
+test_events(order_of_groups_many_suites_no_merge_tests) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}},
@@ -1270,7 +1270,7 @@ test_events(order_of_groups_in_multiple_suites_no_merge_tests) ->
{?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}},
{?eh,stop_logging,[]}];
-test_events(order_of_tests_in_multiple_suites_with_skip_no_merge_tests) ->
+test_events(order_of_tests_many_suites_with_skip_no_merge_tests) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,tc_start,{groups_12_SUITE,testcase_1a}},
{?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}},
@@ -1282,7 +1282,7 @@ test_events(order_of_tests_in_multiple_suites_with_skip_no_merge_tests) ->
{?eh,stop_logging,[]}
];
-test_events(order_of_tests_in_multiple_dirs) ->
+test_events(order_of_tests_many_dirs) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,tc_start,{groups_12_SUITE,testcase_1a}},
{?eh,tc_done,
@@ -1296,7 +1296,7 @@ test_events(order_of_tests_in_multiple_dirs) ->
{?eh,tc_done,{groups_22_SUITE,testcase_1,ok}},
{?eh,stop_logging,[]}
];
-test_events(order_of_tests_in_multiple_suites) ->
+test_events(order_of_tests_many_suites) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,tc_start,{groups_12_SUITE,testcase_1a}},
{?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}},
@@ -1308,7 +1308,7 @@ test_events(order_of_tests_in_multiple_suites) ->
{?eh,tc_done,{groups_11_SUITE,testcase_1,ok}},
{?eh,stop_logging,[]}
];
-test_events(order_of_suites_in_multiple_dirs) ->
+test_events(order_of_suites_many_dirs) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,tc_start,{groups_12_SUITE,init_per_suite}},
{?eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}},
@@ -1325,7 +1325,7 @@ test_events(order_of_suites_in_multiple_dirs) ->
{?eh,tc_start,{groups_22_SUITE,end_per_suite}},
{?eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}},
{?eh,stop_logging,[]}];
-test_events(order_of_groups_in_multiple_dirs) ->
+test_events(order_of_groups_many_dirs) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}},
@@ -1338,7 +1338,7 @@ test_events(order_of_groups_in_multiple_dirs) ->
{?eh,tc_done, {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}},
{?eh,stop_logging,[]}];
-test_events(order_of_groups_in_multiple_suites) ->
+test_events(order_of_groups_many_suites) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}},
@@ -1352,7 +1352,7 @@ test_events(order_of_groups_in_multiple_suites) ->
{?eh,stop_logging,[]}];
-test_events(order_of_tests_in_multiple_suites_with_skip) ->
+test_events(order_of_tests_many_suites_with_skip) ->
[{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,tc_start,{groups_12_SUITE,testcase_1a}},
{?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}},
diff --git a/lib/common_test/test/ct_testspec_2_SUITE.erl b/lib/common_test/test/ct_testspec_2_SUITE.erl
index 9d2dc84ad3..beaba0eaac 100644
--- a/lib/common_test/test/ct_testspec_2_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_2_SUITE.erl
@@ -479,7 +479,7 @@ multiple_specs(_Config) ->
"multiple_specs.1.spec"),
SpecFile2 = ct_test_support:write_testspec(Spec2,SpecDir,
"multiple_specs.2.spec"),
- FileResult = ct_testspec:collect_tests_from_file([SpecFile1,SpecFile2],
+ FileResult = ct_testspec:collect_tests_from_file([[SpecFile1,SpecFile2]],
false),
ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
@@ -490,7 +490,7 @@ multiple_specs(_Config) ->
[{Node2,get_absdir(filename:join(SpecDir,CfgDir))} || CfgDir <- CfgDir2]],
LogDirV = get_absdir(filename:join(SpecDir,"../logs")),
- Verify = #testspec{merge_tests = false,
+ Verify = #testspec{merge_tests = true,
spec_dir = SpecDir,
nodes = [{undefined,Node},{n1,Node1},{n2,Node2}],
alias = [{to1,TO1V},{to2,TO2V}],
@@ -500,22 +500,13 @@ multiple_specs(_Config) ->
logdir = [{Node,LogDirV},{Node1,LogDirV},{Node2,LogDirV},"."],
config = CFGs,
tests = [{{Node1,TO1V},
- [{x_SUITE,[all]}]},
- {{Node1,TO1V},
- [{y_SUITE,[{g1,all},{g2,all}]}]},
- {{Node1,TO1V},
- [{y_SUITE,[tc1,tc2]}]},
- {{Node1,TO1V},
- [{z_SUITE,[{all,{skip,"skipped"}}]}]},
- {{Node2,TO2V},
- [{x_SUITE,[all]}]},
- {{Node2,TO2V},
- [{y_SUITE,[all]}]},
- {{Node2,TO2V},
- [{x_SUITE,[{{g1,all},{skip,"skipped"}},
- {{g2,all},{skip,"skipped"}}]}]},
+ [{x_SUITE,[all]},
+ {y_SUITE,[{g1,all},{g2,all},tc1,tc2]},
+ {z_SUITE,[{all,{skip,"skipped"}}]}]},
{{Node2,TO2V},
- [{y_SUITE,[{tc1,{skip,"skipped"}},
+ [{x_SUITE,[all,{{g1,all},{skip,"skipped"}},
+ {{g2,all},{skip,"skipped"}}]},
+ {y_SUITE,[all,{tc1,{skip,"skipped"}},
{tc2,{skip,"skipped"}}]}]}]},
verify_result(Verify,FileResult,FileResult).
@@ -524,7 +515,7 @@ multiple_specs(_Config) ->
%%%
misc_config_terms(_Config) ->
CfgDir = "../cfgs/to1",
-
+ TODir = "../tests/to1",
Spec =
[{node,x,n1@h1},{node,y,n2@h2},
@@ -554,7 +545,9 @@ misc_config_terms(_Config) ->
{create_priv_dir,[auto_per_tc]},
{create_priv_dir,n1@h1,[manual_per_tc]},
- {create_priv_dir,n2@h2,[auto_per_run]}
+ {create_priv_dir,n2@h2,[auto_per_run]},
+
+ {suites,n1@h1,TODir,[x_SUITE]}
],
{ok,SpecDir} = file:get_cwd(),
@@ -599,7 +592,9 @@ misc_config_terms(_Config) ->
{n2@h2,CSS2}],
create_priv_dir = [{Node,[auto_per_tc]},
{n1@h1,[manual_per_tc]},
- {n2@h2,[auto_per_run]}]
+ {n2@h2,[auto_per_run]}],
+ tests = [{{n1@h1,get_absdir(filename:join(SpecDir,TODir))},
+ [{x_SUITE,[all]}]}]
},
verify_result(Verify,ListResult,FileResult).
@@ -688,10 +683,10 @@ define_names_1(_Config) ->
%%% HELP FUNCTIONS
%%%-----------------------------------------------------------------
-verify_result(Verify,ListResult,FileResult) ->
+verify_result(VerificationRec,ListResult,FileResult) ->
{_,TSLTuples} = rec2proplist(ListResult),
{_,TSFTuples} = rec2proplist(FileResult),
- {_,VTuples} = rec2proplist(Verify),
+ {_,VTuples} = rec2proplist(VerificationRec),
VResult =
(catch lists:foldl(fun({Tag,Val},{[{Tag,Val}|TSL],[{Tag,Val}|TSF]}) ->
{TSL,TSF};
@@ -720,6 +715,8 @@ read_config(S) ->
rec2proplist(E={error,_What}) ->
exit({invalid_testspec_record,E});
+rec2proplist([{Specs,Rec}]) when is_list(Specs) ->
+ rec2proplist(Rec);
rec2proplist(Rec) ->
[RecName|RecList] = tuple_to_list(Rec),
FieldNames =
diff --git a/lib/common_test/test/ct_testspec_3_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE.erl
new file mode 100644
index 0000000000..e2f57c2c64
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE.erl
@@ -0,0 +1,745 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_testspec_1_SUITE
+%%%
+%%% Description:
+%%% Test test specifications
+%%%
+%%% The suites used for the test are located in the data directory.
+%%%-------------------------------------------------------------------
+-module(ct_testspec_3_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Config1 = ct_test_support:init_per_suite(Config),
+ SpecsDir1 = filename:join(DataDir, "specs1"),
+ SpecsDir2 = filename:join(DataDir, "specs2"),
+ [{specs_dir1,SpecsDir1},{specs_dir2,SpecsDir2} | Config1].
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [start_separate,
+ start_join,
+ incl_separate1,
+ incl_separate2,
+ incl_join1,
+ incl_join2,
+ incl_both1,
+ incl_both2,
+ incl_both_and_join1,
+ incl_both_and_join2,
+ rec_incl_separate1,
+ rec_incl_separate2,
+ rec_incl_join1,
+ rec_incl_join2,
+ rec_incl_separate_join1,
+ rec_incl_separate_join2,
+ rec_incl_join_separate1,
+ rec_incl_join_separate2
+ ].
+
+groups() ->
+ [].
+
+init_per_group(_GroupName, Config) ->
+ Config.
+
+end_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+
+start_separate(Config) ->
+ Specs = [fname(specs_dir1, "flat_spec1", Config),
+ fname(specs_dir2, "flat_spec2", Config)],
+ setup_and_execute(start_separate, Specs, [], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+start_join(Config) ->
+ Specs = [fname(specs_dir1, "flat_spec1", Config),
+ fname(specs_dir2, "flat_spec2", Config)],
+ setup_and_execute(start_join, Specs, [{join_specs,true}], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+incl_separate1(Config) ->
+ Specs = [fname(specs_dir1, "spec_sep1", Config),
+ fname(specs_dir2, "spec_sep2", Config)],
+ setup_and_execute(incl_separate1, Specs, [], Config).
+
+incl_separate2(Config) ->
+ Specs = [fname(specs_dir1, "spec_sep1", Config),
+ fname(specs_dir2, "spec_sep2", Config)],
+ setup_and_execute(incl_separate2, Specs, [{join_specs,true}], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+incl_join1(Config) ->
+ Specs = [fname(specs_dir1, "spec_join1", Config),
+ fname(specs_dir2, "spec_join2", Config)],
+ setup_and_execute(incl_join1, Specs, [], Config).
+
+incl_join2(Config) ->
+ Specs = [fname(specs_dir1, "spec_join1", Config),
+ fname(specs_dir2, "spec_join2", Config)],
+ setup_and_execute(incl_join2, Specs, [{join_specs,true}], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+incl_both1(Config) ->
+ Specs = [fname(specs_dir1, "spec_both1", Config),
+ fname(specs_dir2, "spec_both2", Config)],
+ setup_and_execute(incl_both1, Specs, [], Config).
+
+incl_both2(Config) ->
+ Specs = [fname(specs_dir1, "spec_both1", Config),
+ fname(specs_dir2, "spec_both2", Config)],
+ setup_and_execute(incl_both2, Specs, [{join_specs,true}], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+incl_both_and_join1(Config) ->
+ Specs = [fname(specs_dir1, "spec_both_join1", Config),
+ fname(specs_dir2, "spec_both_join2", Config)],
+ setup_and_execute(incl_both_and_join1, Specs, [], Config).
+
+incl_both_and_join2(Config) ->
+ Specs = [fname(specs_dir1, "spec_both_join1", Config),
+ fname(specs_dir2, "spec_both_join2", Config)],
+ setup_and_execute(incl_both_and_join2, Specs, [{join_specs,true}], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+rec_incl_separate1(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_sep1", Config),
+ fname(specs_dir2, "rec_spec_sep2", Config)],
+ setup_and_execute(rec_incl_separate1, Specs, [], Config).
+
+rec_incl_separate2(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_sep1", Config),
+ fname(specs_dir2, "rec_spec_sep2", Config)],
+ setup_and_execute(rec_incl_separate2, Specs, [{join_specs,true}], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+rec_incl_join1(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_join1", Config),
+ fname(specs_dir2, "rec_spec_join2", Config)],
+ setup_and_execute(rec_incl_join1, Specs, [], Config).
+
+rec_incl_join2(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_join1", Config),
+ fname(specs_dir2, "rec_spec_join2", Config)],
+ setup_and_execute(rec_incl_join2, Specs, [{join_specs,true}], Config).
+
+
+%%%-----------------------------------------------------------------
+%%%
+
+rec_incl_separate_join1(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_sep_join1", Config),
+ fname(specs_dir2, "rec_spec_sep_join2", Config)],
+ setup_and_execute(rec_incl_separate_join1, Specs, [], Config).
+
+rec_incl_separate_join2(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_sep_join1", Config),
+ fname(specs_dir2, "rec_spec_sep_join2", Config)],
+ setup_and_execute(rec_incl_separate_join2, Specs,
+ [{join_specs,true}], Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+rec_incl_join_separate1(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_join_sep1", Config),
+ fname(specs_dir2, "rec_spec_join_sep2", Config)],
+ setup_and_execute(rec_incl_join_separate1, Specs, [], Config).
+
+rec_incl_join_separate2(Config) ->
+ Specs = [fname(specs_dir1, "rec_spec_join_sep1", Config),
+ fname(specs_dir2, "rec_spec_join_sep2", Config)],
+ setup_and_execute(rec_incl_join_separate2, Specs,
+ [{join_specs,true}], Config).
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+fname(Tag, File, Config) ->
+ filename:join(?config(Tag, Config), File).
+
+check_parameter(TCID) ->
+ {ok,{config,TCID}}.
+
+read_config(TCID) ->
+ {ok,[{tcname,list_to_atom(TCID)}]}.
+
+setup_and_execute(TCName, Specs, TestOpts, Config) ->
+
+ TestID = {userconfig,{?MODULE,atom_to_list(TCName)}},
+ TestTerms = [TestID,{spec,Specs},{label,TCName}] ++ TestOpts,
+
+ {Opts,ERPid} = setup(TestTerms, Config),
+
+ case ct_test_support:run(Opts, Config) of
+ ok ->
+ ok;
+ Error ->
+ ct:pal("Error executing with opts: ~p", [Opts]),
+ exit(Error)
+ end,
+
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(TCName,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(TCName),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+setup(Test, Config) when is_tuple(Test) ->
+ setup([Test], Config);
+setup(Tests, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ Tests ++ [{event_handler,{?eh,EvHArgs}}],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+%reformat(Events, _EH) ->
+% Events.
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+events_to_check(Test) ->
+ %% 2 tests (ct:run_test + script_start) is default
+ events_to_check(Test, 2).
+
+events_to_check(_, 0) ->
+ [];
+events_to_check(Test, N) ->
+ test_events(Test) ++ events_to_check(Test, N-1).
+
+test_events(start_separate) ->
+ [{?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{3,2,15}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(start_join) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{4,4,20}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{4,8,{4,4}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(incl_separate1) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{3,2,15}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,5}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{3,2,15}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(incl_separate2) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,5}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{3,2,15}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{3,2,15}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(incl_join1) ->
+ [{?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{4,4,20}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{4,8,{4,4}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{4,4,20}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{4,8,{4,4}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(incl_join2) ->
+ [{?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{4,4,20}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{4,8,{4,4}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(incl_both1) ->
+ [{?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{3,2,15}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(incl_both2) ->
+ [{?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{4,4,20}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,ok_tc}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{4,8,{4,4}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{3,2,15}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t12_SUITE,init_per_suite}},
+ {?eh,tc_done,{t12_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t12_SUITE,end_per_suite}},
+ {?eh,tc_done,{t12_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t22_SUITE,init_per_suite}},
+ {?eh,tc_done,{t22_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{3,6,{3,3}}},
+ {?eh,tc_start,{t22_SUITE,end_per_suite}},
+ {?eh,tc_done,{t22_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]},
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{2,2,10}},
+ {?eh,tc_start,{t11_SUITE,init_per_suite}},
+ {?eh,tc_done,{t11_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{1,2,{1,1}}},
+ {?eh,tc_start,{t11_SUITE,end_per_suite}},
+ {?eh,tc_done,{t11_SUITE,end_per_suite,ok}},
+ {?eh,tc_start,{t21_SUITE,init_per_suite}},
+ {?eh,tc_done,{t21_SUITE,init_per_suite,ok}},
+ {?eh,test_stats,{2,4,{2,2}}},
+ {?eh,tc_start,{t21_SUITE,end_per_suite}},
+ {?eh,tc_done,{t21_SUITE,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}];
+
+test_events(incl_both_and_join1) -> [];
+test_events(incl_both_and_join2) -> [];
+test_events(rec_incl_separate1) -> [];
+test_events(rec_incl_separate2) -> [];
+test_events(rec_incl_join1) -> [];
+test_events(rec_incl_join2) -> [];
+test_events(rec_incl_separate_join1) -> [];
+test_events(rec_incl_separate_join2) -> [];
+test_events(rec_incl_join_separate1) -> [];
+test_events(rec_incl_join_separate2) -> [];
+
+test_events(_) ->
+ [].
+
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg11 b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg11
new file mode 100644
index 0000000000..bc672568da
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg11
@@ -0,0 +1 @@
+{file, cfg11}. \ No newline at end of file
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg12 b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg12
new file mode 100644
index 0000000000..30f2cf6857
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg12
@@ -0,0 +1 @@
+{file, cfg12}. \ No newline at end of file
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg13 b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg13
new file mode 100644
index 0000000000..1860ec78e5
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/config1/cfg13
@@ -0,0 +1 @@
+{file, cfg13}. \ No newline at end of file
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/config2/cfg21 b/lib/common_test/test/ct_testspec_3_SUITE_data/config2/cfg21
new file mode 100644
index 0000000000..b18d35443e
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/config2/cfg21
@@ -0,0 +1 @@
+{file, cfg21}. \ No newline at end of file
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/flat_spec1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/flat_spec1
new file mode 100644
index 0000000000..eff87222ea
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/flat_spec1
@@ -0,0 +1,4 @@
+{config, "../config1/cfg11"}.
+{suites, "../tests1", t11_SUITE}.
+{suites, "../tests1", t11_SUITE}.
+{suites, "../tests2", t21_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join1
new file mode 100644
index 0000000000..a3387f48a3
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join1
@@ -0,0 +1,2 @@
+{specs,join,"spec_join1"}.
+{specs,join,"../specs2/spec_join2"}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join_sep1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join_sep1
new file mode 100644
index 0000000000..fe127eb4b9
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_join_sep1
@@ -0,0 +1,5 @@
+{specs,join,"spec_sep1"}.
+{specs,join,"../specs2/spec_sep2"}.
+
+{config, "../config1/cfg13"}.
+{suites, "../tests2", t23_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep1
new file mode 100644
index 0000000000..c778aa68a6
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep1
@@ -0,0 +1,2 @@
+{specs,separate,"spec_sep1"}.
+{specs,separate,"../specs2/spec_sep2"}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep_join1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep_join1
new file mode 100644
index 0000000000..7cb5a05fff
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/rec_spec_sep_join1
@@ -0,0 +1,2 @@
+{specs,separate,"spec_join1"}.
+{specs,separate,"../specs2/spec_join2"}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both1
new file mode 100644
index 0000000000..46111614dc
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both1
@@ -0,0 +1,2 @@
+{specs, join, "../specs1/flat_spec1"}.
+{specs, separate, "../specs2/flat_spec2"}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both_join1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both_join1
new file mode 100644
index 0000000000..f52b3ed030
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_both_join1
@@ -0,0 +1,6 @@
+{specs, join, "../specs1/flat_spec1"}.
+{specs, separate, "../specs2/flat_spec2"}.
+{merge_tests,false}.
+{config, "../config1/cfg12"}.
+{suites, "../tests1", t11_SUITE}.
+{suites, "../tests2", t21_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_join1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_join1
new file mode 100644
index 0000000000..baaaf35be4
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_join1
@@ -0,0 +1 @@
+{specs, join, ["../specs2/flat_spec2", "flat_spec1"]}. \ No newline at end of file
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_sep1 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_sep1
new file mode 100644
index 0000000000..89456c35e0
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs1/spec_sep1
@@ -0,0 +1,3 @@
+{specs, separate, "../specs2/flat_spec2"}.
+{specs, separate, "flat_spec1"}.
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/flat_spec2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/flat_spec2
new file mode 100644
index 0000000000..758d1e2514
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/flat_spec2
@@ -0,0 +1,5 @@
+{merge_tests, false}.
+{config, "../config2/cfg21"}.
+{suites, "../tests1", t12_SUITE}.
+{suites, "../tests1", t12_SUITE}.
+{suites, "../tests2", t22_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join2
new file mode 100644
index 0000000000..19d3a3d8e2
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join2
@@ -0,0 +1,5 @@
+{specs,join,"spec_join2"}.
+{specs,join,"../specs1/spec_join1"}.
+
+{config, "../config1/cfg13"}.
+{suites, "../tests2", t23_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join_sep2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join_sep2
new file mode 100644
index 0000000000..930e68c847
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_join_sep2
@@ -0,0 +1,5 @@
+{specs,join,"spec_sep2"}.
+{specs,join,"../specs1/spec_sep1"}.
+
+{config, "../config1/cfg13"}.
+{suites, "../tests2", t23_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep2
new file mode 100644
index 0000000000..5026f329a7
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep2
@@ -0,0 +1,5 @@
+{specs,separate,"spec_sep2"}.
+{specs,separate,"../specs1/spec_sep1"}.
+
+{config, "../config1/cfg13"}.
+{suites, "../tests2", t23_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep_join2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep_join2
new file mode 100644
index 0000000000..17057088b4
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/rec_spec_sep_join2
@@ -0,0 +1,5 @@
+{specs,separate,"spec_join2"}.
+{specs,separate,"../specs1/spec_join1"}.
+
+{config, "../config1/cfg13"}.
+{suites, "../tests2", t23_SUITE}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both2
new file mode 100644
index 0000000000..4c83115d23
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both2
@@ -0,0 +1,4 @@
+{specs, separate, "../specs1/flat_spec1"}.
+{specs, join, "../specs2/flat_spec2"}.
+{config, "../config1/cfg12"}.
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both_join2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both_join2
new file mode 100644
index 0000000000..ad81bfb4cc
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_both_join2
@@ -0,0 +1,9 @@
+{merge_tests,true}.
+{config, "../config1/cfg12"}.
+{suites, "../tests1", t11_SUITE}.
+{suites, "../tests2", t21_SUITE}.
+{suites, "../tests1", t12_SUITE}.
+{suites, "../tests2", t22_SUITE}.
+
+{specs, separate, "../specs1/flat_spec1"}.
+{specs, join, "../specs2/flat_spec2"}.
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_join2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_join2
new file mode 100644
index 0000000000..d652dbd78f
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_join2
@@ -0,0 +1,5 @@
+{specs, join, ["../specs1/flat_spec1"]}.
+{specs, join, ["flat_spec2"]}.
+{config, "../config1/cfg12"}.
+{suites, "../tests2", t22_SUITE}.
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_sep2 b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_sep2
new file mode 100644
index 0000000000..8d37f508b8
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/specs2/spec_sep2
@@ -0,0 +1,5 @@
+{specs, separate, "../specs1/flat_spec1"}.
+{specs, separate, "flat_spec2"}.
+{config, "../config1/cfg12"}.
+{suites, "../tests2", t22_SUITE}.
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl
new file mode 100644
index 0000000000..bbe79ed3fe
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t11_SUITE.erl
@@ -0,0 +1,175 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(t11_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{require,file},
+ {require,tcname},
+ {timetrap,{seconds,1}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ %% verify that expected config file can be read
+ case {ct:get_config(tcname),ct:get_config(file,undefined,[all])} of
+ {start_separate,[cfg11]} -> ok;
+ {start_join,[cfg11,cfg21]} -> ok;
+ {incl_separate1,[cfg11]} -> ok;
+ {incl_separate2,[cfg11]} -> ok;
+ {incl_join1,[cfg21,cfg11]} -> ok;
+ {incl_join1,[cfg12,cfg11,cfg21]} -> ok;
+ {incl_join2,[cfg21,cfg11,cfg12]} -> ok;
+ {incl_both1,[cfg11]} -> ok;
+ {incl_both2,[cfg11,cfg12,cfg21]} -> ok;
+ {incl_both2,[cfg11]} -> ok;
+ _ -> ok
+
+ end,
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(autoskip_tc, Config) ->
+ exit(kaboom),
+ Config;
+
+init_per_testcase(userskip_tc, Config) ->
+ {skip,"user skipped"};
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc].
+
+%%--------------------------------------------------------------------
+%% @spec TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%% @end
+%%--------------------------------------------------------------------
+ok_tc(_) ->
+ ok.
+
+exit_tc(_) ->
+ exit(kaboom),
+ ok.
+
+to_tc(_) ->
+ ct:timetrap(1),
+ ct:sleep(100),
+ ok.
+
+autoskip_tc(_) ->
+ ok.
+
+userskip_tc(_) ->
+ ok.
+
+
+
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl
new file mode 100644
index 0000000000..810298d348
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests1/t12_SUITE.erl
@@ -0,0 +1,175 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(t12_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{require,file},
+ {require,tcname},
+ {timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ %% verify that expected config file can be read
+ case {ct:get_config(tcname),ct:get_config(file,undefined,[all])} of
+ {start_separate,[cfg21]} -> ok;
+ {start_join,[cfg11,cfg21]} -> ok;
+ {incl_separate1,[cfg21]} -> ok;
+ {incl_separate2,[cfg21]} -> ok;
+ {incl_join1,[cfg21,cfg11]} -> ok;
+ {incl_join1,[cfg12,cfg11,cfg21]} -> ok;
+ {incl_join2,[cfg21,cfg11,cfg12]} -> ok;
+ {incl_both1,[cfg21]} -> ok;
+ {incl_both1,[cfg12,cfg21]} -> ok;
+ {incl_both2,[cfg11,cfg12,cfg21]} -> ok;
+ {incl_both2,[cfg21]} -> ok;
+ _ -> ok
+ end,
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(autoskip_tc, Config) ->
+ exit(kaboom),
+ Config;
+
+init_per_testcase(userskip_tc, Config) ->
+ {skip,"user skipped"};
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc].
+
+%%--------------------------------------------------------------------
+%% @spec TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%% @end
+%%--------------------------------------------------------------------
+ok_tc(_) ->
+ ok.
+
+exit_tc(_) ->
+ exit(kaboom),
+ ok.
+
+to_tc(_) ->
+ ct:timetrap(1),
+ ct:sleep(100),
+ ok.
+
+autoskip_tc(_) ->
+ ok.
+
+userskip_tc(_) ->
+ ok.
+
+
+
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl
new file mode 100644
index 0000000000..9348cd8caf
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t21_SUITE.erl
@@ -0,0 +1,174 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(t21_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{require,file},
+ {require,tcname},
+ {timetrap,{seconds,1}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ %% verify that expected config file can be read
+ case {ct:get_config(tcname),ct:get_config(file,undefined,[all])} of
+ {start_separate,[cfg11]} -> ok;
+ {start_join,[cfg11,cfg21]} -> ok;
+ {incl_separate1,[cfg11]} -> ok;
+ {incl_separate2,[cfg11]} -> ok;
+ {incl_join1,[cfg21,cfg11]} -> ok;
+ {incl_join1,[cfg12,cfg11,cfg21]} -> ok;
+ {incl_join2,[cfg21,cfg11,cfg12]} -> ok;
+ {incl_both1,[cfg11]} -> ok;
+ {incl_both2,[cfg11,cfg12,cfg21]} -> ok;
+ {incl_both2,[cfg11]} -> ok;
+ _ -> ok
+ end,
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(autoskip_tc, Config) ->
+ exit(kaboom),
+ Config;
+
+init_per_testcase(userskip_tc, Config) ->
+ {skip,"user skipped"};
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc].
+
+%%--------------------------------------------------------------------
+%% @spec TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%% @end
+%%--------------------------------------------------------------------
+ok_tc(_) ->
+ ok.
+
+exit_tc(_) ->
+ exit(kaboom),
+ ok.
+
+to_tc(_) ->
+ ct:timetrap(1),
+ ct:sleep(100),
+ ok.
+
+autoskip_tc(_) ->
+ ok.
+
+userskip_tc(_) ->
+ ok.
+
+
+
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl
new file mode 100644
index 0000000000..a92018ec70
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t22_SUITE.erl
@@ -0,0 +1,177 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(t22_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{require,file},
+ {require,tcname},
+ {timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ %% verify that expected config file can be read
+ case {ct:get_config(tcname),ct:get_config(file,undefined,[all])} of
+ {start_separate,[cfg21]} -> ok;
+ {start_join,[cfg11,cfg21]} -> ok;
+ {incl_separate1,[cfg12]} -> ok;
+ {incl_separate1,[cfg21]} -> ok;
+ {incl_separate2,[cfg12]} -> ok;
+ {incl_separate2,[cfg21]} -> ok;
+ {incl_join1,[cfg21,cfg11]} -> ok;
+ {incl_join1,[cfg12,cfg11,cfg21]} -> ok;
+ {incl_join2,[cfg21,cfg11,cfg12]} -> ok;
+ {incl_both1,[cfg21]} -> ok;
+ {incl_both1,[cfg12,cfg21]} -> ok;
+ {incl_both2,[cfg11,cfg12,cfg21]} -> ok;
+ {incl_both2,[cfg21]} -> ok;
+ _ -> ok
+ end,
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(autoskip_tc, Config) ->
+ exit(kaboom),
+ Config;
+
+init_per_testcase(userskip_tc, Config) ->
+ {skip,"user skipped"};
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc].
+
+%%--------------------------------------------------------------------
+%% @spec TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%% @end
+%%--------------------------------------------------------------------
+ok_tc(_) ->
+ ok.
+
+exit_tc(_) ->
+ exit(kaboom),
+ ok.
+
+to_tc(_) ->
+ ct:timetrap(1),
+ ct:sleep(100),
+ ok.
+
+autoskip_tc(_) ->
+ ok.
+
+userskip_tc(_) ->
+ ok.
+
+
+
+
diff --git a/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl
new file mode 100644
index 0000000000..d01fac3144
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_3_SUITE_data/tests2/t23_SUITE.erl
@@ -0,0 +1,158 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(t23_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(autoskip_tc, Config) ->
+ exit(kaboom),
+ Config;
+
+init_per_testcase(userskip_tc, Config) ->
+ {skip,"user skipped"};
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [ok_tc, exit_tc, to_tc, autoskip_tc, userskip_tc].
+
+%%--------------------------------------------------------------------
+%% @spec TestCase(Config0) ->
+%% ok | exit() | {skip,Reason} | {comment,Comment} |
+%% {save_config,Config1} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% Comment = term()
+%% @end
+%%--------------------------------------------------------------------
+ok_tc(_) ->
+ ok.
+
+exit_tc(_) ->
+ exit(kaboom),
+ ok.
+
+to_tc(_) ->
+ ct:timetrap(1),
+ ct:sleep(100),
+ ok.
+
+autoskip_tc(_) ->
+ ok.
+
+userskip_tc(_) ->
+ ok.
+
+
+
+