aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/test
diff options
context:
space:
mode:
Diffstat (limited to 'lib/common_test/test')
-rw-r--r--lib/common_test/test/Makefile6
-rw-r--r--lib/common_test/test/ct_auto_compile_SUITE.erl187
-rw-r--r--lib/common_test/test/ct_auto_compile_SUITE_data/bad_SUITE.erl23
-rw-r--r--lib/common_test/test/ct_auto_compile_SUITE_data/dummy_SUITE.erl130
-rw-r--r--lib/common_test/test/ct_basic_html_SUITE.erl180
-rw-r--r--lib/common_test/test/ct_basic_html_SUITE_data/babbling_SUITE.erl130
-rw-r--r--lib/common_test/test/ct_error_SUITE.erl6
-rw-r--r--lib/common_test/test/ct_misc_1_SUITE.erl2
-rw-r--r--lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl26
-rw-r--r--lib/common_test/test/ct_test_support.erl50
-rw-r--r--lib/common_test/test/ct_testspec_1_SUITE.erl4
-rw-r--r--lib/common_test/test/ct_testspec_2_SUITE.erl751
-rw-r--r--lib/common_test/test/ct_verbosity_SUITE.erl244
-rw-r--r--lib/common_test/test/ct_verbosity_SUITE_data/io_test_SUITE.erl156
14 files changed, 1869 insertions, 26 deletions
diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index 4d85b84b5b..7628ada61a 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -39,6 +39,7 @@ MODULES= \
ct_sequence_1_SUITE \
ct_repeat_1_SUITE \
ct_testspec_1_SUITE \
+ ct_testspec_2_SUITE \
ct_skip_SUITE \
ct_error_SUITE \
ct_test_server_if_1_SUITE \
@@ -46,7 +47,10 @@ MODULES= \
ct_master_SUITE \
ct_misc_1_SUITE \
ct_hooks_SUITE \
- ct_netconfc_SUITE
+ ct_netconfc_SUITE \
+ ct_basic_html_SUITE \
+ ct_auto_compile_SUITE \
+ ct_verbosity_SUITE
ERL_FILES= $(MODULES:%=%.erl)
diff --git a/lib/common_test/test/ct_auto_compile_SUITE.erl b/lib/common_test/test/ct_auto_compile_SUITE.erl
new file mode 100644
index 0000000000..cc546ed30d
--- /dev/null
+++ b/lib/common_test/test/ct_auto_compile_SUITE.erl
@@ -0,0 +1,187 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_auto_compile_SUITE
+%%%
+%%% Description:
+%%%
+%%%
+%%% The suites used for the test are located in the data directory.
+%%%-------------------------------------------------------------------
+-module(ct_auto_compile_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config1 = ct_test_support:init_per_suite(Config),
+ Config1.
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [ac_flag, ac_spec].
+
+groups() ->
+ [].
+
+init_per_group(_GroupName, Config) ->
+ Config.
+
+end_per_group(_GroupName, Config) ->
+ Config.
+
+
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+ac_flag(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ PrivDir = ?config(priv_dir, Config),
+ file:copy(filename:join(DataDir, "bad_SUITE.erl"),
+ filename:join(PrivDir, "bad_SUITE.erl")),
+ Suite = filename:join(DataDir, "dummy_SUITE"),
+ compile:file(Suite, [{outdir,PrivDir}]),
+ {Opts,ERPid} = setup([{dir,PrivDir},
+ {auto_compile,false},
+ {label,"ac_flag"}],
+ Config),
+
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(ac_flag,
+ reformat(Events, ?eh),
+ PrivDir,
+ Opts),
+
+ TestEvents = events_to_check(ac_flag),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+ac_spec(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ PrivDir = ?config(priv_dir, Config),
+ file:copy(filename:join(DataDir, "bad_SUITE.erl"),
+ filename:join(PrivDir, "bad_SUITE.erl")),
+ TestSpec = [{label,ac_spec},
+ {auto_compile,false},
+ {suites,PrivDir,all}],
+ FileName = filename:join(?config(priv_dir, Config),"ac_spec.spec"),
+ {ok,Dev} = file:open(FileName, [write]),
+ [io:format(Dev, "~p.~n", [Term]) || Term <- TestSpec],
+ file:close(Dev),
+
+ {Opts,ERPid} = setup([{spec,FileName}], Config),
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(ac_spec,
+ reformat(Events, ?eh),
+ PrivDir,
+ Opts),
+
+ TestEvents = events_to_check(ac_spec),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+ %reformat(Events, _EH) ->
+ % Events.
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+events_to_check(Test) ->
+ %% 2 tests (ct:run_test + script_start) is default
+ events_to_check(Test, 2).
+
+events_to_check(_, 0) ->
+ [];
+events_to_check(Test, N) ->
+ test_events(Test) ++ events_to_check(Test, N-1).
+
+test_events(ac_flag) ->
+ [
+ {ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {ct_test_support_eh,start_info,{1,1,3}},
+ {ct_test_support_eh,tc_start,{dummy_SUITE,init_per_suite}},
+ {ct_test_support_eh,tc_done,{dummy_SUITE,init_per_suite,ok}},
+ {ct_test_support_eh,test_stats,{1,1,{1,0}}},
+ {ct_test_support_eh,tc_start,{dummy_SUITE,end_per_suite}},
+ {ct_test_support_eh,tc_done,{dummy_SUITE,end_per_suite,ok}},
+ {ct_test_support_eh,test_done,{'DEF','STOP_TIME'}},
+ {ct_test_support_eh,stop_logging,[]}
+ ];
+
+test_events(ac_spec) ->
+ [
+ {ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {ct_test_support_eh,start_info,{1,1,3}},
+ {ct_test_support_eh,tc_start,{dummy_SUITE,init_per_suite}},
+ {ct_test_support_eh,tc_done,{dummy_SUITE,init_per_suite,ok}},
+ {ct_test_support_eh,test_stats,{1,1,{1,0}}},
+ {ct_test_support_eh,tc_start,{dummy_SUITE,end_per_suite}},
+ {ct_test_support_eh,tc_done,{dummy_SUITE,end_per_suite,ok}},
+ {ct_test_support_eh,test_done,{'DEF','STOP_TIME'}},
+ {ct_test_support_eh,stop_logging,[]}
+ ].
diff --git a/lib/common_test/test/ct_auto_compile_SUITE_data/bad_SUITE.erl b/lib/common_test/test/ct_auto_compile_SUITE_data/bad_SUITE.erl
new file mode 100644
index 0000000000..6ebcb3570e
--- /dev/null
+++ b/lib/common_test/test/ct_auto_compile_SUITE_data/bad_SUITE.erl
@@ -0,0 +1,23 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+
+-module(bad_SUITE).
+
+-compile(export_all).
+
+bad_bad_suite
diff --git a/lib/common_test/test/ct_auto_compile_SUITE_data/dummy_SUITE.erl b/lib/common_test/test/ct_auto_compile_SUITE_data/dummy_SUITE.erl
new file mode 100644
index 0000000000..0b1eafc31d
--- /dev/null
+++ b/lib/common_test/test/ct_auto_compile_SUITE_data/dummy_SUITE.erl
@@ -0,0 +1,130 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+
+-module(dummy_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [ok,fail,skip].
+
+
+ok(_Config) ->
+ ok.
+
+fail(Config) ->
+ tuple_to_list(Config),
+ ok.
+
+skip(_Config) ->
+ {skip,"should be skipped"}.
diff --git a/lib/common_test/test/ct_basic_html_SUITE.erl b/lib/common_test/test/ct_basic_html_SUITE.erl
new file mode 100644
index 0000000000..a5f2e6197e
--- /dev/null
+++ b/lib/common_test/test/ct_basic_html_SUITE.erl
@@ -0,0 +1,180 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_basic_html_SUITE
+%%%
+%%% Description:
+%%%
+%%%
+%%% The suites used for the test are located in the data directory.
+%%%-------------------------------------------------------------------
+-module(ct_basic_html_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config1 = ct_test_support:init_per_suite(Config),
+ Config1.
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [basic_flag, basic_spec].
+
+groups() ->
+ [].
+
+init_per_group(_GroupName, Config) ->
+ Config.
+
+end_per_group(_GroupName, Config) ->
+ Config.
+
+
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+basic_flag(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Suites = [filename:join(DataDir, "babbling_SUITE")],
+ {Opts,ERPid} = setup([{suite,Suites},
+ {basic_html,true},
+ {label,"basic_flag"}],
+ Config),
+
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(basic_flag,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(basic_flag),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+basic_spec(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+ TestSpec = [{label,basic_spec},
+ {basic_html,true},
+ {suites,DataDir,babbling_SUITE}],
+ FileName = filename:join(?config(priv_dir, Config),"basic_spec.spec"),
+ {ok,Dev} = file:open(FileName, [write]),
+ [io:format(Dev, "~p.~n", [Term]) || Term <- TestSpec],
+ file:close(Dev),
+
+ {Opts,ERPid} = setup([{spec,FileName}], Config),
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(basic_spec,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(basic_spec),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+ %reformat(Events, _EH) ->
+ % Events.
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+events_to_check(Test) ->
+ %% 2 tests (ct:run_test + script_start) is default
+ events_to_check(Test, 2).
+
+events_to_check(_, 0) ->
+ [];
+events_to_check(Test, N) ->
+ test_events(Test) ++ events_to_check(Test, N-1).
+
+test_events(basic_flag) ->
+ [
+ {ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {ct_test_support_eh,start_info,{1,1,3}},
+ {ct_test_support_eh,tc_start,{babbling_SUITE,init_per_suite}},
+ {ct_test_support_eh,tc_done,{babbling_SUITE,init_per_suite,ok}},
+ {ct_test_support_eh,test_stats,{1,1,{1,0}}},
+ {ct_test_support_eh,tc_start,{babbling_SUITE,end_per_suite}},
+ {ct_test_support_eh,tc_done,{babbling_SUITE,end_per_suite,ok}},
+ {ct_test_support_eh,test_done,{'DEF','STOP_TIME'}},
+ {ct_test_support_eh,stop_logging,[]}
+ ];
+
+test_events(basic_spec) ->
+ [
+ {ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {ct_test_support_eh,start_info,{1,1,3}},
+ {ct_test_support_eh,tc_start,{babbling_SUITE,init_per_suite}},
+ {ct_test_support_eh,tc_done,{babbling_SUITE,init_per_suite,ok}},
+ {ct_test_support_eh,test_stats,{1,1,{1,0}}},
+ {ct_test_support_eh,tc_start,{babbling_SUITE,end_per_suite}},
+ {ct_test_support_eh,tc_done,{babbling_SUITE,end_per_suite,ok}},
+ {ct_test_support_eh,test_done,{'DEF','STOP_TIME'}},
+ {ct_test_support_eh,stop_logging,[]}
+ ].
diff --git a/lib/common_test/test/ct_basic_html_SUITE_data/babbling_SUITE.erl b/lib/common_test/test/ct_basic_html_SUITE_data/babbling_SUITE.erl
new file mode 100644
index 0000000000..d67383c606
--- /dev/null
+++ b/lib/common_test/test/ct_basic_html_SUITE_data/babbling_SUITE.erl
@@ -0,0 +1,130 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+
+-module(babbling_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,{seconds,30}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [ok,fail,skip].
+
+
+ok(_Config) ->
+ ok.
+
+fail(Config) ->
+ tuple_to_list(Config),
+ ok.
+
+skip(_Config) ->
+ {skip,"should be skipped"}.
diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl
index c9ee47e01b..338e76264e 100644
--- a/lib/common_test/test/ct_error_SUITE.erl
+++ b/lib/common_test/test/ct_error_SUITE.erl
@@ -878,11 +878,11 @@ test_events(timetrap_fun) ->
{failed,{timetrap_timeout,{'$approx',1000}}}}},
{?eh,test_stats,{0,5,{0,0}}},
{?eh,tc_start,{timetrap_5_SUITE,tc1}},
- {?eh,tc_done,{undefined,undefined,{user_timetrap_error,
+ {?eh,tc_done,{timetrap_5_SUITE,tc1,{user_timetrap_error,
{kaboom,'_'}}}},
{?eh,test_stats,{0,6,{0,0}}},
{?eh,tc_start,{timetrap_5_SUITE,tc2}},
- {?eh,tc_done,{undefined,undefined,{user_timetrap_error,
+ {?eh,tc_done,{timetrap_5_SUITE,tc2,{user_timetrap_error,
{kaboom,'_'}}}},
{?eh,test_stats,{0,7,{0,0}}},
{?eh,tc_start,{timetrap_5_SUITE,tc3}},
@@ -937,7 +937,7 @@ test_events(timetrap_fun) ->
{?eh,tc_done,{timetrap_5_SUITE,end_per_suite,ok}},
{?eh,tc_start,{timetrap_6_SUITE,init_per_suite}},
- {?eh,tc_done,{undefined,undefined,{user_timetrap_error,
+ {?eh,tc_done,{timetrap_6_SUITE,init_per_suite,{user_timetrap_error,
{kaboom,'_'}}}},
{?eh,tc_auto_skip,{timetrap_6_SUITE,tc0,
{failed,{timetrap_6_SUITE,init_per_suite,
diff --git a/lib/common_test/test/ct_misc_1_SUITE.erl b/lib/common_test/test/ct_misc_1_SUITE.erl
index cb17af9ab5..d2318de445 100644
--- a/lib/common_test/test/ct_misc_1_SUITE.erl
+++ b/lib/common_test/test/ct_misc_1_SUITE.erl
@@ -106,7 +106,7 @@ beam_me_up(Config) when is_list(Config) ->
{Opts,ERPid} = setup([{suite,Suites},{auto_compile,false}], Config),
- ok = ct_test_support:run(ct, run_test, [Opts], Config),
+ {_Ok,_Fail,_Skip} = ct_test_support:run(ct, run_test, [Opts], Config),
Events = ct_test_support:get_events(ERPid, Config),
ct_test_support:log_events(beam_me_up,
diff --git a/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl b/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl
index 423cb2999b..7704a29768 100644
--- a/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl
+++ b/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl
@@ -1,11 +1,21 @@
-%%%-------------------------------------------------------------------
-%%% @author Peter Andersson <[email protected]>
-%%% @copyright (C) 2012, Peter Andersson
-%%% @doc
-%%%
-%%% @end
-%%% Created : 23 Jan 2012 by Peter Andersson <[email protected]>
-%%%-------------------------------------------------------------------
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
-module(priv_dir_SUITE).
-compile(export_all).
diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl
index 02246b5763..80cca4a1cc 100644
--- a/lib/common_test/test/ct_test_support.erl
+++ b/lib/common_test/test/ct_test_support.erl
@@ -35,6 +35,8 @@
verify_events/3, reformat/2, log_events/4,
join_abs_dirs/2]).
+-export([ct_test_halt/1]).
+
-include_lib("kernel/include/file.hrl").
%%%-----------------------------------------------------------------
@@ -229,8 +231,9 @@ run(Opts, Config) when is_list(Opts) ->
%% use ct interface
test_server:format(Level, "~n[RUN #1] Calling ct:run_test(~p) on ~p~n",
[Opts, CTNode]),
- Result1 = rpc:call(CTNode, ct, run_test, [Opts]),
-
+ CtRunTestResult = rpc:call(CTNode, ct, run_test, [Opts]),
+ test_server:format(Level, "~n[RUN #1] Got return value ~p~n",
+ [CtRunTestResult]),
case rpc:call(CTNode, erlang, whereis, [ct_util_server]) of
undefined ->
ok;
@@ -242,17 +245,36 @@ run(Opts, Config) when is_list(Opts) ->
undefined = rpc:call(CTNode, erlang, whereis, [ct_util_server])
end,
%% use run_test interface (simulated)
- test_server:format(Level, "Saving start opts on ~p: ~p~n", [CTNode,Opts]),
- rpc:call(CTNode, application, set_env, [common_test, run_test_start_opts, Opts]),
- test_server:format(Level, "[RUN #2] Calling ct_run:script_start() on ~p~n", [CTNode]),
- Result2 = rpc:call(CTNode, ct_run, script_start, []),
- case {Result1,Result2} of
- {ok,ok} ->
+ Opts1 = [{halt_with,{?MODULE,ct_test_halt}} | Opts],
+ test_server:format(Level, "Saving start opts on ~p: ~p~n",
+ [CTNode, Opts1]),
+ rpc:call(CTNode, application, set_env,
+ [common_test, run_test_start_opts, Opts1]),
+ test_server:format(Level, "[RUN #2] Calling ct_run:script_start() on ~p~n",
+ [CTNode]),
+ ExitStatus = rpc:call(CTNode, ct_run, script_start, []),
+ test_server:format(Level, "[RUN #2] Got exit status value ~p~n",
+ [ExitStatus]),
+ case {CtRunTestResult,ExitStatus} of
+ {{_Ok,Failed,{_UserSkipped,_AutoSkipped}},1} when Failed > 0 ->
+ ok;
+ {{_Ok,0,{_UserSkipped,AutoSkipped}},ExitStatus} when AutoSkipped > 0 ->
+ case proplists:get_value(exit_status, Opts1) of
+ ignore_config when ExitStatus == 1 ->
+ {error,{wrong_exit_status,ExitStatus}};
+ _ ->
+ ok
+ end;
+ {{error,_}=Error,ExitStatus} ->
+ if ExitStatus /= 2 ->
+ {error,{wrong_exit_status,ExitStatus}};
+ ExitStatus == 2 ->
+ Error
+ end;
+ {{_Ok,0,{_UserSkipped,_AutoSkipped}},0} ->
ok;
- {E,_} when E =/= ok ->
- E;
- {_,E} when E =/= ok ->
- E
+ Unexpected ->
+ {error,{unexpected_return_value,Unexpected}}
end.
run(M, F, A, Config) ->
@@ -272,6 +294,10 @@ run({M,F,A}, InitCalls, Config) ->
[M, F, A, CTNode]),
rpc:call(CTNode, M, F, A).
+%% this is the last function that ct_run:script_start() calls, so the
+%% return value here is what rpc:call/4 above returns
+ct_test_halt(ExitStatus) ->
+ ExitStatus.
%%%-----------------------------------------------------------------
%%% wait_for_ct_stop/1
diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl
index b6dcf63fdf..693e8c6567 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE.erl
@@ -621,7 +621,9 @@ setup_and_execute(TCName, TestSpec, Config) ->
ok = ct_test_support:run(Opts, Config),
TestSpec1 = [{logdir,proplists:get_value(logdir,Opts)},
{label,proplists:get_value(label,TestTerms)} | TestSpec],
- ok = ct_test_support:run(ct, run_testspec, [TestSpec1], Config),
+ {_Ok,_Failed,{_USkipped,_ASkipped}} =
+ ct_test_support:run(ct, run_testspec, [TestSpec1], Config),
+
Events = ct_test_support:get_events(ERPid, Config),
ct_test_support:log_events(TCName,
diff --git a/lib/common_test/test/ct_testspec_2_SUITE.erl b/lib/common_test/test/ct_testspec_2_SUITE.erl
new file mode 100644
index 0000000000..93f3520f95
--- /dev/null
+++ b/lib/common_test/test/ct_testspec_2_SUITE.erl
@@ -0,0 +1,751 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_testspec_2_SUITE
+%%%
+%%% Description:
+%%% Test test specifications
+%%%
+%%% The suites used for the test are located in the data directory.
+%%%-------------------------------------------------------------------
+-module(ct_testspec_2_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/src/ct_util.hrl").
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%% suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [basic_compatible_no_nodes,
+ basic_compatible_nodes,
+ unknown_terms,
+ no_merging,
+ multiple_specs,
+ misc_config_terms,
+ define_names_1].
+
+
+%%--------------------------------------------------------------------
+%% VALID TEST SPEC TERMS (R15B02):
+%%
+%% {node,3}
+%% {cover,2}
+%% {cover,3}
+%% {config,2}
+%% {config,3}
+%% {config,4}
+%% {userconfig,2}
+%% {userconfig,3}
+%% {alias,3}
+%% {merge_tests,2}
+%% {logdir,2}
+%% {logdir,3}
+%% {logopts,2}
+%% {logopts,3}
+%% {basic_html,2}
+%% {basic_html,3}
+%% {label,2}
+%% {label,3}
+%% {event_handler,2}
+%% {event_handler,3}
+%% {event_handler,4}
+%% {ct_hooks,2}
+%% {ct_hooks,3}
+%% {enable_builtin_hooks,2}
+%% {noinput,2}
+%% {multiply_timetraps,2}
+%% {multiply_timetraps,3}
+%% {scale_timetraps,2}
+%% {scale_timetraps,3}
+%% {include,2}
+%% {include,3}
+%% {auto_compile,2}
+%% {auto_compile,3}
+%% {stylesheet,2}
+%% {stylesheet,3}
+%% {suites,3}
+%% {suites,4}
+%% {groups,4}
+%% {groups,5}
+%% {groups,6}
+%% {cases,4}
+%% {cases,5}
+%% {skip_suites,4}
+%% {skip_suites,5}
+%% {skip_groups,5}
+%% {skip_groups,6}
+%% {skip_groups,7}
+%% {skip_cases,5}
+%% {skip_cases,6}
+%% {create_priv_dir,2}
+%%
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+basic_compatible_no_nodes(_Config) ->
+
+ AliasDir1 = "../tests/to1",
+ AliasDir2 = "../tests/to2",
+ CfgDir1 = "../cfgs/to1/x.cfg",
+ CfgDir2 = ["../cfgs/to2/x.cfg","../cfgs/to2/y.cfg"],
+ LogDir = "../logs",
+ IncludeDir1 = "../../include",
+ IncludeDir2 = ["../tests/to1/include","../tests/to2/include"],
+
+ Spec =
+ [
+ {label,"basic_compatible_no_nodes"},
+ {alias,to1,AliasDir1},
+ {alias,to2,AliasDir2},
+ {config,CfgDir1},
+ {config,CfgDir2},
+ {userconfig,{?MODULE,"cfg_str1"}},
+ {userconfig,{?MODULE,"cfg_str2"}},
+ {logdir,LogDir},
+ {logopts,[no_nl]},
+ {event_handler,evh1,[1]},
+ {event_handler,[evh2,evh3],[[2,3]]},
+ {ct_hooks,[{cth_mod1,[]}]},
+ {ct_hooks,[{cth_mod2,[]}]},
+ {multiply_timetraps,2},
+ {include,IncludeDir1},
+ {include,IncludeDir2},
+ {suites,to1,[x_SUITE]},
+ {groups,to1,y_SUITE,[g1,g2]},
+ {cases,to1,y_SUITE,[tc1,tc2]},
+ {skip_suites,to1,z_SUITE,"skipped"},
+ {suites,to2,[x_SUITE,y_SUITE]},
+ {skip_groups,to2,x_SUITE,[g1,g2],"skipped"},
+ {skip_cases,to2,y_SUITE,[tc1,tc2],"skipped"}
+ ],
+
+ {ok,SpecDir} = file:get_cwd(),
+
+ ListResult = ct_testspec:collect_tests_from_list(Spec, false),
+ ct:pal("TESTSPEC RECORD FROM LIST:~n~p~n", [rec2proplist(ListResult)]),
+ SpecFile = ct_test_support:write_testspec(Spec,SpecDir,
+ "basic_compatible_no_nodes.spec"),
+ FileResult = ct_testspec:collect_tests_from_file([SpecFile], false),
+ ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
+
+ Node = node(),
+ LogDirV = get_absdir(filename:join(SpecDir,"../logs")),
+ Alias1V = get_absdir(filename:join(SpecDir,AliasDir1)),
+ Alias2V = get_absdir(filename:join(SpecDir,AliasDir2)),
+ CFGs = [{Node,get_absdir(filename:join(SpecDir,CfgDir))} ||
+ CfgDir <- [CfgDir1 | CfgDir2]],
+ Incls = [{Node,get_absdir(filename:join(SpecDir,IncludeDir))} ||
+ IncludeDir <- [IncludeDir1 | IncludeDir2]],
+
+ Verify = #testspec{spec_dir = SpecDir,
+ nodes = [{undefined,Node}],
+ init = [],
+ label = [{Node,"basic_compatible_no_nodes"}],
+ logdir = [{Node,LogDirV},"."],
+ logopts = [{Node,[no_nl]}],
+ basic_html = [],
+ cover = [],
+ config = CFGs,
+ userconfig = [{Node,{?MODULE,"cfg_str1"}},
+ {Node,{?MODULE,"cfg_str2"}}],
+ event_handler = [{Node,evh1,[1]},
+ {Node,evh2,[[2,3]]},
+ {Node,evh3,[[2,3]]}],
+ ct_hooks = [{Node,{cth_mod1,[]}},
+ {Node,{cth_mod2,[]}}],
+ enable_builtin_hooks = true,
+ noinput = false,
+ include = Incls,
+ auto_compile = [],
+ stylesheet = [],
+ multiply_timetraps = [{Node,2}],
+ scale_timetraps = [],
+ create_priv_dir = [],
+ alias = [{to1,Alias1V},{to2,Alias2V}],
+ tests = [{{Node,Alias1V},
+ [{x_SUITE,[all]},
+ {y_SUITE,[{g1,all},{g2,all},tc1,tc2]},
+ {z_SUITE,[{all,{skip,"skipped"}}]}]},
+ {{Node,Alias2V},
+ [{x_SUITE,[all,
+ {{g1,all},{skip,"skipped"}},
+ {{g2,all},{skip,"skipped"}}]},
+ {y_SUITE,[all,
+ {tc1,{skip,"skipped"}},
+ {tc2,{skip,"skipped"}}]}]}],
+ merge_tests = true},
+
+ verify_result(Verify,ListResult,FileResult).
+
+%%%-----------------------------------------------------------------
+%%%
+basic_compatible_nodes(_Config) ->
+
+ Node1 = node1@host1,
+ Node2 = node2@host2,
+ TODir1 = "../tests/to1",
+ TODir2 = "../tests/to2",
+ CfgDir1 = "../cfgs/to1/x.cfg",
+ CfgDir2 = ["../cfgs/to2/x.cfg","../cfgs/to2/y.cfg"],
+ LogDir = "../logs",
+ MasterLogDir = "../master_logs",
+ IncludeDir1 = "../../include",
+ IncludeDir2 = ["../tests/to1/include","../tests/to2/include"],
+
+ Spec =
+ [
+ {node,n1,Node1},
+ {node,n2,Node2},
+ {init,[n1],[{node_start,[{callback_module,cbm}]}]},
+ {init,n2,[{node_start,[]}]},
+ {init,all_nodes,{eval,{mod,func,[]}}},
+ {label,"basic_compatible_nodes"},
+ {label,n1,basic_compatible_nodes_1},
+ {config,n1,CfgDir1},
+ {config,n2,CfgDir2},
+ {userconfig,{?MODULE,"cfg_str1"}},
+ {userconfig,{?MODULE,"cfg_str2"}},
+ {logdir,all_nodes,LogDir},
+ {logdir,master,MasterLogDir},
+ {logopts,node2@host2,[no_nl]},
+ {event_handler,master,evh1,[1]},
+ {event_handler,[n1,n2],[evh2,evh3],[[2,3]]},
+ {ct_hooks,all_nodes,[{cth_mod1,[]}]},
+ {ct_hooks,[{cth_mod2,[]}]},
+ {multiply_timetraps,node1@host1,2},
+ {include,n1,IncludeDir1},
+ {include,[n1,n2],IncludeDir2},
+ {suites,n1,TODir1,[x_SUITE]},
+ {groups,n1,TODir1,y_SUITE,[g1,g2]},
+ {cases,n1,TODir1,y_SUITE,[tc1,tc2]},
+ {skip_suites,n1,TODir1,z_SUITE,"skipped"},
+ {suites,n2,TODir2,[x_SUITE,y_SUITE]},
+ {skip_groups,n2,TODir2,x_SUITE,[g1,g2],"skipped"},
+ {skip_cases,n2,TODir2,y_SUITE,[tc1,tc2],"skipped"}
+ ],
+
+ {ok,SpecDir} = file:get_cwd(),
+
+ ListResult = ct_testspec:collect_tests_from_list(Spec, false),
+ ct:pal("TESTSPEC RECORD FROM LIST:~n~p~n", [rec2proplist(ListResult)]),
+ SpecFile = ct_test_support:write_testspec(Spec,SpecDir,
+ "basic_compatible_nodes.spec"),
+ FileResult = ct_testspec:collect_tests_from_file([SpecFile], false),
+ ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
+
+ Node = node(),
+ LogDirV = get_absdir(filename:join(SpecDir,"../logs")),
+ MasterLogDirV = get_absdir(filename:join(SpecDir,"../master_logs")),
+ TO1V = get_absdir(filename:join(SpecDir,TODir1)),
+ TO2V = get_absdir(filename:join(SpecDir,TODir2)),
+ CFGs = [{Node1,get_absdir(filename:join(SpecDir,CfgDir1))} |
+ [{Node2,get_absdir(filename:join(SpecDir,CfgDir))} || CfgDir <- CfgDir2]],
+ Incls = [{Node1,get_absdir(filename:join(SpecDir,IncludeDir1))} |
+ [{Node1,get_absdir(filename:join(SpecDir,IncludeDir))} ||
+ IncludeDir <- IncludeDir2] ++
+ [{Node2,get_absdir(filename:join(SpecDir,IncludeDir))} ||
+ IncludeDir <- IncludeDir2]],
+
+ Verify = #testspec{spec_dir = SpecDir,
+ nodes = [{undefined,Node},{n1,Node1},{n2,Node2}],
+ init = [{Node1,[{node_start,[{callback_module,cbm}]},
+ {eval,[{mod,func,[]}]}]},
+ {Node2,[{node_start,[{callback_module,ct_slave}]},
+ {eval,[{mod,func,[]}]}]},
+ {Node,[{node_start,[]},
+ {eval,[{mod,func,[]}]}]}],
+ label = [{Node,"basic_compatible_nodes"},
+ {Node2,"basic_compatible_nodes"},
+ {Node1,basic_compatible_nodes_1}],
+ logdir = [{Node,LogDirV},{Node1,LogDirV},{Node2,LogDirV},
+ {master,MasterLogDirV},"."],
+ logopts = [{Node2,[no_nl]}],
+ basic_html = [],
+ cover = [],
+ config = CFGs,
+ userconfig = [{Node,{?MODULE,"cfg_str1"}},
+ {Node1,{?MODULE,"cfg_str1"}},
+ {Node2,{?MODULE,"cfg_str1"}},
+ {Node,{?MODULE,"cfg_str2"}},
+ {Node1,{?MODULE,"cfg_str2"}},
+ {Node2,{?MODULE,"cfg_str2"}}],
+ event_handler = [{master,evh1,[1]},
+ {Node1,evh2,[[2,3]]},
+ {Node1,evh3,[[2,3]]},
+ {Node2,evh2,[[2,3]]},
+ {Node2,evh3,[[2,3]]}],
+ ct_hooks = [{Node,{cth_mod1,[]}},
+ {Node1,{cth_mod1,[]}},
+ {Node2,{cth_mod1,[]}},
+ {Node,{cth_mod2,[]}},
+ {Node1,{cth_mod2,[]}},
+ {Node2,{cth_mod2,[]}}],
+ enable_builtin_hooks = true,
+ noinput = false,
+ include = Incls,
+ auto_compile = [],
+ stylesheet = [],
+ multiply_timetraps = [{Node1,2}],
+ scale_timetraps = [],
+ create_priv_dir = [],
+ tests = [{{Node1,TO1V},
+ [{x_SUITE,[all]},
+ {y_SUITE,[{g1,all},{g2,all},tc1,tc2]},
+ {z_SUITE,[{all,{skip,"skipped"}}]}]},
+ {{Node2,TO2V},
+ [{x_SUITE,[all,
+ {{g1,all},{skip,"skipped"}},
+ {{g2,all},{skip,"skipped"}}]},
+ {y_SUITE,[all,
+ {tc1,{skip,"skipped"}},
+ {tc2,{skip,"skipped"}}]}]}],
+ merge_tests = true},
+
+ verify_result(Verify,ListResult,FileResult).
+
+%%%-----------------------------------------------------------------
+%%%
+unknown_terms(Config) ->
+ PrivDir = ?config(priv_dir, Config),
+
+ Spec1 = [{suites,PrivDir,all},
+ {userdata,"I've got news for you"}],
+ {error,{undefined_term_in_spec,{userdata,_}}} =
+ (catch ct_testspec:collect_tests_from_list(Spec1, false)),
+ true = is_record(ct_testspec:collect_tests_from_list(Spec1, true),
+ testspec),
+
+ Spec2 = [{logdir,{logdir,PrivDir}}],
+ {error,{invalid_directory_name,_}} =
+ (catch ct_testspec:collect_tests_from_list(Spec2, false)),
+
+ Spec3 = [{suite,PrivDir,all}],
+ {error,{undefined_term_in_spec,{suite,_,_}}} =
+ (catch ct_testspec:collect_tests_from_list(Spec3, false)),
+ true = is_record(ct_testspec:collect_tests_from_list(Spec3, true), testspec),
+
+ Spec4 = [{suites,PrivDir,all},
+ {skip_suites,PrivDir,x_SUITE}],
+ {error,{bad_term_in_spec,{skip_suites,_,_}}} =
+ (catch ct_testspec:collect_tests_from_list(Spec4, false)),
+ {error,{bad_term_in_spec,{skip_suites,_,_}}} =
+ (catch ct_testspec:collect_tests_from_list(Spec4, true)),
+
+ Spec5 = [{configs,all_nodes,PrivDir}],
+ {error,{undefined_term_in_spec,{configs,_,_}}} =
+ (catch ct_testspec:collect_tests_from_list(Spec5, false)),
+ true = is_record(ct_testspec:collect_tests_from_list(Spec5, true), testspec),
+
+ ok.
+
+%%%-----------------------------------------------------------------
+%%%
+no_merging(_Config) ->
+ Node1 = node1@host1,
+ Node2 = node2@host2,
+ TODir1 = "../tests/to1",
+ TODir2 = "../tests/to2",
+ Spec =
+ [
+ {merge_tests,false},
+ {node,n1,Node1},
+ {node,n2,Node2},
+ {suites,n1,TODir1,[x_SUITE]},
+ {groups,n1,TODir1,y_SUITE,[g1,g2]},
+ {cases,n1,TODir1,y_SUITE,[tc1,tc2]},
+ {skip_suites,n1,TODir1,z_SUITE,"skipped"},
+ {suites,n2,TODir2,[x_SUITE,y_SUITE]},
+ {skip_groups,n2,TODir2,x_SUITE,[g1,g2],"skipped"},
+ {skip_cases,n2,TODir2,y_SUITE,[tc1,tc2],"skipped"}
+ ],
+
+ {ok,SpecDir} = file:get_cwd(),
+
+ ListResult = ct_testspec:collect_tests_from_list(Spec, false),
+ ct:pal("TESTSPEC RECORD FROM LIST:~n~p~n", [rec2proplist(ListResult)]),
+ SpecFile = ct_test_support:write_testspec(Spec,SpecDir,
+ "no_merging.spec"),
+ FileResult = ct_testspec:collect_tests_from_file([SpecFile], false),
+ ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
+
+ Node = node(),
+ TO1V = get_absdir(filename:join(SpecDir,TODir1)),
+ TO2V = get_absdir(filename:join(SpecDir,TODir2)),
+
+ Verify = #testspec{merge_tests = false,
+ spec_dir = SpecDir,
+ nodes = [{undefined,Node},{n1,Node1},{n2,Node2}],
+ tests = [{{Node1,TO1V},
+ [{x_SUITE,[all]}]},
+ {{Node1,TO1V},
+ [{y_SUITE,[{g1,all},{g2,all}]}]},
+ {{Node1,TO1V},
+ [{y_SUITE,[tc1,tc2]}]},
+ {{Node1,TO1V},
+ [{z_SUITE,[{all,{skip,"skipped"}}]}]},
+ {{Node2,TO2V},
+ [{x_SUITE,[all]}]},
+ {{Node2,TO2V},
+ [{y_SUITE,[all]}]},
+ {{Node2,TO2V},
+ [{x_SUITE,[{{g1,all},{skip,"skipped"}},
+ {{g2,all},{skip,"skipped"}}]}]},
+ {{Node2,TO2V},
+ [{y_SUITE,[{tc1,{skip,"skipped"}},
+ {tc2,{skip,"skipped"}}]}]}]},
+
+ verify_result(Verify,ListResult,FileResult).
+
+%%%-----------------------------------------------------------------
+%%%
+multiple_specs(_Config) ->
+ Node1 = node1@host1,
+ Node2 = node2@host2,
+ TODir1 = "../tests/to1",
+ TODir2 = "../tests/to2",
+ CfgDir1 = "../cfgs/to1/x.cfg",
+ CfgDir2 = ["../cfgs/to2/x.cfg","../cfgs/to2/y.cfg"],
+ LogDir = "../logs",
+ Spec1 =
+ [
+ {node,n1,Node1},
+ {node,n2,Node2},
+ {alias,to1,TODir1},
+ {alias,to2,TODir2},
+ {label,"multiple_specs1"},
+ {config,n1,CfgDir1},
+ {config,n2,CfgDir2},
+ {logdir,all_nodes,LogDir}
+ ],
+ Spec2 =
+ [
+ {merge_tests,false},
+ {label,"multiple_specs2"},
+ {suites,n1,TODir1,[x_SUITE]},
+ {groups,n1,TODir1,y_SUITE,[g1,g2]},
+ {cases,n1,TODir1,y_SUITE,[tc1,tc2]},
+ {skip_suites,n1,TODir1,z_SUITE,"skipped"},
+ {suites,n2,TODir2,[x_SUITE,y_SUITE]},
+ {skip_groups,n2,TODir2,x_SUITE,[g1,g2],"skipped"},
+ {skip_cases,n2,TODir2,y_SUITE,[tc1,tc2],"skipped"}
+ ],
+
+ {ok,SpecDir} = file:get_cwd(),
+ SpecFile1 = ct_test_support:write_testspec(Spec1,SpecDir,
+ "multiple_specs.1.spec"),
+ SpecFile2 = ct_test_support:write_testspec(Spec2,SpecDir,
+ "multiple_specs.2.spec"),
+ FileResult = ct_testspec:collect_tests_from_file([SpecFile1,SpecFile2],
+ false),
+ ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
+
+ Node = node(),
+ TO1V = get_absdir(filename:join(SpecDir,TODir1)),
+ TO2V = get_absdir(filename:join(SpecDir,TODir2)),
+ CFGs = [{Node1,get_absdir(filename:join(SpecDir,CfgDir1))} |
+ [{Node2,get_absdir(filename:join(SpecDir,CfgDir))} || CfgDir <- CfgDir2]],
+ LogDirV = get_absdir(filename:join(SpecDir,"../logs")),
+
+ Verify = #testspec{merge_tests = false,
+ spec_dir = SpecDir,
+ nodes = [{undefined,Node},{n1,Node1},{n2,Node2}],
+ alias = [{to1,TO1V},{to2,TO2V}],
+ label = [{Node,"multiple_specs1"},
+ {Node1,"multiple_specs1"},
+ {Node2,"multiple_specs1"}],
+ logdir = [{Node,LogDirV},{Node1,LogDirV},{Node2,LogDirV},"."],
+ config = CFGs,
+ tests = [{{Node1,TO1V},
+ [{x_SUITE,[all]}]},
+ {{Node1,TO1V},
+ [{y_SUITE,[{g1,all},{g2,all}]}]},
+ {{Node1,TO1V},
+ [{y_SUITE,[tc1,tc2]}]},
+ {{Node1,TO1V},
+ [{z_SUITE,[{all,{skip,"skipped"}}]}]},
+ {{Node2,TO2V},
+ [{x_SUITE,[all]}]},
+ {{Node2,TO2V},
+ [{y_SUITE,[all]}]},
+ {{Node2,TO2V},
+ [{x_SUITE,[{{g1,all},{skip,"skipped"}},
+ {{g2,all},{skip,"skipped"}}]}]},
+ {{Node2,TO2V},
+ [{y_SUITE,[{tc1,{skip,"skipped"}},
+ {tc2,{skip,"skipped"}}]}]}]},
+
+ verify_result(Verify,FileResult,FileResult).
+
+%%%-----------------------------------------------------------------
+%%%
+misc_config_terms(_Config) ->
+ CfgDir = "../cfgs/to1",
+
+ Spec =
+ [{node,x,n1@h1},{node,y,n2@h2},
+
+ {config,CfgDir,"a.cfg"},
+ {config,n1@h1,CfgDir,"b.cfg"},
+ {config,all_nodes,CfgDir,"c.cfg"},
+ {config,all_nodes,filename:join(CfgDir,"d.cfg")},
+
+ {basic_html,true},
+ {basic_html,n1@h1,false},
+ {basic_html,n2@h2,true},
+
+ {enable_builtin_hooks,false},
+
+ {noinput,true},
+
+ {auto_compile,false},
+ {auto_compile,n1@h1,true},
+ {auto_compile,n2@h2,false},
+
+ {stylesheet,"../css"},
+ {stylesheet,n1@h1,"./n1/css"},
+ {stylesheet,n2@h2,"./n2/css"},
+
+ {create_priv_dir,[auto_per_tc]},
+ {create_priv_dir,n1@h1,[manual_per_tc]},
+ {create_priv_dir,n2@h2,[auto_per_run]}
+ ],
+
+ {ok,SpecDir} = file:get_cwd(),
+
+ ListResult = ct_testspec:collect_tests_from_list(Spec, false),
+ ct:pal("TESTSPEC RECORD FROM LIST:~n~p~n", [rec2proplist(ListResult)]),
+ SpecFile = ct_test_support:write_testspec(Spec,SpecDir,
+ "misc_config_terms.spec"),
+ FileResult = ct_testspec:collect_tests_from_file([SpecFile], false),
+ ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
+
+ Node = node(),
+ CfgA = get_absdir(filename:join(filename:join(SpecDir,CfgDir), "a.cfg")),
+ CfgB = get_absdir(filename:join(filename:join(SpecDir,CfgDir), "b.cfg")),
+ CfgC = get_absdir(filename:join(filename:join(SpecDir,CfgDir), "c.cfg")),
+ CfgD = get_absdir(filename:join(filename:join(SpecDir,CfgDir), "d.cfg")),
+ CSS = get_absdir(filename:join(SpecDir,"../css")),
+ CSS1 = get_absdir(filename:join(SpecDir,"./n1/css")),
+ CSS2 = get_absdir(filename:join(SpecDir,"./n2/css")),
+
+ Verify = #testspec{spec_dir = SpecDir,
+ nodes = [{undefined,Node},{x,n1@h1},{y,n2@h2}],
+ basic_html = [{Node,true},{n1@h1,false},{n2@h2,true}],
+ config = [{Node,CfgA},
+ {n1@h1,CfgA},
+ {n2@h2,CfgA},
+ {n1@h1,CfgB},
+ {Node,CfgC},
+ {n1@h1,CfgC},
+ {n2@h2,CfgC},
+ {Node,CfgD},
+ {n1@h1,CfgD},
+ {n2@h2,CfgD}],
+ enable_builtin_hooks = false,
+ noinput = true,
+ auto_compile = [{Node,false},
+ {n1@h1,true},
+ {n2@h2,false}],
+ stylesheet = [{Node,CSS},
+ {n1@h1,CSS1},
+ {n2@h2,CSS2}],
+ create_priv_dir = [{Node,[auto_per_tc]},
+ {n1@h1,[manual_per_tc]},
+ {n2@h2,[auto_per_run]}]
+ },
+
+ verify_result(Verify,ListResult,FileResult).
+
+%%%-----------------------------------------------------------------
+%%%
+define_names_1(_Config) ->
+ Spec =
+ [
+ {define,'HOST','eniac'},
+ {define,'NODE1',testnode1},
+ {define,'NODE2',testnode2},
+ {define,'NODES',['NODE1@HOST',
+ 'NODE2@HOST']},
+ {define,'TOPDIR',".."},
+ {define,'TO1',"to1"},
+ {define,'TO2',"to2"},
+ {define,'LOGDIR',"'TOPDIR'/logdir"},
+ {define,'LOGDIR1',"'TOPDIR'/logdir1"},
+ {define,'LOGDIR2',"'TOPDIR'/logdir2"},
+ {define,'CFGDIR',"'TOPDIR'/cfgs"},
+ {define,'CFGFILES',["cfgX","cfgY"]},
+ {define,'TESTDIR',"'TOPDIR'/test"},
+ {define,'TO1DIR',"'TESTDIR'/'TO1'"},
+ {define,'TO2DIR',"'TESTDIR'/'TO2'"},
+ {define,'EXSUITE',ex_SUITE},
+ {define,'EXGRS',[g1,g2]},
+
+ {logdir,'LOGDIR'},
+ {logdir,'NODE1@HOST','LOGDIR1'},
+ {logdir,'NODE2@HOST','LOGDIR2'},
+
+ {config,["a.cfg","b.cfg"]},
+ {config,'NODES',"./'CFGDIR'/c.cfg"},
+ {config,'CFGDIR',["d.cfg","e.cfg"]},
+ {config,'NODE2@HOST','CFGDIR','CFGFILES'},
+
+ {suites,'NODE1@HOST','TO1DIR',all},
+ {suites,'NODES','TO2DIR',all},
+
+ {groups,'TO1DIR','EXSUITE','EXGRS'}
+ ],
+
+ {ok,SpecDir} = file:get_cwd(),
+
+ ListResult = ct_testspec:collect_tests_from_list(Spec, false),
+ ct:pal("TESTSPEC RECORD FROM LIST:~n~p~n", [rec2proplist(ListResult)]),
+ SpecFile = ct_test_support:write_testspec(Spec,SpecDir,
+ "define_names_1.spec"),
+ FileResult = ct_testspec:collect_tests_from_file([SpecFile], false),
+ ct:pal("TESTSPEC RECORD FROM FILE:~n~p~n", [rec2proplist(FileResult)]),
+
+ N = node(),
+ N1 = testnode1@eniac,
+ N2 = testnode2@eniac,
+ Join = fun(Dir) -> shorten_path(filename:join(SpecDir,Dir),SpecDir) end,
+
+ Verify = #testspec{spec_dir = SpecDir,
+ nodes = [{undefined,N2},
+ {undefined,N1},
+ {undefined,N}],
+ config = [{N2,Join("a.cfg")},{N2,Join("b.cfg")},
+ {N1,Join("a.cfg")},{N1,Join("b.cfg")},
+ {N,Join("a.cfg")},{N,Join("b.cfg")},
+ {N1,Join("../cfgs/c.cfg")},{N2,Join("../cfgs/c.cfg")},
+ {N2,Join("../cfgs/d.cfg")},{N2,Join("../cfgs/e.cfg")},
+ {N1,Join("../cfgs/d.cfg")},{N1,Join("../cfgs/e.cfg")},
+ {N,Join("../cfgs/d.cfg")},{N,Join("../cfgs/e.cfg")},
+ {N2,Join("../cfgs/cfgX")},{N2,Join("../cfgs/cfgY")}],
+ logdir = [{N,Join("../logdir")},
+ {N1,Join("../logdir1")},
+ {N2,Join("../logdir2")},
+ "."],
+ tests = [{{N1,Join("../test/to1")},[{all,[all]}]},
+ {{N1,Join("../test/to2")},[{all,[all]}]},
+ {{N2,Join("../test/to2")},[{all,[all]}]},
+ {{N2,Join("../test/to1")},
+ [{ex_SUITE,[{g1,all},{g2,all}]}]},
+ {{N,Join("../test/to1")},
+ [{ex_SUITE,[{g1,all},{g2,all}]}]}]
+ },
+ verify_result(Verify,ListResult,FileResult).
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+verify_result(Verify,ListResult,FileResult) ->
+ {_,TSLTuples} = rec2proplist(ListResult),
+ {_,TSFTuples} = rec2proplist(FileResult),
+ {_,VTuples} = rec2proplist(Verify),
+ VResult =
+ (catch lists:foldl(fun({Tag,Val},{[{Tag,Val}|TSL],[{Tag,Val}|TSF]}) ->
+ {TSL,TSF};
+ ({Tag,Val},{[{_Tag,TSLVal}|_TSL],[{Tag,Val}|_TSF]}) ->
+ exit({ts_list_mismatch,Tag,Val,TSLVal});
+ ({Tag,Val},{[{Tag,Val}|_TSL],[{_Tag,TSFVal}|_TSF]}) ->
+ exit({ts_file_mismatch,Tag,Val,TSFVal});
+ ({Tag,Val},{_,[{_Tag,TSFVal}|_TSF]}) ->
+ exit({ts_mismatch,Tag,Val,TSFVal})
+ end, {TSLTuples,TSFTuples}, VTuples)),
+ case VResult of
+ {'EXIT',Reason} ->
+ ct:fail(Reason);
+ _ ->
+ ok
+ end,
+ ok.
+
+
+check_parameter(S="cfg_str1") ->
+ {ok,{config,S}};
+check_parameter(S="cfg_str2") ->
+ {ok,{config,S}}.
+read_config(S) ->
+ {ok,[{cfg,S}]}.
+
+rec2proplist(E={error,_What}) ->
+ exit({invalid_testspec_record,E});
+rec2proplist(Rec) ->
+ [RecName|RecList] = tuple_to_list(Rec),
+ FieldNames =
+ if RecName == testspec ->
+ record_info(fields, testspec);
+ true ->
+ undefined
+ end,
+ {RecName,combine_names_and_vals(FieldNames,RecList)}.
+
+combine_names_and_vals([FN|FNs], [V|Vs]) ->
+ [{FN,V} | combine_names_and_vals(FNs, Vs)];
+combine_names_and_vals([], []) ->
+ [];
+combine_names_and_vals(_, _) ->
+ [].
+
+get_absdir(Dir) ->
+ shorten_path(filename:absname(Dir),Dir).
+
+shorten_path(Path,SpecDir) ->
+ case shorten_split_path(filename:split(Path),[]) of
+ [] ->
+ [Root|_] = filename:split(SpecDir),
+ Root;
+ Short ->
+ filename:join(Short)
+ end.
+
+shorten_split_path([".."|Path],SoFar) ->
+ shorten_split_path(Path,tl(SoFar));
+shorten_split_path(["."|Path],SoFar) ->
+ shorten_split_path(Path,SoFar);
+shorten_split_path([Dir|Path],SoFar) ->
+ shorten_split_path(Path,[Dir|SoFar]);
+shorten_split_path([],SoFar) ->
+ lists:reverse(SoFar).
diff --git a/lib/common_test/test/ct_verbosity_SUITE.erl b/lib/common_test/test/ct_verbosity_SUITE.erl
new file mode 100644
index 0000000000..349319de94
--- /dev/null
+++ b/lib/common_test/test/ct_verbosity_SUITE.erl
@@ -0,0 +1,244 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_verbosity_SUITE
+%%%
+%%% Description:
+%%% Test that verbosity levels vs the importance parameter works as
+%%% expected.
+%%%
+%%%-------------------------------------------------------------------
+-module(ct_verbosity_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config1 = ct_test_support:init_per_suite(Config),
+ Config1.
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [
+ no_levels,
+ general_level_low,
+ general_level_std,
+ general_level_hi,
+ change_default,
+ combine_categories,
+ testspec_only,
+ merge_with_testspec
+ ].
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+no_levels(Config) ->
+ TC = no_levels,
+ DataDir = ?config(data_dir, Config),
+ Suite = filename:join(DataDir, "io_test_SUITE"),
+ {Opts,ERPid} = setup([{suite,Suite},{label,TC}], Config),
+ ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+general_level_low(Config) ->
+ TC = general_level_low,
+ DataDir = ?config(data_dir, Config),
+ Suite = filename:join(DataDir, "io_test_SUITE"),
+ {Opts,ERPid} = setup([{suite,Suite},{label,TC},
+ {verbosity,0}], Config),
+ ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+general_level_std(Config) ->
+ TC = general_level_std,
+ DataDir = ?config(data_dir, Config),
+ Suite = filename:join(DataDir, "io_test_SUITE"),
+ {Opts,ERPid} = setup([{suite,Suite},{label,TC},
+ {verbosity,50}], Config),
+ ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+general_level_hi(Config) ->
+ TC = general_level_high,
+ DataDir = ?config(data_dir, Config),
+ Suite = filename:join(DataDir, "io_test_SUITE"),
+ {Opts,ERPid} = setup([{suite,Suite},{label,TC},
+ {verbosity,100}], Config),
+ ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+change_default(Config) ->
+ TC = change_default,
+ DataDir = ?config(data_dir, Config),
+ Suite = filename:join(DataDir, "io_test_SUITE"),
+ {Opts,ERPid} = setup([{suite,Suite},{label,TC},
+ {verbosity,[{default,49}]}], Config),
+ ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+combine_categories(Config) ->
+ TC = combine_categories,
+ DataDir = ?config(data_dir, Config),
+ Suite = filename:join(DataDir, "io_test_SUITE"),
+ {Opts,ERPid} = setup([{suite,Suite},{label,TC},
+ {verbosity,[{error,?HI_VERBOSITY},
+ {default,?LOW_VERBOSITY}]}], Config),
+ ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+testspec_only(Config) ->
+ TC = testspec_only,
+ DataDir = ?config(data_dir, Config),
+ PrivDir = ?config(priv_dir, Config),
+
+ TestSpec = [{verbosity,[{default,1},{error,75},100]},
+ {suites,DataDir,[io_test_SUITE]},
+ {label,TC}],
+
+ TestSpecName = ct_test_support:write_testspec(TestSpec, PrivDir,
+ "verbosity_1_spec"),
+ {Opts,ERPid} = setup([{spec,TestSpecName}], Config),
+
+ ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+merge_with_testspec(Config) ->
+ TC = merge_with_testspec,
+ DataDir = ?config(data_dir, Config),
+ PrivDir = ?config(priv_dir, Config),
+
+ TestSpec = [{verbosity,[{default,100},{error,100}]},
+ {suites,DataDir,[io_test_SUITE]},
+ {label,TC}],
+
+ TestSpecName = ct_test_support:write_testspec(TestSpec, PrivDir,
+ "verbosity_2_spec"),
+
+ %% below should override verbosity categories in testspec
+ {Opts,ERPid} = setup([{spec,TestSpecName},
+ {verbosity,[{default,0},0]}],
+ Config),
+
+ ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+execute(Name, Opts, ERPid, Config) ->
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(Name,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(Name),
+ ct_test_support:verify_events(TestEvents, Events, Config).
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+events_to_check(Test) ->
+ %% 2 tests (ct:run_test + script_start) is default
+ events_to_check(Test, 2).
+
+events_to_check(_, 0) ->
+ [];
+events_to_check(Test, N) ->
+ test_events(Test) ++ events_to_check(Test, N-1).
+
+
+test_events(_) ->
+ [
+ {?eh,tc_done,{io_test_SUITE,tc1,ok}},
+ {?eh,tc_done,{io_test_SUITE,tc2,ok}},
+ {?eh,tc_done,{io_test_SUITE,tc3,ok}},
+
+ {parallel,
+ [
+ {?eh,tc_start,{io_test_SUITE,tc1}},
+ {?eh,tc_start,{io_test_SUITE,tc2}},
+ {?eh,tc_start,{io_test_SUITE,tc3}},
+ {?eh,tc_done,{io_test_SUITE,tc1,ok}},
+ {?eh,tc_done,{io_test_SUITE,tc2,ok}},
+ {?eh,tc_done,{io_test_SUITE,tc3,ok}},
+ {parallel,
+ [
+ {?eh,tc_start,{io_test_SUITE,tc1}},
+ {?eh,tc_start,{io_test_SUITE,tc2}},
+ {?eh,tc_start,{io_test_SUITE,tc3}},
+ {?eh,tc_done,{io_test_SUITE,tc1,ok}},
+ {?eh,tc_done,{io_test_SUITE,tc2,ok}},
+ {?eh,tc_done,{io_test_SUITE,tc3,ok}},
+ {?eh,test_stats,{9,0,{0,0}}}
+ ]}
+ ]},
+
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ].
+
diff --git a/lib/common_test/test/ct_verbosity_SUITE_data/io_test_SUITE.erl b/lib/common_test/test/ct_verbosity_SUITE_data/io_test_SUITE.erl
new file mode 100644
index 0000000000..946e1c1989
--- /dev/null
+++ b/lib/common_test/test/ct_verbosity_SUITE_data/io_test_SUITE.erl
@@ -0,0 +1,156 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(io_test_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+ [{timetrap,{seconds,10}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%% void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%% void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%% repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+ [{g1, [parallel], [tc1,tc2,tc3,{group,g2}]},
+ {g2, [parallel], [tc1,tc2,tc3]}].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() ->
+ [tc1,tc2,tc3,{group,g1}].
+
+tc1(_C) ->
+ io:format("This is an io:format(~p)~n", [[]]),
+ ct:log("ct:log(default)", []),
+ ct:log(?STD_IMPORTANCE, "ct:log(default,~p)", [?STD_IMPORTANCE]),
+ ct:log(error, "ct:log(error)", []),
+ ct:log(error, ?STD_IMPORTANCE, "ct:log(error,~p)", [?STD_IMPORTANCE]),
+ ct:log(1, "ct:log(default,~p)", [1]),
+ ct:log(error, 1, "ct:log(error,~p)", [1]),
+ ct:log(99, "ct:log(default,~p)", [99]),
+ ct:log(error, 99, "ct:log(error,~p)", [99]),
+ ok.
+
+tc2(_C) ->
+ io:format("This is an io:format(~p)~n", [[]]),
+ ct:pal("ct:pal(default)", []),
+ ct:pal(?STD_IMPORTANCE, "ct:pal(default,~p)", [?STD_IMPORTANCE]),
+ ct:pal(error, "ct:pal(error)", []),
+ ct:pal(error, ?STD_IMPORTANCE, "ct:pal(error,~p)", [?STD_IMPORTANCE]),
+ ct:pal(1, "ct:pal(default,~p)", [1]),
+ ct:pal(error, 1, "ct:pal(error,~p)", [1]),
+ ct:pal(99, "ct:pal(default,~p)", [99]),
+ ct:pal(error, 99, "ct:pal(error,~p)", [99]),
+ ok.
+
+tc3(_C) ->
+ io:format("This is an io:format(~p)~n", [[]]),
+ ct:print("ct:print(default)", []),
+ ct:print(?STD_IMPORTANCE, "ct:print(default,~p)", [?STD_IMPORTANCE]),
+ ct:print(error, "ct:print(error)", []),
+ ct:print(error, ?STD_IMPORTANCE, "ct:print(error,~p)", [?STD_IMPORTANCE]),
+ ct:print(1, "ct:print(default,~p)", [1]),
+ ct:print(error, 1, "ct:print(error,~p)", [1]),
+ ct:print(99, "ct:print(default,~p)", [99]),
+ ct:print(error, 99, "ct:print(error,~p)", [99]),
+ ok.