Merge remote branch 'origin/peppe/common_test/verbosity_level' into maint

* origin/peppe/common_test/verbosity_level:
  Fix doc build error
  Implement verbosity levels and parameter for log printout importance

Conflicts:
	lib/common_test/src/ct_logs.erl
	lib/common_test/src/ct_run.erl
	lib/common_test/src/ct_testspec.erl
	lib/common_test/src/ct_util.hrl
	lib/common_test/test/Makefile

OTP-9625
OTP-10067

4 files changed, 420 insertions, 9 deletions

diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index 74b6c16274..7628ada61a 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -49,7 +49,8 @@ MODULES= \
 	ct_hooks_SUITE \
 	ct_netconfc_SUITE \
 	ct_basic_html_SUITE \
-	ct_auto_compile_SUITE
+	ct_auto_compile_SUITE \
+	ct_verbosity_SUITE
 
 ERL_FILES= $(MODULES:%=%.erl)
 
diff --git a/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl b/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl
index 423cb2999b..7704a29768 100644
--- a/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl
+++ b/lib/common_test/test/ct_priv_dir_SUITE_data/priv_dir_SUITE.erl
@@ -1,11 +1,21 @@
-%%%-------------------------------------------------------------------
-%%% @author Peter Andersson <[email protected]>
-%%% @copyright (C) 2012, Peter Andersson
-%%% @doc
-%%%
-%%% @end
-%%% Created : 23 Jan 2012 by Peter Andersson <[email protected]>
-%%%-------------------------------------------------------------------
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
 -module(priv_dir_SUITE).
 
 -compile(export_all).
diff --git a/lib/common_test/test/ct_verbosity_SUITE.erl b/lib/common_test/test/ct_verbosity_SUITE.erl
new file mode 100644
index 0000000000..349319de94
--- /dev/null
+++ b/lib/common_test/test/ct_verbosity_SUITE.erl
@@ -0,0 +1,244 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_verbosity_SUITE
+%%%
+%%% Description: 
+%%% Test that verbosity levels vs the importance parameter works as
+%%% expected.
+%%%
+%%%-------------------------------------------------------------------
+-module(ct_verbosity_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+    Config1 = ct_test_support:init_per_suite(Config),
+    Config1.
+
+end_per_suite(Config) ->
+    ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+    ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+    ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() -> 
+    [
+     no_levels,
+     general_level_low,
+     general_level_std,
+     general_level_hi,
+     change_default,
+     combine_categories,
+     testspec_only,
+     merge_with_testspec
+    ].
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+no_levels(Config) ->
+    TC = no_levels,
+    DataDir = ?config(data_dir, Config),
+    Suite = filename:join(DataDir, "io_test_SUITE"),
+    {Opts,ERPid} = setup([{suite,Suite},{label,TC}], Config),
+    ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+general_level_low(Config) ->
+    TC = general_level_low,
+    DataDir = ?config(data_dir, Config),
+    Suite = filename:join(DataDir, "io_test_SUITE"),
+    {Opts,ERPid} = setup([{suite,Suite},{label,TC},
+			  {verbosity,0}], Config),
+    ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+general_level_std(Config) ->
+    TC = general_level_std,
+    DataDir = ?config(data_dir, Config),
+    Suite = filename:join(DataDir, "io_test_SUITE"),
+    {Opts,ERPid} = setup([{suite,Suite},{label,TC},
+			  {verbosity,50}], Config),
+    ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+general_level_hi(Config) ->
+    TC = general_level_high,
+    DataDir = ?config(data_dir, Config),
+    Suite = filename:join(DataDir, "io_test_SUITE"),
+    {Opts,ERPid} = setup([{suite,Suite},{label,TC},
+			  {verbosity,100}], Config),
+    ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+change_default(Config) ->
+    TC = change_default,
+    DataDir = ?config(data_dir, Config),
+    Suite = filename:join(DataDir, "io_test_SUITE"),
+    {Opts,ERPid} = setup([{suite,Suite},{label,TC},
+			  {verbosity,[{default,49}]}], Config),
+    ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+combine_categories(Config) ->
+    TC = combine_categories,
+    DataDir = ?config(data_dir, Config),
+    Suite = filename:join(DataDir, "io_test_SUITE"),
+    {Opts,ERPid} = setup([{suite,Suite},{label,TC},
+			  {verbosity,[{error,?HI_VERBOSITY},
+				      {default,?LOW_VERBOSITY}]}], Config),
+    ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+testspec_only(Config) ->
+    TC = testspec_only,
+    DataDir = ?config(data_dir, Config),
+    PrivDir = ?config(priv_dir, Config),
+
+    TestSpec = [{verbosity,[{default,1},{error,75},100]},
+		{suites,DataDir,[io_test_SUITE]},
+		{label,TC}],
+
+    TestSpecName = ct_test_support:write_testspec(TestSpec, PrivDir,
+						  "verbosity_1_spec"),
+    {Opts,ERPid} = setup([{spec,TestSpecName}], Config),
+
+    ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+merge_with_testspec(Config) ->
+    TC = merge_with_testspec,
+    DataDir = ?config(data_dir, Config),
+    PrivDir = ?config(priv_dir, Config),
+
+    TestSpec = [{verbosity,[{default,100},{error,100}]},
+		{suites,DataDir,[io_test_SUITE]},
+		{label,TC}],
+
+    TestSpecName = ct_test_support:write_testspec(TestSpec, PrivDir,
+						  "verbosity_2_spec"),
+
+    %% below should override verbosity categories in testspec
+    {Opts,ERPid} = setup([{spec,TestSpecName},
+			  {verbosity,[{default,0},0]}],
+			 Config),
+
+    ok = execute(TC, Opts, ERPid, Config).
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+setup(Test, Config) ->
+    Opts0 = ct_test_support:get_opts(Config),
+    Level = ?config(trace_level, Config),
+    EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+    Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+    ERPid = ct_test_support:start_event_receiver(Config),
+    {Opts,ERPid}.
+
+execute(Name, Opts, ERPid, Config) ->
+    ok = ct_test_support:run(Opts, Config),
+    Events = ct_test_support:get_events(ERPid, Config),
+
+    ct_test_support:log_events(Name, 
+			       reformat(Events, ?eh),
+			       ?config(priv_dir, Config),
+			       Opts),
+
+    TestEvents = events_to_check(Name),
+    ct_test_support:verify_events(TestEvents, Events, Config).
+
+reformat(Events, EH) ->
+    ct_test_support:reformat(Events, EH).
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+events_to_check(Test) ->
+    %% 2 tests (ct:run_test + script_start) is default
+    events_to_check(Test, 2).
+
+events_to_check(_, 0) ->
+    [];
+events_to_check(Test, N) ->
+    test_events(Test) ++ events_to_check(Test, N-1).
+
+
+test_events(_) ->
+    [
+     {?eh,tc_done,{io_test_SUITE,tc1,ok}},
+     {?eh,tc_done,{io_test_SUITE,tc2,ok}},
+     {?eh,tc_done,{io_test_SUITE,tc3,ok}},
+
+     {parallel,
+      [
+       {?eh,tc_start,{io_test_SUITE,tc1}},
+       {?eh,tc_start,{io_test_SUITE,tc2}},
+       {?eh,tc_start,{io_test_SUITE,tc3}},
+       {?eh,tc_done,{io_test_SUITE,tc1,ok}},
+       {?eh,tc_done,{io_test_SUITE,tc2,ok}},
+       {?eh,tc_done,{io_test_SUITE,tc3,ok}},
+       {parallel,
+	[
+	 {?eh,tc_start,{io_test_SUITE,tc1}},
+	 {?eh,tc_start,{io_test_SUITE,tc2}},
+	 {?eh,tc_start,{io_test_SUITE,tc3}},
+	 {?eh,tc_done,{io_test_SUITE,tc1,ok}},
+	 {?eh,tc_done,{io_test_SUITE,tc2,ok}},
+	 {?eh,tc_done,{io_test_SUITE,tc3,ok}},
+	 {?eh,test_stats,{9,0,{0,0}}}
+	]}
+       ]},
+
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]}
+    ].
+
diff --git a/lib/common_test/test/ct_verbosity_SUITE_data/io_test_SUITE.erl b/lib/common_test/test/ct_verbosity_SUITE_data/io_test_SUITE.erl
new file mode 100644
index 0000000000..946e1c1989
--- /dev/null
+++ b/lib/common_test/test/ct_verbosity_SUITE_data/io_test_SUITE.erl
@@ -0,0 +1,156 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2012. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(io_test_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% @spec suite() -> Info
+%% Info = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+suite() ->
+    [{timetrap,{seconds,10}}].
+
+%%--------------------------------------------------------------------
+%% @spec init_per_suite(Config0) ->
+%%     Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+    Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_suite(Config0) -> void() | {save_config,Config1}
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_suite(_Config) ->
+    ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_group(GroupName, Config0) ->
+%%               Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_group(_GroupName, Config) ->
+    Config.
+    
+%%--------------------------------------------------------------------
+%% @spec end_per_group(GroupName, Config0) ->
+%%               void() | {save_config,Config1}
+%% GroupName = atom()
+%% Config0 = Config1 = [tuple()]
+%% @end
+%%--------------------------------------------------------------------
+end_per_group(_GroupName, _Config) ->
+    ok.
+
+%%--------------------------------------------------------------------
+%% @spec init_per_testcase(TestCase, Config0) ->
+%%               Config1 | {skip,Reason} | {skip_and_save,Reason,Config1}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+%%--------------------------------------------------------------------
+%% @spec end_per_testcase(TestCase, Config0) ->
+%%               void() | {save_config,Config1} | {fail,Reason}
+%% TestCase = atom()
+%% Config0 = Config1 = [tuple()]
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
+%%--------------------------------------------------------------------
+%% @spec groups() -> [Group]
+%% Group = {GroupName,Properties,GroupsAndTestCases}
+%% GroupName = atom()
+%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase]
+%% TestCase = atom()
+%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}}
+%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+%%              repeat_until_any_ok | repeat_until_any_fail
+%% N = integer() | forever
+%% @end
+%%--------------------------------------------------------------------
+groups() ->
+    [{g1, [parallel], [tc1,tc2,tc3,{group,g2}]},
+     {g2, [parallel], [tc1,tc2,tc3]}].
+
+%%--------------------------------------------------------------------
+%% @spec all() -> GroupsAndTestCases | {skip,Reason}
+%% GroupsAndTestCases = [{group,GroupName} | TestCase]
+%% GroupName = atom()
+%% TestCase = atom()
+%% Reason = term()
+%% @end
+%%--------------------------------------------------------------------
+all() -> 
+    [tc1,tc2,tc3,{group,g1}].
+
+tc1(_C) ->
+    io:format("This is an io:format(~p)~n", [[]]),
+    ct:log("ct:log(default)", []),
+    ct:log(?STD_IMPORTANCE, "ct:log(default,~p)", [?STD_IMPORTANCE]),
+    ct:log(error, "ct:log(error)", []),
+    ct:log(error, ?STD_IMPORTANCE, "ct:log(error,~p)", [?STD_IMPORTANCE]),
+    ct:log(1, "ct:log(default,~p)", [1]),
+    ct:log(error, 1, "ct:log(error,~p)", [1]),
+    ct:log(99, "ct:log(default,~p)", [99]),
+    ct:log(error, 99, "ct:log(error,~p)", [99]),
+    ok.
+
+tc2(_C) ->
+    io:format("This is an io:format(~p)~n", [[]]),
+    ct:pal("ct:pal(default)", []),
+    ct:pal(?STD_IMPORTANCE, "ct:pal(default,~p)", [?STD_IMPORTANCE]),
+    ct:pal(error, "ct:pal(error)", []),
+    ct:pal(error, ?STD_IMPORTANCE, "ct:pal(error,~p)", [?STD_IMPORTANCE]),
+    ct:pal(1, "ct:pal(default,~p)", [1]),
+    ct:pal(error, 1, "ct:pal(error,~p)", [1]),
+    ct:pal(99, "ct:pal(default,~p)", [99]),
+    ct:pal(error, 99, "ct:pal(error,~p)", [99]),
+    ok.
+
+tc3(_C) ->
+    io:format("This is an io:format(~p)~n", [[]]),
+    ct:print("ct:print(default)", []),
+    ct:print(?STD_IMPORTANCE, "ct:print(default,~p)", [?STD_IMPORTANCE]),
+    ct:print(error, "ct:print(error)", []),
+    ct:print(error, ?STD_IMPORTANCE, "ct:print(error,~p)", [?STD_IMPORTANCE]),
+    ct:print(1, "ct:print(default,~p)", [1]),
+    ct:print(error, 1, "ct:print(error,~p)", [1]),
+    ct:print(99, "ct:print(default,~p)", [99]),
+    ct:print(error, 99, "ct:print(error,~p)", [99]),
+    ok.