aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test/src/ct_run.erl
diff options
context:
space:
mode:
authorPeter Andersson <[email protected]>2012-08-23 16:38:19 +0200
committerPeter Andersson <[email protected]>2012-08-23 16:38:19 +0200
commit182564757661d6cd507d2e92d175c61602a25394 (patch)
treed64c68c681706dba93d1ce6ab16998419b57d184 /lib/common_test/src/ct_run.erl
parent4173064b7044d72f650cf459683793acc9e29352 (diff)
parentb337ca6109902c4713016d2ab366c0bfb6cc1a2e (diff)
downloadotp-182564757661d6cd507d2e92d175c61602a25394.tar.gz
otp-182564757661d6cd507d2e92d175c61602a25394.tar.bz2
otp-182564757661d6cd507d2e92d175c61602a25394.zip
Merge remote branch 'origin/peppe/common_test/verbosity_level' into maint
* origin/peppe/common_test/verbosity_level: Fix doc build error Implement verbosity levels and parameter for log printout importance Conflicts: lib/common_test/src/ct_logs.erl lib/common_test/src/ct_run.erl lib/common_test/src/ct_testspec.erl lib/common_test/src/ct_util.hrl lib/common_test/test/Makefile OTP-9625 OTP-10067
Diffstat (limited to 'lib/common_test/src/ct_run.erl')
-rw-r--r--lib/common_test/src/ct_run.erl201
1 files changed, 165 insertions, 36 deletions
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index ddb0d36c0f..7c31f88f1f 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -39,6 +39,7 @@
%% Misc internal functions
-export([variables_file_name/1,script_start1/2,run_test2/1]).
+-include("ct.hrl").
-include("ct_event.hrl").
-include("ct_util.hrl").
@@ -49,6 +50,9 @@
-define(EXIT_STATUS_TEST_CASE_FAILED, 1).
-define(EXIT_STATUS_TEST_RUN_FAILED, 2).
+-define(default_verbosity, [{default,?MAX_VERBOSITY},
+ {'$unspecified',?MAX_VERBOSITY}]).
+
-record(opts, {label,
profile,
vts,
@@ -59,6 +63,7 @@
logdir,
logopts = [],
basic_html,
+ verbosity = [],
config = [],
event_handlers = [],
ct_hooks = [],
@@ -235,6 +240,7 @@ script_start1(Parent, Args) ->
LogDir = get_start_opt(logdir, fun([LogD]) -> LogD end, Args),
LogOpts = get_start_opt(logopts, fun(Os) -> [list_to_atom(O) || O <- Os] end,
[], Args),
+ Verbosity = verbosity_args2opts(Args),
MultTT = get_start_opt(multiply_timetraps,
fun([MT]) -> list_to_integer(MT) end, 1, Args),
ScaleTT = get_start_opt(scale_timetraps,
@@ -318,6 +324,7 @@ script_start1(Parent, Args) ->
vts = Vts, shell = Shell, cover = Cover,
logdir = LogDir, logopts = LogOpts,
basic_html = BasicHtml,
+ verbosity = Verbosity,
event_handlers = EvHandlers,
ct_hooks = CTHooks,
enable_builtin_hooks = EnableBuiltinHooks,
@@ -391,9 +398,12 @@ script_start2(StartOpts = #opts{vts = undefined,
AllLogOpts = merge_vals([StartOpts#opts.logopts,
SpecStartOpts#opts.logopts]),
-
- Cover = choose_val(StartOpts#opts.cover,
- SpecStartOpts#opts.cover),
+ AllVerbosity =
+ merge_keyvals([StartOpts#opts.verbosity,
+ SpecStartOpts#opts.verbosity]),
+ Cover =
+ choose_val(StartOpts#opts.cover,
+ SpecStartOpts#opts.cover),
MultTT =
choose_val(StartOpts#opts.multiply_timetraps,
SpecStartOpts#opts.multiply_timetraps),
@@ -456,6 +466,7 @@ script_start2(StartOpts = #opts{vts = undefined,
logdir = LogDir,
logopts = AllLogOpts,
basic_html = BasicHtml,
+ verbosity = AllVerbosity,
config = SpecStartOpts#opts.config,
event_handlers = AllEvHs,
ct_hooks = AllCTHooks,
@@ -615,6 +626,7 @@ script_start4(#opts{label = Label, profile = Profile,
event_handlers = EvHandlers,
ct_hooks = CTHooks,
logopts = LogOpts,
+ verbosity = Verbosity,
enable_builtin_hooks = EnableBuiltinHooks,
logdir = LogDir, testspecs = Specs}, _Args) ->
%% label - used by ct_logs
@@ -632,7 +644,8 @@ script_start4(#opts{label = Label, profile = Profile,
{ct_hooks, CTHooks},
{enable_builtin_hooks,EnableBuiltinHooks}]) of
ok ->
- ct_util:start(interactive, LogDir),
+ ct_util:start(interactive, LogDir,
+ add_verbosity_defaults(Verbosity)),
ct_util:set_testdata({logopts, LogOpts}),
log_ts_names(Specs),
io:nl(),
@@ -676,6 +689,7 @@ script_usage() ->
"\n\t[-dir TestDir1 TestDir2 .. TestDirN] |"
"\n\t[-suite Suite [-case Case]]"
"\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]"
+ "\n\t[-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]"
"\n\t[-include InclDir1 InclDir2 .. InclDirN]"
"\n\t[-no_auto_compile]"
"\n\t[-multiply_timetraps N]"
@@ -690,11 +704,12 @@ script_usage() ->
"\n\t[-userconfig CallbackModule ConfigFile1 .. ConfigFileN]"
"\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]"
"\n\t[-logdir LogDir]"
+ "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]"
+ "\n\t[-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]"
"\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]"
- "\n\t[-stylesheet CSSFile]"
+ "\n\t[-stylesheet CSSFile]"
"\n\t[-cover CoverCfgFile]"
"\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]"
- "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]"
"\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]"
"\n\t[-include InclDir1 InclDir2 .. InclDirN]"
"\n\t[-no_auto_compile]"
@@ -710,12 +725,13 @@ script_usage() ->
"\n\t[-config ConfigFile1 ConfigFile2 .. ConfigFileN]"
"\n\t[-decrypt_key Key] | [-decrypt_file KeyFile]"
"\n\t[-logdir LogDir]"
+ "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]"
+ "\n\t[-verbosity GenVLvl | [CategoryVLvl1 .. CategoryVLvlN]]"
"\n\t[-allow_user_terms]"
"\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]"
"\n\t[-stylesheet CSSFile]"
"\n\t[-cover CoverCfgFile]"
"\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]"
- "\n\t[-logopts LogOpt1 LogOpt2 .. LogOptN]"
"\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]"
"\n\t[-include InclDir1 InclDir2 .. InclDirN]"
"\n\t[-no_auto_compile]"
@@ -847,6 +863,19 @@ run_test2(StartOpts) ->
%% logopts
LogOpts = get_start_opt(logopts, value, [], StartOpts),
+ %% verbosity
+ Verbosity =
+ get_start_opt(verbosity,
+ fun(VLvls) when is_list(VLvls) ->
+ lists:map(fun(VLvl = {_Cat,_Lvl}) ->
+ VLvl;
+ (Lvl) ->
+ {'$unspecified',Lvl}
+ end, VLvls);
+ (VLvl) when is_integer(VLvl) ->
+ [{'$unspecified',VLvl}]
+ end, [], StartOpts),
+
%% config & userconfig
CfgFiles = ct_config:get_config_file_list(StartOpts),
@@ -960,6 +989,7 @@ run_test2(StartOpts) ->
cover = Cover, step = Step, logdir = LogDir,
logopts = LogOpts, basic_html = BasicHtml,
config = CfgFiles,
+ verbosity = Verbosity,
event_handlers = EvHandlers,
ct_hooks = CTHooks,
enable_builtin_hooks = EnableBuiltinHooks,
@@ -1011,6 +1041,8 @@ run_spec_file(Relaxed,
SpecOpts#opts.logopts]),
Stylesheet = choose_val(Opts#opts.stylesheet,
SpecOpts#opts.stylesheet),
+ AllVerbosity = merge_keyvals([Opts#opts.verbosity,
+ SpecOpts#opts.verbosity]),
AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]),
Cover = choose_val(Opts#opts.cover,
SpecOpts#opts.cover),
@@ -1025,7 +1057,7 @@ run_spec_file(Relaxed,
AllInclude = merge_vals([Opts#opts.include,
SpecOpts#opts.include]),
AllCTHooks = merge_vals([Opts#opts.ct_hooks,
- SpecOpts#opts.ct_hooks]),
+ SpecOpts#opts.ct_hooks]),
EnableBuiltinHooks = choose_val(Opts#opts.enable_builtin_hooks,
SpecOpts#opts.enable_builtin_hooks),
@@ -1058,6 +1090,7 @@ run_spec_file(Relaxed,
logopts = AllLogOpts,
stylesheet = Stylesheet,
basic_html = BasicHtml,
+ verbosity = AllVerbosity,
config = AllConfig,
event_handlers = AllEvHs,
auto_compile = AutoCompile,
@@ -1320,6 +1353,7 @@ get_data_for_node(#testspec{label = Labels,
logopts = LogOptsList,
basic_html = BHs,
stylesheet = SSs,
+ verbosity = VLvls,
cover = CoverFs,
config = Cfgs,
userconfig = UsrCfgs,
@@ -1343,6 +1377,10 @@ get_data_for_node(#testspec{label = Labels,
end,
BasicHtml = proplists:get_value(Node, BHs),
Stylesheet = proplists:get_value(Node, SSs),
+ Verbosity = case proplists:get_value(Node, VLvls) of
+ undefined -> [];
+ Lvls -> Lvls
+ end,
Cover = proplists:get_value(Node, CoverFs),
MT = proplists:get_value(Node, MTs),
ST = proplists:get_value(Node, STs),
@@ -1359,6 +1397,7 @@ get_data_for_node(#testspec{label = Labels,
logopts = LogOpts,
basic_html = BasicHtml,
stylesheet = Stylesheet,
+ verbosity = Verbosity,
cover = Cover,
config = ConfigFiles,
event_handlers = EvHandlers,
@@ -1406,6 +1445,14 @@ choose_val(V0, _V1) ->
merge_vals(Vs) ->
lists:append(Vs).
+merge_keyvals(Vs) ->
+ make_unique(lists:append(Vs)).
+
+make_unique([Elem={Key,_} | Elems]) ->
+ [Elem | make_unique(proplists:delete(Key, Elems))];
+make_unique([]) ->
+ [].
+
listify([C|_]=Str) when is_integer(C) -> [Str];
listify(L) when is_list(L) -> L;
listify(E) -> [E].
@@ -1515,7 +1562,8 @@ do_run(Tests, Misc, LogDir, LogOpts) when is_list(Misc),
do_run(Tests, [], Opts1#opts{logdir = LogDir}, []);
do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) ->
- #opts{label = Label, profile = Profile, cover = Cover} = Opts,
+ #opts{label = Label, profile = Profile, cover = Cover,
+ verbosity = VLvls} = Opts,
%% label - used by ct_logs
TestLabel =
if Label == undefined -> undefined;
@@ -1557,12 +1605,16 @@ do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) ->
"ct_framework" ->
ok;
Other ->
- erlang:display(list_to_atom("Note: TEST_SERVER_FRAMEWORK = " ++ Other))
+ erlang:display(
+ list_to_atom(
+ "Note: TEST_SERVER_FRAMEWORK = " ++ Other))
end,
- case ct_util:start(Opts#opts.logdir) of
+ Verbosity = add_verbosity_defaults(VLvls),
+ case ct_util:start(Opts#opts.logdir, Verbosity) of
{error,interactive_mode} ->
io:format("CT is started in interactive mode. "
- "To exit this mode, run ct:stop_interactive().\n"
+ "To exit this mode, "
+ "run ct:stop_interactive().\n"
"To enter the interactive mode again, "
"run ct:start_interactive()\n\n",[]),
{error,interactive_mode};
@@ -1603,9 +1655,10 @@ do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) ->
{Tests1,Skip1} = final_tests(Tests,Skip,SavedErrors),
- TestResult = (catch do_run_test(Tests1, Skip1, Opts1)),
-
- case TestResult of
+ R = (catch do_run_test(Tests1, Skip1,
+ Opts1#opts{
+ verbosity=Verbosity})),
+ case R of
{EType,_} = Error when EType == user_error ;
EType == error ->
ct_util:stop(clean),
@@ -1644,11 +1697,13 @@ auto_compile(TestSuites) ->
end,
SuiteMakeErrors =
lists:flatmap(fun({TestDir,Suite} = TS) ->
- case run_make(suites, TestDir, Suite, UserInclude) of
+ case run_make(suites, TestDir,
+ Suite, UserInclude) of
{error,{make_failed,Bad}} ->
[{TS,Bad}];
{error,_} ->
- [{TS,[filename:join(TestDir,"*_SUITE")]}];
+ [{TS,[filename:join(TestDir,
+ "*_SUITE")]}];
_ ->
[]
end
@@ -1689,6 +1744,7 @@ verify_suites(TestSuites) ->
Beam = filename:join(TestDir,
atom_to_list(Suite)++
".beam"),
+
case filelib:is_regular(Beam) of
true ->
{[DS|Found],NotFound};
@@ -1727,8 +1783,8 @@ verify_suites(TestSuites) ->
ActualDir = filename:dirname(SuiteFile),
{[{ActualDir,Suite}|Found],NotFound};
false ->
- io:format(user, "Directory ~s is invalid~n",
- [Dir]),
+ io:format(user, "Directory ~s is "
+ "invalid~n", [Dir]),
Name = filename:join(Dir, atom_to_list(Suite)),
{Found,[{DS,[Name]}|NotFound]}
end
@@ -1742,7 +1798,8 @@ save_make_errors([]) ->
save_make_errors(Errors) ->
Suites = get_bad_suites(Errors,[]),
ct_logs:log("MAKE RESULTS",
- "Error compiling or locating the following suites: ~n~p",[Suites]),
+ "Error compiling or locating the "
+ "following suites: ~n~p",[Suites]),
%% save the info for logger
file:write_file(?missing_suites_info,term_to_binary(Errors)),
Errors.
@@ -1883,9 +1940,11 @@ continue(_MakeErrors) ->
case set_group_leader_same_as_shell() of
true ->
S = self(),
- io:format("Failed to compile or locate one or more test suites\n"
+ io:format("Failed to compile or locate one "
+ "or more test suites\n"
"Press \'c\' to continue or \'a\' to abort.\n"
- "Will continue in 15 seconds if no answer is given!\n"),
+ "Will continue in 15 seconds if no "
+ "answer is given!\n"),
Pid = spawn(fun() ->
case io:get_line('(c/a) ') of
"c\n" ->
@@ -1970,8 +2029,8 @@ do_run_test(Tests, Skip, Opts) ->
"Cross cover: ~w~n"
"Including ~w modules~n"
"Excluding ~w modules",
- [CovFile,CovApp,CovCross,length(CovIncl),
- length(CovExcl)]),
+ [CovFile,CovApp,CovCross,
+ length(CovIncl),length(CovExcl)]),
%% cover export file will be used for export and import
%% between tests so make sure it doesn't exist initially
@@ -1996,7 +2055,8 @@ do_run_test(Tests, Skip, Opts) ->
%% start cover on specified nodes
if (CovNodes /= []) and (CovNodes /= undefined) ->
ct_logs:log("COVER INFO",
- "Nodes included in cover session: ~w",
+ "Nodes included in cover "
+ "session: ~w",
[CovNodes]),
cover:start(CovNodes);
true ->
@@ -2021,17 +2081,27 @@ do_run_test(Tests, Skip, Opts) ->
ct_logs:log("TEST INFO","~w test(s), ~w suite(s)",
[NoOfTests,NoOfSuites]);
true ->
- io:format("~nTEST INFO: ~w test(s), ~w case(s) in ~w suite(s)~n~n",
+ io:format("~nTEST INFO: ~w test(s), ~w case(s) "
+ "in ~w suite(s)~n~n",
[NoOfTests,NoOfCases,NoOfSuites]),
- ct_logs:log("TEST INFO","~w test(s), ~w case(s) in ~w suite(s)",
+ ct_logs:log("TEST INFO","~w test(s), ~w case(s) "
+ "in ~w suite(s)",
[NoOfTests,NoOfCases,NoOfSuites])
end,
-
+ %% if the verbosity level is set lower than ?STD_IMPORTANCE, tell
+ %% test_server to ignore stdout printouts to the test case log file
+ case proplists:get_value(default, Opts#opts.verbosity) of
+ VLvl when is_integer(VLvl), (?STD_IMPORTANCE < (100-VLvl)) ->
+ test_server_ctrl:reject_io_reqs(true);
+ _Lower ->
+ ok
+ end,
test_server_ctrl:multiply_timetraps(Opts#opts.multiply_timetraps),
test_server_ctrl:scale_timetraps(Opts#opts.scale_timetraps),
- test_server_ctrl:create_priv_dir(choose_val(Opts#opts.create_priv_dir,
- auto_per_run)),
+ test_server_ctrl:create_priv_dir(choose_val(
+ Opts#opts.create_priv_dir,
+ auto_per_run)),
ct_event:notify(#event{name=start_info,
node=node(),
data={NoOfTests,NoOfSuites,NoOfCases}}),
@@ -2515,7 +2585,6 @@ parse_cth_args(String) ->
String
end.
-
event_handler_args2opts(Args) ->
case proplists:get_value(event_handler, Args) of
undefined ->
@@ -2538,6 +2607,42 @@ event_handler_init_args2opts([EH, Arg]) ->
event_handler_init_args2opts([]) ->
[].
+verbosity_args2opts(Args) ->
+ case proplists:get_value(verbosity, Args) of
+ undefined ->
+ [];
+ VArgs ->
+ GetVLvls =
+ fun("and", {new,SoFar}) when is_list(SoFar) ->
+ {new,SoFar};
+ ("and", {Lvl,SoFar}) when is_list(SoFar) ->
+ {new,[{'$unspecified',list_to_integer(Lvl)} | SoFar]};
+ (CatOrLvl, {new,SoFar}) when is_list(SoFar) ->
+ {CatOrLvl,SoFar};
+ (Lvl, {Cat,SoFar}) ->
+ {new,[{list_to_atom(Cat),list_to_integer(Lvl)} | SoFar]}
+ end,
+ case lists:foldl(GetVLvls, {new,[]}, VArgs) of
+ {new,Parsed} ->
+ Parsed;
+ {Lvl,Parsed} ->
+ [{'$unspecified',list_to_integer(Lvl)} | Parsed]
+ end
+ end.
+
+add_verbosity_defaults(VLvls) ->
+ case {proplists:get_value('$unspecified', VLvls),
+ proplists:get_value(default, VLvls)} of
+ {undefined,undefined} ->
+ ?default_verbosity ++ VLvls;
+ {Lvl,undefined} ->
+ [{default,Lvl} | VLvls];
+ {undefined,_Lvl} ->
+ [{'$unspecified',?MAX_VERBOSITY} | VLvls];
+ _ ->
+ VLvls
+ end.
+
%% This function reads pa and pz arguments, converts dirs from relative
%% to absolute, and re-inserts them in the code path. The order of the
%% dirs in the code path remain the same. Note however that since this
@@ -2616,10 +2721,14 @@ opts2args(EnvStartOpts) ->
[{userconfig,[atom_to_list(CBM) | CfgStrs]}];
({userconfig,UserCfg}) when is_list(UserCfg) ->
Strs =
- lists:map(fun({CBM,CfgStr=[X|_]}) when is_integer(X) ->
- [atom_to_list(CBM),CfgStr,"and"];
- ({CBM,CfgStrs}) when is_list(CfgStrs) ->
- [atom_to_list(CBM) | CfgStrs] ++ ["and"]
+ lists:map(fun({CBM,CfgStr=[X|_]})
+ when is_integer(X) ->
+ [atom_to_list(CBM),
+ CfgStr,"and"];
+ ({CBM,CfgStrs})
+ when is_list(CfgStrs) ->
+ [atom_to_list(CBM) | CfgStrs] ++
+ ["and"]
end, UserCfg),
[_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)),
[{userconfig,lists:reverse(StrsR)}];
@@ -2667,12 +2776,32 @@ opts2args(EnvStartOpts) ->
({event_handler,{EHs,Arg}}) when is_list(EHs) ->
ArgStr = lists:flatten(io_lib:format("~p", [Arg])),
Strs = lists:map(fun(EH) ->
- [atom_to_list(EH),ArgStr,"and"]
+ [atom_to_list(EH),
+ ArgStr,"and"]
end, EHs),
[_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)),
[{event_handler_init,lists:reverse(StrsR)}];
({logopts,LOs}) when is_list(LOs) ->
[{logopts,[atom_to_list(LO) || LO <- LOs]}];
+ ({verbosity,?default_verbosity}) ->
+ [];
+ ({verbosity,VLvl}) when is_integer(VLvl) ->
+ [{verbosity,[integer_to_list(VLvl)]}];
+ ({verbosity,VLvls}) when is_list(VLvls) ->
+ VLvlArgs =
+ lists:flatmap(fun({'$unspecified',Lvl}) ->
+ [integer_to_list(Lvl),
+ "and"];
+ ({Cat,Lvl}) ->
+ [atom_to_list(Cat),
+ integer_to_list(Lvl),
+ "and"];
+ (Lvl) ->
+ [integer_to_list(Lvl),
+ "and"]
+ end, VLvls),
+ [_LastAnd|VLvlArgsR] = lists:reverse(VLvlArgs),
+ [{verbosity,lists:reverse(VLvlArgsR)}];
({ct_hooks,[]}) ->
[];
({ct_hooks,CTHs}) when is_list(CTHs) ->