diff options
Diffstat (limited to 'lib/common_test')
28 files changed, 1339 insertions, 78 deletions
diff --git a/lib/common_test/doc/src/cover_chapter.xml b/lib/common_test/doc/src/cover_chapter.xml index 803a71de07..b2e64bfff0 100644 --- a/lib/common_test/doc/src/cover_chapter.xml +++ b/lib/common_test/doc/src/cover_chapter.xml @@ -109,6 +109,33 @@ </section> <section> + <marker id="cover_stop"></marker> + <title>Stopping the cover tool when tests are completed</title> + <p>By default the Cover tool is automatically stopped when the + tests are completed. This causes the original (non cover + compiled) modules to be loaded back in to the test node. If a + process at this point is still running old code of any of the + modules that are cover compiled, meaning that it has not done + any fully qualified function call after the cover compilation, + the process will now be killed. To avoid this it is possible to + set the value of the <c>cover_stop</c> option to + <c>false</c>. This means that the modules will stay cover + compiled, and it is therefore only recommended if the erlang + node(s) under test is terminated after the test is completed + or if cover can be manually stopped.</p> + + <p>The option can be set by using the <c>-cover_stop</c> flag with + <c>ct_run</c>, by adding <c>{cover_stop,true|false}</c> to the + Opts argument to <c><seealso + marker="ct#run_test-1">ct:run_test/1</seealso></c>, or by adding + a <c>cover_stop</c> term in your test specification (see chapter + about <seealso + marker="run_test_chapter#test_specifications">test + specifications</seealso>).</p> + + </section> + + <section> <title>The cover specification file</title> <p>These are the terms allowed in a cover specification file:</p> diff --git a/lib/common_test/doc/src/ct_run.xml b/lib/common_test/doc/src/ct_run.xml index 9cc5495af7..da18640df7 100644 --- a/lib/common_test/doc/src/ct_run.xml +++ b/lib/common_test/doc/src/ct_run.xml @@ -104,6 +104,7 @@ [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]] [-stylesheet CSSFile] [-cover CoverCfgFile] + [-cover_stop Bool] [-event_handler EvHandler1 EvHandler2 .. EvHandlerN] | [-event_handler_init EvHandler1 InitArg1 and EvHandler2 InitArg2 and .. EvHandlerN InitArgN] @@ -138,6 +139,7 @@ [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]] [-stylesheet CSSFile] [-cover CoverCfgFile] + [-cover_stop Bool] [-event_handler EvHandler1 EvHandler2 .. EvHandlerN] | [-event_handler_init EvHandler1 InitArg1 and EvHandler2 InitArg2 and .. EvHandlerN InitArgN] diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index ea62df27cc..b5b914d506 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -153,6 +153,8 @@ <item><c><![CDATA[-stylesheet <css_file>]]></c>, points out a user HTML style sheet (see below).</item> <item><c><![CDATA[-cover <cover_cfg_file>]]></c>, to perform code coverage test (see <seealso marker="cover_chapter#cover">Code Coverage Analysis</seealso>).</item> + <item><c><![CDATA[-cover_stop <bool>]]></c>, to specify if the cover tool shall be stopped after the test is completed (see + <seealso marker="cover_chapter#cover_stop">Code Coverage Analysis</seealso>).</item> <item><c><![CDATA[-event_handler <event_handlers>]]></c>, to install <seealso marker="event_handler_chapter#event_handling">event handlers</seealso>.</item> <item><c><![CDATA[-event_handler_init <event_handlers>]]></c>, to install @@ -495,6 +497,9 @@ {cover, CoverSpecFile}. {cover, NodeRefs, CoverSpecFile}. + {cover_stop, Bool}. + {cover_stop, NodeRefs, Bool}. + {include, IncludeDirs}. {include, NodeRefs, IncludeDirs}. diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 5014309c0f..6ecd0a022c 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -148,7 +148,7 @@ run(TestDirs) -> %%% {config,CfgFiles} | {userconfig, UserConfig} | %%% {allow_user_terms,Bool} | {logdir,LogDir} | %%% {silent_connections,Conns} | {stylesheet,CSSFile} | -%%% {cover,CoverSpecFile} | {step,StepOpts} | +%%% {cover,CoverSpecFile} | {cover_stop,Bool} | {step,StepOpts} | %%% {event_handler,EventHandlers} | {include,InclDirs} | %%% {auto_compile,Bool} | {create_priv_dir,CreatePrivDir} | %%% {multiply_timetraps,M} | {scale_timetraps,Bool} | @@ -987,8 +987,9 @@ get_testdata(Key) -> end. %%%----------------------------------------------------------------- -%%% @spec abort_current_testcase(Reason) -> ok | {error,no_testcase_running} +%%% @spec abort_current_testcase(Reason) -> ok | {error,ErrorReason} %%% Reason = term() +%%% ErrorReason = no_testcase_running | parallel_group %%% %%% @doc <p>When calling this function, the currently executing test case will be aborted. %%% It is the user's responsibility to know for sure which test case is currently diff --git a/lib/common_test/src/ct_config.erl b/lib/common_test/src/ct_config.erl index 30bf5925c0..06a8e12f55 100644 --- a/lib/common_test/src/ct_config.erl +++ b/lib/common_test/src/ct_config.erl @@ -532,7 +532,8 @@ do_require(Name,Key) -> case get_key_from_name(Name) of {error,_} -> allocate(Name,Key); - {ok,Key} -> + {ok,NameKey} when NameKey == Key; + is_tuple(Key) andalso element(1,Key) == NameKey -> %% already allocated - check that it has all required subkeys R = make_ref(), case get_config(Key,R,[]) of diff --git a/lib/common_test/src/ct_conn_log_h.erl b/lib/common_test/src/ct_conn_log_h.erl index bf27238121..d7bd18606b 100644 --- a/lib/common_test/src/ct_conn_log_h.erl +++ b/lib/common_test/src/ct_conn_log_h.erl @@ -64,10 +64,16 @@ do_open_files([],Acc) -> handle_event({_Type, GL, _Msg}, State) when node(GL) /= node() -> {ok, State}; handle_event({_Type,_GL,{Pid,{ct_connection,Action,ConnName},Report}},State) -> + %% NOTE: if the format of this event is changed + %% ({ct_connection,Action,ConnName}) then remember to change + %% test_server_h:report_receiver as well!!! Info = conn_info(Pid,#conn_log{name=ConnName,action=Action}), write_report(now(),Info,Report,State), {ok, State}; handle_event({_Type,_GL,{Pid,Info=#conn_log{},Report}},State) -> + %% NOTE: if the format of this event is changed + %% (Info=#conn_log{}) then remember to change + %% test_server_h:report_receiver as well!!! write_report(now(),conn_info(Pid,Info),Report,State), {ok, State}; handle_event({error_report,_,{Pid,_,[{ct_connection,ConnName}|R]}},State) -> diff --git a/lib/common_test/src/ct_master.erl b/lib/common_test/src/ct_master.erl index 99bec3ea09..f29eba605c 100644 --- a/lib/common_test/src/ct_master.erl +++ b/lib/common_test/src/ct_master.erl @@ -51,7 +51,7 @@ %%% {testcase,Cases} | {spec,TestSpecs} | {allow_user_terms,Bool} | %%% {logdir,LogDir} | {event_handler,EventHandlers} | %%% {silent_connections,Conns} | {cover,CoverSpecFile} | -%%% {userconfig, UserCfgFiles} +%%% {cover_stop,Bool} | {userconfig, UserCfgFiles} %%% CfgFiles = string() | [string()] %%% TestDirs = string() | [string()] %%% Suites = atom() | [atom()] diff --git a/lib/common_test/src/ct_master_logs.erl b/lib/common_test/src/ct_master_logs.erl index d76288feef..84f175c0a9 100644 --- a/lib/common_test/src/ct_master_logs.erl +++ b/lib/common_test/src/ct_master_logs.erl @@ -204,7 +204,7 @@ open_ct_master_log(Dir) -> {ok,Fd} = file:open(FullName,[write]), io:put_chars(Fd,header("Common Test Master Log", {[],[1,2],[]})), %% maybe add config info here later - io:put_chars(config_table([])), + io:put_chars(Fd,config_table([])), io:put_chars(Fd, "<style>\n" "div.ct_internal { background:lightgrey; color:black }\n" diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl index 52fe9599ce..294b82bff6 100644 --- a/lib/common_test/src/ct_netconfc.erl +++ b/lib/common_test/src/ct_netconfc.erl @@ -968,7 +968,7 @@ close_session(Client) -> %% @end %%---------------------------------------------------------------------- close_session(Client, Timeout) -> - call(Client,{send_rpc_op, close_session, [], Timeout}). + call(Client,{send_rpc_op, close_session, [], Timeout}, true). %%---------------------------------------------------------------------- @@ -1121,17 +1121,38 @@ close(Client) -> %% Internal functions %%---------------------------------------------------------------------- call(Client, Msg) -> - call(Client, Msg, infinity). -call(Client, Msg, Timeout) -> + call(Client, Msg, infinity, false). +call(Client, Msg, Timeout) when is_integer(Timeout); Timeout==infinity -> + call(Client, Msg, Timeout, false); +call(Client, Msg, WaitStop) when is_boolean(WaitStop) -> + call(Client, Msg, infinity, WaitStop). +call(Client, Msg, Timeout, WaitStop) -> case get_handle(Client) of {ok,Pid} -> case ct_gen_conn:call(Pid,Msg,Timeout) of - {error,{process_down,Client,noproc}} -> + {error,{process_down,Pid,noproc}} -> {error,no_such_client}; - {error,{process_down,Client,normal}} -> + {error,{process_down,Pid,normal}} when WaitStop -> + %% This will happen when server closes connection + %% before clien received rpc-reply on + %% close-session. + ok; + {error,{process_down,Pid,normal}} -> {error,closed}; - {error,{process_down,Client,Reason}} -> + {error,{process_down,Pid,Reason}} -> {error,{closed,Reason}}; + Other when WaitStop -> + MRef = erlang:monitor(process,Pid), + receive + {'DOWN',MRef,process,Pid,Normal} when Normal==normal; + Normal==noproc -> + Other; + {'DOWN',MRef,process,Pid,Reason} -> + {error,{{closed,Reason},Other}} + after Timeout -> + erlang:demonitor(MRef, [flush]), + {error,{timeout,Other}} + end; Other -> Other end; diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 4a6a3cdcac..ed7778c25a 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -58,6 +58,7 @@ vts, shell, cover, + cover_stop, coverspec, step, logdir, @@ -245,6 +246,7 @@ script_start1(Parent, Args) -> Vts = get_start_opt(vts, true, Args), Shell = get_start_opt(shell, true, Args), Cover = get_start_opt(cover, fun([CoverFile]) -> ?abs(CoverFile) end, Args), + CoverStop = get_start_opt(cover_stop, fun([CS]) -> list_to_atom(CS) end, Args), LogDir = get_start_opt(logdir, fun([LogD]) -> LogD end, Args), LogOpts = get_start_opt(logopts, fun(Os) -> [list_to_atom(O) || O <- Os] end, [], Args), @@ -329,7 +331,8 @@ script_start1(Parent, Args) -> end, StartOpts = #opts{label = Label, profile = Profile, - vts = Vts, shell = Shell, cover = Cover, + vts = Vts, shell = Shell, + cover = Cover, cover_stop = CoverStop, logdir = LogDir, logopts = LogOpts, basic_html = BasicHtml, verbosity = Verbosity, @@ -416,6 +419,9 @@ script_start2(StartOpts = #opts{vts = undefined, Cover = choose_val(StartOpts#opts.cover, SpecStartOpts#opts.cover), + CoverStop = + choose_val(StartOpts#opts.cover_stop, + SpecStartOpts#opts.cover_stop), MultTT = choose_val(StartOpts#opts.multiply_timetraps, SpecStartOpts#opts.multiply_timetraps), @@ -475,6 +481,7 @@ script_start2(StartOpts = #opts{vts = undefined, profile = Profile, testspecs = Specs, cover = Cover, + cover_stop = CoverStop, logdir = LogDir, logopts = AllLogOpts, basic_html = BasicHtml, @@ -723,6 +730,7 @@ script_usage() -> "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" + "\n\t[-cover_stop Bool]" "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" @@ -745,6 +753,7 @@ script_usage() -> "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" + "\n\t[-cover_stop Bool]" "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" @@ -938,6 +947,7 @@ run_test2(StartOpts) -> %% code coverage Cover = get_start_opt(cover, fun(CoverFile) -> ?abs(CoverFile) end, StartOpts), + CoverStop = get_start_opt(cover_stop, value, StartOpts), %% timetrap manipulation MultiplyTT = get_start_opt(multiply_timetraps, value, 1, StartOpts), @@ -1000,7 +1010,8 @@ run_test2(StartOpts) -> Step = get_start_opt(step, value, StartOpts), Opts = #opts{label = Label, profile = Profile, - cover = Cover, step = Step, logdir = LogDir, + cover = Cover, cover_stop = CoverStop, + step = Step, logdir = LogDir, logopts = LogOpts, basic_html = BasicHtml, config = CfgFiles, verbosity = Verbosity, @@ -1063,6 +1074,8 @@ run_spec_file(Relaxed, AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]), Cover = choose_val(Opts#opts.cover, SpecOpts#opts.cover), + CoverStop = choose_val(Opts#opts.cover_stop, + SpecOpts#opts.cover_stop), MultTT = choose_val(Opts#opts.multiply_timetraps, SpecOpts#opts.multiply_timetraps), ScaleTT = choose_val(Opts#opts.scale_timetraps, @@ -1103,6 +1116,7 @@ run_spec_file(Relaxed, Opts1 = Opts#opts{label = Label, profile = Profile, cover = Cover, + cover_stop = CoverStop, logdir = which(logdir, LogDir), logopts = AllLogOpts, stylesheet = Stylesheet, @@ -1374,6 +1388,7 @@ get_data_for_node(#testspec{label = Labels, verbosity = VLvls, silent_connections = SilentConnsList, cover = CoverFs, + cover_stop = CoverStops, config = Cfgs, userconfig = UsrCfgs, event_handler = EvHs, @@ -1405,6 +1420,7 @@ get_data_for_node(#testspec{label = Labels, SCs -> SCs end, Cover = proplists:get_value(Node, CoverFs), + CoverStop = proplists:get_value(Node, CoverStops), MT = proplists:get_value(Node, MTs), ST = proplists:get_value(Node, STs), CreatePrivDir = proplists:get_value(Node, PDs), @@ -1423,6 +1439,7 @@ get_data_for_node(#testspec{label = Labels, verbosity = Verbosity, silent_connections = SilentConns, cover = Cover, + cover_stop = CoverStop, config = ConfigFiles, event_handlers = EvHandlers, ct_hooks = FiltCTHooks, @@ -1576,14 +1593,7 @@ do_run(Tests, Misc, LogDir, LogOpts) when is_list(Misc), StepOpts -> #opts{step = StepOpts} end, - Opts1 = - case proplists:get_value(cover, Misc) of - undefined -> - Opts; - CoverFile -> - Opts#opts{cover = CoverFile} - end, - do_run(Tests, [], Opts1#opts{logdir = LogDir}, []); + do_run(Tests, [], Opts#opts{logdir = LogDir}, []); do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) -> #opts{label = Label, profile = Profile, cover = Cover, @@ -1617,7 +1627,13 @@ do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) -> {error,Reason} -> exit({error,Reason}); CoverSpec -> - Opts#opts{coverspec = CoverSpec} + CoverStop = + case Opts#opts.cover_stop of + undefined -> true; + Stop -> Stop + end, + Opts#opts{coverspec = CoverSpec, + cover_stop = CoverStop} end end, %% This env variable is used by test_server to determine @@ -2120,7 +2136,8 @@ do_run_test(Tests, Skip, Opts) -> %% tell test_server which modules should be cover compiled %% note that actual compilation is done when tests start test_server_ctrl:cover(CovApp, CovFile, CovExcl, CovIncl, - CovCross, CovExport, CovLevel), + CovCross, CovExport, CovLevel, + Opts#opts.cover_stop), %% save cover data (used e.g. to add nodes dynamically) ct_util:set_testdata({cover,CovData}), %% start cover on specified nodes @@ -2584,6 +2601,9 @@ merge_arguments([LogDir={logdir,_}|Args], Merged) -> merge_arguments([CoverFile={cover,_}|Args], Merged) -> merge_arguments(Args, handle_arg(replace, CoverFile, Merged)); +merge_arguments([CoverStop={cover_stop,_}|Args], Merged) -> + merge_arguments(Args, handle_arg(replace, CoverStop, Merged)); + merge_arguments([{'case',TC}|Args], Merged) -> merge_arguments(Args, handle_arg(merge, {testcase,TC}, Merged)); diff --git a/lib/common_test/src/ct_slave.erl b/lib/common_test/src/ct_slave.erl index aa3413fa89..cb05423497 100644 --- a/lib/common_test/src/ct_slave.erl +++ b/lib/common_test/src/ct_slave.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010. All Rights Reserved. +%% Copyright Ericsson AB 2010-2012. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -311,6 +311,13 @@ do_start(Host, Node, Options) -> StartupTimeout = Options#options.startup_timeout, Result = case wait_for_node_alive(ENode, BootTimeout) of pong-> + case test_server:is_cover() of + true -> + MainCoverNode = cover:get_main_node(), + rpc:call(MainCoverNode,cover,start,[ENode]); + false -> + ok + end, call_functions(ENode, Functions2), receive {node_started, ENode}-> @@ -423,6 +430,13 @@ wait_for_node_alive(Node, N) -> % call init:stop on a remote node do_stop(ENode) -> + case test_server:is_cover() of + true -> + MainCoverNode = cover:get_main_node(), + rpc:call(MainCoverNode,cover,flush,[ENode]); + false -> + ok + end, spawn(ENode, init, stop, []), wait_for_node_dead(ENode, 5). diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index a8b67d0329..0221b87d49 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -903,6 +903,8 @@ handle_data(logdir,Node,Dir,Spec) -> [{Node,ref2dir(Dir,Spec)}]; handle_data(cover,Node,File,Spec) -> [{Node,get_absfile(File,Spec)}]; +handle_data(cover_stop,Node,Stop,_Spec) -> + [{Node,Stop}]; handle_data(include,Node,Dirs=[D|_],Spec) when is_list(D) -> [{Node,ref2dir(Dir,Spec)} || Dir <- Dirs]; handle_data(include,Node,Dir=[Ch|_],Spec) when is_integer(Ch) -> @@ -1258,6 +1260,8 @@ valid_terms() -> {node,3}, {cover,2}, {cover,3}, + {cover_stop,2}, + {cover_stop,3}, {config,2}, {config,3}, {config,4}, diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index 196b5e46d0..c9c6514fa4 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -38,6 +38,7 @@ verbosity=[], silent_connections=[], cover=[], + cover_stop=[], config=[], userconfig=[], event_handler=[], diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl index 77f57c6195..78ae70f37e 100644 --- a/lib/common_test/src/cth_log_redirect.erl +++ b/lib/common_test/src/cth_log_redirect.erl @@ -54,7 +54,7 @@ post_init_per_group(_Group, _Config, Result, State) -> post_end_per_testcase(_TC, _Config, Result, State) -> %% Make sure that the event queue is flushed %% before ending this test case. - gen_event:call(error_logger, ?MODULE, flush), + gen_event:call(error_logger, ?MODULE, flush, 300000), {Result, State}. pre_end_per_group(Group, Config, {ct_log, Group}) -> diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index 64e2cb6507..3de33688da 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -54,7 +54,8 @@ MODULES= \ ct_shell_SUITE \ ct_system_error_SUITE \ ct_snmp_SUITE \ - ct_group_leader_SUITE + ct_group_leader_SUITE \ + ct_cover_SUITE ERL_FILES= $(MODULES:%=%.erl) @@ -108,7 +109,7 @@ release_spec: opt release_tests_spec: $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(ERL_FILES) $(COVERFILE) "$(RELSYSDIR)" - $(INSTALL_DATA) common_test.spec "$(RELSYSDIR)" + $(INSTALL_DATA) common_test.spec common_test.cover "$(RELSYSDIR)" chmod -R u+w "$(RELSYSDIR)" @tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) diff --git a/lib/common_test/test/common_test.cover b/lib/common_test/test/common_test.cover new file mode 100644 index 0000000000..66697854ea --- /dev/null +++ b/lib/common_test/test/common_test.cover @@ -0,0 +1,10 @@ +%% -*- erlang -*- +{incl_app,common_test,details}. +{cross_apps,common_test,[erl2html2, + test_server, + test_server_ctrl, + test_server_gl, + test_server_h, + test_server_io, + test_server_node, + test_server_sup]}. diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl index 0b1abae757..1e1df908db 100644 --- a/lib/common_test/test/ct_config_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE.erl @@ -251,6 +251,7 @@ expected_events(config_static_SUITE)-> ?sok(test_alias_tclocal_nested,{14,0,{2,1}}), ?sok(test_alias_tclocal_nested_backward_compat,{15,0,{2,1}}), ?sok(test_alias_tclocal_nested_backward_compat_subvals,{16,0,{2,1}}), + ?sok(test_config_same_name_already_in_use,{17,0,{2,1}}), {?eh,tc_start,{config_static_SUITE,end_per_suite}}, {?eh,tc_done,{config_static_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, diff --git a/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl b/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl index 2e1ad651e8..19f1dab4af 100644 --- a/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl @@ -64,7 +64,8 @@ all() -> [test_get_config_simple, test_get_config_nested, test_shadow_all,test_element,test_shadow_all_element, test_internal_deep, test_alias_tclocal_nested, test_alias_tclocal_nested_backward_compat, - test_alias_tclocal_nested_backward_compat_subvals + test_alias_tclocal_nested_backward_compat_subvals, + test_config_same_name_already_in_use ]. init_per_testcase(_,Config) -> @@ -124,6 +125,13 @@ test_config_name_already_in_use2(_) -> ct:fail("Test should've been skipped, you shouldn't see this!"), ok. + +test_config_same_name_already_in_use() -> + []. +test_config_same_name_already_in_use(_) -> + ok = ct:require(x2,{gen_cfg,c}), + ok = ct:require(x2,{gen_cfg,c}). + %% test aliases test_alias_tclocal() -> [{require,newalias,gen_cfg}]. diff --git a/lib/common_test/test/ct_config_info_SUITE.erl b/lib/common_test/test/ct_config_info_SUITE.erl index 40da377ee5..10fe8286dd 100644 --- a/lib/common_test/test/ct_config_info_SUITE.erl +++ b/lib/common_test/test/ct_config_info_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2011. All Rights Reserved. +%% Copyright Ericsson AB 2009-2012. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -123,8 +123,7 @@ test_events(config_info) -> {?eh,tc_done,{config_info_1_SUITE,init_per_suite,ok}}, [{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g1,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {init_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g1,[]}, {failed,{timetrap_timeout,350}}}}, {?eh,tc_auto_skip,{config_info_1_SUITE,t11, {failed,{config_info_1_SUITE,init_per_group,{timetrap_timeout,350}}}}}, @@ -136,14 +135,12 @@ test_events(config_info) -> {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g2,[]},ok}}, {?eh,tc_done,{config_info_1_SUITE,t21,ok}}, {?eh,tc_start,{config_info_1_SUITE,{end_per_group,g2,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {end_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{end_per_group,g2,[]}, {failed,{timetrap_timeout,450}}}}], [{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g3,[]}}}, {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g3,[]},ok}}, [{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g4,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {init_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g4,[]}, {failed,{timetrap_timeout,400}}}}, {?eh,tc_auto_skip,{config_info_1_SUITE,t41, {failed,{config_info_1_SUITE,init_per_group, @@ -164,8 +161,7 @@ test_events(config_info) -> {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g5,[]},ok}}, {?eh,tc_done,{config_info_1_SUITE,t51,ok}}, {?eh,tc_start,{config_info_1_SUITE,{end_per_group,g5,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {end_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{end_per_group,g5,[]}, {failed,{timetrap_timeout,400}}}}], {?eh,tc_start,{config_info_1_SUITE,{end_per_group,g3,[]}}}, {?eh,tc_done,{config_info_1_SUITE,{end_per_group,g3,[]},ok}}], diff --git a/lib/common_test/test/ct_cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE.erl new file mode 100644 index 0000000000..bebfce70d0 --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE.erl @@ -0,0 +1,271 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_cover_SUITE +%%% +%%% Description: +%%% Test code cover analysis support +%%% +%%%------------------------------------------------------------------- +-module(ct_cover_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). +-define(suite, cover_SUITE). +-define(mod, cover_test_mod). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + case test_server:is_cover() of + true -> + {skip,"Test server is running cover already - skipping"}; + false -> + ct_test_support:init_per_suite(Config) + end. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + Node = fullname(existing_node), + case lists:member(Node,nodes()) of + true -> rpc:call(Node,erlang,halt,[]); + false -> ok + end, + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + default, + cover_stop_true, + cover_stop_false, + slave, + slave_start_slave, + cover_node_option, + ct_cover_add_remove_nodes, + otp_9956 + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%% Check that cover is collected from test node +%% Also check that cover is by default stopped after test is completed +default(Config) -> + {ok,Events} = run_test(default,Config), + false = check_cover(Config), + check_calls(Events,1), + ok. + +%% Check that cover is stopped when cover_stop option is set to true +cover_stop_true(Config) -> + {ok,_Events} = run_test(cover_stop_true,[{cover_stop,true}],Config), + false = check_cover(Config). + +%% Check that cover is not stopped when cover_stop option is set to false +cover_stop_false(Config) -> + {ok,_Events} = run_test(cover_stop_false,[{cover_stop,false}],Config), + {true,[],[?mod]} = check_cover(Config), + CTNode = proplists:get_value(ct_node, Config), + ok = rpc:call(CTNode,cover,stop,[]), + false = check_cover(Config), + ok. + +%% Let test node start a slave node - check that cover is collected +%% from both nodes +slave(Config) -> + {ok,Events} = run_test(slave,slave,[],Config), + check_calls(Events,2), + ok. + +%% Let test node start a slave node which in turn starts another slave +%% node - check that cover is collected from all three nodes +slave_start_slave(Config) -> + {ok,Events} = run_test(slave_start_slave,slave_start_slave,[],Config), + check_calls(Events,3), + ok. + +%% Start a slave node before test starts - the node is listed in cover +%% spec file. +%% Check that cover is collected from test node and slave node. +cover_node_option(Config) -> + {ok, HostStr}=inet:gethostname(), + Host = list_to_atom(HostStr), + DataDir = ?config(data_dir,Config), + {ok,Node} = ct_slave:start(Host,existing_node, + [{erl_flags,"-pa " ++ DataDir}]), + false = check_cover(Node), + CoverSpec = default_cover_file_content() ++ [{nodes,[Node]}], + CoverFile = create_cover_file(cover_node_option,CoverSpec,Config), + {ok,Events} = run_test(cover_node_option,cover_node_option, + [{cover,CoverFile}],Config), + check_calls(Events,2), + {ok,Node} = ct_slave:stop(existing_node), + ok. + +%% Test ct_cover:add_nodes/1 and ct_cover:remove_nodes/1 +%% Check that cover is collected from added node +ct_cover_add_remove_nodes(Config) -> + {ok, HostStr}=inet:gethostname(), + Host = list_to_atom(HostStr), + DataDir = ?config(data_dir,Config), + {ok,Node} = ct_slave:start(Host,existing_node, + [{erl_flags,"-pa " ++ DataDir}]), + false = check_cover(Node), + {ok,Events} = run_test(ct_cover_add_remove_nodes,ct_cover_add_remove_nodes, + [],Config), + check_calls(Events,2), + {ok,Node} = ct_slave:stop(existing_node), + ok. + +%% Test that the test suite itself can be cover compiled and that +%% data_dir is set correctly (OTP-9956) +otp_9956(Config) -> + CoverFile = create_cover_file(otp_9956,[{incl_mods,[?suite]}],Config), + {ok,Events} = run_test(otp_9956,otp_9956,[{cover,CoverFile}],Config), + check_calls(Events,{?suite,otp_9956,1},1), + ok. + + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- +run_test(Label,Config) -> + run_test(Label,[],Config). +run_test(Label,ExtraOpts,Config) -> + run_test(Label,default,ExtraOpts,Config). +run_test(Label,Testcase,ExtraOpts,Config) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, ?suite), + CoverFile = + case proplists:get_value(cover,ExtraOpts) of + undefined -> + create_default_cover_file(Label,Config); + CF -> + CF + end, + RestOpts = lists:keydelete(cover,1,ExtraOpts), + {Opts,ERPid} = setup([{suite,Suite},{testcase,Testcase}, + {cover,CoverFile},{label,Label}] ++ RestOpts, Config), + execute(Label, Testcase, Opts, ERPid, Config). + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +execute(Name, Testcase, Opts, ERPid, Config) -> + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Name, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + TestEvents = events_to_check(Testcase), + R = ct_test_support:verify_events(TestEvents, Events, Config), + {R,Events}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +events_to_check(Testcase) -> + OneTest = + [{?eh,start_logging,{'DEF','RUNDIR'}}] ++ + [{?eh,tc_done,{?suite,Testcase,ok}}] ++ + [{?eh,stop_logging,[]}], + + %% 2 tests (ct:run_test + script_start) is default + OneTest ++ OneTest. + +check_cover(Config) when is_list(Config) -> + CTNode = proplists:get_value(ct_node, Config), + check_cover(CTNode); +check_cover(Node) when is_atom(Node) -> + case rpc:call(Node,test_server,is_cover,[]) of + true -> + {true, + rpc:call(Node,cover,which_nodes,[]), + rpc:call(Node,cover,modules,[])}; + false -> + false + end. + +%% Check that each coverlog includes N calls to ?mod:foo/0 +check_calls(Events,N) -> + check_calls(Events,{?mod,foo,0},N). +check_calls(Events,MFA,N) -> + CoverLogs = + [filename:join(filename:dirname(TCLog),"all.coverdata") || + {ct_test_support_eh, + {event,tc_logfile,ct@falco, + {{?suite,init_per_suite},TCLog}}} <- Events], + do_check_logs(CoverLogs,MFA,N). + +do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) -> + {ok,_} = cover:start(), + ok = cover:import(CoverLog), + {ok,Calls} = cover:analyse(Mod,calls,function), + ok = cover:stop(), + {MFA,N} = lists:keyfind(MFA,1,Calls), + do_check_logs(CoverLogs,MFA,N); +do_check_logs([],_,_) -> + ok. + +fullname(Name) -> + {ok,Host} = inet:gethostname(), + list_to_atom(atom_to_list(Name) ++ "@" ++ Host). + +default_cover_file_content() -> + [{incl_mods,[?mod]}]. + +create_default_cover_file(Filename,Config) -> + create_cover_file(Filename,default_cover_file_content(),Config). + +create_cover_file(Filename,Terms,Config) -> + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,Filename) ++ ".cover", + {ok,Fd} = file:open(File,[write]), + lists:foreach(fun(Term) -> + file:write(Fd,io_lib:format("~p.~n",[Term])) + end,Terms), + ok = file:close(Fd), + File. diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl new file mode 100644 index 0000000000..fdc3323f0a --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl @@ -0,0 +1,156 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%%---------------------------------------------------------------------- +%% File: cover_SUITE.erl +%% +%% Description: +%% This file contains the test cases for the code coverage support +%% +%% @author Support +%% @doc Test of code coverage support in common_test +%% @end +%%---------------------------------------------------------------------- +%%---------------------------------------------------------------------- +-module(cover_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +%% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +suite() -> + []. + +all() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + Config. + +init_per_testcase(_Case, Config) -> + Dog = test_server:timetrap(?default_timeout), + [{watchdog, Dog}|Config]. + +end_per_testcase(Case, Config) -> + %% try apply(?MODULE,Case,[cleanup,Config]) + %% catch error:undef -> ok + %% end, + + kill_slaves(Case,nodes()), + Dog=?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +%%%----------------------------------------------------------------- +%%% Test cases +break(_Config) -> + test_server:break(""), + ok. + +default(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + ok. + +slave(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + N1 = nodename(slave,1), + {ok,Node} = ct_slave:start(N1), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), + {ok,Node} = ct_slave:stop(N1), + ok. + +slave_start_slave(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + N1 = nodename(slave_start_slave,1), + N2 = nodename(slave_start_slave,2), + {ok,Node} = ct_slave:start(N1), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), + {ok,Node2} = rpc:call(Node,ct_slave,start,[N2]), + rpc:call(Node2,cover_test_mod,foo,[]), + {ok,Node2} = rpc:call(Node,ct_slave,stop,[N2]), + {ok,Node} = ct_slave:stop(N1), + ok. + +cover_node_option(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + Node = fullname(existing_node), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), + ok. + +ct_cover_add_remove_nodes(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + Node = fullname(existing_node), + Beam = rpc:call(Node,code,which,[cover_test_mod]), + false = (Beam == cover_compiled), + + rpc:call(Node,cover_test_mod,foo,[]), % should not be collected + {ok,[Node]} = ct_cover:add_nodes([Node]), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), % should be collected + ok = ct_cover:remove_nodes([Node]), + rpc:call(Node,cover_test_mod,foo,[]), % should not be collected + + Beam = rpc:call(Node,code,which,[cover_test_mod]), + + ok. + +otp_9956(Config) -> + cover_compiled = code:which(?MODULE), + DataDir = ?config(data_dir,Config), + absolute = filename:pathtype(DataDir), + true = filelib:is_dir(DataDir), + ok. + + +%%%----------------------------------------------------------------- +%%% Internal +nodename(Case,N) -> + list_to_atom(nodeprefix(Case) ++ integer_to_list(N)). + +nodeprefix(Case) -> + atom_to_list(?MODULE) ++ "_" ++ atom_to_list(Case) ++ "_node". + + +fullname(Name) -> + {ok,Host} = inet:gethostname(), + list_to_atom(atom_to_list(Name) ++ "@" ++ Host). + +kill_slaves(Case, [Node|Nodes]) -> + Prefix = nodeprefix(Case), + case lists:prefix(Prefix,atom_to_list(Node)) of + true -> + rpc:call(Node,erlang,halt,[]); + _ -> + ok + end, + kill_slaves(Case,Nodes); +kill_slaves(_,[]) -> + ok. diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl b/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl new file mode 100644 index 0000000000..d4f69452c3 --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl @@ -0,0 +1,4 @@ +-module(cover_test_mod). +-compile(export_all). +foo() -> + ok. diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 338e76264e..6d90b29f41 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -61,7 +61,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [cfg_error, lib_error, no_compile, timetrap_end_conf, timetrap_normal, timetrap_extended, timetrap_parallel, - timetrap_fun, misc_errors]. + timetrap_fun, timetrap_fun_group, misc_errors]. groups() -> []. @@ -251,6 +251,24 @@ timetrap_fun(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% +timetrap_fun_group(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, + Suites = [Join(DataDir, "timetrap_8_SUITE")], + {Opts,ERPid} = setup([{suite,Suites}], Config), + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(timetrap_fun_group, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(timetrap_fun_group), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +%%%----------------------------------------------------------------- +%%% misc_errors(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, @@ -429,8 +447,7 @@ test_events(cfg_error) -> {'EXIT',{init_per_group_fails,g1}}}}}}], [{?eh,tc_start,{cfg_error_8_SUITE,{init_per_group,g2,[]}}}, - {?eh,tc_done,{cfg_error_8_SUITE, - {init_per_group,unknown,[]}, + {?eh,tc_done,{cfg_error_8_SUITE,{init_per_group,g2,[]}, {failed,{timetrap_timeout,2000}}}}, {?eh,tc_auto_skip,{cfg_error_8_SUITE,tc1, {failed,{cfg_error_8_SUITE,init_per_group, @@ -500,7 +517,7 @@ test_events(cfg_error) -> {?eh,tc_done,{cfg_error_8_SUITE,tc1,ok}}, {?eh,test_stats,{9,0,{0,14}}}, {?eh,tc_start,{cfg_error_8_SUITE,{end_per_group,g12,[]}}}, - {?eh,tc_done,{cfg_error_8_SUITE,{end_per_group,unknown,[]}, + {?eh,tc_done,{cfg_error_8_SUITE,{end_per_group,g12,[]}, {failed,{timetrap_timeout,2000}}}}], {?eh,tc_start,{cfg_error_8_SUITE,end_per_suite}}, @@ -971,11 +988,423 @@ test_events(timetrap_fun) -> {?eh,stop_logging,[]} ]; +test_events(timetrap_fun_group) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,58}}, + {?eh,tc_start,{timetrap_8_SUITE,init_per_suite}}, + {?eh,tc_done,{timetrap_8_SUITE,init_per_suite,ok}}, + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g0,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g0,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,2,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g0,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g0,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g1,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,3,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,4,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g1,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g2,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,5,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,6,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g2,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g3,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g3,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc4}}, + {?eh,tc_done,{timetrap_8_SUITE,tc4, + {failed,{timetrap_timeout,{'$approx',2000}}}}}, + {?eh,test_stats,{0,7,{0,0}}}, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g1,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,8,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,9,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g1,[]},ok}}], + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g2,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,10,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,11,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g2,[]},ok}}], + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g3,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g3,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g4,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g4,[]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,1}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,2}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g5,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g5,[]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,3}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,4}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g6,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g6,[]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{0,11,{0,5}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{0,11,{0,6}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g7,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g7,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{1,11,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g7,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g7,[]}, + {user_timetrap_error,{kaboom,'_'}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g8,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g8,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{2,11,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g8,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g8,[]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g9,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g9,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{3,11,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,test_stats,{3,12,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g9,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g9,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g10,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g10,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,test_stats,{3,13,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{4,13,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g10,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g10,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g11,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g11,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc3}}, + {?eh,tc_done,{timetrap_8_SUITE,tc3, + {failed,{timetrap_timeout,{'$approx',4000}}}}}, + {?eh,test_stats,{4,14,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,15,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g11,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g11,[]},ok}}], + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg0,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg0,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,16,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,17,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg0,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg0,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,18,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,19,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,20,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,21,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg3,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg3,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc4}}, + {?eh,tc_done,{timetrap_8_SUITE,tc4, + {failed,{timetrap_timeout,{'$approx',2000}}}}}, + {?eh,test_stats,{4,22,{0,6}}}, + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]}, + ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,23,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,24,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]}, + ok}}]}, + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]}, + ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,25,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,26,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]}, + ok}}]}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg3,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg3,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg4,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg4,[parallel]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,7}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,8}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg5,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg5,[parallel]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,9}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,10}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg6,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg6,[parallel]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{4,26,{0,11}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{4,26,{0,12}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg7,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg7,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{5,26,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg7,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg7,[parallel]}, + {user_timetrap_error,{kaboom,'_'}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg8,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg8,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{6,26,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg8,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg8,[parallel]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg9,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg9,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + %% Due to parallelism only checking final test stat in group + {?eh,test_stats,{7,27,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg9,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg9,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg10,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg10,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + %% Due to parallelism only checking final test stat in group + {?eh,test_stats,{8,28,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg10,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg10,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg11,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg11,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc3}}, + {?eh,tc_done,{timetrap_8_SUITE,tc3, + {failed,{timetrap_timeout,{'$approx',4000}}}}}, + {?eh,test_stats,{8,29,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{8,30,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg11,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg11,[parallel]},ok}}]}, + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,sg1,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,sg1,[sequence]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{9,30,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,test_stats,{9,31,{0,12}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc1, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{9,31,{0,13}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{9,31,{0,14}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,sg1,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,sg1,[sequence]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,sg2,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,sg2,[sequence]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{10,31,{0,14}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{10,32,{0,14}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc1, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{10,32,{0,15}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{10,32,{0,16}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,sg2,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,sg2,[sequence]},ok}}], + + {?eh,tc_start,{timetrap_8_SUITE,end_per_suite}}, + {?eh,tc_done,{timetrap_8_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + test_events(misc_errors) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{1,1,7}}, + {?eh,start_info,{1,1,9}}, {?eh,tc_start,{misc_error_1_SUITE,ct_fail_1}}, {?eh,tc_done,{misc_error_1_SUITE,ct_fail_1, {failed,{error,{test_case_failed,{error,this_is_expected}}}}}}, @@ -1002,7 +1431,12 @@ test_events(misc_errors) -> {?eh,tc_start,{misc_error_1_SUITE,killed_by_signal_2}}, {?eh,tc_done,{misc_error_1_SUITE,killed_by_signal_2, {failed,testcase_aborted_or_killed}}}, - {?eh,test_stats,{0,7,{0,0}}}, + {parallel, + [{?eh,tc_start,{misc_error_1_SUITE,p1}}, + {?eh,tc_done,{misc_error_1_SUITE,p1,ok}}, + {?eh,tc_start,{misc_error_1_SUITE,p2}}, + {?eh,tc_done,{misc_error_1_SUITE,p2,ok}}]}, + {?eh,test_stats,{2,7,{0,0}}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl index 99c3ed05ec..61f3fa7e59 100644 --- a/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl @@ -96,7 +96,7 @@ end_per_testcase(_TestCase, _Config) -> %% N = integer() | forever %%-------------------------------------------------------------------- groups() -> - []. + [{p,[parallel],[p1,p2]}]. %%-------------------------------------------------------------------- %% Function: all() -> GroupsAndTestCases | {skip,Reason} @@ -107,7 +107,8 @@ groups() -> %%-------------------------------------------------------------------- all() -> [ct_fail_1, ct_fail_2, ct_fail_3, ts_fail_1, ts_fail_2, - killed_by_signal_1, killed_by_signal_2]. + killed_by_signal_1, killed_by_signal_2, + {group,p}]. ct_fail_1(_) -> ct:fail({error,this_is_expected}), @@ -152,3 +153,10 @@ killed_by_signal_2(_) -> end), ct:sleep(1000), exit(this_should_not_be_seen). + +p1(_) -> + {error,parallel_group} = ct:abort_current_testcase(aborted), + ok. + +p2(_) -> + receive after 1000 -> ok end. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl new file mode 100644 index 0000000000..ff138f38b5 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl @@ -0,0 +1,258 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_8_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(TO, 4). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,?TO}]}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(G6, Config) when G6==g6; G6==pg6 -> + ct:sleep({seconds,1}), + Config; +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(G7or8, _Config) when G7or8==g7; G7or8==pg7; G7or8==g8; G7or8==pg8 -> + ct:sleep({seconds,5}), + ok; +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + [ + {g0,[],[tc0,tc2]}, % group override suite and tc overrides group + {g1,[],[tc0,tc2]}, % group override suite and tc overrides group + {g2,[],[tc1,tc2]}, % tc override group + {g3,[],[tc4,{group,g1},{group,g2}]}, % subgroup override group + {g4,[],[tc0,tc2]}, % exit during init_per_group + {g5,[],[tc0,tc2]}, % exit during init_per_group + {g6,[],[tc0,tc2]}, % timeout during init_per_group + {g7,[],[tc5]}, % exit during end_per_group + {g8,[],[tc5]}, % timeout during end_per_group + {g9,[],[tc5,tc0]}, % exit during testcase + {g10,[],[tc0,tc5]}, % exit during testcase + {g11,[],[tc3,tc2]}, % suite is valid if nothing else is specified + {pg0,[parallel],[tc0,tc2]}, % group override suite and tc overrides group + {pg1,[parallel],[tc0,tc2]}, % group override suite and tc overrides group + {pg2,[parallel],[tc1,tc2]}, % tc override group + {pg3,[parallel],[tc4,{group,pg1},{group,pg2}]}, % subgroup override group + {pg4,[parallel],[tc0,tc2]}, % exit during init_per_group + {pg5,[parallel],[tc0,tc2]}, % exit during init_per_group + {pg6,[parallel],[tc0,tc2]}, % timeout during init_per_group + {pg7,[parallel],[tc5]}, % exit during end_per_group + {pg8,[parallel],[tc5]}, % timeout during end_per_group + {pg9,[parallel],[tc5,tc0]}, % exit during testcase + {pg10,[parallel],[tc0,tc5]},% exit during testcase + {pg11,[parallel],[tc3,tc2]},% suite is valid if nothing else is specified + {sg1,[sequence],[tc5,tc0,tc1,tc2]}, % exit during sequencial testcase + {sg2,[sequence],[tc5,tc0,tc1,tc2]}].% timeout during sequencial testcase + +group(g0) -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}]; +group(g1) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(1000) end}]; +group(g2) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(3000) end}]; +group(g3) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(2000) end}]; +group(g4) -> + [{timetrap,{timetrap_utils,timetrap_exit,[kaboom]}}]; +group(g5) -> + [{timetrap,fun() -> exit(kaboom) end}]; +group(g6) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(g7) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(g8) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(g9) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(g10) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(g11) -> + []; +group(pg0) -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}]; +group(pg1) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(1000) end}]; +group(pg2) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(3000) end}]; +group(pg3) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(2000) end}]; +group(pg4) -> + [{timetrap,{timetrap_utils,timetrap_exit,[kaboom]}}]; +group(pg5) -> + [{timetrap,fun() -> exit(kaboom) end}]; +group(pg6) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(pg7) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(pg8) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(pg9) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(pg10) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(pg11) -> + []; +group(sg1) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(sg2) -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}]. + + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [ + {group,g0}, + {group,g1}, + {group,g2}, + {group,g3}, + {group,g4}, + {group,g5}, + {group,g6}, + {group,g7}, + {group,g8}, + {group,g9}, + {group,g10}, + {group,g11}, + {group,pg0}, + {group,pg1}, + {group,pg2}, + {group,pg3}, + {group,pg4}, + {group,pg5}, + {group,pg6}, + {group,pg7}, + {group,pg8}, + {group,pg9}, + {group,pg10}, + {group,pg11}, + {group,sg1}, + {group,sg2}]. + + + +tc0(_) -> + ct:comment("TO set by group"), + ct:sleep({seconds,5}), + ok. + +tc1() -> + [{timetrap,{timetrap_utils,timetrap_val,[1000]}}]. +tc1(_) -> + ct:comment("TO after 1 sec"), + ct:sleep({seconds,2}), + ok. + +tc2() -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(500) end}]. +tc2(_) -> + ct:comment("TO after 0.5 sec"), + ct:sleep({seconds,2}), + ok. + +tc3(_) -> + ct:comment(io_lib:format("TO after ~w sec", [?TO])), + ct:sleep({seconds,5}), + ok. + +tc4(_) -> + ct:comment("TO set by group"), + ct:sleep({seconds,5}), + ok. + +tc5(_) -> + ct:comment("No TO in this testcase, maybe later"), + ok. diff --git a/lib/common_test/test/ct_netconfc_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE.erl index e6e8d5b09c..3042a924fe 100644 --- a/lib/common_test/test/ct_netconfc_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE.erl @@ -44,7 +44,12 @@ %%-------------------------------------------------------------------- init_per_suite(Config) -> Config1 = ct_test_support:init_per_suite(Config), - Config1. + case application:load(crypto) of + {error,Reason} -> + {skip, Reason}; + _ -> + Config1 + end. end_per_suite(Config) -> ct_test_support:end_per_suite(Config). @@ -108,7 +113,7 @@ reformat(Events, EH) -> %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- -events_to_check(Test,Config) -> +events_to_check(default,Config) -> {module,_} = code:load_abs(filename:join(?config(data_dir,Config), netconfc1_SUITE)), TCs = netconfc1_SUITE:all(), diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 80cca4a1cc..901e53c71d 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -117,7 +117,10 @@ end_per_suite(Config) -> CTNode = proplists:get_value(ct_node, Config), PrivDir = proplists:get_value(priv_dir, Config), true = rpc:call(CTNode, code, del_path, [filename:join(PrivDir,"")]), - cover:stop(CTNode), + case test_server:is_cover() of + true -> cover:flush(CTNode); + false -> ok + end, slave:stop(CTNode), ok. @@ -149,7 +152,10 @@ end_per_testcase(_TestCase, Config) -> case wait_for_ct_stop(CTNode) of %% Common test was not stopped to we restart node. false -> - cover:stop(CTNode), + case test_server:is_cover() of + true -> cover:flush(CTNode); + false -> ok + end, slave:stop(CTNode), start_slave(Config,proplists:get_value(trace_level,Config)), {fail, "Could not stop common_test"}; @@ -612,8 +618,11 @@ locate({parallel,TEvs}, Node, Evs, Config) -> fun({EH,#event{name=tc_auto_skip, node=EvNode, data={Mod,end_per_group,Reason}}}) when - EH == TEH, EvNode == Node, Mod == M, Reason == R -> - false; + EH == TEH, EvNode == Node, Mod == M -> + case match_data(R, Reason) of + match -> false; + _ -> true + end; ({EH,#event{name=stop_logging, node=EvNode,data=_}}) when EH == TEH, EvNode == Node -> @@ -627,23 +636,12 @@ locate({parallel,TEvs}, Node, Evs, Config) -> [_AutoSkip | RemEvs2] -> {Done,RemEvs2,length(RemEvs2)} end; - %% match other event than test case - (TEv={TEH,N,D}, Acc) when D == '_' -> - case [E || E={EH,#event{name=Name, - node=EvNode, - data=_}} <- Evs1, - EH == TEH, EvNode == Node, Name == N] of - [] -> - exit({unmatched,TEv}); - _ -> - test_server:format("Found ~p!", [TEv]), - Acc - end; (TEv={TEH,N,D}, Acc) -> case [E || E={EH,#event{name=Name, node=EvNode, data=Data}} <- Evs1, - EH == TEH, EvNode == Node, Name == N, Data == D] of + EH == TEH, EvNode == Node, Name == N, + match == match_data(D,Data)] of [] -> exit({unmatched,TEv}); _ -> @@ -1002,33 +1000,39 @@ locate({TEH,Name,Data}, Node, [{TEH,#event{name=Name, data = EvData, node = Node}}|Evs], Config) -> - try match_data(Data, EvData) of + case match_data(Data, EvData) of match -> - {Config,Evs} - catch _:_ -> + {Config,Evs}; + _ -> nomatch end; locate({_TEH,_Name,_Data}, _Node, [_|_Evs], _Config) -> nomatch. -match_data(D,D) -> +match_data(Data, EvData) -> + try do_match_data(Data, EvData) + catch _:_ -> + nomatch + end. + +do_match_data(D,D) -> match; -match_data('_',_) -> +do_match_data('_',_) -> match; -match_data(Fun,Data) when is_function(Fun) -> +do_match_data(Fun,Data) when is_function(Fun) -> Fun(Data); -match_data('$proplist',Proplist) -> - match_data( +do_match_data('$proplist',Proplist) -> + do_match_data( fun(List) -> lists:foreach(fun({_,_}) -> ok end,List) end,Proplist); -match_data([H1|MatchT],[H2|ValT]) -> - match_data(H1,H2), - match_data(MatchT,ValT); -match_data(Tuple1,Tuple2) when is_tuple(Tuple1),is_tuple(Tuple2) -> - match_data(tuple_to_list(Tuple1),tuple_to_list(Tuple2)); -match_data([],[]) -> +do_match_data([H1|MatchT],[H2|ValT]) -> + do_match_data(H1,H2), + do_match_data(MatchT,ValT); +do_match_data(Tuple1,Tuple2) when is_tuple(Tuple1),is_tuple(Tuple2) -> + do_match_data(tuple_to_list(Tuple1),tuple_to_list(Tuple2)); +do_match_data([],[]) -> match. result_match({SkipOrFail,{ErrorInd,{Why,'_'}}}, @@ -1043,6 +1047,9 @@ result_match({failed,{timetrap_timeout,{'$approx',Num}}}, Value =< trunc(Num+0.02*Num) -> true; true -> false end; +result_match({user_timetrap_error,{Why,'_'}}, + {user_timetrap_error,{Why,_Stack}}) -> + true; result_match(Result, Result) -> true; result_match(_, _) -> |