diff options
Diffstat (limited to 'lib/common_test')
34 files changed, 3689 insertions, 513 deletions
diff --git a/lib/common_test/doc/src/cover_chapter.xml b/lib/common_test/doc/src/cover_chapter.xml index 803a71de07..b2e64bfff0 100644 --- a/lib/common_test/doc/src/cover_chapter.xml +++ b/lib/common_test/doc/src/cover_chapter.xml @@ -109,6 +109,33 @@ </section> <section> + <marker id="cover_stop"></marker> + <title>Stopping the cover tool when tests are completed</title> + <p>By default the Cover tool is automatically stopped when the + tests are completed. This causes the original (non cover + compiled) modules to be loaded back in to the test node. If a + process at this point is still running old code of any of the + modules that are cover compiled, meaning that it has not done + any fully qualified function call after the cover compilation, + the process will now be killed. To avoid this it is possible to + set the value of the <c>cover_stop</c> option to + <c>false</c>. This means that the modules will stay cover + compiled, and it is therefore only recommended if the erlang + node(s) under test is terminated after the test is completed + or if cover can be manually stopped.</p> + + <p>The option can be set by using the <c>-cover_stop</c> flag with + <c>ct_run</c>, by adding <c>{cover_stop,true|false}</c> to the + Opts argument to <c><seealso + marker="ct#run_test-1">ct:run_test/1</seealso></c>, or by adding + a <c>cover_stop</c> term in your test specification (see chapter + about <seealso + marker="run_test_chapter#test_specifications">test + specifications</seealso>).</p> + + </section> + + <section> <title>The cover specification file</title> <p>These are the terms allowed in a cover specification file:</p> diff --git a/lib/common_test/doc/src/ct_run.xml b/lib/common_test/doc/src/ct_run.xml index 9cc5495af7..0750f560b3 100644 --- a/lib/common_test/doc/src/ct_run.xml +++ b/lib/common_test/doc/src/ct_run.xml @@ -90,7 +90,7 @@ <pre> ct_run [-dir TestDir1 TestDir2 .. TestDirN] | [[-dir TestDir] -suite Suite1 Suite2 .. SuiteN - [[-group Group1 Group2 .. GroupN] [-case Case1 Case2 .. CaseN]]] + [[-group Groups1 Groups2 .. GroupsN] [-case Case1 Case2 .. CaseN]]] [-step [config | keep_inactive]] [-config ConfigFile1 ConfigFile2 .. ConfigFileN] [-userconfig CallbackModule1 ConfigString1 and CallbackModule2 @@ -104,6 +104,7 @@ [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]] [-stylesheet CSSFile] [-cover CoverCfgFile] + [-cover_stop Bool] [-event_handler EvHandler1 EvHandler2 .. EvHandlerN] | [-event_handler_init EvHandler1 InitArg1 and EvHandler2 InitArg2 and .. EvHandlerN InitArgN] @@ -138,6 +139,7 @@ [-silent_connections [ConnType1 ConnType2 .. ConnTypeN]] [-stylesheet CSSFile] [-cover CoverCfgFile] + [-cover_stop Bool] [-event_handler EvHandler1 EvHandler2 .. EvHandlerN] | [-event_handler_init EvHandler1 InitArg1 and EvHandler2 InitArg2 and .. EvHandlerN InitArgN] diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml index ea62df27cc..b804f134c6 100644 --- a/lib/common_test/doc/src/run_test_chapter.xml +++ b/lib/common_test/doc/src/run_test_chapter.xml @@ -119,7 +119,7 @@ <item><c><![CDATA[ct_run -userconfig <callbackmodulename> <configfilenames> -suite <suiteswithfullpath>]]></c> </item> <item><c><![CDATA[ct_run -config <configfilenames> -suite <suitewithfullpath> - -group <groupnames> -case <casenames>]]></c></item> + -group <groups> -case <casenames>]]></c></item> </list> <p>Examples:</p> <p><c>$ ct_run -config $CFGS/sys1.cfg $CFGS/sys2.cfg -dir $SYS1_TEST $SYS2_TEST</c></p> @@ -137,6 +137,8 @@ <p><c>$ ct_run -suite ./testdir/x_SUITE ./testdir/y_SUITE</c></p> + <p>For more details on <seealso marker="run_test_chapter#group_execution">test case group execution</seealso>, please see below.</p> + <p>Other flags that may be used with <c>ct_run</c>:</p> <list> <item><c><![CDATA[-logdir <dir>]]></c>, specifies where the HTML log files are to be written.</item> @@ -153,6 +155,8 @@ <item><c><![CDATA[-stylesheet <css_file>]]></c>, points out a user HTML style sheet (see below).</item> <item><c><![CDATA[-cover <cover_cfg_file>]]></c>, to perform code coverage test (see <seealso marker="cover_chapter#cover">Code Coverage Analysis</seealso>).</item> + <item><c><![CDATA[-cover_stop <bool>]]></c>, to specify if the cover tool shall be stopped after the test is completed (see + <seealso marker="cover_chapter#cover_stop">Code Coverage Analysis</seealso>).</item> <item><c><![CDATA[-event_handler <event_handlers>]]></c>, to install <seealso marker="event_handler_chapter#event_handling">event handlers</seealso>.</item> <item><c><![CDATA[-event_handler_init <event_handlers>]]></c>, to install @@ -267,6 +271,163 @@ <c><seealso marker="ct#run_test-1">ct</seealso></c> manual page.</p> </section> + <marker id="group_execution"></marker> + <section> + <title>Test case group execution</title> + + <p>With the <c>ct_run</c> flag, or <c>ct:run_test/1</c> option <c>group</c>, + one or more test case groups can be specified, optionally in combination + with specific test cases. The syntax for specifying groups is as follows + (on the command line):</p> + + <pre> + <![CDATA[$ ct_run -group <group_names_or_paths> [-case <cases>]]]></pre> + <p>or (in the Erlang shell):</p> + <pre> + <![CDATA[1> ct:run_test([{group,GroupsNamesOrPaths}, {case,Cases}]).]]></pre> + + <p>The <c>group_names_or_paths</c> parameter specifies either one + or more group names and/or one or more group paths. At start up, + Common Test will search for matching groups in the group definitions + tree (i.e. the list returned from <c>Suite:groups/0</c>, please see the + <seealso marker="write_test_chapter#test_case_groups">Test case groups</seealso> + chapter for details). + Given a group name, say <c>g</c>, Common Test will search for all paths + that lead to <c>g</c>. By path here we mean a sequence of nested groups, + all of which have to be followed in order to get from the top level + group to <c>g</c>. Actually, what Common Test needs to do in order to + execute the test cases in group <c>g</c>, is to call the + <c>init_per_group/2</c> function for each group in the path to + <c>g</c>, as well as all corresponding <c>end_per_group/2</c> + functions afterwards. The obvious reason for this is that the configuration + of a test case in <c>g</c> (and its <c>Config</c> input data) depends on + <c>init_per_testcase(TestCase, Config)</c> and its return value, which + in turn depends on <c>init_per_group(g, Config)</c> and its return value, + which in turn depends on <c>init_per_group/2</c> of the group above + <c>g</c>, etc, all the way up to the top level group.</p> + + <p>As you may have already realized, this means that if there is more than + one way to locate a group (and its test cases) in a path, the result of the + group search operation is a number of tests, all of which will be performed. + Common Test actually interprets a group specification that consists of a + single name this way:</p> + + <p>"Search and find all paths in the group definitions tree that lead + to the specified group and, for each path, create a test which (1) executes + all configuration functions in the path to the specified group, then (2) + executes all - or all matching - test cases in this group, as well as (3) + all - or all matching - test cases in all sub groups of the group". + </p> + + <p>It is also possible for the user to specify a specific group path with + the <c>group_names_or_paths</c> parameter. With this type of specification it's + possible to avoid execution of unwanted groups (in otherwise matching paths), + and/or the execution of sub groups. The syntax of the group path is a list of + group names in the path, e.g. on the command line: + </p> + <p><c>$ ct_run -suite "./x_SUITE" -group [g1,g3,g4] -case tc1 tc5</c></p> + <p>or similarly in the Erlang shell (requires a list within the groups list):</p> + <p><c>1> ct:run_test([{suite,"./x_SUITE"}, {group,[[g1,g3,g4]]}, {testcase,[tc1,tc5]}]).</c></p> + + <p>The last group in the specified path will be the terminating group in + the test, i.e. no sub groups following this group will be executed. In the + example above, <c>g4</c> is the terminating group, hence Common Test will + execute a test that calls all init configuration functions in the path to + <c>g4</c>, i.e. <c>g1..g3..g4</c>. It will then call test cases <c>tc1</c> + and <c>tc5</c> in <c>g4</c> and finally all end configuration functions in order + <c>g4..g3..g1</c>.</p> + + <p>Note that the group path specification doesn't necessarily + have to include <em>all</em> groups in the path to the terminating group. + Common Test will search for all matching paths if given an incomplete group + path.</p> + + <p>Note also that it's possible to combine group names and group paths with the + <c>group_names_or_paths</c> parameter. Each element is treated as + an individual specification in combination with the <c>cases</c> parameter. + See examples below.</p> + + <p>Examples:</p> + <pre> + -module(x_SUITE). + ... + %% The group definitions: + groups() -> + [{top1,[],[tc11,tc12, + {sub11,[],[tc12,tc13]}, + {sub12,[],[tc14,tc15, + {sub121,[],[tc12,tc16]}]}]}, + + {top2,[],[{group,sub21},{group,sub22}]}, + {sub21,[],[tc21,{group,sub2X2}]}, + {sub22,[],[{group,sub221},tc21,tc22,{group,sub2X2}]}, + {sub221,[],[tc21,tc23]}, + {sub2X2,[],[tc21,tc24]}]. + </pre> + <br></br> + <p><c>$ ct_run -suite "x_SUITE" -group all</c></p> + <p><c>1> ct:run_test([{suite,"x_SUITE"}, {group,all}]).</c></p> + <p>Two tests will be executed, one for all cases and all sub groups under <c>top1</c>, + and one for all under <c>top2</c>. (We would get the same result with + <c>-group top1 top2</c>, or <c>{group,[top1,top2]}</c>.</p> + <br></br> + <p><c>$ ct_run -suite "x_SUITE" -group top1</c></p> + <p><c>1> ct:run_test([{suite,"x_SUITE"}, {group,[top1]}]).</c></p> + <p>This will execute one test for all cases and sub groups under <c>top1</c>.</p> + <br></br> + <p><c>$ ct_run -suite "x_SUITE" -group top1 -case tc12</c></p> + <p><c>1> ct:run_test([{suite,"x_SUITE"}, {group,[top1]}, {testcase,[tc12]}]).</c></p> + <p>This will run a test that executes <c>tc12</c> in <c>top1</c> and any sub group + under <c>top1</c> where it can be found (<c>sub11</c> and <c>sub121</c>).</p> + <br></br> + <p><c>$ ct_run -suite "x_SUITE" -group [top1] -case tc12</c></p> + <p><c>1> ct:run_test([{suite,"x_SUITE"}, {group,[[top1]]}, {testcase,[tc12]}]).</c></p> + <p>This will execute <c>tc12</c> <em>only</em> in group <c>top1</c>.</p> + <br></br> + <p><c>$ ct_run -suite "x_SUITE" -group top1 -case tc16</c></p> + <p><c>1> ct:run_test([{suite,"x_SUITE"}, {group,[top1]}, {testcase,[tc16]}]).</c></p> + <p>This will search <c>top1</c> and all its sub groups for <c>tc16</c> and the result + will be that this test case executes in group <c>sub121</c>. (The specific path: + <c>-group [sub121]</c> or <c>{group,[[sub121]]}</c>, would have given + us the same result in this example).</p> + <br></br> + <p><c>$ ct_run -suite "x_SUITE" -group sub12 [sub12]</c></p> + <p><c>1> ct:run_test([{suite,"x_SUITE"}, {group,[sub12,[sub12]]}]).</c></p> + <p>This will execute two tests, one that includes all cases and sub groups under + <c>sub12</c>, and one with <em>only</em> the test cases in <c>sub12</c>.</p> + <br></br> + <p><c>$ ct_run -suite "x_SUITE" -group sub2X2</c></p> + <p><c>1> ct:run_test([{suite,"x_SUITE"}, {group,[sub2X2]}]).</c></p> + <p>In this example, Common Test will find and execute two tests, one for the path from + <c>top2</c> to <c>sub2X2</c> via <c>sub21</c>, and one from <c>top2</c> to <c>sub2X2</c> + via <c>sub22</c>.</p> + <br></br> + <p><c>$ ct_run -suite "x_SUITE" -group [sub21,sub2X2]</c></p> + <p><c>1> ct:run_test([{suite,"x_SUITE"}, {group,[[sub21,sub2X2]]}]).</c></p> + <p>Here, by specifying the unique path: <c>top2 -> sub21 -> sub2X2</c>, only one test + is executed. The second possible path from <c>top2</c> to <c>sub2X2</c> (above) + will be discarded.</p> + <br></br> + <p><c>$ ct_run -suite "x_SUITE" -group [sub22] -case tc22 tc21</c></p> + <p><c>1> ct:run_test([{suite,"x_SUITE"}, {group,[[sub22]]}, {testcase,[tc22,tc21]}]).</c></p> + <p>In this example only the test cases for <c>sub22</c> will be executed, and in + reverse order compared to the group definition.</p> + <br></br> + + <p>If a test case that belongs to a group (according to the group definition), is executed + without a group specification, i.e. simply by means of (command line):</p> + <p><c>$ ct_run -suite "my_SUITE" -case my_tc</c></p> + <p>or (Erlang shell):</p> + <p><c>1> ct:run_test([{suite,"my_SUITE"}, {testcase,my_tc}]).</c></p> + <p>then Common Test ignores the group definition and executes the test case in the scope of the + test suite only (no group configuration functions are called).</p> + + <p>The group specification feature, exactly as it has been presented in this section, can also + be used in <seealso marker="run_test_chapter#test_specifications">Test + Specifications</seealso> (with some extra features added). Please see below.</p> + </section> + + <section> <title>Running the interactive shell mode</title> @@ -398,8 +559,8 @@ terms (e.g. log directory, label, style sheet, auto compilation).</p> <p>With test specification terms it is possible to state exactly which tests should run and in which order. A test term specifies - either one or more suites, one or more test case groups, or one - or more test cases in a group or suite.</p> + either one or more suites, one or more test case groups (possibly nested), + or one or more test cases in a group (or in multiple groups) or in a suite.</p> <p>An arbitrary number of test terms may be declared in sequence. Common Test will by default compile the terms into one or more tests to be performed in one resulting test run. Note that a term that @@ -418,28 +579,32 @@ are not executed and show up in the HTML log files as SKIPPED.</p> <p>When a test case group is specified, the resulting test - executes the - <c>init_per_group</c> function, followed by all test cases and - sub groups (including their configuration functions), and + executes the <c>init_per_group</c> function, followed by all test + cases and sub groups (including their configuration functions), and finally the <c>end_per_group</c> function. Also if particular test cases in a group are specified, <c>init_per_group</c> and <c>end_per_group</c> for the group in question are called. If a group which is defined (in <c>Suite:group/0</c>) to - be a sub group of another group, is specified (or particular test + be a sub group of another group, is specified (or if particular test cases of a sub group are), Common Test will call the configuration functions for the top level groups as well as for the sub group in question (making it possible to pass configuration data all the way from <c>init_per_suite</c> down to the test cases in the sub group).</p> - - <p>With the <c>GroupSpec</c> element (below) it's possible to specify - group execution properties that will override those specified in the + <p>The test specification utilizes the same mechanism for specifying + test case groups by means of names and paths, as explained in the + <seealso marker="run_test_chapter#group_execution">Group Execution</seealso> + section above, with the addition of the <c>GroupSpec</c> element + described next.</p> + <p>The <c>GroupSpec</c> element makes it possible to specify + group execution properties that will override those in the group definition (i.e. in <c>groups/0</c>). Execution properties for sub-groups may be overridden as well. This feature makes it possible to change properties of groups at the time of execution, - without even having to edit the test suite. More detailed documentation, - and examples, can be found in the - <seealso marker="write_test_chapter#test_case_groups"> + without even having to edit the test suite. The very same + feature is available for <c>group</c> elements in the <c>Suite:all/0</c> + list. Therefore, more detailed documentation, and examples, can be + found in the <seealso marker="write_test_chapter#test_case_groups"> Test case groups</seealso> chapter.</p> <p>Below is the test specification syntax. Test specifications can @@ -495,6 +660,9 @@ {cover, CoverSpecFile}. {cover, NodeRefs, CoverSpecFile}. + {cover_stop, Bool}. + {cover_stop, NodeRefs, Bool}. + {include, IncludeDirs}. {include, NodeRefs, IncludeDirs}. @@ -541,8 +709,8 @@ {groups, Dir, Suite, Groups}. {groups, NodeRefs, Dir, Suite, Groups}. - {groups, Dir, Suite, GroupSpec, {cases,Cases}}. - {groups, NodeRefs, Dir, Suite, GroupSpec, {cases,Cases}}. + {groups, Dir, Suite, Groups, {cases,Cases}}. + {groups, NodeRefs, Dir, Suite, Groups, {cases,Cases}}. {cases, Dir, Suite, Cases}. {cases, NodeRefs, Dir, Suite, Cases}. @@ -590,7 +758,8 @@ Dir = string() Suites = atom() | [atom()] | all Suite = atom() - Groups = GroupSpec | [GroupSpec] | all + Groups = GroupPath | [GroupPath] | GroupSpec | [GroupSpec] | all + GroupPath = [GroupName] GroupSpec = GroupName | {GroupName,Properties} | {GroupName,Properties,GroupSpec} GroupName = atom() GroupNames = GroupName | [GroupName] diff --git a/lib/common_test/src/Makefile b/lib/common_test/src/Makefile index f7dce195d7..dd2923ece9 100644 --- a/lib/common_test/src/Makefile +++ b/lib/common_test/src/Makefile @@ -73,7 +73,8 @@ MODULES= \ cth_surefire \ ct_netconfc \ ct_conn_log_h \ - cth_conn_log + cth_conn_log \ + ct_groups TARGET_MODULES= $(MODULES:%=$(EBIN)/%) BEAM_FILES= $(MODULES:%=$(EBIN)/%.$(EMULATOR)) diff --git a/lib/common_test/src/ct.erl b/lib/common_test/src/ct.erl index 5014309c0f..8eafdff29f 100644 --- a/lib/common_test/src/ct.erl +++ b/lib/common_test/src/ct.erl @@ -148,7 +148,7 @@ run(TestDirs) -> %%% {config,CfgFiles} | {userconfig, UserConfig} | %%% {allow_user_terms,Bool} | {logdir,LogDir} | %%% {silent_connections,Conns} | {stylesheet,CSSFile} | -%%% {cover,CoverSpecFile} | {step,StepOpts} | +%%% {cover,CoverSpecFile} | {cover_stop,Bool} | {step,StepOpts} | %%% {event_handler,EventHandlers} | {include,InclDirs} | %%% {auto_compile,Bool} | {create_priv_dir,CreatePrivDir} | %%% {multiply_timetraps,M} | {scale_timetraps,Bool} | @@ -161,7 +161,8 @@ run(TestDirs) -> %%% TestDirs = [string()] | string() %%% Suites = [string()] | [atom()] | string() | atom() %%% Cases = [atom()] | atom() -%%% Groups = [atom()] | atom() +%%% Groups = GroupNameOrPath | [GroupNameOrPath] +%%% GroupNameOrPath = [atom()] | atom() | all %%% TestSpecs = [string()] | string() %%% Label = string() | atom() %%% CfgFiles = [string()] | string() @@ -987,8 +988,9 @@ get_testdata(Key) -> end. %%%----------------------------------------------------------------- -%%% @spec abort_current_testcase(Reason) -> ok | {error,no_testcase_running} +%%% @spec abort_current_testcase(Reason) -> ok | {error,ErrorReason} %%% Reason = term() +%%% ErrorReason = no_testcase_running | parallel_group %%% %%% @doc <p>When calling this function, the currently executing test case will be aborted. %%% It is the user's responsibility to know for sure which test case is currently diff --git a/lib/common_test/src/ct_config.erl b/lib/common_test/src/ct_config.erl index 30bf5925c0..b1d709bc75 100644 --- a/lib/common_test/src/ct_config.erl +++ b/lib/common_test/src/ct_config.erl @@ -171,7 +171,7 @@ process_default_configs(Opts) -> lists:flatmap(fun({config,[_|_] = FileOrFiles}) -> case {io_lib:printable_list(FileOrFiles), io_lib:printable_list(hd(FileOrFiles))} of - {true,true} -> + {false,true} -> FileOrFiles; {true,false} -> [FileOrFiles]; @@ -532,7 +532,8 @@ do_require(Name,Key) -> case get_key_from_name(Name) of {error,_} -> allocate(Name,Key); - {ok,Key} -> + {ok,NameKey} when NameKey == Key; + is_tuple(Key) andalso element(1,Key) == NameKey -> %% already allocated - check that it has all required subkeys R = make_ref(), case get_config(Key,R,[]) of diff --git a/lib/common_test/src/ct_conn_log_h.erl b/lib/common_test/src/ct_conn_log_h.erl index bf27238121..d7bd18606b 100644 --- a/lib/common_test/src/ct_conn_log_h.erl +++ b/lib/common_test/src/ct_conn_log_h.erl @@ -64,10 +64,16 @@ do_open_files([],Acc) -> handle_event({_Type, GL, _Msg}, State) when node(GL) /= node() -> {ok, State}; handle_event({_Type,_GL,{Pid,{ct_connection,Action,ConnName},Report}},State) -> + %% NOTE: if the format of this event is changed + %% ({ct_connection,Action,ConnName}) then remember to change + %% test_server_h:report_receiver as well!!! Info = conn_info(Pid,#conn_log{name=ConnName,action=Action}), write_report(now(),Info,Report,State), {ok, State}; handle_event({_Type,_GL,{Pid,Info=#conn_log{},Report}},State) -> + %% NOTE: if the format of this event is changed + %% (Info=#conn_log{}) then remember to change + %% test_server_h:report_receiver as well!!! write_report(now(),conn_info(Pid,Info),Report,State), {ok, State}; handle_event({error_report,_,{Pid,_,[{ct_connection,ConnName}|R]}},State) -> diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index bec3368869..c1abf27e9f 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -32,8 +32,6 @@ -export([error_in_suite/1, init_per_suite/1, end_per_suite/1, init_per_group/2, end_per_group/2]). --export([make_all_conf/3, make_conf/5]). - -include("ct_event.hrl"). -include("ct_util.hrl"). @@ -876,13 +874,13 @@ get_suite(Mod, all) -> {'EXIT',_} -> get_all(Mod, []); GroupDefs when is_list(GroupDefs) -> - case catch find_groups(Mod, all, all, GroupDefs) of + case catch ct_groups:find_groups(Mod, all, all, GroupDefs) of {error,_} = Error -> %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Error properly [{?MODULE,error_in_suite,[[Error]]}]; ConfTests -> - get_all(Mod, ConfTests) + get_all(Mod, ConfTests) end; _ -> E = "Bad return value from "++atom_to_list(Mod)++":groups/0", @@ -901,7 +899,7 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> {'EXIT',_} -> [Group]; GroupDefs when is_list(GroupDefs) -> - case catch find_groups(Mod, Name, TCs, GroupDefs) of + case catch ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of {error,_} = Error -> %% this makes test_server call error_in_suite as first %% (and only) test case so we can report Error properly @@ -916,12 +914,13 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> %% init/end functions for top groups will be executed case catch ?val(name, element(2, hd(ConfTests))) of Name -> % top group - delete_subs(ConfTests, ConfTests); + ct_groups:delete_subs(ConfTests, ConfTests); _ -> [] end; false -> - ConfTests1 = delete_subs(ConfTests, ConfTests), + ConfTests1 = ct_groups:delete_subs(ConfTests, + ConfTests), case ?val(override, Props) of undefined -> ConfTests1; @@ -930,9 +929,9 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> ORSpec -> ORSpec1 = if is_tuple(ORSpec) -> [ORSpec]; true -> ORSpec end, - search_and_override(ConfTests1, - ORSpec1, Mod) - end + ct_groups:search_and_override(ConfTests1, + ORSpec1, Mod) + end end end; _ -> @@ -976,234 +975,6 @@ get_all_cases1(_, []) -> %%%----------------------------------------------------------------- -find_groups(Mod, Name, TCs, GroupDefs) -> - Found = find(Mod, Name, TCs, GroupDefs, [], GroupDefs, false), - trim(Found). - -find(Mod, all, _TCs, [{Name,Props,Tests} | Gs], Known, Defs, _) - when is_atom(Name), is_list(Props), is_list(Tests) -> - cyclic_test(Mod, Name, Known), - [make_conf(Mod, Name, Props, - find(Mod, all, all, Tests, [Name | Known], Defs, true)) | - find(Mod, all, all, Gs, [], Defs, true)]; - -find(Mod, Name, TCs, [{Name,Props,Tests} | _Gs], Known, Defs, false) - when is_atom(Name), is_list(Props), is_list(Tests) -> - cyclic_test(Mod, Name, Known), - case TCs of - all -> - [make_conf(Mod, Name, Props, - find(Mod, Name, TCs, Tests, [Name | Known], Defs, true))]; - _ -> - Tests1 = [TC || TC <- TCs, - lists:member(TC, Tests) == true], - [make_conf(Mod, Name, Props, Tests1)] - end; - -find(Mod, Name, TCs, [{Name1,Props,Tests} | Gs], Known, Defs, false) - when is_atom(Name1), is_list(Props), is_list(Tests) -> - cyclic_test(Mod, Name1, Known), - [make_conf(Mod,Name1,Props, - find(Mod, Name, TCs, Tests, [Name1 | Known], Defs, false)) | - find(Mod, Name, TCs, Gs, [], Defs, false)]; - -find(Mod, Name, _TCs, [{Name,_Props,_Tests} | _Gs], _Known, _Defs, true) - when is_atom(Name) -> - E = "Duplicate groups named "++atom_to_list(Name)++" in "++ - atom_to_list(Mod)++":groups/0", - throw({error,list_to_atom(E)}); - -find(Mod, Name, all, [{Name1,Props,Tests} | Gs], Known, Defs, true) - when is_atom(Name1), is_list(Props), is_list(Tests) -> - cyclic_test(Mod, Name1, Known), - [make_conf(Mod, Name1, Props, - find(Mod, Name, all, Tests, [Name1 | Known], Defs, true)) | - find(Mod, Name, all, Gs, [], Defs, true)]; - -find(Mod, Name, TCs, [{group,Name1} | Gs], Known, Defs, Found) - when is_atom(Name1) -> - find(Mod, Name, TCs, [expand(Mod, Name1, Defs) | Gs], Known, Defs, Found); - -%% Undocumented remote group feature, use with caution -find(Mod, Name, TCs, [{group, ExtMod, ExtGrp} | Gs], Known, Defs, true) - when is_atom(ExtMod), is_atom(ExtGrp) -> - ExternalDefs = ExtMod:groups(), - ExternalTCs = find(ExtMod, ExtGrp, TCs, [{group, ExtGrp}], - [], ExternalDefs, false), - ExternalTCs ++ find(Mod, Name, TCs, Gs, Known, Defs, true); - -find(Mod, Name, TCs, [{Name1,Tests} | Gs], Known, Defs, Found) - when is_atom(Name1), is_list(Tests) -> - find(Mod, Name, TCs, [{Name1,[],Tests} | Gs], Known, Defs, Found); - -find(Mod, Name, TCs, [_TC | Gs], Known, Defs, false) -> - find(Mod, Name, TCs, Gs, Known, Defs, false); - -find(Mod, Name, TCs, [TC | Gs], Known, Defs, true) when is_atom(TC) -> - [{Mod, TC} | find(Mod, Name, TCs, Gs, Known, Defs, true)]; - -find(Mod, Name, TCs, [{ExternalTC, Case} = TC | Gs], Known, Defs, true) - when is_atom(ExternalTC), - is_atom(Case) -> - [TC | find(Mod, Name, TCs, Gs, Known, Defs, true)]; - -find(Mod, _Name, _TCs, [BadTerm | _Gs], Known, _Defs, _Found) -> - Where = if length(Known) == 0 -> - atom_to_list(Mod)++":groups/0"; - true -> - "group "++atom_to_list(lists:last(Known))++ - " in "++atom_to_list(Mod)++":groups/0" - end, - Term = io_lib:format("~p", [BadTerm]), - E = "Bad term "++lists:flatten(Term)++" in "++Where, - throw({error,list_to_atom(E)}); - -find(_Mod, _Name, _TCs, [], _Known, _Defs, false) -> - ['$NOMATCH']; - -find(_Mod, _Name, _TCs, [], _Known, _Defs, _Found) -> - []. - -delete_subs([{conf, _,_,_,_} = Conf | Confs], All) -> - All1 = delete_conf(Conf, All), - case is_sub(Conf, All1) of - true -> - delete_subs(Confs, All1); - false -> - delete_subs(Confs, All) - end; -delete_subs([_Else | Confs], All) -> - delete_subs(Confs, All); -delete_subs([], All) -> - All. - -delete_conf({conf,Props,_,_,_}, Confs) -> - Name = ?val(name, Props), - [Conf || Conf = {conf,Props0,_,_,_} <- Confs, - Name =/= ?val(name, Props0)]. - -is_sub({conf,Props,_,_,_}=Conf, [{conf,_,_,Tests,_} | Confs]) -> - Name = ?val(name, Props), - case lists:any(fun({conf,Props0,_,_,_}) -> - case ?val(name, Props0) of - N when N == Name -> - true; - _ -> - false - end; - (_) -> - false - end, Tests) of - true -> - true; - false -> - is_sub(Conf, Tests) or is_sub(Conf, Confs) - end; - -is_sub(Conf, [_TC | Tests]) -> - is_sub(Conf, Tests); - -is_sub(_Conf, []) -> - false. - -trim(['$NOMATCH' | Tests]) -> - trim(Tests); - -trim([{conf,Props,Init,Tests,End} | Confs]) -> - case trim(Tests) of - [] -> - trim(Confs); - Trimmed -> - [{conf,Props,Init,Trimmed,End} | trim(Confs)] - end; - -trim([TC | Tests]) -> - [TC | trim(Tests)]; - -trim([]) -> - []. - -cyclic_test(Mod, Name, Names) -> - case lists:member(Name, Names) of - true -> - E = "Cyclic reference to group "++atom_to_list(Name)++ - " in "++atom_to_list(Mod)++":groups/0", - throw({error,list_to_atom(E)}); - false -> - ok - end. - -expand(Mod, Name, Defs) -> - case lists:keysearch(Name, 1, Defs) of - {value,Def} -> - Def; - false -> - E = "Invalid group "++atom_to_list(Name)++ - " in "++atom_to_list(Mod)++":groups/0", - throw({error,list_to_atom(E)}) - end. - -make_all_conf(Dir, Mod, _Props) -> - case code:is_loaded(Mod) of - false -> - code:load_abs(filename:join(Dir,atom_to_list(Mod))); - _ -> - ok - end, - make_all_conf(Mod). - -make_all_conf(Mod) -> - case catch apply(Mod, groups, []) of - {'EXIT',_} -> - {error,{invalid_group_definition,Mod}}; - GroupDefs when is_list(GroupDefs) -> - case catch find_groups(Mod, all, all, GroupDefs) of - {error,_} = Error -> - %% this makes test_server call error_in_suite as first - %% (and only) test case so we can report Error properly - [{?MODULE,error_in_suite,[[Error]]}]; - [] -> - {error,{invalid_group_spec,Mod}}; - ConfTests -> - [{conf,Props,Init,all,End} || - {conf,Props,Init,_,End} - <- delete_subs(ConfTests, ConfTests)] - end - end. - -make_conf(Dir, Mod, Name, Props, TestSpec) -> - case code:is_loaded(Mod) of - false -> - code:load_abs(filename:join(Dir,atom_to_list(Mod))); - _ -> - ok - end, - make_conf(Mod, Name, Props, TestSpec). - -make_conf(Mod, Name, Props, TestSpec) -> - case code:is_loaded(Mod) of - false -> - code:load_file(Mod); - _ -> - ok - end, - {InitConf,EndConf,ExtraProps} = - case erlang:function_exported(Mod,init_per_group,2) of - true -> - {{Mod,init_per_group},{Mod,end_per_group},[]}; - false -> - ct_logs:log("TEST INFO", "init_per_group/2 and " - "end_per_group/2 missing for group " - "~p in ~p, using default.", - [Name,Mod]), - {{?MODULE,init_per_group}, - {?MODULE,end_per_group}, - [{suite,Mod}]} - end, - {conf,[{name,Name}|Props++ExtraProps],InitConf,TestSpec,EndConf}. - -%%%----------------------------------------------------------------- - get_all(Mod, ConfTests) -> case catch apply(Mod, all, []) of {'EXIT',_} -> @@ -1218,133 +989,24 @@ get_all(Mod, ConfTests) -> [{?MODULE,error_in_suite,[[{error,What}]]}]; SeqsAndTCs -> %% expand group references in all() using ConfTests - case catch expand_groups(SeqsAndTCs, ConfTests, Mod) of + case catch ct_groups:expand_groups(SeqsAndTCs, + ConfTests, + Mod) of {error,_} = Error -> [{?MODULE,error_in_suite,[[Error]]}]; Tests -> - delete_subs(Tests, Tests) + ct_groups:delete_subs(Tests, Tests) end end; Skip = {skip,_Reason} -> Skip; _ -> Reason = - list_to_atom("Bad return value from "++atom_to_list(Mod)++":all/0"), + list_to_atom("Bad return value from "++ + atom_to_list(Mod)++":all/0"), [{?MODULE,error_in_suite,[[{error,Reason}]]}] end. -expand_groups([H | T], ConfTests, Mod) -> - [expand_groups(H, ConfTests, Mod) | expand_groups(T, ConfTests, Mod)]; -expand_groups([], _ConfTests, _Mod) -> - []; -expand_groups({group,Name}, ConfTests, Mod) -> - expand_groups({group,Name,default,[]}, ConfTests, Mod); -expand_groups({group,Name,default}, ConfTests, Mod) -> - expand_groups({group,Name,default,[]}, ConfTests, Mod); -expand_groups({group,Name,ORProps}, ConfTests, Mod) when is_list(ORProps) -> - expand_groups({group,Name,ORProps,[]}, ConfTests, Mod); -expand_groups({group,Name,ORProps,SubORSpec}, ConfTests, Mod) -> - FindConf = - fun(Conf = {conf,Props,Init,Ts,End}) -> - case ?val(name, Props) of - Name when ORProps == default -> - [Conf]; - Name -> - [{conf,[{name,Name}|ORProps],Init,Ts,End}]; - _ -> - [] - end - end, - case lists:flatmap(FindConf, ConfTests) of - [] -> - throw({error,invalid_ref_msg(Name, Mod)}); - Matching when SubORSpec == [] -> - Matching; - Matching -> - override_props(Matching, SubORSpec, Name,Mod) - end; -expand_groups(SeqOrTC, _ConfTests, _Mod) -> - SeqOrTC. - -%% search deep for the matching conf test and modify it and any -%% sub tests according to the override specification -search_and_override([Conf = {conf,Props,Init,Tests,End}], ORSpec, Mod) -> - Name = ?val(name, Props), - case lists:keysearch(Name, 1, ORSpec) of - {value,{Name,default}} -> - [Conf]; - {value,{Name,ORProps}} -> - [{conf,[{name,Name}|ORProps],Init,Tests,End}]; - {value,{Name,default,[]}} -> - [Conf]; - {value,{Name,default,SubORSpec}} -> - override_props([Conf], SubORSpec, Name,Mod); - {value,{Name,ORProps,SubORSpec}} -> - override_props([{conf,[{name,Name}|ORProps], - Init,Tests,End}], SubORSpec, Name,Mod); - _ -> - [{conf,Props,Init,search_and_override(Tests,ORSpec,Mod),End}] - end. - -%% Modify the Tests element according to the override specification -override_props([{conf,Props,Init,Tests,End} | Confs], SubORSpec, Name,Mod) -> - {Subs,SubORSpec1} = override_sub_props(Tests, [], SubORSpec, Mod), - [{conf,Props,Init,Subs,End} | override_props(Confs, SubORSpec1, Name,Mod)]; -override_props([], [], _,_) -> - []; -override_props([], SubORSpec, Name,Mod) -> - Es = [invalid_ref_msg(Name, element(1,Spec), Mod) || Spec <- SubORSpec], - throw({error,Es}). - -override_sub_props([], New, ORSpec, _) -> - {?rev(New),ORSpec}; -override_sub_props([T = {conf,Props,Init,Tests,End} | Ts], - New, ORSpec, Mod) -> - Name = ?val(name, Props), - case lists:keysearch(Name, 1, ORSpec) of - {value,Spec} -> % group found in spec - Props1 = - case element(2, Spec) of - default -> Props; - ORProps -> [{name,Name} | ORProps] - end, - case catch element(3, Spec) of - Undef when Undef == [] ; 'EXIT' == element(1, Undef) -> - override_sub_props(Ts, [{conf,Props1,Init,Tests,End} | New], - lists:keydelete(Name, 1, ORSpec), Mod); - SubORSpec when is_list(SubORSpec) -> - case override_sub_props(Tests, [], SubORSpec, Mod) of - {Subs,[]} -> - override_sub_props(Ts, [{conf,Props1,Init, - Subs,End} | New], - lists:keydelete(Name, 1, ORSpec), - Mod); - {_,NonEmptySpec} -> - Es = [invalid_ref_msg(Name, element(1, GrRef), - Mod) || GrRef <- NonEmptySpec], - throw({error,Es}) - end; - BadGrSpec -> - throw({error,{invalid_form,BadGrSpec}}) - end; - _ -> % not a group in spec - override_sub_props(Ts, [T | New], ORSpec, Mod) - end; -override_sub_props([TC | Ts], New, ORSpec, Mod) -> - override_sub_props(Ts, [TC | New], ORSpec, Mod). - -invalid_ref_msg(Name, Mod) -> - E = "Invalid reference to group "++ - atom_to_list(Name)++" in "++ - atom_to_list(Mod)++":all/0", - list_to_atom(E). - -invalid_ref_msg(Name0, Name1, Mod) -> - E = "Invalid reference to group "++ - atom_to_list(Name1)++" from "++atom_to_list(Name0)++ - " in "++atom_to_list(Mod)++":all/0", - list_to_atom(E). - %%!============================================================ %%! The support for sequences by means of using sequences/0 %%! will be removed in OTP R15. The code below is only kept diff --git a/lib/common_test/src/ct_groups.erl b/lib/common_test/src/ct_groups.erl new file mode 100644 index 0000000000..74ab5e5439 --- /dev/null +++ b/lib/common_test/src/ct_groups.erl @@ -0,0 +1,599 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2004-2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%% @doc Common Test Framework callback module. +%%% +%%% <p>This module contains CT internal help functions for searching +%%% through groups specification trees and producing resulting +%%% tests.</p> + +-module(ct_groups). + +-export([find_groups/4]). +-export([make_all_conf/3, make_all_conf/4, make_conf/5]). +-export([delete_subs/2]). +-export([expand_groups/3, search_and_override/3]). + +-define(val(Key, List), proplists:get_value(Key, List)). +-define(val(Key, List, Def), proplists:get_value(Key, List, Def)). +-define(rev(L), lists:reverse(L)). + +find_groups(Mod, GrNames, TCs, GroupDefs) when is_atom(GrNames) ; + (length(GrNames) == 1) -> + find_groups1(Mod, GrNames, TCs, GroupDefs); + +find_groups(Mod, Groups, TCs, GroupDefs) when Groups /= [] -> + lists:append([find_groups1(Mod, [GrNames], TCs, GroupDefs) || + GrNames <- Groups]); + +find_groups(_Mod, [], _TCs, _GroupDefs) -> + []. + +%% GrNames == atom(): Single group name, perform full search +%% GrNames == list(): List of groups, find all matching paths +%% GrNames == [list()]: Search path terminated by last group in GrNames +find_groups1(Mod, GrNames, TCs, GroupDefs) -> + {GrNames1,FindAll} = + case GrNames of + Name when is_atom(Name), Name /= all -> + {[Name],true}; + [Path] when is_list(Path) -> + {Path,false}; + Path -> + {Path,true} + end, + TCs1 = if (is_atom(TCs) and (TCs /= all)) or is_tuple(TCs) -> + [TCs]; + true -> + TCs + end, + Found = find(Mod, GrNames1, TCs1, GroupDefs, [], + GroupDefs, FindAll), + [Conf || Conf <- Found, Conf /= 'NOMATCH']. + +%% Locate all groups +find(Mod, all, all, [{Name,Props,Tests} | Gs], Known, Defs, _) + when is_atom(Name), is_list(Props), is_list(Tests) -> + cyclic_test(Mod, Name, Known), + trim(make_conf(Mod, Name, Props, + find(Mod, all, all, Tests, [Name | Known], + Defs, true))) ++ + find(Mod, all, all, Gs, Known, Defs, true); + +%% Locate particular TCs in all groups +find(Mod, all, TCs, [{Name,Props,Tests} | Gs], Known, Defs, _) + when is_atom(Name), is_list(Props), is_list(Tests) -> + cyclic_test(Mod, Name, Known), + Tests1 = rm_unwanted_tcs(Tests, TCs, []), + trim(make_conf(Mod, Name, Props, + find(Mod, all, TCs, Tests1, [Name | Known], + Defs, true))) ++ + find(Mod, all, TCs, Gs, Known, Defs, true); + +%% Found next group is in search path +find(Mod, [Name|GrNames]=SPath, TCs, [{Name,Props,Tests} | Gs], Known, + Defs, FindAll) when is_atom(Name), is_list(Props), is_list(Tests) -> + cyclic_test(Mod, Name, Known), + Tests1 = rm_unwanted_tcs(Tests, TCs, GrNames), + trim(make_conf(Mod, Name, Props, + find(Mod, GrNames, TCs, Tests1, [Name|Known], + Defs, FindAll))) ++ + find(Mod, SPath, TCs, Gs, Known, Defs, FindAll); + +%% Group path terminated, stop the search +find(Mod, [], TCs, Tests, _Known, _Defs, false) -> + Cases = lists:flatmap(fun(TC) when is_atom(TC), TCs == all -> + [{Mod,TC}]; + ({group,_}) -> + []; + ({_,_}=TC) when TCs == all -> + [TC]; + (TC) -> + if is_atom(TC) -> + Tuple = {Mod,TC}, + case lists:member(Tuple, TCs) of + true -> + [Tuple]; + false -> + case lists:member(TC, TCs) of + true -> [{Mod,TC}]; + false -> [] + end + end; + true -> + [] + end + end, Tests), + if Cases == [] -> ['NOMATCH']; + true -> Cases + end; + +%% No more groups +find(_Mod, [_|_], _TCs, [], _Known, _Defs, _) -> + ['NOMATCH']; + +%% Found group not next in search path +find(Mod, GrNames, TCs, [{Name,Props,Tests} | Gs], Known, + Defs, FindAll) when is_atom(Name), is_list(Props), is_list(Tests) -> + cyclic_test(Mod, Name, Known), + Tests1 = rm_unwanted_tcs(Tests, TCs, GrNames), + trim(make_conf(Mod, Name, Props, + find(Mod, GrNames, TCs, Tests1, [Name|Known], + Defs, FindAll))) ++ + find(Mod, GrNames, TCs, Gs, Known, Defs, FindAll); + +%% A nested group defined on top level found +find(Mod, GrNames, TCs, [{group,Name1} | Gs], Known, Defs, FindAll) + when is_atom(Name1) -> + find(Mod, GrNames, TCs, [expand(Mod, Name1, Defs) | Gs], Known, + Defs, FindAll); + +%% Undocumented remote group feature, use with caution +find(Mod, GrNames, TCs, [{group, ExtMod, ExtGrp} | Gs], Known, + Defs, FindAll) when is_atom(ExtMod), is_atom(ExtGrp) -> + ExternalDefs = ExtMod:groups(), + ExternalTCs = find(ExtMod, ExtGrp, TCs, [{group, ExtGrp}], + [], ExternalDefs, FindAll), + ExternalTCs ++ find(Mod, GrNames, TCs, Gs, Known, Defs, FindAll); + +%% Group definition without properties, add an empty property list +find(Mod, GrNames, TCs, [{Name1,Tests} | Gs], Known, Defs, FindAll) + when is_atom(Name1), is_list(Tests) -> + find(Mod, GrNames, TCs, [{Name1,[],Tests} | Gs], Known, Defs, FindAll); + +%% Save, and keep searching +find(Mod, GrNames, TCs, [{ExternalTC, Case} = TC | Gs], Known, + Defs, FindAll) when is_atom(ExternalTC), + is_atom(Case) -> + [TC | find(Mod, GrNames, TCs, Gs, Known, Defs, FindAll)]; + +%% Save test case +find(Mod, GrNames, all, [TC | Gs], Known, + Defs, FindAll) when is_atom(TC) -> + [{Mod,TC} | find(Mod, GrNames, all, Gs, Known, Defs, FindAll)]; + +%% Save test case +find(Mod, GrNames, all, [{M,TC} | Gs], Known, + Defs, FindAll) when is_atom(M), M /= group, is_atom(TC) -> + [{M,TC} | find(Mod, GrNames, all, Gs, Known, Defs, FindAll)]; + +%% Check if test case should be saved +find(Mod, GrNames, TCs, [TC | Gs], Known, + Defs, FindAll) when is_atom(TC) orelse + ((size(TC) == 2) and (element(1,TC) /= group)) -> + Case = + if is_atom(TC) -> + Tuple = {Mod,TC}, + case lists:member(Tuple, TCs) of + true -> + Tuple; + false -> + case lists:member(TC, TCs) of + true -> {Mod,TC}; + false -> [] + end + end; + true -> + case lists:member(TC, TCs) of + true -> {Mod,TC}; + false -> [] + end + end, + if Case == [] -> + find(Mod, GrNames, TCs, Gs, Known, Defs, FindAll); + true -> + [Case | find(Mod, GrNames, TCs, Gs, Known, Defs, FindAll)] + end; + +%% Unexpeted term in group list +find(Mod, _GrNames, _TCs, [BadTerm | _Gs], Known, _Defs, _FindAll) -> + Where = if length(Known) == 0 -> + atom_to_list(Mod)++":groups/0"; + true -> + "group "++atom_to_list(lists:last(Known))++ + " in "++atom_to_list(Mod)++":groups/0" + end, + Term = io_lib:format("~p", [BadTerm]), + E = "Bad term "++lists:flatten(Term)++" in "++Where, + throw({error,list_to_atom(E)}); + +%% No more groups +find(_Mod, _GrNames, _TCs, [], _Known, _Defs, _) -> + []. + +%%%----------------------------------------------------------------- + +%% We have to always search bottom up to only remove a branch +%% if there's 'NOMATCH' in the leaf (otherwise, the branch should +%% be kept) + +trim({conf,Props,Init,Tests,End}) -> + try trim(Tests) of + [] -> []; + Tests1 -> [{conf,Props,Init,Tests1,End}] + catch + throw:_ -> [] + end; + +trim(Tests) when is_list(Tests) -> + %% we need to compare the result of trimming each test on this + %% level, and only let a 'NOMATCH' fail the level if no + %% successful sub group can be found + Tests1 = + lists:flatmap(fun(Test) -> + IsConf = case Test of + {conf,_,_,_,_} -> + true; + _ -> + false + end, + try trim_test(Test) of + [] -> []; + Test1 when IsConf -> [{conf,Test1}]; + Test1 -> [Test1] + catch + throw:_ -> ['NOMATCH'] + end + end, Tests), + case lists:keymember(conf, 1, Tests1) of + true -> % at least one successful group + lists:flatmap(fun({conf,Test}) -> [Test]; + ('NOMATCH') -> []; % ignore any 'NOMATCH' + (Test) -> [Test] + end, Tests1); + false -> + case lists:member('NOMATCH', Tests1) of + true -> + throw('NOMATCH'); + false -> + Tests1 + end + end. + +trim_test({conf,Props,Init,Tests,End}) -> + case trim(Tests) of + [] -> + []; + Tests1 -> + {conf,Props,Init,Tests1,End} + end; + +trim_test('NOMATCH') -> + throw('NOMATCH'); + +trim_test(Test) -> + Test. + +%% GrNames is [] if the terminating group has been found. From +%% that point, all specified test should be included (as well as +%% sub groups for deeper search). +rm_unwanted_tcs(Tests, all, []) -> + Tests; + +rm_unwanted_tcs(Tests, TCs, []) -> + sort_tests(lists:flatmap(fun(Test) when is_tuple(Test), + (size(Test) > 2) -> + [Test]; + (Test={group,_}) -> + [Test]; + (Test={_M,TC}) -> + case lists:member(TC, TCs) of + true -> [Test]; + false -> [] + end; + (Test) when is_atom(Test) -> + case lists:keysearch(Test, 2, TCs) of + {value,_} -> + [Test]; + _ -> + case lists:member(Test, TCs) of + true -> [Test]; + false -> [] + end + end; + (Test) -> [Test] + end, Tests), TCs); + +rm_unwanted_tcs(Tests, _TCs, _) -> + [Test || Test <- Tests, not is_atom(Test)]. + +%% make sure the order of tests is according to the order in TCs +sort_tests(Tests, TCs) when is_list(TCs)-> + lists:sort(fun(T1, T2) -> + case {is_tc(T1),is_tc(T2)} of + {true,true} -> + (position(T1, TCs) =< + position(T2, TCs)); + {false,true} -> + (position(T2, TCs) == (length(TCs)+1)); + _ -> true + + end + end, Tests); +sort_tests(Tests, _) -> + Tests. + +is_tc(T) when is_atom(T) -> true; +is_tc({group,_}) -> false; +is_tc({_M,T}) when is_atom(T) -> true; +is_tc(_) -> false. + +position(T, TCs) -> + position(T, TCs, 1). + +position(T, [T|_TCs], Pos) -> + Pos; +position(T, [{_,T}|_TCs], Pos) -> + Pos; +position({M,T}, [T|_TCs], Pos) when M /= group -> + Pos; +position(T, [_|TCs], Pos) -> + position(T, TCs, Pos+1); +position(_, [], Pos) -> + Pos. + +%%%----------------------------------------------------------------- + +delete_subs([{conf, _,_,_,_} = Conf | Confs], All) -> + All1 = delete_conf(Conf, All), + case is_sub(Conf, All1) of + true -> + delete_subs(Confs, All1); + false -> + delete_subs(Confs, All) + end; +delete_subs([_Else | Confs], All) -> + delete_subs(Confs, All); +delete_subs([], All) -> + All. + +delete_conf({conf,Props,_,_,_}, Confs) -> + Name = ?val(name, Props), + [Conf || Conf = {conf,Props0,_,_,_} <- Confs, + Name =/= ?val(name, Props0)]. + +is_sub({conf,Props,_,_,_}=Conf, [{conf,_,_,Tests,_} | Confs]) -> + Name = ?val(name, Props), + case lists:any(fun({conf,Props0,_,_,_}) -> + case ?val(name, Props0) of + N when N == Name -> + true; + _ -> + false + end; + (_) -> + false + end, Tests) of + true -> + true; + false -> + is_sub(Conf, Tests) orelse is_sub(Conf, Confs) + end; + +is_sub(Conf, [_TC | Tests]) -> + is_sub(Conf, Tests); + +is_sub(_Conf, []) -> + false. + + +cyclic_test(Mod, Name, Names) -> + case lists:member(Name, Names) of + true -> + E = "Cyclic reference to group "++atom_to_list(Name)++ + " in "++atom_to_list(Mod)++":groups/0", + throw({error,list_to_atom(E)}); + false -> + ok + end. + +expand(Mod, Name, Defs) -> + case lists:keysearch(Name, 1, Defs) of + {value,Def} -> + Def; + false -> + E = "Invalid group "++atom_to_list(Name)++ + " in "++atom_to_list(Mod)++":groups/0", + throw({error,list_to_atom(E)}) + end. + +make_all_conf(Dir, Mod, Props, TestSpec) -> + case code:is_loaded(Mod) of + false -> + code:load_abs(filename:join(Dir,atom_to_list(Mod))); + _ -> + ok + end, + make_all_conf(Mod, Props, TestSpec). + +make_all_conf(Mod, Props, TestSpec) -> + case catch apply(Mod, groups, []) of + {'EXIT',_} -> + exit({invalid_group_definition,Mod}); + GroupDefs when is_list(GroupDefs) -> + case catch find_groups(Mod, all, TestSpec, GroupDefs) of + {error,_} = Error -> + %% this makes test_server call error_in_suite as first + %% (and only) test case so we can report Error properly + [{ct_framework,error_in_suite,[[Error]]}]; + [] -> + exit({invalid_group_spec,Mod}); + _ConfTests -> + make_conf(Mod, all, Props, TestSpec) + end + end. + +make_conf(Dir, Mod, Name, Props, TestSpec) -> + case code:is_loaded(Mod) of + false -> + code:load_abs(filename:join(Dir,atom_to_list(Mod))); + _ -> + ok + end, + make_conf(Mod, Name, Props, TestSpec). + +make_conf(Mod, Name, Props, TestSpec) -> + case code:is_loaded(Mod) of + false -> + code:load_file(Mod); + _ -> + ok + end, + {InitConf,EndConf,ExtraProps} = + case erlang:function_exported(Mod,init_per_group,2) of + true -> + {{Mod,init_per_group},{Mod,end_per_group},[]}; + false -> + ct_logs:log("TEST INFO", "init_per_group/2 and " + "end_per_group/2 missing for group " + "~p in ~p, using default.", + [Name,Mod]), + {{ct_framework,init_per_group}, + {ct_framework,end_per_group}, + [{suite,Mod}]} + end, + {conf,[{name,Name}|Props++ExtraProps],InitConf,TestSpec,EndConf}. + +%%%----------------------------------------------------------------- + +expand_groups([H | T], ConfTests, Mod) -> + [expand_groups(H, ConfTests, Mod) | expand_groups(T, ConfTests, Mod)]; +expand_groups([], _ConfTests, _Mod) -> + []; +expand_groups({group,Name}, ConfTests, Mod) -> + expand_groups({group,Name,default,[]}, ConfTests, Mod); +expand_groups({group,Name,default}, ConfTests, Mod) -> + expand_groups({group,Name,default,[]}, ConfTests, Mod); +expand_groups({group,Name,ORProps}, ConfTests, Mod) when is_list(ORProps) -> + expand_groups({group,Name,ORProps,[]}, ConfTests, Mod); +expand_groups({group,Name,ORProps,SubORSpec}, ConfTests, Mod) -> + FindConf = + fun(Conf = {conf,Props,Init,Ts,End}) -> + case ?val(name, Props) of + Name when ORProps == default -> + [Conf]; + Name -> + Props1 = case ?val(suite, Props) of + undefined -> + ORProps; + SuiteName -> + [{suite,SuiteName}|ORProps] + end, + [{conf,[{name,Name}|Props1],Init,Ts,End}]; + _ -> + [] + end + end, + case lists:flatmap(FindConf, ConfTests) of + [] -> + throw({error,invalid_ref_msg(Name, Mod)}); + Matching when SubORSpec == [] -> + Matching; + Matching -> + override_props(Matching, SubORSpec, Name,Mod) + end; +expand_groups(SeqOrTC, _ConfTests, _Mod) -> + SeqOrTC. + +%% search deep for the matching conf test and modify it and any +%% sub tests according to the override specification +search_and_override([Conf = {conf,Props,Init,Tests,End}], ORSpec, Mod) -> + InsProps = fun(GrName, undefined, Ps) -> + [{name,GrName} | Ps]; + (GrName, Suite, Ps) -> + [{name,GrName}, {suite,Suite} | Ps] + end, + Name = ?val(name, Props), + Suite = ?val(suite, Props), + case lists:keysearch(Name, 1, ORSpec) of + {value,{Name,default}} -> + [Conf]; + {value,{Name,ORProps}} -> + [{conf,InsProps(Name,Suite,ORProps),Init,Tests,End}]; + {value,{Name,default,[]}} -> + [Conf]; + {value,{Name,default,SubORSpec}} -> + override_props([Conf], SubORSpec, Name,Mod); + {value,{Name,ORProps,SubORSpec}} -> + override_props([{conf,InsProps(Name,Suite,ORProps), + Init,Tests,End}], SubORSpec, Name,Mod); + _ -> + [{conf,Props,Init,search_and_override(Tests,ORSpec,Mod),End}] + end. + +%% Modify the Tests element according to the override specification +override_props([{conf,Props,Init,Tests,End} | Confs], SubORSpec, Name,Mod) -> + {Subs,SubORSpec1} = override_sub_props(Tests, [], SubORSpec, Mod), + [{conf,Props,Init,Subs,End} | override_props(Confs, SubORSpec1, Name,Mod)]; +override_props([], [], _,_) -> + []; +override_props([], SubORSpec, Name,Mod) -> + Es = [invalid_ref_msg(Name, element(1,Spec), Mod) || Spec <- SubORSpec], + throw({error,Es}). + +override_sub_props([], New, ORSpec, _) -> + {?rev(New),ORSpec}; +override_sub_props([T = {conf,Props,Init,Tests,End} | Ts], + New, ORSpec, Mod) -> + Name = ?val(name, Props), + Suite = ?val(suite, Props), + case lists:keysearch(Name, 1, ORSpec) of + {value,Spec} -> % group found in spec + Props1 = + case element(2, Spec) of + default -> Props; + ORProps when Suite == undefined -> [{name,Name} | ORProps]; + ORProps -> [{name,Name}, {suite,Suite} | ORProps] + end, + case catch element(3, Spec) of + Undef when Undef == [] ; 'EXIT' == element(1, Undef) -> + override_sub_props(Ts, [{conf,Props1,Init,Tests,End} | New], + lists:keydelete(Name, 1, ORSpec), Mod); + SubORSpec when is_list(SubORSpec) -> + case override_sub_props(Tests, [], SubORSpec, Mod) of + {Subs,[]} -> + override_sub_props(Ts, [{conf,Props1,Init, + Subs,End} | New], + lists:keydelete(Name, 1, ORSpec), + Mod); + {_,NonEmptySpec} -> + Es = [invalid_ref_msg(Name, element(1, GrRef), + Mod) || GrRef <- NonEmptySpec], + throw({error,Es}) + end; + BadGrSpec -> + throw({error,{invalid_form,BadGrSpec}}) + end; + _ -> % not a group in spec + override_sub_props(Ts, [T | New], ORSpec, Mod) + end; +override_sub_props([TC | Ts], New, ORSpec, Mod) -> + override_sub_props(Ts, [TC | New], ORSpec, Mod). + +invalid_ref_msg(Name, Mod) -> + E = "Invalid reference to group "++ + atom_to_list(Name)++" in "++ + atom_to_list(Mod)++":all/0", + list_to_atom(E). + +invalid_ref_msg(Name0, Name1, Mod) -> + E = "Invalid reference to group "++ + atom_to_list(Name1)++" from "++atom_to_list(Name0)++ + " in "++atom_to_list(Mod)++":all/0", + list_to_atom(E). diff --git a/lib/common_test/src/ct_master.erl b/lib/common_test/src/ct_master.erl index 99bec3ea09..f29eba605c 100644 --- a/lib/common_test/src/ct_master.erl +++ b/lib/common_test/src/ct_master.erl @@ -51,7 +51,7 @@ %%% {testcase,Cases} | {spec,TestSpecs} | {allow_user_terms,Bool} | %%% {logdir,LogDir} | {event_handler,EventHandlers} | %%% {silent_connections,Conns} | {cover,CoverSpecFile} | -%%% {userconfig, UserCfgFiles} +%%% {cover_stop,Bool} | {userconfig, UserCfgFiles} %%% CfgFiles = string() | [string()] %%% TestDirs = string() | [string()] %%% Suites = atom() | [atom()] diff --git a/lib/common_test/src/ct_master_logs.erl b/lib/common_test/src/ct_master_logs.erl index d76288feef..84f175c0a9 100644 --- a/lib/common_test/src/ct_master_logs.erl +++ b/lib/common_test/src/ct_master_logs.erl @@ -204,7 +204,7 @@ open_ct_master_log(Dir) -> {ok,Fd} = file:open(FullName,[write]), io:put_chars(Fd,header("Common Test Master Log", {[],[1,2],[]})), %% maybe add config info here later - io:put_chars(config_table([])), + io:put_chars(Fd,config_table([])), io:put_chars(Fd, "<style>\n" "div.ct_internal { background:lightgrey; color:black }\n" diff --git a/lib/common_test/src/ct_netconfc.erl b/lib/common_test/src/ct_netconfc.erl index 52fe9599ce..294b82bff6 100644 --- a/lib/common_test/src/ct_netconfc.erl +++ b/lib/common_test/src/ct_netconfc.erl @@ -968,7 +968,7 @@ close_session(Client) -> %% @end %%---------------------------------------------------------------------- close_session(Client, Timeout) -> - call(Client,{send_rpc_op, close_session, [], Timeout}). + call(Client,{send_rpc_op, close_session, [], Timeout}, true). %%---------------------------------------------------------------------- @@ -1121,17 +1121,38 @@ close(Client) -> %% Internal functions %%---------------------------------------------------------------------- call(Client, Msg) -> - call(Client, Msg, infinity). -call(Client, Msg, Timeout) -> + call(Client, Msg, infinity, false). +call(Client, Msg, Timeout) when is_integer(Timeout); Timeout==infinity -> + call(Client, Msg, Timeout, false); +call(Client, Msg, WaitStop) when is_boolean(WaitStop) -> + call(Client, Msg, infinity, WaitStop). +call(Client, Msg, Timeout, WaitStop) -> case get_handle(Client) of {ok,Pid} -> case ct_gen_conn:call(Pid,Msg,Timeout) of - {error,{process_down,Client,noproc}} -> + {error,{process_down,Pid,noproc}} -> {error,no_such_client}; - {error,{process_down,Client,normal}} -> + {error,{process_down,Pid,normal}} when WaitStop -> + %% This will happen when server closes connection + %% before clien received rpc-reply on + %% close-session. + ok; + {error,{process_down,Pid,normal}} -> {error,closed}; - {error,{process_down,Client,Reason}} -> + {error,{process_down,Pid,Reason}} -> {error,{closed,Reason}}; + Other when WaitStop -> + MRef = erlang:monitor(process,Pid), + receive + {'DOWN',MRef,process,Pid,Normal} when Normal==normal; + Normal==noproc -> + Other; + {'DOWN',MRef,process,Pid,Reason} -> + {error,{{closed,Reason},Other}} + after Timeout -> + erlang:demonitor(MRef, [flush]), + {error,{timeout,Other}} + end; Other -> Other end; diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 4a6a3cdcac..eb05c90ba8 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -58,6 +58,7 @@ vts, shell, cover, + cover_stop, coverspec, step, logdir, @@ -245,6 +246,7 @@ script_start1(Parent, Args) -> Vts = get_start_opt(vts, true, Args), Shell = get_start_opt(shell, true, Args), Cover = get_start_opt(cover, fun([CoverFile]) -> ?abs(CoverFile) end, Args), + CoverStop = get_start_opt(cover_stop, fun([CS]) -> list_to_atom(CS) end, Args), LogDir = get_start_opt(logdir, fun([LogD]) -> LogD end, Args), LogOpts = get_start_opt(logopts, fun(Os) -> [list_to_atom(O) || O <- Os] end, [], Args), @@ -329,7 +331,8 @@ script_start1(Parent, Args) -> end, StartOpts = #opts{label = Label, profile = Profile, - vts = Vts, shell = Shell, cover = Cover, + vts = Vts, shell = Shell, + cover = Cover, cover_stop = CoverStop, logdir = LogDir, logopts = LogOpts, basic_html = BasicHtml, verbosity = Verbosity, @@ -416,6 +419,9 @@ script_start2(StartOpts = #opts{vts = undefined, Cover = choose_val(StartOpts#opts.cover, SpecStartOpts#opts.cover), + CoverStop = + choose_val(StartOpts#opts.cover_stop, + SpecStartOpts#opts.cover_stop), MultTT = choose_val(StartOpts#opts.multiply_timetraps, SpecStartOpts#opts.multiply_timetraps), @@ -475,6 +481,7 @@ script_start2(StartOpts = #opts{vts = undefined, profile = Profile, testspecs = Specs, cover = Cover, + cover_stop = CoverStop, logdir = LogDir, logopts = AllLogOpts, basic_html = BasicHtml, @@ -723,6 +730,7 @@ script_usage() -> "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" + "\n\t[-cover_stop Bool]" "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" @@ -745,6 +753,7 @@ script_usage() -> "\n\t[-silent_connections [ConnType1 ConnType2 .. ConnTypeN]]" "\n\t[-stylesheet CSSFile]" "\n\t[-cover CoverCfgFile]" + "\n\t[-cover_stop Bool]" "\n\t[-event_handler EvHandler1 EvHandler2 .. EvHandlerN]" "\n\t[-ct_hooks CTHook1 CTHook2 .. CTHookN]" "\n\t[-include InclDir1 InclDir2 .. InclDirN]" @@ -938,6 +947,7 @@ run_test2(StartOpts) -> %% code coverage Cover = get_start_opt(cover, fun(CoverFile) -> ?abs(CoverFile) end, StartOpts), + CoverStop = get_start_opt(cover_stop, value, StartOpts), %% timetrap manipulation MultiplyTT = get_start_opt(multiply_timetraps, value, 1, StartOpts), @@ -1000,7 +1010,8 @@ run_test2(StartOpts) -> Step = get_start_opt(step, value, StartOpts), Opts = #opts{label = Label, profile = Profile, - cover = Cover, step = Step, logdir = LogDir, + cover = Cover, cover_stop = CoverStop, + step = Step, logdir = LogDir, logopts = LogOpts, basic_html = BasicHtml, config = CfgFiles, verbosity = Verbosity, @@ -1063,6 +1074,8 @@ run_spec_file(Relaxed, AllConfig = merge_vals([CfgFiles, SpecOpts#opts.config]), Cover = choose_val(Opts#opts.cover, SpecOpts#opts.cover), + CoverStop = choose_val(Opts#opts.cover_stop, + SpecOpts#opts.cover_stop), MultTT = choose_val(Opts#opts.multiply_timetraps, SpecOpts#opts.multiply_timetraps), ScaleTT = choose_val(Opts#opts.scale_timetraps, @@ -1103,6 +1116,7 @@ run_spec_file(Relaxed, Opts1 = Opts#opts{label = Label, profile = Profile, cover = Cover, + cover_stop = CoverStop, logdir = which(logdir, LogDir), logopts = AllLogOpts, stylesheet = Stylesheet, @@ -1272,7 +1286,8 @@ run_dir(Opts = #opts{logdir = LogDir, reformat_result(catch do_run(tests(Dir2, Mod), [], Opts1, StartOpts)); _ -> - reformat_result(catch do_run(tests(Dir2, Mod, GsAndCs), + reformat_result(catch do_run(tests(Dir2, Mod, + GsAndCs), [], Opts1, StartOpts)) end; @@ -1281,7 +1296,8 @@ run_dir(Opts = #opts{logdir = LogDir, [_,_|_] when GsAndCs /= [] -> exit({error,multiple_suites_and_cases}); [{Dir2,Mod}] when GsAndCs /= [] -> - reformat_result(catch do_run(tests(Dir2, Mod, GsAndCs), + reformat_result(catch do_run(tests(Dir2, Mod, + GsAndCs), [], Opts1, StartOpts)); DirMods -> reformat_result(catch do_run(tests(DirMods), @@ -1374,6 +1390,7 @@ get_data_for_node(#testspec{label = Labels, verbosity = VLvls, silent_connections = SilentConnsList, cover = CoverFs, + cover_stop = CoverStops, config = Cfgs, userconfig = UsrCfgs, event_handler = EvHs, @@ -1405,6 +1422,7 @@ get_data_for_node(#testspec{label = Labels, SCs -> SCs end, Cover = proplists:get_value(Node, CoverFs), + CoverStop = proplists:get_value(Node, CoverStops), MT = proplists:get_value(Node, MTs), ST = proplists:get_value(Node, STs), CreatePrivDir = proplists:get_value(Node, PDs), @@ -1423,6 +1441,7 @@ get_data_for_node(#testspec{label = Labels, verbosity = Verbosity, silent_connections = SilentConns, cover = Cover, + cover_stop = CoverStop, config = ConfigFiles, event_handlers = EvHandlers, ct_hooks = FiltCTHooks, @@ -1536,17 +1555,36 @@ groups_and_cases(Gs, Cs) when ((Gs == undefined) or (Gs == [])) and ((Cs == undefined) or (Cs == [])) -> []; groups_and_cases(Gs, Cs) when Gs == undefined ; Gs == [] -> - [ensure_atom(C) || C <- listify(Cs)]; -groups_and_cases(Gs, Cs) when Cs == undefined ; Cs == [] -> - [{ensure_atom(G),all} || G <- listify(Gs)]; -groups_and_cases(G, Cs) when is_atom(G) -> - [{G,[ensure_atom(C) || C <- listify(Cs)]}]; -groups_and_cases([G], Cs) -> - [{ensure_atom(G),[ensure_atom(C) || C <- listify(Cs)]}]; -groups_and_cases([_,_|_] , Cs) when Cs =/= [] -> - {error,multiple_groups_and_cases}; -groups_and_cases(_Gs, _Cs) -> - {error,incorrect_group_or_case_option}. + if (Cs == all) or (Cs == [all]) or (Cs == ["all"]) -> all; + true -> [ensure_atom(C) || C <- listify(Cs)] + end; +groups_and_cases(GOrGs, Cs) when (is_atom(GOrGs) orelse + (is_list(GOrGs) andalso + (is_atom(hd(GOrGs)) orelse + (is_list(hd(GOrGs)) andalso + is_atom(hd(hd(GOrGs))))))) -> + if (Cs == undefined) or (Cs == []) or + (Cs == all) or (Cs == [all]) or (Cs == ["all"]) -> + [{GOrGs,all}]; + true -> + [{GOrGs,[ensure_atom(C) || C <- listify(Cs)]}] + end; +groups_and_cases(Gs, Cs) when is_integer(hd(hd(Gs))) -> + %% if list of strings, this comes from 'ct_run -group G1 G2 ...' and + %% we need to parse the strings + Gs1 = + if (Gs == [all]) or (Gs == ["all"]) -> + all; + true -> + lists:map(fun(G) -> + {ok,Ts,_} = erl_scan:string(G++"."), + {ok,Term} = erl_parse:parse_term(Ts), + Term + end, Gs) + end, + groups_and_cases(Gs1, Cs); +groups_and_cases(Gs, Cs) -> + {error,{incorrect_group_or_case_option,Gs,Cs}}. tests(TestDir, Suites, []) when is_list(TestDir), is_integer(hd(TestDir)) -> [{?testdir(TestDir,Suites),ensure_atom(Suites),all}]; @@ -1576,14 +1614,7 @@ do_run(Tests, Misc, LogDir, LogOpts) when is_list(Misc), StepOpts -> #opts{step = StepOpts} end, - Opts1 = - case proplists:get_value(cover, Misc) of - undefined -> - Opts; - CoverFile -> - Opts#opts{cover = CoverFile} - end, - do_run(Tests, [], Opts1#opts{logdir = LogDir}, []); + do_run(Tests, [], Opts#opts{logdir = LogDir}, []); do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) -> #opts{label = Label, profile = Profile, cover = Cover, @@ -1617,7 +1648,13 @@ do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) -> {error,Reason} -> exit({error,Reason}); CoverSpec -> - Opts#opts{coverspec = CoverSpec} + CoverStop = + case Opts#opts.cover_stop of + undefined -> true; + Stop -> Stop + end, + Opts#opts{coverspec = CoverSpec, + cover_stop = CoverStop} end end, %% This env variable is used by test_server to determine @@ -1687,11 +1724,15 @@ compile_and_run(Tests, Skip, Opts, Args) -> SavedErrors = save_make_errors(SuiteMakeErrors), ct_repeat:log_loop_info(Args), - {Tests1,Skip1} = final_tests(Tests,Skip,SavedErrors), - - ReleaseSh = proplists:get_value(release_shell, Args), - ct_util:set_testdata({release_shell,ReleaseSh}), - possibly_spawn(ReleaseSh == true, Tests1, Skip1, Opts); + try final_tests(Tests,Skip,SavedErrors) of + {Tests1,Skip1} -> + ReleaseSh = proplists:get_value(release_shell, Args), + ct_util:set_testdata({release_shell,ReleaseSh}), + possibly_spawn(ReleaseSh == true, Tests1, Skip1, Opts) + catch + _:BadFormat -> + {error,BadFormat} + end; false -> io:nl(), ct_util:stop(clean), @@ -1961,22 +2002,21 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when %% for now, only flat group defs are allowed as %% start options and test spec terms fun({all,all}) -> - ct_framework:make_all_conf(TestDir, - Suite, []); + [ct_groups:make_conf(TestDir, Suite, all, [], all)]; ({skipped,Group,TCs}) -> - [ct_framework:make_conf(TestDir, Suite, - Group, [skipped], TCs)]; - ({GrSpec = {Group,_},TCs}) -> + [ct_groups:make_conf(TestDir, Suite, + Group, [skipped], TCs)]; + ({GrSpec = {GroupName,_},TCs}) -> Props = [{override,GrSpec}], - [ct_framework:make_conf(TestDir, Suite, - Group, Props, TCs)]; - ({GrSpec = {Group,_,_},TCs}) -> + [ct_groups:make_conf(TestDir, Suite, + GroupName, Props, TCs)]; + ({GrSpec = {GroupName,_,_},TCs}) -> Props = [{override,GrSpec}], - [ct_framework:make_conf(TestDir, Suite, - Group, Props, TCs)]; - ({Group,TCs}) -> - [ct_framework:make_conf(TestDir, Suite, - Group, [], TCs)]; + [ct_groups:make_conf(TestDir, Suite, + GroupName, Props, TCs)]; + ({GroupOrGroups,TCs}) -> + [ct_groups:make_conf(TestDir, Suite, + GroupOrGroups, [], TCs)]; (TC) -> [TC] end, GrsOrCs), @@ -1988,12 +2028,12 @@ final_tests1([], Final, Skip, _Bad) -> {lists:reverse(Final),Skip}. final_skip([{TestDir,Suite,{all,all},Reason}|Skips], Final) -> - SkipConf = ct_framework:make_conf(TestDir, Suite, all, [], all), + SkipConf = ct_groups:make_conf(TestDir, Suite, all, [], all), Skip = {TestDir,Suite,SkipConf,Reason}, final_skip(Skips, [Skip|Final]); final_skip([{TestDir,Suite,{Group,TCs},Reason}|Skips], Final) -> - Conf = ct_framework:make_conf(TestDir, Suite, Group, [], TCs), + Conf = ct_groups:make_conf(TestDir, Suite, Group, [], TCs), Skip = {TestDir,Suite,Conf,Reason}, final_skip(Skips, [Skip|Final]); @@ -2120,7 +2160,8 @@ do_run_test(Tests, Skip, Opts) -> %% tell test_server which modules should be cover compiled %% note that actual compilation is done when tests start test_server_ctrl:cover(CovApp, CovFile, CovExcl, CovIncl, - CovCross, CovExport, CovLevel), + CovCross, CovExport, CovLevel, + Opts#opts.cover_stop), %% save cover data (used e.g. to add nodes dynamically) ct_util:set_testdata({cover,CovData}), %% start cover on specified nodes @@ -2265,9 +2306,11 @@ add_jobs([{TestDir,all,_}|Tests], Skip, Opts, CleanUp) -> wait_for_idle(), add_jobs(Tests, Skip, Opts, CleanUp) end; -add_jobs([{TestDir,[Suite],all}|Tests], Skip, Opts, CleanUp) when is_atom(Suite) -> +add_jobs([{TestDir,[Suite],all}|Tests], Skip, + Opts, CleanUp) when is_atom(Suite) -> add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp); -add_jobs([{TestDir,Suites,all}|Tests], Skip, Opts, CleanUp) when is_list(Suites) -> +add_jobs([{TestDir,Suites,all}|Tests], Skip, + Opts, CleanUp) when is_list(Suites) -> Name = get_name(TestDir) ++ ".suites", case catch test_server_ctrl:add_module_with_skip(Name, Suites, skiplist(TestDir,Skip)) of @@ -2282,7 +2325,8 @@ add_jobs([{TestDir,Suite,all}|Tests], Skip, Opts, CleanUp) -> ok -> Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite), case catch test_server_ctrl:add_module_with_skip(Name, [Suite], - skiplist(TestDir,Skip)) of + skiplist(TestDir, + Skip)) of {'EXIT',_} -> CleanUp; _ -> @@ -2305,15 +2349,24 @@ add_jobs([{TestDir,Suite,Confs}|Tests], Skip, Opts, CleanUp) when GrTestName = case Confs of [Conf] -> - "." ++ atom_to_list(Group(Conf)) ++ TCTestName(TestCases(Conf)); + case Group(Conf) of + GrName when is_atom(GrName) -> + "." ++ atom_to_list(GrName) ++ + TCTestName(TestCases(Conf)); + _ -> + ".groups" ++ TCTestName(TestCases(Conf)) + end; _ -> ".groups" end, TestName = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ GrTestName, case maybe_interpret(Suite, init_per_group, Opts) of ok -> - case catch test_server_ctrl:add_conf_with_skip(TestName, Suite, Confs, - skiplist(TestDir,Skip)) of + case catch test_server_ctrl:add_conf_with_skip(TestName, + Suite, + Confs, + skiplist(TestDir, + Skip)) of {'EXIT',_} -> CleanUp; _ -> @@ -2325,18 +2378,21 @@ add_jobs([{TestDir,Suite,Confs}|Tests], Skip, Opts, CleanUp) when end; %% test case -add_jobs([{TestDir,Suite,[Case]}|Tests], Skip, Opts, CleanUp) when is_atom(Case) -> +add_jobs([{TestDir,Suite,[Case]}|Tests], + Skip, Opts, CleanUp) when is_atom(Case) -> add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp); -add_jobs([{TestDir,Suite,Cases}|Tests], Skip, Opts, CleanUp) when is_list(Cases) -> +add_jobs([{TestDir,Suite,Cases}|Tests], + Skip, Opts, CleanUp) when is_list(Cases) -> Cases1 = lists:map(fun({GroupName,_}) when is_atom(GroupName) -> GroupName; (Case) -> Case end, Cases), case maybe_interpret(Suite, Cases1, Opts) of ok -> - Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ ".cases", + Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ ".cases", case catch test_server_ctrl:add_cases_with_skip(Name, Suite, Cases1, - skiplist(TestDir,Skip)) of + skiplist(TestDir, + Skip)) of {'EXIT',_} -> CleanUp; _ -> @@ -2352,7 +2408,8 @@ add_jobs([{TestDir,Suite,Case}|Tests], Skip, Opts, CleanUp) when is_atom(Case) - Name = get_name(TestDir) ++ "." ++ atom_to_list(Suite) ++ "." ++ atom_to_list(Case), case catch test_server_ctrl:add_case_with_skip(Name, Suite, Case, - skiplist(TestDir,Skip)) of + skiplist(TestDir, + Skip)) of {'EXIT',_} -> CleanUp; _ -> @@ -2387,7 +2444,8 @@ skiplist(Dir, [{Dir,all,Cmt}|Skip]) -> %% we need to turn 'all' into list of modules since %% test_server doesn't do skips on Dir level Ss = filelib:wildcard(filename:join(Dir, "*_SUITE.beam")), - [{list_to_atom(filename:basename(S,".beam")),Cmt} || S <- Ss] ++ skiplist(Dir,Skip); + [{list_to_atom(filename:basename(S,".beam")),Cmt} || S <- Ss] ++ + skiplist(Dir,Skip); skiplist(Dir, [{Dir,S,Cmt}|Skip]) -> [{S,Cmt} | skiplist(Dir, Skip)]; skiplist(Dir, [{Dir,S,C,Cmt}|Skip]) -> @@ -2447,8 +2505,10 @@ run_make(Targets, TestDir0, Mod, UserInclude) -> FileTest = fun(F, suites) -> is_suite(F); (F, helpmods) -> not is_suite(F) end, - Files = lists:flatmap(fun({F,out_of_date}) -> - case FileTest(F, Targets) of + Files = + lists:flatmap(fun({F,out_of_date}) -> + case FileTest(F, + Targets) of true -> [F]; false -> [] end; @@ -2584,6 +2644,9 @@ merge_arguments([LogDir={logdir,_}|Args], Merged) -> merge_arguments([CoverFile={cover,_}|Args], Merged) -> merge_arguments(Args, handle_arg(replace, CoverFile, Merged)); +merge_arguments([CoverStop={cover_stop,_}|Args], Merged) -> + merge_arguments(Args, handle_arg(replace, CoverStop, Merged)); + merge_arguments([{'case',TC}|Args], Merged) -> merge_arguments(Args, handle_arg(merge, {testcase,TC}, Merged)); @@ -2792,11 +2855,14 @@ opts2args(EnvStartOpts) -> lists:flatmap(fun({exit_status,ExitStatusOpt}) when is_atom(ExitStatusOpt) -> [{exit_status,[atom_to_list(ExitStatusOpt)]}]; ({halt_with,{HaltM,HaltF}}) -> - [{halt_with,[atom_to_list(HaltM),atom_to_list(HaltF)]}]; + [{halt_with,[atom_to_list(HaltM), + atom_to_list(HaltF)]}]; ({interactive_mode,true}) -> [{shell,[]}]; - ({config,CfgFiles}) -> - [{ct_config,[CfgFiles]}]; + ({config,CfgFile}) when is_integer(hd(CfgFile)) -> + [{ct_config,[CfgFile]}]; + ({config,CfgFiles}) when is_list(hd(CfgFiles)) -> + [{ct_config,CfgFiles}]; ({userconfig,{CBM,CfgStr=[X|_]}}) when is_integer(X) -> [{userconfig,[atom_to_list(CBM),CfgStr]}]; ({userconfig,{CBM,CfgStrs}}) when is_list(CfgStrs) -> @@ -2814,6 +2880,12 @@ opts2args(EnvStartOpts) -> end, UserCfg), [_LastAnd|StrsR] = lists:reverse(lists:flatten(Strs)), [{userconfig,lists:reverse(StrsR)}]; + ({group,G}) when is_atom(G) -> + [{group,[atom_to_list(G)]}]; + ({group,Gs}) when is_list(Gs) -> + LOfGStrs = [lists:flatten(io_lib:format("~w",[G])) || + G <- Gs], + [{group,LOfGStrs}]; ({testcase,Case}) when is_atom(Case) -> [{'case',[atom_to_list(Case)]}]; ({testcase,Cases}) -> diff --git a/lib/common_test/src/ct_slave.erl b/lib/common_test/src/ct_slave.erl index 77e44f7d6a..58633b7de6 100644 --- a/lib/common_test/src/ct_slave.erl +++ b/lib/common_test/src/ct_slave.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010. All Rights Reserved. +%% Copyright Ericsson AB 2010-2012. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -320,6 +320,13 @@ do_start(Host, Node, Options) -> StartupTimeout = Options#options.startup_timeout, Result = case wait_for_node_alive(ENode, BootTimeout) of pong-> + case test_server:is_cover() of + true -> + MainCoverNode = cover:get_main_node(), + rpc:call(MainCoverNode,cover,start,[ENode]); + false -> + ok + end, call_functions(ENode, Functions2), receive {node_started, ENode}-> @@ -442,6 +449,13 @@ wait_for_node_alive(Node, N) -> % call init:stop on a remote node do_stop(ENode) -> + case test_server:is_cover() of + true -> + MainCoverNode = cover:get_main_node(), + rpc:call(MainCoverNode,cover,flush,[ENode]); + false -> + ok + end, spawn(ENode, init, stop, []), wait_for_node_dead(ENode, 5). diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index a8b67d0329..202d8f9373 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -903,6 +903,8 @@ handle_data(logdir,Node,Dir,Spec) -> [{Node,ref2dir(Dir,Spec)}]; handle_data(cover,Node,File,Spec) -> [{Node,get_absfile(File,Spec)}]; +handle_data(cover_stop,Node,Stop,_Spec) -> + [{Node,Stop}]; handle_data(include,Node,Dirs=[D|_],Spec) when is_list(D) -> [{Node,ref2dir(Dir,Spec)} || Dir <- Dirs]; handle_data(include,Node,Dir=[Ch|_],Spec) when is_integer(Ch) -> @@ -1026,20 +1028,24 @@ insert_groups(Node,Dir,Suite,Group,Cases,Tests,MergeTests) insert_groups(Node,Dir,Suite,[Group],Cases,Tests,MergeTests); insert_groups(Node,Dir,Suite,Groups,Cases,Tests,false) when ((Cases == all) or is_list(Cases)) and is_list(Groups) -> - Groups1 = [{Gr,Cases} || Gr <- Groups], + Groups1 = [if is_list(Gr) -> % preserve group path + {[Gr],Cases}; + true -> + {Gr,Cases} end || Gr <- Groups], append({{Node,Dir},[{Suite,Groups1}]},Tests); insert_groups(Node,Dir,Suite,Groups,Cases,Tests,true) when ((Cases == all) or is_list(Cases)) and is_list(Groups) -> + Groups1 = [if is_list(Gr) -> % preserve group path + {[Gr],Cases}; + true -> + {Gr,Cases} end || Gr <- Groups], case lists:keysearch({Node,Dir},1,Tests) of {value,{{Node,Dir},[{all,_}]}} -> Tests; {value,{{Node,Dir},Suites0}} -> - Suites1 = insert_groups1(Suite, - [{Gr,Cases} || Gr <- Groups], - Suites0), + Suites1 = insert_groups1(Suite,Groups1,Suites0), insert_in_order({{Node,Dir},Suites1},Tests); false -> - Groups1 = [{Gr,Cases} || Gr <- Groups], insert_in_order({{Node,Dir},[{Suite,Groups1}]},Tests) end; insert_groups(Node,Dir,Suite,Groups,Case,Tests, MergeTests) @@ -1062,13 +1068,13 @@ insert_groups1(Suite,Groups,Suites0) -> insert_groups2(_Groups,all) -> all; -insert_groups2([Group={GrName,Cases}|Groups],GrAndCases) -> - case lists:keysearch(GrName,1,GrAndCases) of - {value,{GrName,all}} -> +insert_groups2([Group={Gr,Cases}|Groups],GrAndCases) -> + case lists:keysearch(Gr,1,GrAndCases) of + {value,{Gr,all}} -> GrAndCases; - {value,{GrName,Cases0}} -> + {value,{Gr,Cases0}} -> Cases1 = insert_in_order(Cases,Cases0), - insert_groups2(Groups,insert_in_order({GrName,Cases1},GrAndCases)); + insert_groups2(Groups,insert_in_order({Gr,Cases1},GrAndCases)); false -> insert_groups2(Groups,insert_in_order(Group,GrAndCases)) end; @@ -1258,6 +1264,8 @@ valid_terms() -> {node,3}, {cover,2}, {cover,3}, + {cover_stop,2}, + {cover_stop,3}, {config,2}, {config,3}, {config,4}, diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl index 196b5e46d0..c9c6514fa4 100644 --- a/lib/common_test/src/ct_util.hrl +++ b/lib/common_test/src/ct_util.hrl @@ -38,6 +38,7 @@ verbosity=[], silent_connections=[], cover=[], + cover_stop=[], config=[], userconfig=[], event_handler=[], diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl index 77f57c6195..78ae70f37e 100644 --- a/lib/common_test/src/cth_log_redirect.erl +++ b/lib/common_test/src/cth_log_redirect.erl @@ -54,7 +54,7 @@ post_init_per_group(_Group, _Config, Result, State) -> post_end_per_testcase(_TC, _Config, Result, State) -> %% Make sure that the event queue is flushed %% before ending this test case. - gen_event:call(error_logger, ?MODULE, flush), + gen_event:call(error_logger, ?MODULE, flush, 300000), {Result, State}. pre_end_per_group(Group, Config, {ct_log, Group}) -> diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index 64e2cb6507..df816f9a61 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -54,7 +54,9 @@ MODULES= \ ct_shell_SUITE \ ct_system_error_SUITE \ ct_snmp_SUITE \ - ct_group_leader_SUITE + ct_group_leader_SUITE \ + ct_cover_SUITE \ + ct_groups_search_SUITE ERL_FILES= $(MODULES:%=%.erl) @@ -108,7 +110,7 @@ release_spec: opt release_tests_spec: $(INSTALL_DIR) "$(RELSYSDIR)" $(INSTALL_DATA) $(ERL_FILES) $(COVERFILE) "$(RELSYSDIR)" - $(INSTALL_DATA) common_test.spec "$(RELSYSDIR)" + $(INSTALL_DATA) common_test.spec common_test.cover "$(RELSYSDIR)" chmod -R u+w "$(RELSYSDIR)" @tar cf - *_SUITE_data | (cd "$(RELSYSDIR)"; tar xf -) diff --git a/lib/common_test/test/common_test.cover b/lib/common_test/test/common_test.cover new file mode 100644 index 0000000000..66697854ea --- /dev/null +++ b/lib/common_test/test/common_test.cover @@ -0,0 +1,10 @@ +%% -*- erlang -*- +{incl_app,common_test,details}. +{cross_apps,common_test,[erl2html2, + test_server, + test_server_ctrl, + test_server_gl, + test_server_h, + test_server_io, + test_server_node, + test_server_sup]}. diff --git a/lib/common_test/test/ct_config_SUITE.erl b/lib/common_test/test/ct_config_SUITE.erl index 0b1abae757..d92be9ec6e 100644 --- a/lib/common_test/test/ct_config_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE.erl @@ -88,8 +88,8 @@ require(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), run_test(config_static_SUITE, Config, - [{config, filename:join(DataDir, "config/shadow.txt")}, - {config, filename:join(DataDir, "config/config.txt")}], + [{config, [filename:join(DataDir, "config/shadow.txt"), + filename:join(DataDir, "config/config.txt")]}], ["config_static_SUITE"]). install_config(Config) when is_list(Config) -> @@ -174,6 +174,7 @@ run_test(Name, Config, CTConfig, SuiteNames)-> Joiner = fun(Suite) -> filename:join(DataDir, "config/test/"++Suite) end, Suites = lists:map(Joiner, SuiteNames), {Opts,ERPid} = setup_env({suite,Suites}, Config, CTConfig), + ok = ct_test_support:run(Opts, Config), TestEvents = ct_test_support:get_events(ERPid, Config), ct_test_support:log_events(Name, @@ -251,6 +252,7 @@ expected_events(config_static_SUITE)-> ?sok(test_alias_tclocal_nested,{14,0,{2,1}}), ?sok(test_alias_tclocal_nested_backward_compat,{15,0,{2,1}}), ?sok(test_alias_tclocal_nested_backward_compat_subvals,{16,0,{2,1}}), + ?sok(test_config_same_name_already_in_use,{17,0,{2,1}}), {?eh,tc_start,{config_static_SUITE,end_per_suite}}, {?eh,tc_done,{config_static_SUITE,end_per_suite,ok}}, {?eh,test_done,{'DEF','STOP_TIME'}}, diff --git a/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl b/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl index 2e1ad651e8..19f1dab4af 100644 --- a/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl +++ b/lib/common_test/test/ct_config_SUITE_data/config/test/config_static_SUITE.erl @@ -64,7 +64,8 @@ all() -> [test_get_config_simple, test_get_config_nested, test_shadow_all,test_element,test_shadow_all_element, test_internal_deep, test_alias_tclocal_nested, test_alias_tclocal_nested_backward_compat, - test_alias_tclocal_nested_backward_compat_subvals + test_alias_tclocal_nested_backward_compat_subvals, + test_config_same_name_already_in_use ]. init_per_testcase(_,Config) -> @@ -124,6 +125,13 @@ test_config_name_already_in_use2(_) -> ct:fail("Test should've been skipped, you shouldn't see this!"), ok. + +test_config_same_name_already_in_use() -> + []. +test_config_same_name_already_in_use(_) -> + ok = ct:require(x2,{gen_cfg,c}), + ok = ct:require(x2,{gen_cfg,c}). + %% test aliases test_alias_tclocal() -> [{require,newalias,gen_cfg}]. diff --git a/lib/common_test/test/ct_config_info_SUITE.erl b/lib/common_test/test/ct_config_info_SUITE.erl index 40da377ee5..10fe8286dd 100644 --- a/lib/common_test/test/ct_config_info_SUITE.erl +++ b/lib/common_test/test/ct_config_info_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2011. All Rights Reserved. +%% Copyright Ericsson AB 2009-2012. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -123,8 +123,7 @@ test_events(config_info) -> {?eh,tc_done,{config_info_1_SUITE,init_per_suite,ok}}, [{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g1,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {init_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g1,[]}, {failed,{timetrap_timeout,350}}}}, {?eh,tc_auto_skip,{config_info_1_SUITE,t11, {failed,{config_info_1_SUITE,init_per_group,{timetrap_timeout,350}}}}}, @@ -136,14 +135,12 @@ test_events(config_info) -> {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g2,[]},ok}}, {?eh,tc_done,{config_info_1_SUITE,t21,ok}}, {?eh,tc_start,{config_info_1_SUITE,{end_per_group,g2,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {end_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{end_per_group,g2,[]}, {failed,{timetrap_timeout,450}}}}], [{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g3,[]}}}, {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g3,[]},ok}}, [{?eh,tc_start,{config_info_1_SUITE,{init_per_group,g4,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {init_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g4,[]}, {failed,{timetrap_timeout,400}}}}, {?eh,tc_auto_skip,{config_info_1_SUITE,t41, {failed,{config_info_1_SUITE,init_per_group, @@ -164,8 +161,7 @@ test_events(config_info) -> {?eh,tc_done,{config_info_1_SUITE,{init_per_group,g5,[]},ok}}, {?eh,tc_done,{config_info_1_SUITE,t51,ok}}, {?eh,tc_start,{config_info_1_SUITE,{end_per_group,g5,[]}}}, - {?eh,tc_done,{config_info_1_SUITE, - {end_per_group,unknown,[]}, + {?eh,tc_done,{config_info_1_SUITE,{end_per_group,g5,[]}, {failed,{timetrap_timeout,400}}}}], {?eh,tc_start,{config_info_1_SUITE,{end_per_group,g3,[]}}}, {?eh,tc_done,{config_info_1_SUITE,{end_per_group,g3,[]},ok}}], diff --git a/lib/common_test/test/ct_cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE.erl new file mode 100644 index 0000000000..bebfce70d0 --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE.erl @@ -0,0 +1,271 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_cover_SUITE +%%% +%%% Description: +%%% Test code cover analysis support +%%% +%%%------------------------------------------------------------------- +-module(ct_cover_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). +-define(suite, cover_SUITE). +-define(mod, cover_test_mod). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + case test_server:is_cover() of + true -> + {skip,"Test server is running cover already - skipping"}; + false -> + ct_test_support:init_per_suite(Config) + end. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + Node = fullname(existing_node), + case lists:member(Node,nodes()) of + true -> rpc:call(Node,erlang,halt,[]); + false -> ok + end, + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + default, + cover_stop_true, + cover_stop_false, + slave, + slave_start_slave, + cover_node_option, + ct_cover_add_remove_nodes, + otp_9956 + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +%% Check that cover is collected from test node +%% Also check that cover is by default stopped after test is completed +default(Config) -> + {ok,Events} = run_test(default,Config), + false = check_cover(Config), + check_calls(Events,1), + ok. + +%% Check that cover is stopped when cover_stop option is set to true +cover_stop_true(Config) -> + {ok,_Events} = run_test(cover_stop_true,[{cover_stop,true}],Config), + false = check_cover(Config). + +%% Check that cover is not stopped when cover_stop option is set to false +cover_stop_false(Config) -> + {ok,_Events} = run_test(cover_stop_false,[{cover_stop,false}],Config), + {true,[],[?mod]} = check_cover(Config), + CTNode = proplists:get_value(ct_node, Config), + ok = rpc:call(CTNode,cover,stop,[]), + false = check_cover(Config), + ok. + +%% Let test node start a slave node - check that cover is collected +%% from both nodes +slave(Config) -> + {ok,Events} = run_test(slave,slave,[],Config), + check_calls(Events,2), + ok. + +%% Let test node start a slave node which in turn starts another slave +%% node - check that cover is collected from all three nodes +slave_start_slave(Config) -> + {ok,Events} = run_test(slave_start_slave,slave_start_slave,[],Config), + check_calls(Events,3), + ok. + +%% Start a slave node before test starts - the node is listed in cover +%% spec file. +%% Check that cover is collected from test node and slave node. +cover_node_option(Config) -> + {ok, HostStr}=inet:gethostname(), + Host = list_to_atom(HostStr), + DataDir = ?config(data_dir,Config), + {ok,Node} = ct_slave:start(Host,existing_node, + [{erl_flags,"-pa " ++ DataDir}]), + false = check_cover(Node), + CoverSpec = default_cover_file_content() ++ [{nodes,[Node]}], + CoverFile = create_cover_file(cover_node_option,CoverSpec,Config), + {ok,Events} = run_test(cover_node_option,cover_node_option, + [{cover,CoverFile}],Config), + check_calls(Events,2), + {ok,Node} = ct_slave:stop(existing_node), + ok. + +%% Test ct_cover:add_nodes/1 and ct_cover:remove_nodes/1 +%% Check that cover is collected from added node +ct_cover_add_remove_nodes(Config) -> + {ok, HostStr}=inet:gethostname(), + Host = list_to_atom(HostStr), + DataDir = ?config(data_dir,Config), + {ok,Node} = ct_slave:start(Host,existing_node, + [{erl_flags,"-pa " ++ DataDir}]), + false = check_cover(Node), + {ok,Events} = run_test(ct_cover_add_remove_nodes,ct_cover_add_remove_nodes, + [],Config), + check_calls(Events,2), + {ok,Node} = ct_slave:stop(existing_node), + ok. + +%% Test that the test suite itself can be cover compiled and that +%% data_dir is set correctly (OTP-9956) +otp_9956(Config) -> + CoverFile = create_cover_file(otp_9956,[{incl_mods,[?suite]}],Config), + {ok,Events} = run_test(otp_9956,otp_9956,[{cover,CoverFile}],Config), + check_calls(Events,{?suite,otp_9956,1},1), + ok. + + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- +run_test(Label,Config) -> + run_test(Label,[],Config). +run_test(Label,ExtraOpts,Config) -> + run_test(Label,default,ExtraOpts,Config). +run_test(Label,Testcase,ExtraOpts,Config) -> + DataDir = ?config(data_dir, Config), + Suite = filename:join(DataDir, ?suite), + CoverFile = + case proplists:get_value(cover,ExtraOpts) of + undefined -> + create_default_cover_file(Label,Config); + CF -> + CF + end, + RestOpts = lists:keydelete(cover,1,ExtraOpts), + {Opts,ERPid} = setup([{suite,Suite},{testcase,Testcase}, + {cover,CoverFile},{label,Label}] ++ RestOpts, Config), + execute(Label, Testcase, Opts, ERPid, Config). + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +execute(Name, Testcase, Opts, ERPid, Config) -> + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Name, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + TestEvents = events_to_check(Testcase), + R = ct_test_support:verify_events(TestEvents, Events, Config), + {R,Events}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +events_to_check(Testcase) -> + OneTest = + [{?eh,start_logging,{'DEF','RUNDIR'}}] ++ + [{?eh,tc_done,{?suite,Testcase,ok}}] ++ + [{?eh,stop_logging,[]}], + + %% 2 tests (ct:run_test + script_start) is default + OneTest ++ OneTest. + +check_cover(Config) when is_list(Config) -> + CTNode = proplists:get_value(ct_node, Config), + check_cover(CTNode); +check_cover(Node) when is_atom(Node) -> + case rpc:call(Node,test_server,is_cover,[]) of + true -> + {true, + rpc:call(Node,cover,which_nodes,[]), + rpc:call(Node,cover,modules,[])}; + false -> + false + end. + +%% Check that each coverlog includes N calls to ?mod:foo/0 +check_calls(Events,N) -> + check_calls(Events,{?mod,foo,0},N). +check_calls(Events,MFA,N) -> + CoverLogs = + [filename:join(filename:dirname(TCLog),"all.coverdata") || + {ct_test_support_eh, + {event,tc_logfile,ct@falco, + {{?suite,init_per_suite},TCLog}}} <- Events], + do_check_logs(CoverLogs,MFA,N). + +do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) -> + {ok,_} = cover:start(), + ok = cover:import(CoverLog), + {ok,Calls} = cover:analyse(Mod,calls,function), + ok = cover:stop(), + {MFA,N} = lists:keyfind(MFA,1,Calls), + do_check_logs(CoverLogs,MFA,N); +do_check_logs([],_,_) -> + ok. + +fullname(Name) -> + {ok,Host} = inet:gethostname(), + list_to_atom(atom_to_list(Name) ++ "@" ++ Host). + +default_cover_file_content() -> + [{incl_mods,[?mod]}]. + +create_default_cover_file(Filename,Config) -> + create_cover_file(Filename,default_cover_file_content(),Config). + +create_cover_file(Filename,Terms,Config) -> + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,Filename) ++ ".cover", + {ok,Fd} = file:open(File,[write]), + lists:foreach(fun(Term) -> + file:write(Fd,io_lib:format("~p.~n",[Term])) + end,Terms), + ok = file:close(Fd), + File. diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl new file mode 100644 index 0000000000..fdc3323f0a --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE.erl @@ -0,0 +1,156 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%%---------------------------------------------------------------------- +%% File: cover_SUITE.erl +%% +%% Description: +%% This file contains the test cases for the code coverage support +%% +%% @author Support +%% @doc Test of code coverage support in common_test +%% @end +%%---------------------------------------------------------------------- +%%---------------------------------------------------------------------- +-module(cover_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +%% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +suite() -> + []. + +all() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + Config. + +init_per_testcase(_Case, Config) -> + Dog = test_server:timetrap(?default_timeout), + [{watchdog, Dog}|Config]. + +end_per_testcase(Case, Config) -> + %% try apply(?MODULE,Case,[cleanup,Config]) + %% catch error:undef -> ok + %% end, + + kill_slaves(Case,nodes()), + Dog=?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +%%%----------------------------------------------------------------- +%%% Test cases +break(_Config) -> + test_server:break(""), + ok. + +default(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + ok. + +slave(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + N1 = nodename(slave,1), + {ok,Node} = ct_slave:start(N1), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), + {ok,Node} = ct_slave:stop(N1), + ok. + +slave_start_slave(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + N1 = nodename(slave_start_slave,1), + N2 = nodename(slave_start_slave,2), + {ok,Node} = ct_slave:start(N1), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), + {ok,Node2} = rpc:call(Node,ct_slave,start,[N2]), + rpc:call(Node2,cover_test_mod,foo,[]), + {ok,Node2} = rpc:call(Node,ct_slave,stop,[N2]), + {ok,Node} = ct_slave:stop(N1), + ok. + +cover_node_option(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + Node = fullname(existing_node), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), + ok. + +ct_cover_add_remove_nodes(Config) -> + cover_compiled = code:which(cover_test_mod), + cover_test_mod:foo(), + Node = fullname(existing_node), + Beam = rpc:call(Node,code,which,[cover_test_mod]), + false = (Beam == cover_compiled), + + rpc:call(Node,cover_test_mod,foo,[]), % should not be collected + {ok,[Node]} = ct_cover:add_nodes([Node]), + cover_compiled = rpc:call(Node,code,which,[cover_test_mod]), + rpc:call(Node,cover_test_mod,foo,[]), % should be collected + ok = ct_cover:remove_nodes([Node]), + rpc:call(Node,cover_test_mod,foo,[]), % should not be collected + + Beam = rpc:call(Node,code,which,[cover_test_mod]), + + ok. + +otp_9956(Config) -> + cover_compiled = code:which(?MODULE), + DataDir = ?config(data_dir,Config), + absolute = filename:pathtype(DataDir), + true = filelib:is_dir(DataDir), + ok. + + +%%%----------------------------------------------------------------- +%%% Internal +nodename(Case,N) -> + list_to_atom(nodeprefix(Case) ++ integer_to_list(N)). + +nodeprefix(Case) -> + atom_to_list(?MODULE) ++ "_" ++ atom_to_list(Case) ++ "_node". + + +fullname(Name) -> + {ok,Host} = inet:gethostname(), + list_to_atom(atom_to_list(Name) ++ "@" ++ Host). + +kill_slaves(Case, [Node|Nodes]) -> + Prefix = nodeprefix(Case), + case lists:prefix(Prefix,atom_to_list(Node)) of + true -> + rpc:call(Node,erlang,halt,[]); + _ -> + ok + end, + kill_slaves(Case,Nodes); +kill_slaves(_,[]) -> + ok. diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE_data/cover_SUITE_data/.gitignore diff --git a/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl b/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl new file mode 100644 index 0000000000..d4f69452c3 --- /dev/null +++ b/lib/common_test/test/ct_cover_SUITE_data/cover_test_mod.erl @@ -0,0 +1,4 @@ +-module(cover_test_mod). +-compile(export_all). +foo() -> + ok. diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index 338e76264e..6d90b29f41 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -61,7 +61,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [cfg_error, lib_error, no_compile, timetrap_end_conf, timetrap_normal, timetrap_extended, timetrap_parallel, - timetrap_fun, misc_errors]. + timetrap_fun, timetrap_fun_group, misc_errors]. groups() -> []. @@ -251,6 +251,24 @@ timetrap_fun(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% +timetrap_fun_group(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, + Suites = [Join(DataDir, "timetrap_8_SUITE")], + {Opts,ERPid} = setup([{suite,Suites}], Config), + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(timetrap_fun_group, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + + TestEvents = events_to_check(timetrap_fun_group), + ok = ct_test_support:verify_events(TestEvents, Events, Config). + +%%%----------------------------------------------------------------- +%%% misc_errors(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), Join = fun(D, S) -> filename:join(D, "error/test/"++S) end, @@ -429,8 +447,7 @@ test_events(cfg_error) -> {'EXIT',{init_per_group_fails,g1}}}}}}], [{?eh,tc_start,{cfg_error_8_SUITE,{init_per_group,g2,[]}}}, - {?eh,tc_done,{cfg_error_8_SUITE, - {init_per_group,unknown,[]}, + {?eh,tc_done,{cfg_error_8_SUITE,{init_per_group,g2,[]}, {failed,{timetrap_timeout,2000}}}}, {?eh,tc_auto_skip,{cfg_error_8_SUITE,tc1, {failed,{cfg_error_8_SUITE,init_per_group, @@ -500,7 +517,7 @@ test_events(cfg_error) -> {?eh,tc_done,{cfg_error_8_SUITE,tc1,ok}}, {?eh,test_stats,{9,0,{0,14}}}, {?eh,tc_start,{cfg_error_8_SUITE,{end_per_group,g12,[]}}}, - {?eh,tc_done,{cfg_error_8_SUITE,{end_per_group,unknown,[]}, + {?eh,tc_done,{cfg_error_8_SUITE,{end_per_group,g12,[]}, {failed,{timetrap_timeout,2000}}}}], {?eh,tc_start,{cfg_error_8_SUITE,end_per_suite}}, @@ -971,11 +988,423 @@ test_events(timetrap_fun) -> {?eh,stop_logging,[]} ]; +test_events(timetrap_fun_group) -> + [ + {?eh,start_logging,{'DEF','RUNDIR'}}, + {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, + {?eh,start_info,{1,1,58}}, + {?eh,tc_start,{timetrap_8_SUITE,init_per_suite}}, + {?eh,tc_done,{timetrap_8_SUITE,init_per_suite,ok}}, + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g0,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g0,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,1,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,2,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g0,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g0,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g1,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,3,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,4,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g1,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g2,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,5,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,6,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g2,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g3,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g3,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc4}}, + {?eh,tc_done,{timetrap_8_SUITE,tc4, + {failed,{timetrap_timeout,{'$approx',2000}}}}}, + {?eh,test_stats,{0,7,{0,0}}}, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g1,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,8,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,9,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g1,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g1,[]},ok}}], + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g2,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{0,10,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{0,11,{0,0}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g2,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g2,[]},ok}}], + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g3,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g3,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g4,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g4,[]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,1}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,2}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g5,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g5,[]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,3}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{0,11,{0,4}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g6,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g6,[]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{0,11,{0,5}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{0,11,{0,6}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g7,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g7,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{1,11,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g7,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g7,[]}, + {user_timetrap_error,{kaboom,'_'}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g8,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g8,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{2,11,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g8,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g8,[]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g9,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g9,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{3,11,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,test_stats,{3,12,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g9,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g9,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g10,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g10,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,test_stats,{3,13,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{4,13,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g10,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g10,[]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,g11,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,g11,[]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc3}}, + {?eh,tc_done,{timetrap_8_SUITE,tc3, + {failed,{timetrap_timeout,{'$approx',4000}}}}}, + {?eh,test_stats,{4,14,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,15,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,g11,[]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,g11,[]},ok}}], + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg0,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg0,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,16,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,17,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg0,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg0,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,18,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,19,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,20,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,21,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg3,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg3,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc4}}, + {?eh,tc_done,{timetrap_8_SUITE,tc4, + {failed,{timetrap_timeout,{'$approx',2000}}}}}, + {?eh,test_stats,{4,22,{0,6}}}, + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg1,[parallel]}, + ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,23,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,24,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg1,[parallel]}, + ok}}]}, + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg2,[parallel]}, + ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc1}}, + {?eh,tc_done,{timetrap_8_SUITE,tc1, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{4,25,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{4,26,{0,6}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg2,[parallel]}, + ok}}]}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg3,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg3,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg4,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg4,[parallel]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,7}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,8}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg5,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg5,[parallel]}, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,9}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}, + {?eh,test_stats,{4,26,{0,10}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {user_timetrap_error,{kaboom,'_'}}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg6,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg6,[parallel]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc0, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{4,26,{0,11}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}, + {?eh,test_stats,{4,26,{0,12}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,end_per_group, + {failed,{timetrap_8_SUITE,init_per_group, + {timetrap_timeout,'_'}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg7,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg7,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{5,26,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg7,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg7,[parallel]}, + {user_timetrap_error,{kaboom,'_'}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg8,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg8,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{6,26,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg8,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg8,[parallel]}, + {failed,{timetrap_timeout,{'$approx',500}}}}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg9,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg9,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + %% Due to parallelism only checking final test stat in group + {?eh,test_stats,{7,27,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg9,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg9,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg10,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg10,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + %% Due to parallelism only checking final test stat in group + {?eh,test_stats,{8,28,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg10,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg10,[parallel]},ok}}]}, + + {parallel, + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,pg11,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,pg11,[parallel]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc3}}, + {?eh,tc_done,{timetrap_8_SUITE,tc3, + {failed,{timetrap_timeout,{'$approx',4000}}}}}, + {?eh,test_stats,{8,29,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc2}}, + {?eh,tc_done,{timetrap_8_SUITE,tc2, + {failed,{timetrap_timeout,{'$approx',500}}}}}, + {?eh,test_stats,{8,30,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,pg11,[parallel]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,pg11,[parallel]},ok}}]}, + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,sg1,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,sg1,[sequence]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{9,30,{0,12}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {user_timetrap_error,{kaboom,'_'}}}}, + {?eh,test_stats,{9,31,{0,12}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc1, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{9,31,{0,13}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{9,31,{0,14}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,sg1,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,sg1,[sequence]},ok}}], + + [{?eh,tc_start,{timetrap_8_SUITE,{init_per_group,sg2,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{init_per_group,sg2,[sequence]},ok}}, + {?eh,tc_start,{timetrap_8_SUITE,tc5}}, + {?eh,tc_done,{timetrap_8_SUITE,tc5,ok}}, + {?eh,test_stats,{10,31,{0,14}}}, + {?eh,tc_start,{timetrap_8_SUITE,tc0}}, + {?eh,tc_done,{timetrap_8_SUITE,tc0, + {failed,{timetrap_timeout,{'$approx',1000}}}}}, + {?eh,test_stats,{10,32,{0,14}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc1, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{10,32,{0,15}}}, + {?eh,tc_auto_skip,{timetrap_8_SUITE,tc2, + {failed,{timetrap_8_SUITE,tc0}}}}, + {?eh,test_stats,{10,32,{0,16}}}, + {?eh,tc_start,{timetrap_8_SUITE,{end_per_group,sg2,[sequence]}}}, + {?eh,tc_done,{timetrap_8_SUITE,{end_per_group,sg2,[sequence]},ok}}], + + {?eh,tc_start,{timetrap_8_SUITE,end_per_suite}}, + {?eh,tc_done,{timetrap_8_SUITE,end_per_suite,ok}}, + {?eh,test_done,{'DEF','STOP_TIME'}}, + {?eh,stop_logging,[]} + ]; + test_events(misc_errors) -> [ {?eh,start_logging,{'DEF','RUNDIR'}}, {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, - {?eh,start_info,{1,1,7}}, + {?eh,start_info,{1,1,9}}, {?eh,tc_start,{misc_error_1_SUITE,ct_fail_1}}, {?eh,tc_done,{misc_error_1_SUITE,ct_fail_1, {failed,{error,{test_case_failed,{error,this_is_expected}}}}}}, @@ -1002,7 +1431,12 @@ test_events(misc_errors) -> {?eh,tc_start,{misc_error_1_SUITE,killed_by_signal_2}}, {?eh,tc_done,{misc_error_1_SUITE,killed_by_signal_2, {failed,testcase_aborted_or_killed}}}, - {?eh,test_stats,{0,7,{0,0}}}, + {parallel, + [{?eh,tc_start,{misc_error_1_SUITE,p1}}, + {?eh,tc_done,{misc_error_1_SUITE,p1,ok}}, + {?eh,tc_start,{misc_error_1_SUITE,p2}}, + {?eh,tc_done,{misc_error_1_SUITE,p2,ok}}]}, + {?eh,test_stats,{2,7,{0,0}}}, {?eh,test_done,{'DEF','STOP_TIME'}}, {?eh,stop_logging,[]} ]. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl index 99c3ed05ec..61f3fa7e59 100644 --- a/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/misc_error_1_SUITE.erl @@ -96,7 +96,7 @@ end_per_testcase(_TestCase, _Config) -> %% N = integer() | forever %%-------------------------------------------------------------------- groups() -> - []. + [{p,[parallel],[p1,p2]}]. %%-------------------------------------------------------------------- %% Function: all() -> GroupsAndTestCases | {skip,Reason} @@ -107,7 +107,8 @@ groups() -> %%-------------------------------------------------------------------- all() -> [ct_fail_1, ct_fail_2, ct_fail_3, ts_fail_1, ts_fail_2, - killed_by_signal_1, killed_by_signal_2]. + killed_by_signal_1, killed_by_signal_2, + {group,p}]. ct_fail_1(_) -> ct:fail({error,this_is_expected}), @@ -152,3 +153,10 @@ killed_by_signal_2(_) -> end), ct:sleep(1000), exit(this_should_not_be_seen). + +p1(_) -> + {error,parallel_group} = ct:abort_current_testcase(aborted), + ok. + +p2(_) -> + receive after 1000 -> ok end. diff --git a/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl new file mode 100644 index 0000000000..ff138f38b5 --- /dev/null +++ b/lib/common_test/test/ct_error_SUITE_data/error/test/timetrap_8_SUITE.erl @@ -0,0 +1,258 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(timetrap_8_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + +-define(TO, 4). + +%%-------------------------------------------------------------------- +%% Function: suite() -> Info +%% Info = [tuple()] +%%-------------------------------------------------------------------- +suite() -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,?TO}]}}]. + +%%-------------------------------------------------------------------- +%% Function: init_per_suite(Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_suite(Config0) -> void() | {save_config,Config1} +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_suite(_Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_group(GroupName, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_group(G6, Config) when G6==g6; G6==pg6 -> + ct:sleep({seconds,1}), + Config; +init_per_group(_GroupName, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_group(GroupName, Config0) -> +%% void() | {save_config,Config1} +%% GroupName = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_group(G7or8, _Config) when G7or8==g7; G7or8==pg7; G7or8==g8; G7or8==pg8 -> + ct:sleep({seconds,5}), + ok; +end_per_group(_GroupName, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: init_per_testcase(TestCase, Config0) -> +%% Config1 | {skip,Reason} | {skip_and_save,Reason,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%% Reason = term() +%%-------------------------------------------------------------------- +init_per_testcase(_, Config) -> + Config. + +%%-------------------------------------------------------------------- +%% Function: end_per_testcase(TestCase, Config0) -> +%% void() | {save_config,Config1} +%% TestCase = atom() +%% Config0 = Config1 = [tuple()] +%%-------------------------------------------------------------------- +end_per_testcase(_, _Config) -> + ok. + +%%-------------------------------------------------------------------- +%% Function: groups() -> [Group] +%% Group = {GroupName,Properties,GroupsAndTestCases} +%% GroupName = atom() +%% Properties = [parallel | sequence | Shuffle | {RepeatType,N}] +%% GroupsAndTestCases = [Group | {group,GroupName} | TestCase] +%% TestCase = atom() +%% Shuffle = shuffle | {shuffle,{integer(),integer(),integer()}} +%% RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | +%% repeat_until_any_ok | repeat_until_any_fail +%% N = integer() | forever +%%-------------------------------------------------------------------- +groups() -> + [ + {g0,[],[tc0,tc2]}, % group override suite and tc overrides group + {g1,[],[tc0,tc2]}, % group override suite and tc overrides group + {g2,[],[tc1,tc2]}, % tc override group + {g3,[],[tc4,{group,g1},{group,g2}]}, % subgroup override group + {g4,[],[tc0,tc2]}, % exit during init_per_group + {g5,[],[tc0,tc2]}, % exit during init_per_group + {g6,[],[tc0,tc2]}, % timeout during init_per_group + {g7,[],[tc5]}, % exit during end_per_group + {g8,[],[tc5]}, % timeout during end_per_group + {g9,[],[tc5,tc0]}, % exit during testcase + {g10,[],[tc0,tc5]}, % exit during testcase + {g11,[],[tc3,tc2]}, % suite is valid if nothing else is specified + {pg0,[parallel],[tc0,tc2]}, % group override suite and tc overrides group + {pg1,[parallel],[tc0,tc2]}, % group override suite and tc overrides group + {pg2,[parallel],[tc1,tc2]}, % tc override group + {pg3,[parallel],[tc4,{group,pg1},{group,pg2}]}, % subgroup override group + {pg4,[parallel],[tc0,tc2]}, % exit during init_per_group + {pg5,[parallel],[tc0,tc2]}, % exit during init_per_group + {pg6,[parallel],[tc0,tc2]}, % timeout during init_per_group + {pg7,[parallel],[tc5]}, % exit during end_per_group + {pg8,[parallel],[tc5]}, % timeout during end_per_group + {pg9,[parallel],[tc5,tc0]}, % exit during testcase + {pg10,[parallel],[tc0,tc5]},% exit during testcase + {pg11,[parallel],[tc3,tc2]},% suite is valid if nothing else is specified + {sg1,[sequence],[tc5,tc0,tc1,tc2]}, % exit during sequencial testcase + {sg2,[sequence],[tc5,tc0,tc1,tc2]}].% timeout during sequencial testcase + +group(g0) -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}]; +group(g1) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(1000) end}]; +group(g2) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(3000) end}]; +group(g3) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(2000) end}]; +group(g4) -> + [{timetrap,{timetrap_utils,timetrap_exit,[kaboom]}}]; +group(g5) -> + [{timetrap,fun() -> exit(kaboom) end}]; +group(g6) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(g7) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(g8) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(g9) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(g10) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(g11) -> + []; +group(pg0) -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}]; +group(pg1) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(1000) end}]; +group(pg2) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(3000) end}]; +group(pg3) -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(2000) end}]; +group(pg4) -> + [{timetrap,{timetrap_utils,timetrap_exit,[kaboom]}}]; +group(pg5) -> + [{timetrap,fun() -> exit(kaboom) end}]; +group(pg6) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(pg7) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(pg8) -> + [{timetrap,{timetrap_utils,timetrap_val,[500]}}]; +group(pg9) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(pg10) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(pg11) -> + []; +group(sg1) -> + [{timetrap,fun() -> ct:sleep(1000),exit(kaboom) end}]; +group(sg2) -> + [{timetrap,{timetrap_utils,timetrap_val,[{seconds,1}]}}]. + + +%%-------------------------------------------------------------------- +%% Function: all() -> GroupsAndTestCases | {skip,Reason} +%% GroupsAndTestCases = [{group,GroupName} | TestCase] +%% GroupName = atom() +%% TestCase = atom() +%% Reason = term() +%%-------------------------------------------------------------------- +all() -> + [ + {group,g0}, + {group,g1}, + {group,g2}, + {group,g3}, + {group,g4}, + {group,g5}, + {group,g6}, + {group,g7}, + {group,g8}, + {group,g9}, + {group,g10}, + {group,g11}, + {group,pg0}, + {group,pg1}, + {group,pg2}, + {group,pg3}, + {group,pg4}, + {group,pg5}, + {group,pg6}, + {group,pg7}, + {group,pg8}, + {group,pg9}, + {group,pg10}, + {group,pg11}, + {group,sg1}, + {group,sg2}]. + + + +tc0(_) -> + ct:comment("TO set by group"), + ct:sleep({seconds,5}), + ok. + +tc1() -> + [{timetrap,{timetrap_utils,timetrap_val,[1000]}}]. +tc1(_) -> + ct:comment("TO after 1 sec"), + ct:sleep({seconds,2}), + ok. + +tc2() -> + [{timetrap,fun() -> timetrap_utils:timetrap_val(500) end}]. +tc2(_) -> + ct:comment("TO after 0.5 sec"), + ct:sleep({seconds,2}), + ok. + +tc3(_) -> + ct:comment(io_lib:format("TO after ~w sec", [?TO])), + ct:sleep({seconds,5}), + ok. + +tc4(_) -> + ct:comment("TO set by group"), + ct:sleep({seconds,5}), + ok. + +tc5(_) -> + ct:comment("No TO in this testcase, maybe later"), + ok. diff --git a/lib/common_test/test/ct_groups_search_SUITE.erl b/lib/common_test/test/ct_groups_search_SUITE.erl new file mode 100644 index 0000000000..6b1c1f4634 --- /dev/null +++ b/lib/common_test/test/ct_groups_search_SUITE.erl @@ -0,0 +1,1245 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: +%%% +%%% Description: +%%% +%%% +%%% The suites used for the test are located in the data directory. +%%% +%%% The group(s) and case(s) are specified according to this: +%%% +%%% Tests = ct_groups:find_groups(Mod, GroupPaths, TestCases, GroupDef) +%%% +%%% GroupPaths = GroupPath | [GroupPath] +%%% GroupPath = atom() | [atom()] +%%% +%%% CT will find all paths that include GroupPath. GroupPath can be a +%%% single group, or a list of groups along the path to TestCases. +%%% If GroupPath is the latter, the last group in the list must be +%%% the "terminating" group in the path, or it will be impossible to +%%% execute test cases in higher level groups *only*, as in this case: +%%% groups() -> [{g1,[],[tc1,{g2,[],[tc2]}]}]. +%%% Compare: find_groups(x, g1, all, groups()), and +%%% find_groups(x, [[g1]], all, groups()) +%%% +%%% Some examples: +%%% +%%% GroupPaths = g1, means find all paths with g1 included +%%% GroupPaths = [g1], -''- +%%% GroupPaths = [g1,g2], search twice - once for g1 and once for g2 +%%% GroupPaths = [[g1,g2]], find cases under group g1 and sub group g2 +%%% GroupPaths = [[g1,g2],[g1,g3]], find cases for g1-g2 AND g1-g3 +%%% +%%% TestCases = all | atom() | [atom()] +%%% +%%%------------------------------------------------------------------- + +-module(ct_groups_search_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/src/ct_util.hrl"). + + +-define(eh, ct_test_support_eh). + +-define(M1, groups_search_dummy_1_SUITE). +-define(M2, groups_search_dummy_2_SUITE). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +init_per_suite(Config) -> + DataDir = proplists:get_value(data_dir, Config), + code:add_patha(DataDir), + M1Erl = filename:join(DataDir, atom_to_list(?M1)++".erl"), + M2Erl = filename:join(DataDir, atom_to_list(?M2)++".erl"), + {ok,?M1} = compile:file(M1Erl, [{outdir,DataDir}]), + {ok,?M2} = compile:file(M2Erl, [{outdir,DataDir}]), + {module,?M1} = code:load_file(?M1), + {module,?M2} = code:load_file(?M2), + + Config1 = ct_test_support:init_per_suite(Config), + Config1. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +groups() -> + [ + {find_groups,[],[all_groups, + testcases_in_all_groups, + all_in_top_group1, + all_in_top_group2, + all_in_sub_group1, + all_in_sub_group2, + testcase_in_top_group1, + testcase_in_top_group2, + testcase_in_sub_group1, + testcase_in_sub_group2, + testcase_in_top_groups1, + testcase_in_top_groups2, + testcase_in_top_groups3, + testcase_in_top_groups4, + testcase_in_top_groups5, + testcase_in_top_groups6, + testcase_in_top_groups7, + testcase_in_sub_groups1, + testcase_in_sub_groups2, + testcase_in_sub_groups3, + testcase_in_sub_groups4, + testcase_in_sub_groups5, + testcase_in_sub_groups6, + testcase_in_sub_groups7, + testcase_in_sub_groups8, + testcase_in_sub_groups9, + testcase_in_sub_groups10, + testcase_in_sub_groups11, + testcase_in_sub_groups12, + testcase_in_sub_groups13, + bad_testcase_in_sub_groups1]}, + + {run_groups,[sequence],[run_groups_with_options, + run_groups_with_testspec]} + ]. + +all() -> + [{group,find_groups,[parallel]}, + {group,run_groups}]. + + + +%%-------------------------------------------------------------------- +%% TEST CASES CHECKING RETURN VALUE ONLY +%%-------------------------------------------------------------------- + +all_groups(_) -> + GPath = all, TCs = all, + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + Top1 = ct_groups:find_groups(?M1, top1, TCs, groups1()), + Top2 = ct_groups:find_groups(?M1, top2, TCs, groups1()), + + All = Top1 ++ Top2 ++ [{conf,[{name,sub2}], + {?M1,init_per_group}, + [{?M1,sub2_tc1},{?M1,sub2_tc2}], + {?M1,end_per_group}}], + + All = Found, + + {?M1,GPath,TCs,Top1++Top2}. + +%%%----------------------------------------------------------------- +%%% +testcases_in_all_groups(_) -> + GPath = all, TCs = [tc3,sub_tc2], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [Top1 = + {conf,[{name,top1}],{?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub11}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}], + {?M2,end_per_group}}, + {conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,tc3},{?M2,sub_tc2}, + {conf,[{name,sub121}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + Top2 = + {conf,[{name,top2}],{?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub21}], + {?M2,init_per_group}, + [{?M2,tc3},{?M2,sub_tc2}, + {conf,[{name,sub2xx}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{?M2,tc3},{?M2,sub_tc2}, + {conf,[{name,sub221}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}], + {?M2,end_per_group}}, + {conf,[{name,sub2xx}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub21}], + {?M2,init_per_group}, + [{?M2,tc3},{?M2,sub_tc2}, + {conf,[{name,sub2xx}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{?M2,tc3},{?M2,sub_tc2}, + {conf,[{name,sub221}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}}, + {conf,[{name,sub2xx}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub221}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}}, + + {conf,[{name,sub2xx}], + {?M2,init_per_group},[{?M2,tc3},{?M2,sub_tc2}],{?M2,end_per_group}}] + + = Found, + + {?M2,GPath,TCs,[Top1,Top2]}. + +%%%----------------------------------------------------------------- +%%% +all_in_top_group1(_) -> + GPath= top1, TCs = all, + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [{conf,[{name,top1}], + {?M1,init_per_group}, + [{?M1,top1_tc1},{?M1,top1_tc2}, + {conf,[{name,sub1}], + {?M1,init_per_group}, + [{?M1,sub1_tc1},{?M1,sub1_tc2}], + {?M1,end_per_group}}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +all_in_top_group2(_) -> + GPath= top2, TCs = all, + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [{conf,[{name,top2}], + {?M1,init_per_group}, + [{conf,[{name,sub2}], + {?M1,init_per_group}, + [{?M1,sub2_tc1},{?M1,sub2_tc2}], + {?M1,end_per_group}}, + {?M1,top2_tc1},{?M1,top2_tc2}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +all_in_sub_group1(_) -> + GPath = sub1, TCs = all, + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [{conf,[{name,top1}], + {?M1,init_per_group}, + [{conf,[{name,sub1}], + {?M1,init_per_group}, + [{?M1,sub1_tc1},{?M1,sub1_tc2}], + {?M1,end_per_group}}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +all_in_sub_group2(_) -> + GPath = sub2, TCs = all, + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [Top2 = + {conf,[{name,top2}], + {?M1,init_per_group}, + [{conf,[{name,sub2}], + {?M1,init_per_group}, + [{?M1,sub2_tc1},{?M1,sub2_tc2}], + {?M1,end_per_group}}], + {?M1,end_per_group}}, + + {conf,[{name,sub2}], + {?M1,init_per_group}, + [{?M1,sub2_tc1},{?M1,sub2_tc2}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Top2}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_group1(_) -> + GPath = top1, TCs = [top1_tc2], + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [{conf,[{name,top1}], + {?M1,init_per_group}, + [{?M1,top1_tc2}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_group2(_) -> + GPath = top2, TCs = [top2_tc2], + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [{conf,[{name,top2}], + {?M1,init_per_group}, + [{?M1,top2_tc2}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_group1(_) -> + GPath = sub1, TCs = [sub1_tc2], + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [{conf,[{name,top1}], + {?M1,init_per_group}, + [{conf,[{name,sub1}], + {?M1,init_per_group}, + [{?M1,sub1_tc2}], + {?M1,end_per_group}}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_group2(_) -> + GPath = sub2, TCs = [sub2_tc2], + + Found = ct_groups:find_groups(?M1, GPath, TCs, groups1()), + + [Top2 = + {conf,[{name,top2}], + {?M1,init_per_group}, + [{conf,[{name,sub2}], + {?M1,init_per_group}, + [{?M1,sub2_tc2}], + {?M1,end_per_group}}], + {?M1,end_per_group}}, + + {conf,[{name,sub2}], + {?M1,init_per_group}, + [{?M1,sub2_tc2}], + {?M1,end_per_group}}] = Found, + + {?M1,GPath,TCs,Top2}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_groups1(_) -> + GPath = [top1,top2], TCs = all, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{?M2,top1_tc1},{?M2,top_tc2},{?M2,tc3}, + {conf,[{name,sub11}], + {?M2,init_per_group}, + [{?M2,sub11_tc1},{?M2,sub_tc2},{?M2,tc3}], + {?M2,end_per_group}}, + {conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,sub12_tc1},{?M2,sub_tc2},{?M2,tc3}, + {conf,[{name,sub121}], + {?M2,init_per_group}, + [{?M2,sub121_tc1},{?M2,sub_tc2},{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub21}], + {?M2,init_per_group}, + [{?M2,sub21_tc1},{?M2,sub_tc2},{?M2,tc3}, + {conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub2xx_tc1},{?M2,sub_tc2},{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + {?M2,top2_tc1},{?M2,top_tc2},{?M2,tc3}, + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub221}], + {?M2,init_per_group}, + [{?M2,sub221_tc1},{?M2,sub_tc2},{?M2,tc3}], + {?M2,end_per_group}}, + {?M2,sub22_tc1},{?M2,sub_tc2},{?M2,tc3}, + {conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub2xx_tc1},{?M2,sub_tc2},{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_groups2(_) -> + GPath = [top1,top2], TCs = tc3, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub11}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}, + {conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub121}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,top2}], + {?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub21}], + {?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub221}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}, + {conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_groups3(_) -> + GPath = [top1,top2], TCs = top1_tc1, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{?M2,top1_tc1}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_groups4(_) -> + GPath = [top1,top2], TCs = sub2xx_tc1, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub21}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub2xx_tc1}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub2xx_tc1}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_groups5(_) -> + GPath = [top1,top2], TCs = [sub21_tc1,sub22_tc1], + + Found = ct_groups:find_groups(?M2, [top1,top2], [sub21_tc1,sub22_tc1], + groups2()), + + [{conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub21}], + {?M2,init_per_group}, + [{?M2,sub21_tc1}], + {?M2,end_per_group}}, + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{?M2,sub22_tc1}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_groups6(_) -> + GPath = [[top1],[top2]], TCs = tc3, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}, + {conf,[{name,top2}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_top_groups7(_) -> + GPath = [[top1],[top2]], TCs = all, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{?M2,top1_tc1}, + {?M2,top_tc2}, + {?M2,tc3}], + {?M2,end_per_group}}, + {conf,[{name,top2}], + {?M2,init_per_group}, + [{?M2,top2_tc1}, + {?M2,top_tc2}, + {?M2,tc3}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups1(_) -> + GPath = [sub121], TCs = tc3, + + Found = ct_groups:find_groups(?M2, sub121, tc3, groups2()), + Found = ct_groups:find_groups(?M2, [sub121], tc3, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{conf,[{name,sub121}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups2(_) -> + GPath = sub12, TCs = tc3, + + Found = ct_groups:find_groups(?M2, sub12, tc3, groups2()), + Found = ct_groups:find_groups(?M2, [sub12], tc3, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,tc3}, + {conf,[{name,sub121}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + FoundX = ct_groups:find_groups(?M2, [[sub12]], tc3, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = FoundX, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups3(_) -> + GPath = [sub121,sub221], TCs = all, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [Top1 = + {conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{conf,[{name,sub121}], + {?M2,init_per_group}, + [{?M2,sub121_tc1}, + {?M2,sub_tc2}, + {?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + Top2 = + {conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub221}], + {?M2,init_per_group}, + [{?M2,sub221_tc1}, + {?M2,sub_tc2}, + {?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub221}], + {?M2,init_per_group}, + [{?M2,sub221_tc1}, + {?M2,sub_tc2}, + {?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub221}], + {?M2,init_per_group}, + [{?M2,sub221_tc1}, + {?M2,sub_tc2}, + {?M2,tc3}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,[Top1,Top2]}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups4(_) -> + GPath = [top1,sub21], TCs = sub_tc2, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [Top1 = + {conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub11}], + {?M2,init_per_group}, + [{?M2,sub_tc2}], + {?M2,end_per_group}}, + {conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,sub_tc2}, + {conf,[{name,sub121}], + {?M2,init_per_group}, + [{?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + Top2 = + {conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub21}], + {?M2,init_per_group}, + [{?M2,sub_tc2}, + {conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub21}], + {?M2,init_per_group}, + [{?M2,sub_tc2}, + {conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,[Top1,Top2]}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups5(_) -> + GPath = [[top1,sub12]], TCs = sub12_tc1, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,sub12_tc1}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups6(_) -> + GPath = [[top1,sub12]], TCs = [sub_tc2], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups7(_) -> + GPath = [[top1,sub12]], TCs = [sub12_tc1,sub_tc2], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{?M2,sub12_tc1}, + {?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups8(_) -> + GPath = [[top2,sub22]], TCs = [sub22_tc1,sub_tc2], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub22}], + {?M2,init_per_group}, + [{?M2,sub22_tc1}, + {?M2,sub_tc2}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups9(_) -> + GPath = [[sub2xx]], TCs = tc3, + + Found = ct_groups:find_groups(?M2, sub2xx, tc3, groups2()), + Found = ct_groups:find_groups(?M2, [[sub2xx]], tc3, groups2()), + + [Top2 = + {conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub21}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub21}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Top2}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups10(_) -> + GPath = [[sub22,sub2xx]], TCs = tc3, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [Top2 = + {conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Top2}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups11(_) -> + GPath = [[top1,sub12,sub121]], TCs = all, + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top1}], + {?M2,init_per_group}, + [{conf,[{name,sub12}], + {?M2,init_per_group}, + [{conf,[{name,sub121}], + {?M2,init_per_group}, + [{?M2,sub121_tc1}, + {?M2,sub_tc2}, + {?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups12(_) -> + GPath = [[top2,sub2xx]], TCs = [sub2xx_tc1,tc3], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub21}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub2xx_tc1}, + {?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}, + {conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub2xx_tc1}, + {?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +testcase_in_sub_groups13(_) -> + GPath = [[top2,sub22,sub2xx]], TCs = [top2_tc1,sub2xx_tc1,tc3], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [{conf,[{name,top2}], + {?M2,init_per_group}, + [{conf,[{name,sub22}], + {?M2,init_per_group}, + [{conf,[{name,sub2xx}], + {?M2,init_per_group}, + [{?M2,sub2xx_tc1}, + {?M2,tc3}], + {?M2,end_per_group}}], + {?M2,end_per_group}}], + {?M2,end_per_group}}] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +bad_testcase_in_sub_groups1(_) -> + GPath = [sub2xx], TCs = [top2_tc1], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% +bad_testcase_in_sub_groups2(_) -> + GPath = [sub12,sub2xx], TCs = [top1_tc1,top2_tc1], + + Found = ct_groups:find_groups(?M2, GPath, TCs, groups2()), + + [] = Found, + + {?M2,GPath,TCs,Found}. + +%%%----------------------------------------------------------------- +%%% CASES EXECUTING THE TESTS +%%%----------------------------------------------------------------- + +run_groups_with_options(Config) -> + DataDir = ?config(data_dir, Config), + + {M1All,M1Rest,M2All,M2Rest} = get_all_groups_and_cases(Config), + + M1AllGrs = lists:flatmap(fun({Path,_,_}) when is_atom(hd(Path)) -> Path; + ({Path,_,_}) when is_list(hd(Path)) -> Path; + ({Path,_,_}) -> [Path] + end, M1All), + + %% ct:pal("NOW RUNNING M1 TEST: ~p", [M1All]), + + {OptsM11,ERPidM11} = setup([{dir,DataDir},{suite,?M1}, + {group,M1AllGrs},{label,m1_all_cases}], Config), + M1AllGrInfo = {M1AllGrs,lists:flatten([Found || {_,_,Found} <- M1All])}, + ok = execute(m1_all_cases, M1AllGrInfo, OptsM11, ERPidM11, Config), + + lists:foldl( + fun({GrPath,TCs,Found}, N) -> + TestName = list_to_atom("m1_spec_cases_" ++ integer_to_list(N)), + %% ct:pal("NOW RUNNING M1 TEST ~p: ~p + ~p", + %% [TestName,GrPath,TCs]), + {OptsM12,ERPidM12} = setup([{dir,DataDir},{suite,?M1}, + {group,GrPath},{testcase,TCs}, + {label,TestName}], Config), + ok = execute(TestName, {GrPath,TCs,Found}, + OptsM12, ERPidM12, Config), + N+1 + end, 1, M1Rest), + + %% ct:pal("NOW RUNNING M2 TEST: ~p", [M2All]), + + M2AllGrs = lists:flatmap(fun({Path,_,_}) when is_atom(hd(Path)) -> Path; + ({Path,_,_}) when is_list(hd(Path)) -> Path; + ({Path,_,_}) -> [Path] + end, M2All), + + + {OptsM21,ERPidM21} = setup([{dir,DataDir},{suite,?M2}, + {group,M2AllGrs},{testcase,all}, + {label,m2_all_cases}], Config), + M2AllGrInfo = {M2AllGrs,lists:flatten([Found || {_,_,Found} <- M2All])}, + ok = execute(m2_all_cases, M2AllGrInfo, OptsM21, ERPidM21, Config), + + lists:foldl( + fun({GrPath,TCs,Found}, N) -> + TestName = list_to_atom("m2_spec_cases_" ++ integer_to_list(N)), + %% ct:pal("NOW RUNNING M2 TEST ~p: ~p + ~p", [TestName,GrPath,TCs]), + {OptsM22,ERPidM22} = setup([{dir,DataDir},{suite,?M2}, + {group,GrPath},{testcase,TCs}, + {label,TestName}], Config), + ok = execute(TestName, {GrPath,TCs,Found}, + OptsM22, ERPidM22, Config), + N+1 + end, 1, M2Rest), + ok. + + +%%%----------------------------------------------------------------- +%%% +run_groups_with_testspec(Config) -> + Name = run_groups_with_testspec, + DataDir = ?config(data_dir, Config), + PrivDir = ?config(priv_dir, Config), + + {M1All,M1Rest,M2All,M2Rest} = get_all_groups_and_cases(Config), + + M1AllGrs = lists:flatmap(fun({Path,_,_}) when is_atom(hd(Path)) -> Path; + ({Path,_,_}) when is_list(hd(Path)) -> Path; + ({Path,_,_}) -> [Path] + end, M1All), + M1AllTerm = {groups,DataDir,?M1,M1AllGrs}, + + M1RestTerms = lists:map( + fun({GrPath,TCs,_}) -> + {groups,DataDir,?M1,GrPath,{cases,TCs}} + end, M1Rest), + + M2AllGrs = lists:flatmap(fun({Path,_,_}) when is_atom(hd(Path)) -> Path; + ({Path,_,_}) when is_list(hd(Path)) -> Path; + ({Path,_,_}) -> [Path] + end, M2All), + M2AllTerm = {groups,DataDir,?M2,M2AllGrs,{cases,all}}, + + M2RestTerms = lists:map( + fun({GrPath,TCs,_}) -> + {groups,DataDir,?M2,GrPath,{cases,TCs}} + end, M2Rest), + + GroupTerms = lists:flatten([M1AllTerm, + M1RestTerms, + M2AllTerm, + M2RestTerms]), + + TestSpec = [{merge_tests,false}, + {label,Name}] ++ GroupTerms, + + ct:pal("Here's the test spec:~n~p", [TestSpec]), + + TestSpecName = ct_test_support:write_testspec(TestSpec, PrivDir, + "groups_search_spec"), + + {Opts,ERPid} = setup([{spec,TestSpecName}], Config), + GroupInfo = + [{M1AllTerm,lists:flatten([Found || {_,_,Found} <- M1All])} | + M1Rest] ++ + [{M2AllTerm,lists:flatten([Found || {_,_,Found} <- M2All])} | + M2Rest], + ok = execute(Name, GroupInfo, Opts, ERPid, Config). + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- + +groups1() -> + [{top1,[],[top1_tc1,top1_tc2,{sub1,[],[sub1_tc1,sub1_tc2]}]}, + {top2,[],[{group,sub2},top2_tc1,top2_tc2]}, + {sub2,[],[sub2_tc1,sub2_tc2]}]. + +groups2() -> + [{top1,[],[top1_tc1,top_tc2,tc3, + {sub11,[],[sub11_tc1,sub_tc2,tc3]}, + {sub12,[],[sub12_tc1,sub_tc2,tc3, + {sub121,[],[sub121_tc1,sub_tc2,tc3]}]}]}, + {top2,[],[{group,sub21},top2_tc1,top_tc2,tc3,{group,sub22}]}, + {sub21,[],[sub21_tc1,sub_tc2,tc3,{group,sub2xx}]}, + {sub22,[],[{group,sub221},sub22_tc1,sub_tc2,tc3,{group,sub2xx}]}, + {sub221,[],[sub221_tc1,sub_tc2,tc3]}, + {sub2xx,[],[sub2xx_tc1,sub_tc2,tc3]}]. + +get_all_groups_and_cases(Config) -> + {value,{_,_,FindGrTCs}} = lists:keysearch(find_groups, 1, groups()), + + MGTFs = [apply(?MODULE, TC, [Config]) || TC <- FindGrTCs], + + ct:pal("Extracted data from ~p test cases", [length(MGTFs)]), + + lists:foldr(fun({M,Gs,TCs,F}, + {M11,M12,M21,M22}) -> + case {M,Gs,TCs} of + {?M1,all,_} -> {M11,[{Gs,TCs,F}|M12],M21,M22}; + {?M1,_,all} -> {[{Gs,all,F}|M11],M12,M21,M22}; + {?M1,_,_} -> {M11,[{Gs,TCs,F}|M12],M21,M22}; + {?M2,all,_} -> {M11,M12,M21,[{Gs,TCs,F}|M22]}; + {?M2,_,all} -> {M11,M12,[{Gs,all,F}|M21],M22}; + {?M2,_,_} -> {M11,M12,M21,[{Gs,TCs,F}|M22]} + end + end, {[],[],[],[]}, MGTFs). + +%%%----------------------------------------------------------------- + +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +execute(Name, TestParams, Opts, ERPid, Config) -> + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + Events1 = reformat(Events, ?eh), + ct_test_support:log_events(Name, + Events1, + ?config(priv_dir, Config), + Opts), + verify_events(Name, TestParams, Events1). + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +%%%----------------------------------------------------------------- +%%% TEST EVENTS +verify_events(Name, Params, Events) -> + %% 2 tests (ct:run_test + script_start) is default + verify_events(Name, Params, Events, 2). + +verify_events(_, _, _, 0) -> + ok; +verify_events(Name, Params, Events, N) -> + test_events(Name, Params, Events), + verify_events(Name, Params, Events, N-1). + +%%%----------------------------------------------------------------- +%%% check run_groups_with_options + +test_events(TestName, {GrPath,Found}, Events) -> + test_events(TestName, {GrPath,all,Found}, Events); + +test_events(TestName, {GrPath,TCs,Found}, Events) + when TestName /= run_groups_with_testspec -> + try check_events(Events, flatten_tests(Found)) of + ok -> ok + catch + throw:Reason -> + ct:pal("Test failed for ~p with group path ~p and cases ~p" + "~nReason: ~p", [TestName,GrPath,TCs,Reason]), + throw(failed) + end; + +%%%----------------------------------------------------------------- +%%% check run_groups_with_testspec + +test_events(run_groups_with_testspec, Params, Events) -> + AllFound = lists:flatmap(fun({_All,Found}) when is_tuple(Found) -> + [Found]; + ({_All,Found}) -> + Found; + ({_Gr,_TCs,Found}) when is_tuple(Found) -> + [Found]; + ({_Gr,_TCs,Found}) -> + Found + end, Params), + try check_events(Events, flatten_tests(AllFound)) of + ok -> ok + catch + throw:Reason -> + ct:pal("Test failed for run_groups_with_testspec." + "~nReason: ~p", [Reason]), + throw(failed) + end. + +flatten_tests({conf,[{name,G}|_],{Mod,_I},Tests,_E}) -> + lists:flatten([{group,Mod,G} | flatten_tests(Tests)]); +flatten_tests([{conf,[{name,G}|_],{Mod,_I},Tests,_E} | Confs]) -> + lists:flatten([{group,Mod,G} | flatten_tests(Tests)]) ++ + lists:flatten(flatten_tests(Confs)); +flatten_tests([{_Mod,_TC} = Case | Tests]) -> + lists:flatten([Case | flatten_tests(Tests)]); +flatten_tests([]) -> + []. + +check_events([{_,tc_start,{Mod,{init_per_group,G,_}}} | Evs], + [{group,Mod,G} | Check]) -> + check_events(Evs, Check); +check_events([{_,tc_start,{Mod,TC}} | Evs], + [{Mod,TC} | Check]) when is_atom(TC) -> + check_events(Evs, Check); +check_events([{_,tc_start,{Mod,{init_per_group,G,_}}} | _Evs], Check) -> + ct:pal("CHECK FAILED!~nGroup ~p in ~p not found in ~p.", + [G,Mod,Check]), + throw({test_not_found,{Mod,G}}); +check_events([{_,tc_start,{Mod,TC}} | _Evs], Check) + when is_atom(TC), TC /= init_per_suite, TC /= end_per_suite -> + ct:pal("CHECK FAILED!~nCase ~p in ~p not found in ~p.", + [TC,Mod,Check]), + throw({test_not_found,{Mod,TC}}); +check_events([Group | Evs], Check) when is_list(Group) -> + Check1 = check_events(Group, Check), + check_events(Evs, Check1); +check_events(_, []) -> + ok; +check_events([Elem | Evs], Check) when is_tuple(Elem) -> + check_events(Evs, Check); +check_events([], Check = [_|_]) -> + ct:pal("CHECK FAILED!~nTests remain: ~p", [Check]), + throw({tests_remain,Check}); +check_events([Wut | _],_) -> + throw({unexpected,Wut}). + diff --git a/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_1_SUITE.erl b/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_1_SUITE.erl new file mode 100644 index 0000000000..1c5b572f92 --- /dev/null +++ b/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_1_SUITE.erl @@ -0,0 +1,83 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(groups_search_dummy_1_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + + +all() -> + [{group,top1}, + {group,top2}]. + +groups() -> + [{top1,[],[top1_tc1,top1_tc2,{sub1,[],[sub1_tc1,sub1_tc2]}]}, + {top2,[],[{group,sub2},top2_tc1,top2_tc2]}, + {sub2,[],[sub2_tc1,sub2_tc2]}]. + +%%%----------------------------------------------------------------- +%%% CONFIG FUNCS +%%%----------------------------------------------------------------- + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +init_per_group(_, Config) -> + Config. + +end_per_group(_, _Config) -> + ok. + +%%%----------------------------------------------------------------- +%%% TEST CASES +%%%----------------------------------------------------------------- + +top1_tc1(_) -> + ok. + +top1_tc2(_) -> + ok. + +sub1_tc1(_) -> + ok. + +sub1_tc2(_) -> + ok. + +top2_tc1(_) -> + ok. + +top2_tc2(_) -> + ok. + +sub2_tc1(_) -> + ok. + +sub2_tc2(_) -> + ok. diff --git a/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_2_SUITE.erl b/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_2_SUITE.erl new file mode 100644 index 0000000000..060012de29 --- /dev/null +++ b/lib/common_test/test/ct_groups_search_SUITE_data/groups_search_dummy_2_SUITE.erl @@ -0,0 +1,102 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2009-2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(groups_search_dummy_2_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). + + +all() -> + [{group,top1}, + {group,top2}]. + +groups() -> + [{top1,[],[top1_tc1,top_tc2,tc3, + {sub11,[],[sub11_tc1,sub_tc2,tc3]}, + {sub12,[],[sub12_tc1,sub_tc2,tc3, + {sub121,[],[sub121_tc1,sub_tc2,tc3]}]}]}, + + {top2,[],[{group,sub21},top2_tc1,top_tc2,tc3,{group,sub22}]}, + {sub21,[],[sub21_tc1,sub_tc2,tc3,{group,sub2xx}]}, + {sub22,[],[{group,sub221},sub22_tc1,sub_tc2,tc3,{group,sub2xx}]}, + {sub221,[],[sub221_tc1,sub_tc2,tc3]}, + {sub2xx,[],[sub2xx_tc1,sub_tc2,tc3]}]. + +%%%----------------------------------------------------------------- +%%% CONFIG FUNCS +%%%----------------------------------------------------------------- + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(_TestCase, Config) -> + Config. + +end_per_testcase(_TestCase, _Config) -> + ok. + +init_per_group(_, Config) -> + Config. + +end_per_group(_, _Config) -> + ok. + +%%%------------------------------------------------------------------ +%%% TEST CASES +%%%------------------------------------------------------------------ + +top1_tc1(_) -> + ok. + +top_tc2(_) -> + ok. + +tc3(_) -> + ok. + +sub_tc2(_) -> + ok. + +sub11_tc1(_) -> + ok. + +sub12_tc1(_) -> + ok. + +sub121_tc1(_) -> + ok. + +top2_tc1(_) -> + ok. + +sub21_tc1(_) -> + ok. + +sub22_tc1(_) -> + ok. + +sub221_tc1(_) -> + ok. + +sub2xx_tc1(_) -> + ok. diff --git a/lib/common_test/test/ct_netconfc_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE.erl index e6e8d5b09c..3042a924fe 100644 --- a/lib/common_test/test/ct_netconfc_SUITE.erl +++ b/lib/common_test/test/ct_netconfc_SUITE.erl @@ -44,7 +44,12 @@ %%-------------------------------------------------------------------- init_per_suite(Config) -> Config1 = ct_test_support:init_per_suite(Config), - Config1. + case application:load(crypto) of + {error,Reason} -> + {skip, Reason}; + _ -> + Config1 + end. end_per_suite(Config) -> ct_test_support:end_per_suite(Config). @@ -108,7 +113,7 @@ reformat(Events, EH) -> %%%----------------------------------------------------------------- %%% TEST EVENTS %%%----------------------------------------------------------------- -events_to_check(Test,Config) -> +events_to_check(default,Config) -> {module,_} = code:load_abs(filename:join(?config(data_dir,Config), netconfc1_SUITE)), TCs = netconfc1_SUITE:all(), diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index 48c6e1c0ed..e5e2e68fcb 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -117,7 +117,10 @@ end_per_suite(Config) -> CTNode = proplists:get_value(ct_node, Config), PrivDir = proplists:get_value(priv_dir, Config), true = rpc:call(CTNode, code, del_path, [filename:join(PrivDir,"")]), - cover:stop(CTNode), + case test_server:is_cover() of + true -> cover:flush(CTNode); + false -> ok + end, slave:stop(CTNode), ok. @@ -149,7 +152,10 @@ end_per_testcase(_TestCase, Config) -> case wait_for_ct_stop(CTNode) of %% Common test was not stopped to we restart node. false -> - cover:stop(CTNode), + case test_server:is_cover() of + true -> cover:flush(CTNode); + false -> ok + end, slave:stop(CTNode), start_slave(Config,proplists:get_value(trace_level,Config)), {fail, "Could not stop common_test"}; @@ -620,8 +626,11 @@ locate({parallel,TEvs}, Node, Evs, Config) -> fun({EH,#event{name=tc_auto_skip, node=EvNode, data={Mod,end_per_group,Reason}}}) when - EH == TEH, EvNode == Node, Mod == M, Reason == R -> - false; + EH == TEH, EvNode == Node, Mod == M -> + case match_data(R, Reason) of + match -> false; + _ -> true + end; ({EH,#event{name=stop_logging, node=EvNode,data=_}}) when EH == TEH, EvNode == Node -> @@ -635,23 +644,12 @@ locate({parallel,TEvs}, Node, Evs, Config) -> [_AutoSkip | RemEvs2] -> {Done,RemEvs2,length(RemEvs2)} end; - %% match other event than test case - (TEv={TEH,N,D}, Acc) when D == '_' -> - case [E || E={EH,#event{name=Name, - node=EvNode, - data=_}} <- Evs1, - EH == TEH, EvNode == Node, Name == N] of - [] -> - exit({unmatched,TEv}); - _ -> - test_server:format("Found ~p!", [TEv]), - Acc - end; (TEv={TEH,N,D}, Acc) -> case [E || E={EH,#event{name=Name, node=EvNode, data=Data}} <- Evs1, - EH == TEH, EvNode == Node, Name == N, Data == D] of + EH == TEH, EvNode == Node, Name == N, + match == match_data(D,Data)] of [] -> exit({unmatched,TEv}); _ -> @@ -1010,33 +1008,39 @@ locate({TEH,Name,Data}, Node, [{TEH,#event{name=Name, data = EvData, node = Node}}|Evs], Config) -> - try match_data(Data, EvData) of + case match_data(Data, EvData) of match -> - {Config,Evs} - catch _:_ -> + {Config,Evs}; + _ -> nomatch end; locate({_TEH,_Name,_Data}, _Node, [_|_Evs], _Config) -> nomatch. -match_data(D,D) -> +match_data(Data, EvData) -> + try do_match_data(Data, EvData) + catch _:_ -> + nomatch + end. + +do_match_data(D,D) -> match; -match_data('_',_) -> +do_match_data('_',_) -> match; -match_data(Fun,Data) when is_function(Fun) -> +do_match_data(Fun,Data) when is_function(Fun) -> Fun(Data); -match_data('$proplist',Proplist) -> - match_data( +do_match_data('$proplist',Proplist) -> + do_match_data( fun(List) -> lists:foreach(fun({_,_}) -> ok end,List) end,Proplist); -match_data([H1|MatchT],[H2|ValT]) -> - match_data(H1,H2), - match_data(MatchT,ValT); -match_data(Tuple1,Tuple2) when is_tuple(Tuple1),is_tuple(Tuple2) -> - match_data(tuple_to_list(Tuple1),tuple_to_list(Tuple2)); -match_data([],[]) -> +do_match_data([H1|MatchT],[H2|ValT]) -> + do_match_data(H1,H2), + do_match_data(MatchT,ValT); +do_match_data(Tuple1,Tuple2) when is_tuple(Tuple1),is_tuple(Tuple2) -> + do_match_data(tuple_to_list(Tuple1),tuple_to_list(Tuple2)); +do_match_data([],[]) -> match. result_match({SkipOrFail,{ErrorInd,{Why,'_'}}}, @@ -1051,6 +1055,9 @@ result_match({failed,{timetrap_timeout,{'$approx',Num}}}, Value =< trunc(Num+0.02*Num) -> true; true -> false end; +result_match({user_timetrap_error,{Why,'_'}}, + {user_timetrap_error,{Why,_Stack}}) -> + true; result_match(Result, Result) -> true; result_match(_, _) -> |