diff options
| author | Siri Hansen <[email protected]> | 2017-03-06 08:52:34 +0100 | 
|---|---|---|
| committer | Siri Hansen <[email protected]> | 2017-03-06 08:52:34 +0100 | 
| commit | ff965ce109ea10ec69bbac47b3a3f9b51c43f236 (patch) | |
| tree | f97628a75d7303ff334b2ca40bc43ca2103587a2 /lib/common_test | |
| parent | b14e556b334069ca60b1be1a1ed45374daf8e1fc (diff) | |
| parent | 0e0155410734e04556ceecb1222dd2ef956b4177 (diff) | |
| download | otp-ff965ce109ea10ec69bbac47b3a3f9b51c43f236.tar.gz otp-ff965ce109ea10ec69bbac47b3a3f9b51c43f236.tar.bz2 otp-ff965ce109ea10ec69bbac47b3a3f9b51c43f236.zip | |
Merge branch 'maint'
Diffstat (limited to 'lib/common_test')
51 files changed, 2905 insertions, 742 deletions
| diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml index 48ffe653e4..d407a0a53f 100644 --- a/lib/common_test/doc/src/common_test_app.xml +++ b/lib/common_test/doc/src/common_test_app.xml @@ -224,7 +224,9 @@  	</type>  	<desc>	 -	  <p>OPTIONAL</p> +	  <p>OPTIONAL; if this function is defined, then <seealso +	  marker="#Module:end_per_suite-1"><c>end_per_suite/1</c></seealso> +	  must also be defined.</p>  	  <p>This configuration function is called as the first function in the  	  suite. It typically contains initializations that are common for @@ -256,7 +258,9 @@        </type>        <desc>	 -	<p>OPTIONAL</p> +	<p>OPTIONAL; if this function is defined, then <seealso +	  marker="#Module:init_per_suite-1"><c>init_per_suite/1</c></seealso> +	  must also be defined.</p>  	<p>This function is called as the last test case in the  	  suite. It is meant to be used for cleaning up after @@ -360,7 +364,9 @@  	</type>  	<desc>	 -	  <p>OPTIONAL</p> +	  <p>OPTIONAL; if this function is defined, then <seealso +	    marker="#Module:end_per_group-2"><c>end_per_group/2</c></seealso> +	    must also be defined.</p>  	  <p>This configuration function is called before execution of a  	    test case group. It typically contains initializations that are  @@ -396,7 +402,9 @@  	</type>  	<desc>	 -	  <p>OPTIONAL</p> +	  <p>OPTIONAL; if this function is defined, then <seealso +	    marker="#Module:init_per_group-2"><c>init_per_group/2</c></seealso> +	    must also be defined.</p>  	  <p>This function is called after the execution of a test case group  	    is finished. It is meant to be used for cleaning up after @@ -427,7 +435,10 @@  	</type>  	<desc> -	  <p>OPTIONAL</p> +	  <p>OPTIONAL; if this function is defined, +	    then <seealso marker="#Module:end_per_testcase-2"> +	      <c>end_per_testcase/2</c></seealso> must also be +	    defined.</p>  	  <p>This function is called before each test case. Argument  	  <c>TestCase</c> is the test case name, and @@ -454,7 +465,10 @@  	</type>  	<desc>	 -	  <p>OPTIONAL</p> +	  <p>OPTIONAL; if this function is defined, +	    then <seealso marker="#Module:init_per_testcase-2"> +	      <c>init_per_testcase/2</c></seealso> must also be +	    defined.</p>  	  <p>This function is called after each test case, and can be used  	    to clean up after diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml index c2cf29c530..a085f30262 100644 --- a/lib/common_test/doc/src/ct_hooks.xml +++ b/lib/common_test/doc/src/ct_hooks.xml @@ -208,9 +208,10 @@      </func>      <func> -      <name>Module:pre_init_per_group(GroupName, InitData, CTHState) -> Result</name> +      <name>Module:pre_init_per_group(SuiteName, GroupName, InitData, CTHState) -> Result</name>        <fsummary>Called before init_per_group.</fsummary>        <type> +        <v>SuiteName = atom()</v>          <v>GroupName = atom()</v>          <v>InitData = Config | SkipOrFail</v>          <v>Config = NewConfig = [{Key,Value}]</v> @@ -231,13 +232,19 @@            but for function            <seealso marker="common_test#Module:init_per_group-2"><c>init_per_group</c></seealso>            instead.</p> + +	<p>If <c>Module:pre_init_per_group/4</c> is not exported, common_test +	  will attempt to call <c>Module:pre_init_per_group(GroupName, +	  InitData, CTHState)</c> instead. This is for backwards +	  compatibility.</p>        </desc>      </func>      <func> -      <name>Module:post_init_per_group(GroupName, Config, Return, CTHState) -> Result</name> +      <name>Module:post_init_per_group(SuiteName, GroupName, Config, Return, CTHState) -> Result</name>        <fsummary>Called after init_per_group.</fsummary>        <type> +        <v>SuiteName = atom()</v>          <v>GroupName = atom()</v>          <v>Config = [{Key,Value}]</v>          <v>Return = NewReturn = Config | SkipOrFail | term()</v> @@ -258,13 +265,19 @@            but for function            <seealso marker="common_test#Module:init_per_group-2"><c>init_per_group</c></seealso>            instead.</p> + +	<p>If <c>Module:post_init_per_group/5</c> is not exported, common_test +	  will attempt to call <c>Module:post_init_per_group(GroupName, +	  Config, Return, CTHState)</c> instead. This is for backwards +	  compatibility.</p>        </desc>      </func>      <func> -      <name>Module:pre_init_per_testcase(TestcaseName, InitData, CTHState) -> Result</name> +      <name>Module:pre_init_per_testcase(SuiteName, TestcaseName, InitData, CTHState) -> Result</name>        <fsummary>Called before init_per_testcase.</fsummary>        <type> +        <v>SuiteName = atom()</v>          <v>TestcaseName = atom()</v>          <v>InitData = Config | SkipOrFail</v>          <v>Config = NewConfig = [{Key,Value}]</v> @@ -286,6 +299,11 @@            <seealso marker="common_test#Module:init_per_testcase-2"><c>init_per_testcase</c></seealso>            instead.</p> +	<p>If <c>Module:pre_init_per_testcase/4</c> is not exported, common_test +	  will attempt to call <c>Module:pre_init_per_testcase(TestcaseName, +	  InitData, CTHState)</c> instead. This is for backwards +	  compatibility.</p> +          <p>CTHs cannot be added here right now. That feature may be added in            a later release, but it would right now break backwards            compatibility.</p> @@ -293,9 +311,10 @@      </func>      <func> -      <name>Module:post_init_per_testcase(TestcaseName, Config, Return, CTHState) -> Result</name> +      <name>Module:post_init_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -> Result</name>        <fsummary>Called after init_per_testcase.</fsummary>        <type> +        <v>SuiteName = atom()</v>          <v>TestcaseName = atom()</v>          <v>Config = [{Key,Value}]</v>          <v>Return = NewReturn = Config | SkipOrFail | term()</v> @@ -316,15 +335,21 @@            but for function            <seealso marker="common_test#Module:init_per_testcase-2"><c>init_per_testcase</c></seealso>            instead.</p> + +	<p>If <c>Module:post_init_per_testcase/5</c> is not exported, common_test +	  will attempt to call <c>Module:post_init_per_testcase(TestcaseName, +	  Config, Return, CTHState)</c> instead. This is for backwards +	  compatibility.</p>        </desc>      </func>      <func> -      <name>Module:pre_end_per_testcase(TestcaseName, InitData, CTHState) -> Result</name> +      <name>Module:pre_end_per_testcase(SuiteName, TestcaseName, EndData, CTHState) -> Result</name>        <fsummary>Called before end_per_testcase.</fsummary>        <type> +        <v>SuiteName = atom()</v>          <v>TestcaseName = atom()</v> -        <v>InitData = Config</v> +        <v>EndData = Config</v>          <v>Config = NewConfig = [{Key,Value}]</v>          <v>CTHState = NewCTHState = term()</v>          <v>Result = {NewConfig, NewCTHState}</v> @@ -345,14 +370,20 @@          <p>This function can not change the result of the test case by returning skip or fail  	tuples, but it may insert items in <c>Config</c> that can be read in -	<c>end_per_testcase/2</c> or in <c>post_end_per_testcase/4</c>.</p> +	<c>end_per_testcase/2</c> or in <c>post_end_per_testcase/5</c>.</p> + +	<p>If <c>Module:pre_end_per_testcase/4</c> is not exported, common_test +	  will attempt to call <c>Module:pre_end_per_testcase(TestcaseName, +	  EndData, CTHState)</c> instead. This is for backwards +	  compatibility.</p>        </desc>      </func>      <func> -      <name>Module:post_end_per_testcase(TestcaseName, Config, Return, CTHState) -> Result</name> +      <name>Module:post_end_per_testcase(SuiteName, TestcaseName, Config, Return, CTHState) -> Result</name>        <fsummary>Called after end_per_testcase.</fsummary>        <type> +        <v>SuiteName = atom()</v>          <v>TestcaseName = atom()</v>          <v>Config = [{Key,Value}]</v>          <v>Return = NewReturn = Config | SkipOrFail | term()</v> @@ -373,13 +404,19 @@            but for function            <seealso marker="common_test#Module:end_per_testcase-2"><c>end_per_testcase</c></seealso>            instead.</p> + +	<p>If <c>Module:post_end_per_testcase/5</c> is not exported, common_test +	  will attempt to call <c>Module:post_end_per_testcase(TestcaseName, +	  Config, Return, CTHState)</c> instead. This is for backwards +	  compatibility.</p>        </desc>      </func>      <func> -      <name>Module:pre_end_per_group(GroupName, EndData, CTHState) -> Result</name> +      <name>Module:pre_end_per_group(SuiteName, GroupName, EndData, CTHState) -> Result</name>        <fsummary>Called before end_per_group.</fsummary>        <type> +        <v>SuiteName = atom()</v>          <v>GroupName = atom()</v>          <v>EndData = Config | SkipOrFail</v>          <v>Config = NewConfig = [{Key,Value}]</v> @@ -400,13 +437,19 @@            but for function            <seealso marker="common_test#Module:end_per_group-2"><c>end_per_group</c></seealso>            instead.</p> + +	<p>If <c>Module:pre_end_per_group/4</c> is not exported, common_test +	  will attempt to call <c>Module:pre_end_per_group(GroupName, +	  EndData, CTHState)</c> instead. This is for backwards +	  compatibility.</p>        </desc>      </func>      <func> -      <name>Module:post_end_per_group(GroupName, Config, Return, CTHState) -> Result</name> +      <name>Module:post_end_per_group(SuiteName, GroupName, Config, Return, CTHState) -> Result</name>        <fsummary>Called after end_per_group.</fsummary>        <type> +        <v>SuiteName = atom()</v>          <v>GroupName = atom()</v>          <v>Config = [{Key,Value}]</v>          <v>Return = NewReturn = Config | SkipOrFail | term()</v> @@ -427,6 +470,11 @@            but for function            <seealso marker="common_test#Module:end_per_group-2">end_per_group</seealso>            instead.</p> + +	<p>If <c>Module:post_end_per_group/5</c> is not exported, common_test +	  will attempt to call <c>Module:post_end_per_group(GroupName, +	  Config, Return, CTHState)</c> instead. This is for backwards +	  compatibility.</p>        </desc>      </func> @@ -485,9 +533,10 @@      </func>      <func> -      <name>Module:on_tc_fail(TestName, Reason, CTHState) -> NewCTHState</name> +      <name>Module:on_tc_fail(SuiteName, TestName, Reason, CTHState) -> NewCTHState</name>        <fsummary>Called after the CTH scope ends.</fsummary>        <type> +        <v>SuiteName = atom()</v>          <v>TestName = init_per_suite | end_per_suite | {init_per_group,GroupName} | {end_per_group,GroupName} | {FuncName,GroupName} | FuncName</v>          <v>FuncName = atom()</v>          <v>GroupName = atom()</v> @@ -505,7 +554,7 @@            <item><p>If <c>init_per_suite</c> fails, this function is called after              <seealso marker="#Module:post_init_per_suite-4"><c>post_init_per_suite</c></seealso>.</p></item>            <item><p>If a test case fails, this funcion is called after -          <seealso marker="#Module:post_end_per_testcase-4"><c>post_end_per_testcase</c></seealso>.</p></item> +          <seealso marker="#Module:post_end_per_testcase-5"><c>post_end_per_testcase</c></seealso>.</p></item>          </list>           <p>If the failed test case belongs to a test case group, the first @@ -519,13 +568,19 @@            For details, see section            <seealso marker="event_handler_chapter#events">Event Handling</seealso>            in the User's Guide.</p> + +	<p>If <c>Module:on_tc_fail/4</c> is not exported, common_test +	  will attempt to call <c>Module:on_tc_fail(TestName, Reason, +	  CTHState)</c> instead. This is for backwards +	  compatibility.</p>        </desc>      </func>      <func> -      <name>Module:on_tc_skip(TestName, Reason, CTHState) -> NewCTHState</name> +      <name>Module:on_tc_skip(SuiteName, TestName, Reason, CTHState) -> NewCTHState</name>        <fsummary>Called after the CTH scope ends.</fsummary>        <type> +        <v>SuiteName = atom()</v>          <v>TestName = init_per_suite | end_per_suite | {init_per_group,GroupName} | {end_per_group,GroupName} | {FuncName,GroupName} | FuncName</v>          <v>FuncName = atom()</v>          <v>GroupName = atom()</v> @@ -542,9 +597,9 @@          <list type="bulleted">            <item><p>If <c>init_per_group</c> is skipped, this function is              called after  -            <seealso marker="#Module:post_init_per_group-4"><c>post_init_per_group</c></seealso>.</p></item> +            <seealso marker="#Module:post_init_per_group-5"><c>post_init_per_group</c></seealso>.</p></item>            <item><p>If a test case is skipped, this function is called after -          <seealso marker="#Module:post_end_per_testcase-4"><c>post_end_per_testcase</c></seealso>.</p></item> +          <seealso marker="#Module:post_end_per_testcase-5"><c>post_end_per_testcase</c></seealso>.</p></item>          </list>          <p>If the skipped test case belongs to a test case group, the first @@ -559,6 +614,11 @@            For details, see section            <seealso marker="event_handler_chapter#events">Event Handling</seealso>            in the User's Guide.</p> + +	<p>If <c>Module:on_tc_skip/4</c> is not exported, common_test +	  will attempt to call <c>Module:on_tc_skip(TestName, Reason, +	  CTHState)</c> instead. This is for backwards +	  compatibility.</p>        </desc>      </func> diff --git a/lib/common_test/doc/src/ct_hooks_chapter.xml b/lib/common_test/doc/src/ct_hooks_chapter.xml index 0e4c35e11f..bfad96e489 100644 --- a/lib/common_test/doc/src/ct_hooks_chapter.xml +++ b/lib/common_test/doc/src/ct_hooks_chapter.xml @@ -38,7 +38,7 @@        extensions of the default behavior of <c>Common Test</c> using hooks         before and after all test suite calls. CTHs allow advanced <c>Common Test</c>        users to abstract out behavior that is common to multiple test suites -      without littering all test suites with library calls. this can be used  +      without littering all test suites with library calls. This can be used        for logging, starting, and monitoring external systems,         building C files needed by the tests, and so on.</p> @@ -175,10 +175,10 @@        <row>  	<cell><seealso marker="common_test#Module:init_per_group-2">  	  init_per_group/2</seealso></cell> -	<cell><seealso marker="ct_hooks#Module:post_init_per_group-4"> -	    post_init_per_group/4</seealso> is called</cell> -	<cell><seealso marker="ct_hooks#Module:post_end_per_suite-4"> -	  post_end_per_group/4</seealso> has been called for that group</cell> +	<cell><seealso marker="ct_hooks#Module:post_init_per_group-5"> +	    post_init_per_group/5</seealso> is called</cell> +	<cell><seealso marker="ct_hooks#Module:post_end_per_group-5"> +	  post_end_per_group/5</seealso> has been called for that group</cell>        </row>      <tcaption>Scope of a CTH</tcaption>      </table> @@ -245,16 +245,18 @@       </list>          <p> -	This is done in the CTH functions called pre_<name of function>. -	These functions take the same three arguments, <c>Name</c>,  +	This is done in the CTH functions called <c>pre_<name of function></c>. +	These functions take the arguments <c>SuiteName</c>, <c>Name</c> (group or test case name, if applicable),   	<c>Config</c>, and <c>CTHState</c>. The return value of the CTH function  	is always a combination of a result for the suite/group/test and an   	updated <c>CTHState</c>.</p>  	<p>To let the test suite continue on executing, return the configuration  -	list that you want the test to use as the result. To skip or  -	fail the test, return a tuple with <c>skip</c> or <c>fail</c>, and a reason  -	as the result.</p> +	list that you want the test to use as the result.</p> + +        <p>All pre hooks, except <c>pre_end_per_testcase/4</c>, can +        skip or fail the test by returning a tuple with <c>skip</c> or +        <c>fail</c>, and a reason as the result.</p>  	<p><em>Example:</em></p>  	<code> @@ -290,7 +292,7 @@       <p>        This is done in the CTH functions called <c>post_<name of function></c>.  -      These functions take the same four arguments, <c>Name</c>,  +      These functions take the arguments <c>SuiteName</c>, <c>Name</c> (group or test case name, if applicable),        <c>Config</c>, <c>Return</c>, and <c>CTHState</c>. <c>Config</c> in this        case is the same <c>Config</c> as the testcase is called with.         <c>Return</c> is the value returned by the testcase. If the testcase  @@ -308,7 +310,7 @@  	<p><em>Example:</em></p>  	<code> - post_end_per_testcase(_TC, Config, {'EXIT',{_,_}}, CTHState) -> + post_end_per_testcase(_Suite, _TC, Config, {'EXIT',{_,_}}, CTHState) ->     case db:check_consistency() of       true ->         %% DB is good, pass the test. @@ -317,7 +319,7 @@         %% DB is not good, mark as skipped instead of failing         {{skip, "DB is inconsisten!"}, CTHState}     end; - post_end_per_testcase(_TC, Config, Return, CTHState) -> + post_end_per_testcase(_Suite, _TC, Config, Return, CTHState) ->     %% Do nothing if tc does not crash.     {Return, CTHState}.</code> @@ -331,8 +333,8 @@        <title>Skip and Fail Hooks</title>        <p>  	After any post hook has been executed for all installed CTHs,  -	<seealso marker="ct_hooks#Module:on_tc_fail-3">on_tc_fail</seealso> -	or <seealso marker="ct_hooks#Module:on_tc_skip-3">on_tc_skip</seealso>  +	<seealso marker="ct_hooks#Module:on_tc_fail-4">on_tc_fail</seealso> +	or <seealso marker="ct_hooks#Module:on_tc_skip-4">on_tc_skip</seealso>  	is called if the testcase failed or was skipped, respectively.   	You cannot affect the outcome of the tests any further at this point.         </p> @@ -389,18 +391,18 @@   -export([pre_end_per_suite/3]).   -export([post_end_per_suite/4]). - -export([pre_init_per_group/3]). - -export([post_init_per_group/4]). - -export([pre_end_per_group/3]). - -export([post_end_per_group/4]). + -export([pre_init_per_group/4]). + -export([post_init_per_group/5]). + -export([pre_end_per_group/4]). + -export([post_end_per_group/5]). - -export([pre_init_per_testcase/3]). - -export([post_init_per_testcase/4]). - -export([pre_end_per_testcase/3]). - -export([post_end_per_testcase/4]). + -export([pre_init_per_testcase/4]). + -export([post_init_per_testcase/5]). + -export([pre_end_per_testcase/4]). + -export([post_end_per_testcase/5]). - -export([on_tc_fail/3]). - -export([on_tc_skip/3]). + -export([on_tc_fail/4]). + -export([on_tc_skip/4]).   -export([terminate/1]). @@ -435,46 +437,46 @@                             total = State#state.total + State#state.suite_total } }.   %% @doc Called before each init_per_group. - pre_init_per_group(Group,Config,State) -> + pre_init_per_group(Suite,Group,Config,State) ->       {Config, State}.   %% @doc Called after each init_per_group. - post_init_per_group(Group,Config,Return,State) -> + post_init_per_group(Suite,Group,Config,Return,State) ->       {Return, State}.   %% @doc Called before each end_per_group.  - pre_end_per_group(Group,Config,State) -> + pre_end_per_group(Suite,Group,Config,State) ->       {Config, State}.   %% @doc Called after each end_per_group.  - post_end_per_group(Group,Config,Return,State) -> + post_end_per_group(Suite,Group,Config,Return,State) ->       {Return, State}.   %% @doc Called before each init_per_testcase. - pre_init_per_testcase(TC,Config,State) -> + pre_init_per_testcase(Suite,TC,Config,State) ->       {Config, State#state{ ts = now(), total = State#state.suite_total + 1 } }.   %% Called after each init_per_testcase (immediately before the test case). - post_init_per_testcase(TC,Config,Return,State) -> + post_init_per_testcase(Suite,TC,Config,Return,State) ->       {Return, State}  %% @doc Called before each end_per_testcase (immediately after the test case). - pre_end_per_testcase(TC,Config,State) -> + pre_end_per_testcase(Suite,TC,Config,State) ->       {Config, State}.   %% @doc Called after each end_per_testcase. - post_end_per_testcase(TC,Config,Return,State) -> -     TCInfo = {testcase, TC, Return, timer:now_diff(now(), State#state.ts)}, + post_end_per_testcase(Suite,TC,Config,Return,State) -> +     TCInfo = {testcase, Suite, TC, Return, timer:now_diff(now(), State#state.ts)},       {Return, State#state{ ts = undefined, tcs = [TCInfo | State#state.tcs] } }.   %% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,   %% post_end_per_group and post_end_per_testcase if the suite, group or test case failed. - on_tc_fail(TC, Reason, State) -> + on_tc_fail(Suite, TC, Reason, State) ->       State.   %% @doc Called when a test case is skipped by either user action   %% or due to an init function failing.   - on_tc_skip(TC, Reason, State) -> + on_tc_skip(Suite, TC, Reason, State) ->       State.   %% @doc Called when the scope of the CTH is done diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml index f70bdb16c5..6a0d87bcaf 100644 --- a/lib/common_test/doc/src/write_test_chapter.xml +++ b/lib/common_test/doc/src/write_test_chapter.xml @@ -566,7 +566,7 @@      for the test cases in the group. After execution of the group is finished, function      <seealso marker="common_test#Module:end_per_group-2"><c>end_per_group(GroupName, Config)</c></seealso>       is called. This function is meant to be used for cleaning up after  -    <c>init_per_group/2</c>.</p> +    <c>init_per_group/2</c>. If the init function is defined, so must the end function be.</p>      <p>Whenever a group is executed, if <c>init_per_group</c> and        <c>end_per_group</c> do not exist in the suite, <c>Common Test</c> calls diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 291a4d716c..43f1c9de0f 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -52,6 +52,10 @@  %%%  %%% @doc Test server framework callback, called by the test_server  %%% when a new test case is started. +init_tc(_,{end_per_testcase_not_run,_},[Config]) -> +    %% Testcase is completed (skipped or failed), but end_per_testcase +    %% is not run - don't call pre-hook. +    {ok,[Config]};  init_tc(Mod,EPTC={end_per_testcase,_},[Config]) ->      %% in case Mod == ct_framework, lookup the suite name      Suite = get_suite_name(Mod, Config), @@ -62,7 +66,7 @@ init_tc(Mod,EPTC={end_per_testcase,_},[Config]) ->  	    Other      end; -init_tc(Mod,Func0,Args) ->     +init_tc(Mod,Func0,Args) ->      %% in case Mod == ct_framework, lookup the suite name      Suite = get_suite_name(Mod, Args),      {Func,HookFunc} = case Func0 of @@ -84,12 +88,15 @@ init_tc(Mod,Func0,Args) ->  	andalso Func=/=end_per_group  	andalso ct_util:get_testdata(skip_rest) of  	true -> +            initialize(false,Mod,Func,Args),  	    {auto_skip,"Repeated test stopped by force_stop option"};  	_ ->  	    case ct_util:get_testdata(curr_tc) of  		{Suite,{suite0_failed,{require,Reason}}} -> +                    initialize(false,Mod,Func,Args),  		    {auto_skip,{require_failed_in_suite0,Reason}};  		{Suite,{suite0_failed,_}=Failure} -> +                    initialize(false,Mod,Func,Args),  		    {fail,Failure};  		_ ->  		    ct_util:update_testdata(curr_tc, @@ -118,16 +125,14 @@ init_tc(Mod,Func0,Args) ->  			    end,  			    init_tc1(Mod,Suite,Func,HookFunc,Args);  			{failed,Seq,BadFunc} -> -			    {auto_skip,{sequence_failed,Seq,BadFunc}} +                            initialize(false,Mod,Func,Args), +                            {auto_skip,{sequence_failed,Seq,BadFunc}}  		    end  	    end      end.  init_tc1(?MODULE,_,error_in_suite,_,[Config0]) when is_list(Config0) -> -    ct_logs:init_tc(false), -    ct_event:notify(#event{name=tc_start, -			   node=node(), -			   data={?MODULE,error_in_suite}}), +    initialize(false,?MODULE,error_in_suite),      _ = ct_suite_init(?MODULE,error_in_suite,[],Config0),      case ?val(error,Config0) of  	undefined -> @@ -177,27 +182,21 @@ init_tc1(Mod,Suite,Func,HookFunc,[Config0]) when is_list(Config0) ->  		ct_config:delete_default_config(testcase),  		HookFunc  	end, -    Initialize = fun() ->  -			 ct_logs:init_tc(false), -			 ct_event:notify(#event{name=tc_start, -						node=node(), -						data={Mod,FuncSpec}}) -		 end,      case add_defaults(Mod,Func,AllGroups) of  	Error = {suite0_failed,_} -> -	    Initialize(), +	    initialize(false,Mod,FuncSpec),  	    ct_util:set_testdata({curr_tc,{Suite,Error}}),  	    {error,Error};  	Error = {group0_failed,_} -> -	    Initialize(), +	    initialize(false,Mod,FuncSpec),  	    {auto_skip,Error};  	Error = {testcase0_failed,_} -> -	    Initialize(), +	    initialize(false,Mod,FuncSpec),  	    {auto_skip,Error};  	{SuiteInfo,MergeResult} ->  	    case MergeResult of  		{error,Reason} -> -		    Initialize(), +		    initialize(false,Mod,FuncSpec),  		    {fail,Reason};  		_ ->  		    init_tc2(Mod,Suite,Func,HookFunc1, @@ -236,11 +235,8 @@ init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) ->  	Conns ->  	    ct_util:silence_connections(Conns)      end, -    ct_logs:init_tc(Func == init_per_suite),      FuncSpec = group_or_func(Func,Config), -    ct_event:notify(#event{name=tc_start, -			   node=node(), -			   data={Mod,FuncSpec}}), +    initialize((Func==init_per_suite),Mod,FuncSpec),      case catch configure(MergedInfo,MergedInfo,SuiteInfo,  			 FuncSpec,[],Config) of @@ -268,6 +264,18 @@ init_tc2(Mod,Suite,Func,HookFunc,SuiteInfo,MergeResult,Config) ->  	    end      end. +initialize(RefreshLogs,Mod,Func,[Config]) when is_list(Config) -> +    initialize(RefreshLogs,Mod,group_or_func(Func,Config)); +initialize(RefreshLogs,Mod,Func,_) -> +    initialize(RefreshLogs,Mod,Func). + +initialize(RefreshLogs,Mod,FuncSpec) -> +    ct_logs:init_tc(RefreshLogs), +    ct_event:notify(#event{name=tc_start, +			   node=node(), +			   data={Mod,FuncSpec}}). + +  ct_suite_init(Suite,HookFunc,PostInitHook,Config) when is_list(Config) ->      case ct_hooks:init_tc(Suite,HookFunc,Config) of  	NewConfig when is_list(NewConfig) -> @@ -675,22 +683,35 @@ end_tc(Mod,Func,{Result,[Args]}, Return) ->      end_tc(Mod,Func,self(),Result,Args,Return).  end_tc(Mod,IPTC={init_per_testcase,_Func},_TCPid,Result,Args,Return) -> -    %% in case Mod == ct_framework, lookup the suite name -    Suite = get_suite_name(Mod, Args), -    case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of -	    '$ct_no_change' -> -		ok; -	    HookResult -> -		HookResult +    case end_hook_func(IPTC,Return,IPTC) of +        undefined -> ok; +        _ -> +            %% in case Mod == ct_framework, lookup the suite name +            Suite = get_suite_name(Mod, Args), +            case ct_hooks:end_tc(Suite,IPTC,Args,Result,Return) of +                '$ct_no_change' -> +                    ok; +                HookResult -> +                    HookResult +            end      end;  end_tc(Mod,Func0,TCPid,Result,Args,Return) ->      %% in case Mod == ct_framework, lookup the suite name      Suite = get_suite_name(Mod, Args), -    {EPTC,Func} = case Func0 of -		      {end_per_testcase,F} -> {true,F}; -		      _                    -> {false,Func0} -		  end, +    {Func,FuncSpec,HookFunc} = +        case Func0 of +            {end_per_testcase_not_run,F} -> +                %% Testcase is completed (skipped or failed), but +                %% end_per_testcase is not run - don't call post-hook. +                {F,F,undefined}; +            {end_per_testcase,F} -> +                {F,F,Func0}; +            _ -> +                FS = group_or_func(Func0,Args), +                HF = end_hook_func(Func0,Return,FS), +                {Func0,FS,HF} +        end,      test_server:timetrap_cancel(), @@ -717,20 +738,18 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) ->      end,      ct_util:delete_suite_data(last_saved_config), -    {FuncSpec,HookFunc} = -	if not EPTC -> -		FS = group_or_func(Func,Args), -		{FS,FS}; -	   true -> -		{Func,Func0} -	end,      {Result1,FinalNotify} = -	case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of -	    '$ct_no_change' -> -		{ok,Result}; -	    HookResult -> -		{HookResult,HookResult} -	end, +        case HookFunc of +            undefined -> +                {ok,Result}; +            _ -> +                case ct_hooks:end_tc(Suite,HookFunc,Args,Result,Return) of +                    '$ct_no_change' -> +                        {ok,Result}; +                    HookResult -> +                        {HookResult,HookResult} +                end +        end,      FinalResult =  	case get('$test_server_framework_test') of  	    undefined -> @@ -821,6 +840,34 @@ end_tc(Mod,Func0,TCPid,Result,Args,Return) ->      end,      FinalResult.	     +%% This is to make sure that no post_init_per_* is ever called if the +%% corresponding pre_init_per_* was not called. +%% The skip or fail reasons are those that can be returned from +%% init_tc above in situations where we never came to call +%% ct_hooks:init_tc/3, e.g. if suite/0 fails, then we never call +%% ct_hooks:init_tc for init_per_suite, and thus we must not call +%% ct_hooks:end_tc for init_per_suite either. +end_hook_func({init_per_testcase,_},{auto_skip,{sequence_failed,_,_}},_) -> +    undefined; +end_hook_func({init_per_testcase,_},{auto_skip,"Repeated test stopped by force_stop option"},_) -> +    undefined; +end_hook_func({init_per_testcase,_},{fail,{config_name_already_in_use,_}},_) -> +    undefined; +end_hook_func({init_per_testcase,_},{auto_skip,{InfoFuncError,_}},_) +  when InfoFuncError==testcase0_failed; +       InfoFuncError==require_failed -> +    undefined; +end_hook_func(init_per_group,{auto_skip,{InfoFuncError,_}},_) +  when InfoFuncError==group0_failed; +       InfoFuncError==require_failed -> +    undefined; +end_hook_func(init_per_suite,{auto_skip,{require_failed_in_suite0,_}},_) -> +    undefined; +end_hook_func(init_per_suite,{auto_skip,{failed,{error,{suite0_failed,_}}}},_) -> +    undefined; +end_hook_func(_,_,Default) -> +    Default. +  %% {error,Reason} | {skip,Reason} | {timetrap_timeout,TVal} |   %% {testcase_aborted,Reason} | testcase_aborted_or_killed |   %% {'EXIT',Reason} | {fail,Reason} | {failed,Reason} | @@ -1339,25 +1386,25 @@ report(What,Data) ->  	    ok;  	tc_done ->  	    {Suite,{Func,GrName},Result} = Data, -	    Data1 = if GrName == undefined -> {Suite,Func,Result}; -		       true                -> Data -	    end, +            FuncSpec = if GrName == undefined -> Func; +                          true                -> {Func,GrName} +                       end,  	    %% Register the group leader for the process calling the report  	    %% function, making it possible for a hook function to print  	    %% in the test case log file  	    ReportingPid = self(),  	    ct_logs:register_groupleader(ReportingPid, group_leader()),  	    case Result of -		{failed, _} -> -		    ct_hooks:on_tc_fail(What, Data1); -		{skipped,{failed,{_,init_per_testcase,_}}} -> -		    ct_hooks:on_tc_skip(tc_auto_skip, Data1); -		{skipped,{require_failed,_}} -> -		    ct_hooks:on_tc_skip(tc_auto_skip, Data1); -		{skipped,_} -> -		    ct_hooks:on_tc_skip(tc_user_skip, Data1); -		{auto_skipped,_} -> -		    ct_hooks:on_tc_skip(tc_auto_skip, Data1); +		{failed, Reason} -> +		    ct_hooks:on_tc_fail(What, {Suite,FuncSpec,Reason}); +		{skipped,{failed,{_,init_per_testcase,_}}=Reason} -> +		    ct_hooks:on_tc_skip(tc_auto_skip,  {Suite,FuncSpec,Reason}); +		{skipped,{require_failed,_}=Reason} -> +		    ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason}); +		{skipped,Reason} -> +		    ct_hooks:on_tc_skip(tc_user_skip, {Suite,FuncSpec,Reason}); +		{auto_skipped,Reason} -> +		    ct_hooks:on_tc_skip(tc_auto_skip, {Suite,FuncSpec,Reason});  		_Else ->  		    ok  	    end, diff --git a/lib/common_test/src/ct_groups.erl b/lib/common_test/src/ct_groups.erl index 1375e7dcc7..1c9faf6a70 100644 --- a/lib/common_test/src/ct_groups.erl +++ b/lib/common_test/src/ct_groups.erl @@ -442,17 +442,21 @@ make_conf(Mod, Name, Props, TestSpec) ->  	    ok      end,      {InitConf,EndConf,ExtraProps} = -	case erlang:function_exported(Mod,init_per_group,2) of -	    true -> -		{{Mod,init_per_group},{Mod,end_per_group},[]}; -	    false -> +	case {erlang:function_exported(Mod,init_per_group,2), +              erlang:function_exported(Mod,end_per_group,2)} of +	    {false,false} ->  		ct_logs:log("TEST INFO", "init_per_group/2 and "  			    "end_per_group/2 missing for group "  			    "~w in ~w, using default.",  			    [Name,Mod]),  		{{ct_framework,init_per_group},  		 {ct_framework,end_per_group}, -		 [{suite,Mod}]} +		 [{suite,Mod}]}; +	    _ -> +                %% If any of these exist, the other should too +                %% (required and documented). If it isn't, it will fail +                %% with reason 'undef'. +		{{Mod,init_per_group},{Mod,end_per_group},[]}  	end,      {conf,[{name,Name}|Props++ExtraProps],InitConf,TestSpec,EndConf}. diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl index c9a4abb5ee..60d1ea2b1c 100644 --- a/lib/common_test/src/ct_hooks.erl +++ b/lib/common_test/src/ct_hooks.erl @@ -92,15 +92,17 @@ init_tc(Mod, end_per_suite, Config) ->      call(fun call_generic/3, Config, [pre_end_per_suite, Mod]);  init_tc(Mod, {init_per_group, GroupName, Properties}, Config) ->      maybe_start_locker(Mod, GroupName, Properties), -    call(fun call_generic/3, Config, [pre_init_per_group, GroupName]); -init_tc(_Mod, {end_per_group, GroupName, _}, Config) -> -    call(fun call_generic/3, Config, [pre_end_per_group, GroupName]); -init_tc(_Mod, {init_per_testcase,TC}, Config) -> -    call(fun call_generic/3, Config, [pre_init_per_testcase, TC]); -init_tc(_Mod, {end_per_testcase,TC}, Config) -> -    call(fun call_generic/3, Config, [pre_end_per_testcase, TC]); -init_tc(_Mod, TC = error_in_suite, Config) -> -    call(fun call_generic/3, Config, [pre_init_per_testcase, TC]). +    call(fun call_generic_fallback/3, Config, +         [pre_init_per_group, Mod, GroupName]); +init_tc(Mod, {end_per_group, GroupName, _}, Config) -> +    call(fun call_generic_fallback/3, Config, +         [pre_end_per_group, Mod, GroupName]); +init_tc(Mod, {init_per_testcase,TC}, Config) -> +    call(fun call_generic_fallback/3, Config, [pre_init_per_testcase, Mod, TC]); +init_tc(Mod, {end_per_testcase,TC}, Config) -> +    call(fun call_generic_fallback/3, Config, [pre_end_per_testcase, Mod, TC]); +init_tc(Mod, TC = error_in_suite, Config) -> +    call(fun call_generic_fallback/3, Config, [pre_init_per_testcase, Mod, TC]).  %% @doc Called as each test case is completed. This includes all configuration  %% tests. @@ -126,23 +128,23 @@ end_tc(Mod, init_per_suite, Config, _Result, Return) ->  end_tc(Mod, end_per_suite, Config, Result, _Return) ->      call(fun call_generic/3, Result, [post_end_per_suite, Mod, Config],  	'$ct_no_change'); -end_tc(_Mod, {init_per_group, GroupName, _}, Config, _Result, Return) -> -    call(fun call_generic/3, Return, [post_init_per_group, GroupName, Config], -	 '$ct_no_change'); +end_tc(Mod, {init_per_group, GroupName, _}, Config, _Result, Return) -> +    call(fun call_generic_fallback/3, Return, +         [post_init_per_group, Mod, GroupName, Config], '$ct_no_change');  end_tc(Mod, {end_per_group, GroupName, Properties}, Config, Result, _Return) -> -    Res = call(fun call_generic/3, Result, -	       [post_end_per_group, GroupName, Config], '$ct_no_change'), +    Res = call(fun call_generic_fallback/3, Result, +	       [post_end_per_group, Mod, GroupName, Config], '$ct_no_change'),      maybe_stop_locker(Mod, GroupName, Properties),      Res; -end_tc(_Mod, {init_per_testcase,TC}, Config, Result, _Return) -> -    call(fun call_generic/3, Result, [post_init_per_testcase, TC, Config], -	'$ct_no_change'); -end_tc(_Mod, {end_per_testcase,TC}, Config, Result, _Return) -> -    call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config], -	 '$ct_no_change'); -end_tc(_Mod, TC = error_in_suite, Config, Result, _Return) -> -    call(fun call_generic/3, Result, [post_end_per_testcase, TC, Config], -	'$ct_no_change'). +end_tc(Mod, {init_per_testcase,TC}, Config, Result, _Return) -> +    call(fun call_generic_fallback/3, Result, +         [post_init_per_testcase, Mod, TC, Config], '$ct_no_change'); +end_tc(Mod, {end_per_testcase,TC}, Config, Result, _Return) -> +    call(fun call_generic_fallback/3, Result, +         [post_end_per_testcase, Mod, TC, Config], '$ct_no_change'); +end_tc(Mod, TC = error_in_suite, Config, Result, _Return) -> +    call(fun call_generic_fallback/3, Result, +         [post_end_per_testcase, Mod, TC, Config], '$ct_no_change').  %% Case = TestCase | {TestCase,GroupName} @@ -181,15 +183,21 @@ call_terminate(#ct_hook_config{ module = Mod, state = State} = Hook, _, _) ->      {[],Hook}.  call_cleanup(#ct_hook_config{ module = Mod, state = State} = Hook, -	     Reason, [Function, _Suite | Args]) -> +	     Reason, [Function | Args]) ->      NewState = catch_apply(Mod,Function, Args ++ [Reason, State], -			   State), +			   State, true),      {Reason, Hook#ct_hook_config{ state = NewState } }. -call_generic(#ct_hook_config{ module = Mod, state = State} = Hook, -	     Value, [Function | Args]) -> +call_generic(Hook, Value, Meta) -> +    do_call_generic(Hook, Value, Meta, false). + +call_generic_fallback(Hook, Value, Meta) -> +    do_call_generic(Hook, Value, Meta, true). + +do_call_generic(#ct_hook_config{ module = Mod, state = State} = Hook, +                Value, [Function | Args], Fallback) ->      {NewValue, NewState} = catch_apply(Mod, Function, Args ++ [Value, State], -				       {Value,State}), +				       {Value,State}, Fallback),      {NewValue, Hook#ct_hook_config{ state = NewState } }.  %% Generic call function @@ -257,15 +265,15 @@ remove(Key,List) when is_list(List) ->  remove(_, Else) ->      Else. -%% Translate scopes, i.e. init_per_group,group1 -> end_per_group,group1 etc -scope([pre_init_per_testcase, TC|_]) -> -    [post_init_per_testcase, TC]; -scope([pre_end_per_testcase, TC|_]) -> -    [post_end_per_testcase, TC]; -scope([pre_init_per_group, GroupName|_]) -> -    [post_end_per_group, GroupName]; -scope([post_init_per_group, GroupName|_]) -> -    [post_end_per_group, GroupName]; +%% Translate scopes, i.e. is_tuplenit_per_group,group1 -> end_per_group,group1 etc +scope([pre_init_per_testcase, SuiteName, TC|_]) -> +    [post_init_per_testcase, SuiteName, TC]; +scope([pre_end_per_testcase, SuiteName, TC|_]) -> +    [post_end_per_testcase, SuiteName, TC]; +scope([pre_init_per_group, SuiteName, GroupName|_]) -> +    [post_end_per_group, SuiteName, GroupName]; +scope([post_init_per_group, SuiteName, GroupName|_]) -> +    [post_end_per_group, SuiteName, GroupName];  scope([pre_init_per_suite, SuiteName|_]) ->      [post_end_per_suite, SuiteName];  scope([post_init_per_suite, SuiteName|_]) -> @@ -273,14 +281,29 @@ scope([post_init_per_suite, SuiteName|_]) ->  scope(init) ->      none. -terminate_if_scope_ends(HookId, [on_tc_skip,_Suite,{end_per_group,Name}],  +strip_config([post_init_per_testcase, SuiteName, TC|_]) -> +    [post_init_per_testcase, SuiteName, TC]; +strip_config([post_end_per_testcase, SuiteName, TC|_]) -> +    [post_end_per_testcase, SuiteName, TC]; +strip_config([post_init_per_group, SuiteName, GroupName|_]) -> +    [post_init_per_group, SuiteName, GroupName]; +strip_config([post_end_per_group, SuiteName, GroupName|_]) -> +    [post_end_per_group, SuiteName, GroupName]; +strip_config([post_init_per_suite, SuiteName|_]) -> +    [post_init_per_suite, SuiteName]; +strip_config([post_end_per_suite, SuiteName|_]) -> +    [post_end_per_suite, SuiteName]; +strip_config(Other) -> +    Other. + + +terminate_if_scope_ends(HookId, [on_tc_skip,Suite,{end_per_group,Name}],  			Hooks) -> -    terminate_if_scope_ends(HookId, [post_end_per_group, Name], Hooks); +    terminate_if_scope_ends(HookId, [post_end_per_group, Suite, Name], Hooks);  terminate_if_scope_ends(HookId, [on_tc_skip,Suite,end_per_suite], Hooks) ->      terminate_if_scope_ends(HookId, [post_end_per_suite, Suite], Hooks); -terminate_if_scope_ends(HookId, [Function,Tag|T], Hooks) when T =/= [] -> -    terminate_if_scope_ends(HookId,[Function,Tag],Hooks); -terminate_if_scope_ends(HookId, Function, Hooks) -> +terminate_if_scope_ends(HookId, Function0, Hooks) -> +    Function = strip_config(Function0),      case lists:keyfind(HookId, #ct_hook_config.id, Hooks) of          #ct_hook_config{ id = HookId, scope = Function} = Hook ->              terminate([Hook]), @@ -384,21 +407,29 @@ pos(Id,[_|Rest],Num) ->  catch_apply(M,F,A, Default) -> +    catch_apply(M,F,A,Default,false). +catch_apply(M,F,A, Default, Fallback) -> +    not erlang:module_loaded(M) andalso (catch M:module_info()), +    case erlang:function_exported(M,F,length(A)) of +        false when Fallback -> +            catch_apply(M,F,tl(A),Default,false); +        false -> +            Default; +        true -> +            catch_apply(M,F,A) +    end. + +catch_apply(M,F,A) ->      try -	erlang:apply(M,F,A) +        erlang:apply(M,F,A)      catch _:Reason -> -	    case erlang:get_stacktrace() of -		%% Return the default if it was the CTH module which did not have the function. -		[{M,F,A,_}|_] when Reason == undef -> -		    Default; -		Trace -> -		    ct_logs:log("Suite Hook","Call to CTH failed: ~w:~p", -				[error,{Reason,Trace}]), -		    throw({error_in_cth_call, -			   lists:flatten( -			     io_lib:format("~w:~w/~w CTH call failed", -					   [M,F,length(A)]))}) -	    end +            Trace = erlang:get_stacktrace(), +            ct_logs:log("Suite Hook","Call to CTH failed: ~w:~p", +                            [error,{Reason,Trace}]), +            throw({error_in_cth_call, +                   lists:flatten( +                     io_lib:format("~w:~w/~w CTH call failed", +                                   [M,F,length(A)]))})      end. diff --git a/lib/common_test/src/cth_conn_log.erl b/lib/common_test/src/cth_conn_log.erl index 883da0da0a..ce8852b3ea 100644 --- a/lib/common_test/src/cth_conn_log.erl +++ b/lib/common_test/src/cth_conn_log.erl @@ -54,8 +54,8 @@  -include_lib("common_test/include/ct.hrl").  -export([init/2, -	 pre_init_per_testcase/3, -	 post_end_per_testcase/4]). +	 pre_init_per_testcase/4, +	 post_end_per_testcase/5]).  %%----------------------------------------------------------------------  %% Exported types @@ -104,7 +104,7 @@ get_log_opts(Mod,Opts) ->      Hosts = proplists:get_value(hosts,Opts,[]),      {LogType,Hosts}. -pre_init_per_testcase(TestCase,Config,CthState) -> +pre_init_per_testcase(_Suite,TestCase,Config,CthState) ->      Logs =  	lists:map(  	  fun({ConnMod,{LogType,Hosts}}) ->		   @@ -158,7 +158,7 @@ pre_init_per_testcase(TestCase,Config,CthState) ->      ct_util:update_testdata(?MODULE, Update, [create]),      {Config,CthState}. -post_end_per_testcase(TestCase,_Config,Return,CthState) -> +post_end_per_testcase(_Suite,TestCase,_Config,Return,CthState) ->      Update =  	fun(PrevUsers) ->  		case lists:delete(TestCase, PrevUsers) of diff --git a/lib/common_test/src/cth_log_redirect.erl b/lib/common_test/src/cth_log_redirect.erl index 6d77d7ee9e..eda090d4f5 100644 --- a/lib/common_test/src/cth_log_redirect.erl +++ b/lib/common_test/src/cth_log_redirect.erl @@ -28,10 +28,10 @@  %% CTH Callbacks  -export([id/1, init/2,  	 pre_init_per_suite/3, pre_end_per_suite/3, post_end_per_suite/4, -	 pre_init_per_group/3, post_init_per_group/4, -	 pre_end_per_group/3, post_end_per_group/4, -	 pre_init_per_testcase/3, post_init_per_testcase/4, -	 pre_end_per_testcase/3, post_end_per_testcase/4]). +	 pre_init_per_group/4, post_init_per_group/5, +	 pre_end_per_group/4, post_end_per_group/5, +	 pre_init_per_testcase/4, post_init_per_testcase/5, +	 pre_end_per_testcase/4, post_end_per_testcase/5]).  %% Event handler Callbacks  -export([init/1, @@ -71,11 +71,11 @@ post_end_per_suite(_Suite, Config, Return, State) ->      set_curr_func(undefined, Config),      {Return, State}. -pre_init_per_group(Group, Config, State) -> +pre_init_per_group(_Suite, Group, Config, State) ->      set_curr_func({group,Group,init_per_group}, Config),      {Config, State}. -post_init_per_group(Group, Config, Result, tc_log_async) when is_list(Config) -> +post_init_per_group(_Suite, Group, Config, Result, tc_log_async) when is_list(Config) ->      case lists:member(parallel,proplists:get_value(  				 tc_group_properties,Config,[])) of  	true -> @@ -83,33 +83,33 @@ post_init_per_group(Group, Config, Result, tc_log_async) when is_list(Config) ->  	false ->  	    {Result, tc_log_async}      end; -post_init_per_group(_Group, _Config, Result, State) -> +post_init_per_group(_Suite, _Group, _Config, Result, State) ->      {Result, State}. -pre_init_per_testcase(TC, Config, State) -> +pre_init_per_testcase(_Suite, TC, Config, State) ->      set_curr_func(TC, Config),      {Config, State}. -post_init_per_testcase(_TC, _Config, Return, State) -> +post_init_per_testcase(_Suite, _TC, _Config, Return, State) ->      {Return, State}. -pre_end_per_testcase(_TC, Config, State) -> +pre_end_per_testcase(_Suite, _TC, Config, State) ->      {Config, State}. -post_end_per_testcase(_TC, _Config, Result, State) -> +post_end_per_testcase(_Suite, _TC, _Config, Result, State) ->      %% Make sure that the event queue is flushed      %% before ending this test case.      gen_event:call(error_logger, ?MODULE, flush, 300000),      {Result, State}. -pre_end_per_group(Group, Config, {tc_log, Group}) -> +pre_end_per_group(_Suite, Group, Config, {tc_log, Group}) ->      set_curr_func({group,Group,end_per_group}, Config),      {Config, set_log_func(tc_log_async)}; -pre_end_per_group(Group, Config, State) -> +pre_end_per_group(_Suite, Group, Config, State) ->      set_curr_func({group,Group,end_per_group}, Config),      {Config, State}. -post_end_per_group(_Group, Config, Return, State) -> +post_end_per_group(_Suite, _Group, Config, Return, State) ->      set_curr_func({group,undefined}, Config),      {Return, State}. diff --git a/lib/common_test/src/cth_surefire.erl b/lib/common_test/src/cth_surefire.erl index 59b916851e..c4941948cc 100644 --- a/lib/common_test/src/cth_surefire.erl +++ b/lib/common_test/src/cth_surefire.erl @@ -33,16 +33,16 @@  -export([pre_end_per_suite/3]).  -export([post_end_per_suite/4]). --export([pre_init_per_group/3]). --export([post_init_per_group/4]). --export([pre_end_per_group/3]). --export([post_end_per_group/4]). +-export([pre_init_per_group/4]). +-export([post_init_per_group/5]). +-export([pre_end_per_group/4]). +-export([post_end_per_group/5]). --export([pre_init_per_testcase/3]). --export([post_end_per_testcase/4]). +-export([pre_init_per_testcase/4]). +-export([post_end_per_testcase/5]). --export([on_tc_fail/3]). --export([on_tc_skip/3]). +-export([on_tc_fail/4]). +-export([on_tc_skip/4]).  -export([terminate/1]). @@ -116,29 +116,29 @@ pre_end_per_suite(_Suite,Config,State) ->  post_end_per_suite(_Suite,Config,Result,State) ->      {Result, end_tc(end_per_suite,Config,Result,State)}. -pre_init_per_group(Group,Config,State) -> +pre_init_per_group(_Suite,Group,Config,State) ->      {Config, init_tc(State#state{ curr_group = [Group|State#state.curr_group]},  		     Config)}. -post_init_per_group(_Group,Config,Result,State) -> +post_init_per_group(_Suite,_Group,Config,Result,State) ->      {Result, end_tc(init_per_group,Config,Result,State)}. -pre_end_per_group(_Group,Config,State) -> +pre_end_per_group(_Suite,_Group,Config,State) ->      {Config, init_tc(State, Config)}. -post_end_per_group(_Group,Config,Result,State) -> +post_end_per_group(_Suite,_Group,Config,Result,State) ->      NewState = end_tc(end_per_group, Config, Result, State),      {Result, NewState#state{ curr_group = tl(NewState#state.curr_group)}}. -pre_init_per_testcase(_TC,Config,State) -> +pre_init_per_testcase(_Suite,_TC,Config,State) ->      {Config, init_tc(State, Config)}. -post_end_per_testcase(TC,Config,Result,State) -> +post_end_per_testcase(_Suite,TC,Config,Result,State) ->      {Result, end_tc(TC,Config, Result,State)}. -on_tc_fail(_TC, _Res, State = #state{test_cases = []}) -> +on_tc_fail(_Suite,_TC, _Res, State = #state{test_cases = []}) ->      State; -on_tc_fail(_TC, Res, State) -> +on_tc_fail(_Suite,_TC, Res, State) ->      TCs = State#state.test_cases,      TC = hd(TCs),      NewTC = TC#testcase{ @@ -146,10 +146,9 @@ on_tc_fail(_TC, Res, State) ->  		  {fail,lists:flatten(io_lib:format("~p",[Res]))} },      State#state{ test_cases = [NewTC | tl(TCs)]}. -on_tc_skip({ConfigFunc,_GrName},{Type,_Reason} = Res, State0) -  when Type == tc_auto_skip; Type == tc_user_skip -> -    on_tc_skip(ConfigFunc, Res, State0); -on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip -> +on_tc_skip(Suite,{ConfigFunc,_GrName}, Res, State) -> +    on_tc_skip(Suite,ConfigFunc, Res, State); +on_tc_skip(Suite,Tc, Res, State0) ->      TcStr = atom_to_list(Tc),      State =  	case State0#state.test_cases of @@ -158,11 +157,7 @@ on_tc_skip(Tc,{Type,_Reason} = Res, State0) when Type == tc_auto_skip ->  	    _ ->  		State0  	end, -    do_tc_skip(Res, end_tc(Tc,[],Res,init_tc(State,[]))); -on_tc_skip(_Tc, _Res, State = #state{test_cases = []}) -> -    State; -on_tc_skip(_Tc, Res, State) -> -    do_tc_skip(Res, State). +    do_tc_skip(Res, end_tc(Tc,[],Res,init_tc(set_suite(Suite,State),[]))).  do_tc_skip(Res, State) ->      TCs = State#state.test_cases, @@ -209,6 +204,12 @@ end_tc(Name, _Config, _Res, State = #state{ curr_suite = Suite,  					  result = passed }|  			       State#state.test_cases],  		 tc_log = ""}. % so old tc_log is not set if next is on_tc_skip + +set_suite(Suite,#state{curr_suite=undefined}=State) -> +    State#state{curr_suite=Suite, curr_suite_ts=?now}; +set_suite(_,State) -> +    State. +  close_suite(#state{ test_cases = [] } = State) ->      State;  close_suite(#state{ test_cases = TCs, url_base = UrlBase } = State) -> @@ -228,7 +229,8 @@ close_suite(#state{ test_cases = TCs, url_base = UrlBase } = State) ->  			testcases = lists:reverse(TCs),  			log = SuiteLog,  			url = SuiteUrl}, -    State#state{ test_cases = [], +    State#state{ curr_suite = undefined, +                 test_cases = [],  		 test_suites = [Suite | State#state.test_suites]}.  terminate(State = #state{ test_cases = [] }) -> diff --git a/lib/common_test/src/test_server.erl b/lib/common_test/src/test_server.erl index 924086f2bd..be49191f2e 100644 --- a/lib/common_test/src/test_server.erl +++ b/lib/common_test/src/test_server.erl @@ -778,9 +778,9 @@ spawn_fw_call(Mod,IPTC={init_per_testcase,Func},CurrConf,Pid,  		%% if init_per_testcase fails, the test case  		%% should be skipped  		try begin do_end_tc_call(Mod,IPTC, {Pid,Skip,[CurrConf]}, Why), -			  do_init_tc_call(Mod,{end_per_testcase,Func}, +			  do_init_tc_call(Mod,{end_per_testcase_not_run,Func},  					  [CurrConf],{ok,[CurrConf]}), -			  do_end_tc_call(Mod,{end_per_testcase,Func},  +			  do_end_tc_call(Mod,{end_per_testcase_not_run,Func},  					 {Pid,Skip,[CurrConf]}, Why) end of  		    _ -> ok  		catch @@ -1151,14 +1151,14 @@ do_end_tc_call(Mod, IPTC={init_per_testcase,Func}, Res, Return) ->  			 Args  		 end,  	     EPTCInitRes = -		 case do_init_tc_call(Mod,{end_per_testcase,Func}, +		 case do_init_tc_call(Mod,{end_per_testcase_not_run,Func},  				      IPTCEndRes,Return) of  		     {ok,EPTCInitConfig} when is_list(EPTCInitConfig) ->  			 {Return,EPTCInitConfig};  		     _ -> -			 Return +                         {Return,IPTCEndRes}  		 end, -	     do_end_tc_call1(Mod, {end_per_testcase,Func}, +	     do_end_tc_call1(Mod, {end_per_testcase_not_run,Func},  			     EPTCInitRes, Return);  	 _Ok ->  	     do_end_tc_call1(Mod, IPTC, Res, Return) diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl index b52e4bef9b..39c523f8b3 100644 --- a/lib/common_test/src/test_server_ctrl.erl +++ b/lib/common_test/src/test_server_ctrl.erl @@ -2051,17 +2051,21 @@ add_init_and_end_per_suite([], _LastMod, skipped_suite, _FwMod) ->  add_init_and_end_per_suite([], LastMod, LastRef, FwMod) ->      %% we'll add end_per_suite here even if it's not exported      %% (and simply let the call fail if it's missing) -    case erlang:function_exported(LastMod, end_per_suite, 1) of -	true -> -	    [{conf,LastRef,[],{LastMod,end_per_suite}}]; -	false -> +    case {erlang:function_exported(LastMod, end_per_suite, 1), +          erlang:function_exported(LastMod, init_per_suite, 1)} of +	{false,false} ->  	    %% let's call a "fake" end_per_suite if it exists			  	    case erlang:function_exported(FwMod, end_per_suite, 1) of  		true ->					  		    [{conf,LastRef,[{suite,LastMod}],{FwMod,end_per_suite}}];  		false ->		  		    [{conf,LastRef,[],{LastMod,end_per_suite}}] -	    end +	    end; +	_ -> +            %% If any of these exist, the other should too +            %% (required and documented). If it isn't, it will fail +            %% with reason 'undef'. +	    [{conf,LastRef,[],{LastMod,end_per_suite}}]      end.      do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) -> @@ -2070,11 +2074,9 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->  	_ -> ok      end,      {Init,NextMod,NextRef} = -	case erlang:function_exported(Mod, init_per_suite, 1) of -	    true -> -		Ref = make_ref(), -		{[{conf,Ref,[],{Mod,init_per_suite}}],Mod,Ref}; -	    false -> +	case {erlang:function_exported(Mod, init_per_suite, 1), +              erlang:function_exported(Mod, end_per_suite, 1)} of +	    {false,false} ->  		%% let's call a "fake" init_per_suite if it exists  		case erlang:function_exported(FwMod, init_per_suite, 1) of  		    true -> @@ -2083,8 +2085,13 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->  			   {FwMod,init_per_suite}}],Mod,Ref};  		    false ->  			{[],Mod,undefined} -		end - +		end; +	    _ -> +                %% If any of these exist, the other should too +                %% (required and documented). If it isn't, it will fail +                %% with reason 'undef'. +		Ref = make_ref(), +		{[{conf,Ref,[],{Mod,init_per_suite}}],Mod,Ref}  	end,      Cases =  	if LastRef==undefined -> @@ -2094,10 +2101,9 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->  	   true ->  		%% we'll add end_per_suite here even if it's not exported  		%% (and simply let the call fail if it's missing) -		case erlang:function_exported(LastMod, end_per_suite, 1) of -		    true -> -			[{conf,LastRef,[],{LastMod,end_per_suite}}|Init]; -		    false -> +		case {erlang:function_exported(LastMod, end_per_suite, 1), +                      erlang:function_exported(LastMod, init_per_suite, 1)} of +		    {false,false} ->  			%% let's call a "fake" end_per_suite if it exists  			case erlang:function_exported(FwMod, end_per_suite, 1) of  			    true ->				 @@ -2105,8 +2111,13 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->  				  {FwMod,end_per_suite}}|Init];  			    false ->  				[{conf,LastRef,[],{LastMod,end_per_suite}}|Init] -			end -		end +			end; +		    _ -> +                        %% If any of these exist, the other should too +                        %% (required and documented). If it isn't, it will fail +                        %% with reason 'undef'. +			[{conf,LastRef,[],{LastMod,end_per_suite}}|Init] +                end  	end,      {Cases,NextMod,NextRef}. @@ -2115,11 +2126,9 @@ do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod) ->  	No when No==undefined ; No==skipped_suite ->  	    {[],Mod,skipped_suite};  	_Ref -> -	    case erlang:function_exported(LastMod, end_per_suite, 1) of -		true -> -		    {[{conf,LastRef,[],{LastMod,end_per_suite}}], -		     Mod,skipped_suite}; -		false -> +	    case {erlang:function_exported(LastMod, end_per_suite, 1), +                  erlang:function_exported(LastMod, init_per_suite, 1)} of +		{false,false} ->  		    case erlang:function_exported(FwMod, end_per_suite, 1) of  			true ->				  			    %% let's call "fake" end_per_suite if it exists @@ -2128,7 +2137,13 @@ do_add_end_per_suite_and_skip(LastMod, LastRef, Mod, FwMod) ->  			false ->  			    {[{conf,LastRef,[],{LastMod,end_per_suite}}],  			     Mod,skipped_suite} -		    end +		    end; +		_ -> +                    %% If any of these exist, the other should too +                    %% (required and documented). If it isn't, it will fail +                    %% with reason 'undef'. +		    {[{conf,LastRef,[],{LastMod,end_per_suite}}], +		     Mod,skipped_suite}  	    end    	          end. @@ -2924,22 +2939,21 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status)  	    exit(framework_error);  	%% sequential execution of test case finished  	{Time,RetVal,_} -> +            RetTag = +                if is_tuple(RetVal) -> element(1,RetVal); +                   true -> undefined +                end,  	    {Failed,Status1} = -		case Time of -		    died -> -			{true,update_status(failed, Mod, Func, Status)}; -		    _ when is_tuple(RetVal) -> -			case element(1, RetVal) of -			    R when R=='EXIT'; R==failed -> -				{true,update_status(failed, Mod, Func, Status)}; -			    R when R==skip; R==skipped -> -				{false,update_status(skipped, Mod, Func, Status)}; -			    _ -> -				{false,update_status(ok, Mod, Func, Status)} -			end; -		    _ -> -			{false,update_status(ok, Mod, Func, Status)} -		end, +                case RetTag of +                    Skip when Skip==skip; Skip==skipped -> +                        {false,update_status(skipped, Mod, Func, Status)}; +                    Fail when Fail=='EXIT'; Fail==failed -> +                        {true,update_status(failed, Mod, Func, Status)}; +                    _ when Time==died, RetVal=/=ok -> +                        {true,update_status(failed, Mod, Func, Status)}; +                    _ -> +                        {false,update_status(ok, Mod, Func, Status)} +                end,  	    case check_prop(sequence, Mode) of  		false ->  		    stop_minor_log_file(), @@ -3794,7 +3808,15 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit,  	    {died,{timetrap_timeout,TimetrapTimeout}} ->  		progress(failed, Num, Mod, Func, GrName, Loc,  			 timetrap_timeout, TimetrapTimeout, Comment, Style); -	    {died,Reason} -> +	    {died,{Skip,Reason}} when Skip==skip; Skip==skipped -> +                %% died in init_per_testcase +		progress(skip, Num, Mod, Func, GrName, Loc, Reason, +			 Time, Comment, Style); +	    {died,Reason} when Reason=/=ok -> +                %% (If Reason==ok it means that process died in +                %% end_per_testcase after successfully completing the +                %% test case itself - then we shall not fail, but a +                %% warning will be issued in the comment field.)  		progress(failed, Num, Mod, Func, GrName, Loc, Reason,  			 Time, Comment, Style);  	    {_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped; @@ -3943,6 +3965,9 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time,  	  [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]),      test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},  						     {ReportTag,Reason1}}]), +    TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; +			       true -> "~w" +			    end, [Time]),      ReasonStr = escape_chars(reason_to_string(Reason1)),      ReasonStr1 = lists:flatten([string:strip(S,left) ||  				S <- string:tokens(ReasonStr,[$\n])]), @@ -3957,10 +3982,10 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time,  		   _ -> xhtml("<br>(","<br />(") ++ to_string(Comment) ++ ")"  	       end,      print(html, -	  "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>" +	  "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>"  	  "<td><font color=\"~ts\">SKIPPED</font></td>"  	  "<td>~ts~ts</td></tr>\n", -	  [Time,Color,ReasonStr2,Comment1]), +	  [TimeStr,Color,ReasonStr2,Comment1]),      FormatLoc = test_server_sup:format_loc(Loc),      print(minor, "=== Location: ~ts", [FormatLoc]),      print(minor, "=== Reason: ~ts", [ReasonStr1]), @@ -4098,6 +4123,9 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time,  	 Comment0, {St0,St1}) ->      print(minor, "successfully completed test case", []),      test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},ok}]), +    TimeStr = io_lib:format(if is_float(Time) -> "~.3fs"; +			       true -> "~w" +			    end, [Time]),      Comment =  	case RetVal of  	    {comment,RetComment} -> @@ -4116,10 +4144,10 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time,  	end,      print(major, "=elapsed       ~p", [Time]),      print(html, -	  "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>" +	  "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>"  	  "<td><font color=\"green\">Ok</font></td>"  	  "~ts</tr>\n", -	  [Time,Comment]), +	  [TimeStr,Comment]),      print(minor,  	  escape_chars(io_lib:format("=== Returned value: ~tp", [RetVal])),  	  []), diff --git a/lib/common_test/test/ct_error_SUITE.erl b/lib/common_test/test/ct_error_SUITE.erl index fae23484e6..621f3b6d2d 100644 --- a/lib/common_test/test/ct_error_SUITE.erl +++ b/lib/common_test/test/ct_error_SUITE.erl @@ -1531,17 +1531,17 @@ test_events(config_func_errors) ->       {?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}},       {?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}}, -     {?eh,test_stats,{0,1,{0,0}}}, +     {?eh,test_stats,{0,0,{0,1}}},       {?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}},       {?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}}, -     {?eh,test_stats,{0,2,{0,0}}}, +     {?eh,test_stats,{1,0,{0,1}}},       [{?eh,tc_start,{config_func_error_1_SUITE,{init_per_group,g1,[]}}},        {?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g1,[]},ok}},        {?eh,tc_start,{config_func_error_1_SUITE,exit_in_iptc}},        {?eh,tc_done,{config_func_error_1_SUITE,exit_in_iptc,'_'}}, -      {?eh,test_stats,{0,3,{0,0}}}, +      {?eh,test_stats,{1,0,{0,2}}},        {?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g1,[]}}},        {?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g1,[]},ok}}], @@ -1549,7 +1549,7 @@ test_events(config_func_errors) ->        {?eh,tc_done,{config_func_error_1_SUITE,{init_per_group,g2,[]},ok}},        {?eh,tc_start,{config_func_error_1_SUITE,exit_in_eptc}},        {?eh,tc_done,{config_func_error_1_SUITE,exit_in_eptc,'_'}}, -      {?eh,test_stats,{0,4,{0,0}}}, +      {?eh,test_stats,{2,0,{0,2}}},        {?eh,tc_start,{config_func_error_1_SUITE,{end_per_group,g2,[]}}},        {?eh,tc_done,{config_func_error_1_SUITE,{end_per_group,g2,[]},ok}}], diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl index bc716fb5e3..93bcb8fe52 100644 --- a/lib/common_test/test/ct_hooks_SUITE.erl +++ b/lib/common_test/test/ct_hooks_SUITE.erl @@ -82,10 +82,13 @@ all(suite) ->         scope_suite_state_cth,         fail_pre_suite_cth, double_fail_pre_suite_cth,         fail_post_suite_cth, skip_pre_suite_cth, skip_pre_end_cth, +       skip_pre_init_tc_cth,         skip_post_suite_cth, recover_post_suite_cth, update_config_cth,         state_update_cth, options_cth, same_id_cth,         fail_n_skip_with_minimal_cth, prio_cth, no_config, -       data_dir, cth_log +       no_init_suite_config, no_init_config, no_end_config, +       failed_sequence, repeat_force_stop, config_clash, +       callbacks_on_skip, fallback, data_dir, cth_log        ]      ). @@ -190,6 +193,10 @@ skip_post_suite_cth(Config) when is_list(Config) ->      do_test(skip_post_suite_cth, "ct_cth_empty_SUITE.erl",  	    [skip_post_suite_cth],Config). +skip_pre_init_tc_cth(Config) -> +    do_test(skip_pre_init_tc_cth, "ct_cth_empty_SUITE.erl", +	    [skip_pre_init_tc_cth],Config). +  recover_post_suite_cth(Config) when is_list(Config) ->      do_test(recover_post_suite_cth, "ct_cth_fail_per_suite_SUITE.erl",  	    [recover_post_suite_cth],Config). @@ -223,6 +230,16 @@ no_config(Config) when is_list(Config) ->      do_test(no_config, "ct_no_config_SUITE.erl",  	    [verify_config_cth],Config). +no_init_suite_config(Config) when is_list(Config) -> +    do_test(no_init_suite_config, "ct_no_init_suite_config_SUITE.erl", +            [empty_cth],Config). + +no_init_config(Config) when is_list(Config) -> +    do_test(no_init_config, "ct_no_init_config_SUITE.erl",[empty_cth],Config). + +no_end_config(Config) when is_list(Config) -> +    do_test(no_end_config, "ct_no_end_config_SUITE.erl",[empty_cth],Config). +  data_dir(Config) when is_list(Config) ->      do_test(data_dir, "ct_data_dir_SUITE.erl",  	    [verify_data_dir_cth],Config). @@ -254,24 +271,53 @@ cth_log(Config) when is_list(Config) ->        end, UnexpIoLogs),      ok. +%% OTP-10599 adds the Suite argument as first argument to all hook +%% callbacks that did not have a Suite argument from before. This test +%% checks that ct_hooks will fall back to old versions of callbacks if +%% new versions are not exported. +fallback(Config) -> +    do_test(fallback, "all_hook_callbacks_SUITE.erl",[fallback_cth], Config). + +%% Test that expected callbacks, and only those, are called when tests +%% are skipped in different ways +callbacks_on_skip(Config) -> +    do_test(callbacks_on_skip, {spec,"skip.spec"},[skip_cth], Config). + +%% Test that expected callbacks, and only those, are called when tests +%% are skipped due to failed sequence +failed_sequence(Config) -> +    do_test(failed_sequence, "seq_SUITE.erl", [skip_cth], Config). + +%% Test that expected callbacks, and only those, are called when tests +%% are skipped due to {force_stop,skip_rest} option +repeat_force_stop(Config) -> +    do_test(repeat_force_stop, "repeat_SUITE.erl", [skip_cth], Config, ok, 2, +            [{force_stop,skip_rest},{duration,"000009"}]). + +%% Test that expected callbacks, and only those, are called when a test +%% are fails due to clash in config alias names +config_clash(Config) -> +    do_test(config_clash, "config_clash_SUITE.erl", [skip_cth], Config).  %%%-----------------------------------------------------------------  %%% HELP FUNCTIONS  %%%----------------------------------------------------------------- -do_test(Tag, SWC, CTHs, Config) -> -    do_test(Tag, SWC, CTHs, Config, ok). -do_test(Tag, SWC, CTHs, Config, {error,_} = Res) -> -    do_test(Tag, SWC, CTHs, Config, Res, 1); -do_test(Tag, SWC, CTHs, Config, Res) -> -    do_test(Tag, SWC, CTHs, Config, Res, 2). +do_test(Tag, WTT, CTHs, Config) -> +    do_test(Tag, WTT, CTHs, Config, ok). +do_test(Tag, WTT, CTHs, Config, {error,_} = Res) -> +    do_test(Tag, WTT, CTHs, Config, Res, 1,[]); +do_test(Tag, WTT, CTHs, Config, Res) -> +    do_test(Tag, WTT, CTHs, Config, Res, 2,[]). -do_test(Tag, SuiteWildCard, CTHs, Config, Res, EC) -> +do_test(Tag, WhatToTest, CTHs, Config, Res, EC, ExtraOpts) when is_list(WhatToTest) -> +    do_test(Tag, {suite,WhatToTest}, CTHs, Config, Res, EC, ExtraOpts); +do_test(Tag, {WhatTag,Wildcard}, CTHs, Config, Res, EC, ExtraOpts) ->      DataDir = ?config(data_dir, Config), -    Suites = filelib:wildcard( -	       filename:join([DataDir,"cth/tests",SuiteWildCard])), -    {Opts,ERPid} = setup([{suite,Suites}, -			  {ct_hooks,CTHs},{label,Tag}], Config), +    Files = filelib:wildcard( +               filename:join([DataDir,"cth/tests",Wildcard])), +    {Opts,ERPid} = +        setup([{WhatTag,Files},{ct_hooks,CTHs},{label,Tag}|ExtraOpts], Config),      Res = ct_test_support:run(Opts, Config),      Events = ct_test_support:get_events(ERPid, Config), @@ -323,10 +369,10 @@ test_events(one_empty_cth) ->       {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}},       {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, -     {?eh,cth,{empty_cth,pre_init_per_testcase,[test_case,'$proplist',[]]}}, -     {?eh,cth,{empty_cth,post_init_per_testcase,[test_case,'$proplist','_',[]]}}, -     {?eh,cth,{empty_cth,pre_end_per_testcase,[test_case,'$proplist',[]]}}, -     {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[]]}}, +     {?eh,cth,{empty_cth,pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[]]}}, +     {?eh,cth,{empty_cth,pre_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[]]}},       {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}},       {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, @@ -355,10 +401,10 @@ test_events(two_empty_cth) ->       {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}},       {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, -     {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, -     {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, -     {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, -     {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, +     {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, +     {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, +     {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}}, +     {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}},       {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}},       {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, @@ -423,8 +469,8 @@ test_events(minimal_and_maximal_cth) ->       {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}},       {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, -     {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, -     {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, +     {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, +     {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}},       {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}},       {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, @@ -512,8 +558,8 @@ test_events(scope_per_suite_cth) ->       {?eh,tc_done,{ct_scope_per_suite_cth_SUITE,init_per_suite,ok}},       {?eh,tc_start,{ct_scope_per_suite_cth_SUITE,test_case}}, -     {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, -     {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, +     {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_suite_cth_SUITE,test_case,'$proplist',[]]}}, +     {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_suite_cth_SUITE,test_case,'$proplist',ok,[]]}},       {?eh,tc_done,{ct_scope_per_suite_cth_SUITE,test_case,ok}},       {?eh,tc_start,{ct_scope_per_suite_cth_SUITE,end_per_suite}}, @@ -538,8 +584,8 @@ test_events(scope_suite_cth) ->       {?eh,tc_done,{ct_scope_suite_cth_SUITE,init_per_suite,ok}},       {?eh,tc_start,{ct_scope_suite_cth_SUITE,test_case}}, -     {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, -     {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, +     {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_suite_cth_SUITE,test_case,'$proplist',[]]}}, +     {?eh,cth,{'_',post_end_per_testcase,[ct_scope_suite_cth_SUITE,test_case,'$proplist',ok,[]]}},       {?eh,tc_done,{ct_scope_suite_cth_SUITE,test_case,ok}},       {?eh,tc_start,{ct_scope_suite_cth_SUITE,end_per_suite}}, @@ -561,17 +607,17 @@ test_events(scope_per_group_cth) ->       [{?eh,tc_start,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]}}},        {?eh,cth,{'_',id,[[]]}},        {?eh,cth,{'_',init,['_',[]]}}, -      {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[]]}}, +      {?eh,cth,{'_',post_init_per_group,[ct_scope_per_group_cth_SUITE,group1, '$proplist','$proplist',[]]}},        {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]},ok}},        {?eh,tc_start,{ct_scope_per_group_cth_SUITE,test_case}}, -      {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, -      {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, +      {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',[]]}}, +      {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',ok,[]]}},        {?eh,tc_done,{ct_scope_per_group_cth_SUITE,test_case,ok}},        {?eh,tc_start,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}}}, -      {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[]]}}, -      {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[]]}}, +      {?eh,cth,{'_',pre_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist',[]]}}, +      {?eh,cth,{'_',post_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist','_',[]]}},        {?eh,cth,{'_',terminate,[[]]}},        {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]},ok}}], @@ -592,8 +638,8 @@ test_events(scope_per_suite_state_cth) ->       {?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,init_per_suite,ok}},       {?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,test_case}}, -     {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, -     {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, +     {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_suite_state_cth_SUITE,test_case,'$proplist',[test]]}}, +     {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_suite_state_cth_SUITE,test_case,'$proplist',ok,[test]]}},       {?eh,tc_done,{ct_scope_per_suite_state_cth_SUITE,test_case,ok}},       {?eh,tc_start,{ct_scope_per_suite_state_cth_SUITE,end_per_suite}}, @@ -618,8 +664,8 @@ test_events(scope_suite_state_cth) ->       {?eh,tc_done,{ct_scope_suite_state_cth_SUITE,init_per_suite,ok}},       {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,test_case}}, -     {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, -     {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, +     {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_suite_state_cth_SUITE,test_case,'$proplist',[test]]}}, +     {?eh,cth,{'_',post_end_per_testcase,[ct_scope_suite_state_cth_SUITE,test_case,'$proplist',ok,[test]]}},       {?eh,tc_done,{ct_scope_suite_state_cth_SUITE,test_case,ok}},       {?eh,tc_start,{ct_scope_suite_state_cth_SUITE,end_per_suite}}, @@ -641,17 +687,17 @@ test_events(scope_per_group_state_cth) ->       [{?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,{init_per_group,group1,[]}}},        {?eh,cth,{'_',id,[[test]]}},        {?eh,cth,{'_',init,['_',[test]]}}, -      {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[test]]}}, +      {?eh,cth,{'_',post_init_per_group,[ct_scope_per_group_state_cth_SUITE,group1,'$proplist','$proplist',[test]]}},        {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,{init_per_group,group1,[]},ok}},        {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,test_case}}, -      {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[test]]}}, -      {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[test]]}}, +      {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_group_state_cth_SUITE,test_case,'$proplist',[test]]}}, +      {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_group_state_cth_SUITE,test_case,'$proplist',ok,[test]]}},        {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,test_case,ok}},        {?eh,tc_start,{ct_scope_per_group_state_cth_SUITE,{end_per_group,group1,[]}}}, -      {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[test]]}}, -      {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[test]]}}, +      {?eh,cth,{'_',pre_end_per_group,[ct_scope_per_group_state_cth_SUITE,group1,'$proplist',[test]]}}, +      {?eh,cth,{'_',post_end_per_group,[ct_scope_per_group_state_cth_SUITE,group1,'$proplist','_',[test]]}},        {?eh,cth,{'_',terminate,[[test]]}},        {?eh,tc_done,{ct_scope_per_group_state_cth_SUITE,{end_per_group,group1,[]},ok}}], @@ -674,14 +720,14 @@ test_events(fail_pre_suite_cth) ->       {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,                     {failed, {error,"Test failure"}}}},       {?eh,cth,{'_',on_tc_fail, -	       [init_per_suite,{failed,"Test failure"},[]]}}, +	       [ct_cth_empty_SUITE,init_per_suite,"Test failure",[]]}},       {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,                          {failed,{ct_cth_empty_SUITE,init_per_suite,  				 {failed,"Test failure"}}}}},       {?eh,cth,{'_',on_tc_skip, -	       [test_case, {tc_auto_skip, +	       [ct_cth_empty_SUITE,test_case, {tc_auto_skip,  			    {failed, {ct_cth_empty_SUITE, init_per_suite,  				     {failed, "Test failure"}}}},[]]}}, @@ -690,7 +736,7 @@ test_events(fail_pre_suite_cth) ->                           {failed, {ct_cth_empty_SUITE, init_per_suite,  				   {failed, "Test failure"}}}}},       {?eh,cth,{'_',on_tc_skip, -	       [end_per_suite, {tc_auto_skip, +	       [ct_cth_empty_SUITE,end_per_suite, {tc_auto_skip,  				{failed, {ct_cth_empty_SUITE, init_per_suite,  					  {failed, "Test failure"}}}},[]]}}, @@ -727,17 +773,17 @@ test_events(fail_post_suite_cth) ->       {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}},       {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,  		   {failed,{error,"Test failure"}}}}, -     {?eh,cth,{'_',on_tc_fail,[init_per_suite, {failed,"Test failure"}, []]}}, +     {?eh,cth,{'_',on_tc_fail,[ct_cth_empty_SUITE,init_per_suite, "Test failure", []]}},       {?eh,tc_auto_skip,{ct_cth_empty_SUITE,test_case,                          {failed,{ct_cth_empty_SUITE,init_per_suite,  				 {failed,"Test failure"}}}}}, -     {?eh,cth,{'_',on_tc_skip,[test_case,{tc_auto_skip,'_'},[]]}}, +     {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,test_case,{tc_auto_skip,'_'},[]]}},       {?eh,tc_auto_skip, {ct_cth_empty_SUITE, end_per_suite,                           {failed, {ct_cth_empty_SUITE, init_per_suite,  				   {failed, "Test failure"}}}}}, -     {?eh,cth,{'_',on_tc_skip,[end_per_suite,{tc_auto_skip,'_'},[]]}}, +     {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,end_per_suite,{tc_auto_skip,'_'},[]]}},       {?eh,test_done,{'DEF','STOP_TIME'}},       {?eh,cth, {'_',terminate,[[]]}}, @@ -754,10 +800,10 @@ test_events(skip_pre_suite_cth) ->       {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist',{skip,"Test skip"},[]]}},       {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,{skipped,"Test skip"}}},       {?eh,cth,{'_',on_tc_skip, -	       [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, +	       [ct_cth_empty_SUITE,init_per_suite,{tc_user_skip,"Test skip"},[]]}},       {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, -     {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}}, +     {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,test_case,{tc_user_skip,"Test skip"},[]]}},       {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, @@ -776,27 +822,29 @@ test_events(skip_pre_end_cth) ->       [{?eh,tc_start,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]}}},        {?eh,cth,{'_',id,[[]]}},        {?eh,cth,{'_',init,['_',[]]}}, -      {?eh,cth,{'_',post_init_per_group,[group1,'$proplist','$proplist',[]]}}, +      {?eh,cth,{'_',post_init_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist','$proplist',[]]}},        {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{init_per_group,group1,[]},ok}},        {?eh,tc_start,{ct_scope_per_group_cth_SUITE,test_case}}, -      {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, -      {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, +      {?eh,cth,{'_',pre_init_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',[]]}}, +      {?eh,cth,{'_',post_end_per_testcase,[ct_scope_per_group_cth_SUITE,test_case,'$proplist',ok,[]]}},        {?eh,tc_done,{ct_scope_per_group_cth_SUITE,test_case,ok}},        {?eh,tc_start,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]}}}, -      {?eh,cth,{'_',pre_end_per_group,[group1,'$proplist',[]]}}, -      {?eh,cth,{'_',post_end_per_group,[group1,'$proplist','_',[]]}}, +      {?eh,cth,{'_',pre_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist',[]]}}, +      {?eh,cth,{'_',post_end_per_group,[ct_scope_per_group_cth_SUITE,group1,'$proplist','_',[]]}},        {?eh,tc_done,{ct_scope_per_group_cth_SUITE,{end_per_group,group1,[]},  		    {skipped,"Test skip"}}}], -      {?eh,cth,{'_',on_tc_skip,[{end_per_group,group1}, -				{tc_user_skip,{skipped,"Test skip"}}, +      {?eh,cth,{'_',on_tc_skip,[ct_scope_per_group_cth_SUITE, +                                {end_per_group,group1}, +				{tc_user_skip,"Test skip"},  				[]]}},       {?eh,tc_start,{ct_scope_per_group_cth_SUITE,end_per_suite}},       {?eh,tc_done,{ct_scope_per_group_cth_SUITE,end_per_suite,  		   {skipped,"Test skip"}}}, -     {?eh,cth,{'_',on_tc_skip,[end_per_suite, -			       {tc_user_skip,{skipped,"Test skip"}}, +     {?eh,cth,{'_',on_tc_skip,[ct_scope_per_group_cth_SUITE, +                               end_per_suite, +			       {tc_user_skip,"Test skip"},  			       []]}},       {?eh,test_done,{'DEF','STOP_TIME'}},       {?eh,cth,{'_',terminate,[[]]}}, @@ -814,10 +862,10 @@ test_events(skip_post_suite_cth) ->       {?eh,cth,{'_',post_init_per_suite,[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}},       {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,{skipped,"Test skip"}}},       {?eh,cth,{'_',on_tc_skip, -	       [init_per_suite,{tc_user_skip,{skipped,"Test skip"}},[]]}}, +	       [ct_cth_empty_SUITE,init_per_suite,{tc_user_skip,"Test skip"},[]]}},       {?eh,tc_user_skip,{ct_cth_empty_SUITE,test_case,"Test skip"}}, -     {?eh,cth,{'_',on_tc_skip,[test_case,{tc_user_skip,"Test skip"},[]]}}, +     {?eh,cth,{'_',on_tc_skip,[ct_cth_empty_SUITE,test_case,{tc_user_skip,"Test skip"},[]]}},       {?eh,tc_user_skip, {ct_cth_empty_SUITE, end_per_suite,"Test skip"}}, @@ -826,6 +874,41 @@ test_events(skip_post_suite_cth) ->       {?eh,stop_logging,[]}      ]; +test_events(skip_pre_init_tc_cth) -> +    [ +     {?eh,start_logging,{'DEF','RUNDIR'}}, +     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, +     {?eh,cth,{empty_cth,init,['_',[]]}}, +     {?eh,start_info,{1,1,1}}, +     {?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}}, +     {?eh,cth,{empty_cth,pre_init_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_init_per_suite, +               [ct_cth_empty_SUITE,'$proplist','$proplist',[]]}}, +     {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}, +     {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, +     {?eh,cth,{empty_cth,pre_init_per_testcase, +               [ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_init_per_testcase, +               [ct_cth_empty_SUITE,test_case,'$proplist', +                {skip,"Skipped in pre_init_per_testcase"}, +                []]}}, +     {?eh,tc_done,{ct_cth_empty_SUITE,test_case, +                   {skipped,"Skipped in pre_init_per_testcase"}}}, +     {?eh,cth,{empty_cth,on_tc_skip, +               [ct_cth_empty_SUITE,test_case, +                {tc_user_skip,"Skipped in pre_init_per_testcase"}, +                []]}}, +     {?eh,test_stats,{0,0,{1,0}}}, +     {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, +     {?eh,cth,{empty_cth,pre_end_per_suite,[ct_cth_empty_SUITE,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_end_per_suite, +               [ct_cth_empty_SUITE,'$proplist',ok,[]]}}, +     {?eh,tc_done,{ct_cth_empty_SUITE,end_per_suite,ok}}, +     {?eh,test_done,{'DEF','STOP_TIME'}}, +     {?eh,cth,{empty_cth,terminate,[[]]}}, +     {?eh,stop_logging,[]} +    ]; +  test_events(recover_post_suite_cth) ->      Suite = ct_cth_fail_per_suite_SUITE,      [ @@ -840,9 +923,9 @@ test_events(recover_post_suite_cth) ->       {?eh,tc_start,{Suite,test_case}},       {?eh,cth,{'_',pre_init_per_testcase, -	       [test_case, not_contains([tc_status]),[]]}}, +	       [Suite,test_case, not_contains([tc_status]),[]]}},       {?eh,cth,{'_',post_end_per_testcase, -	       [test_case, contains([tc_status]),'_',[]]}}, +	       [Suite,test_case, contains([tc_status]),'_',[]]}},       {?eh,tc_done,{Suite,test_case,ok}},       {?eh,tc_start,{Suite,end_per_suite}}, @@ -876,13 +959,15 @@ test_events(update_config_cth) ->       {?eh,tc_start,{ct_update_config_SUITE, {init_per_group,group1,[]}}},       {?eh,cth,{'_',pre_init_per_group, -	       [group1,contains( +	       [ct_update_config_SUITE, +                group1,contains(  			 [post_init_per_suite,  			  init_per_suite,  			  pre_init_per_suite]),  		[]]}},       {?eh,cth,{'_',post_init_per_group, -	       [group1, +	       [ct_update_config_SUITE, +                group1,  		contains(  		  [post_init_per_suite,  		   init_per_suite, @@ -898,7 +983,8 @@ test_events(update_config_cth) ->       {?eh,tc_start,{ct_update_config_SUITE,test_case}},       {?eh,cth,{'_',pre_init_per_testcase, -	       [test_case,contains( +	       [ct_update_config_SUITE, +                test_case,contains(  			    [post_init_per_group,  			     init_per_group,  			     pre_init_per_group, @@ -907,7 +993,8 @@ test_events(update_config_cth) ->  			     pre_init_per_suite]),  		[]]}},       {?eh,cth,{'_',post_end_per_testcase, -	       [test_case,contains( +	       [ct_update_config_SUITE, +                test_case,contains(  			    [init_per_testcase,  			     pre_init_per_testcase,  			     post_init_per_group, @@ -921,7 +1008,8 @@ test_events(update_config_cth) ->       {?eh,tc_start,{ct_update_config_SUITE, {end_per_group,group1,[]}}},       {?eh,cth,{'_',pre_end_per_group, -	       [group1,contains( +	       [ct_update_config_SUITE, +                group1,contains(  			 [post_init_per_group,  			  init_per_group,  			  pre_init_per_group, @@ -930,7 +1018,8 @@ test_events(update_config_cth) ->  			  pre_init_per_suite]),  		[]]}},       {?eh,cth,{'_',post_end_per_group, -	       [group1, +	       [ct_update_config_SUITE, +                group1,  		contains(  		  [pre_end_per_group,  		   post_init_per_group, @@ -1018,8 +1107,8 @@ test_events(options_cth) ->       {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}},       {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, -     {?eh,cth,{empty_cth,pre_init_per_testcase,[test_case,'$proplist',[test]]}}, -     {?eh,cth,{empty_cth,post_end_per_testcase,[test_case,'$proplist','_',[test]]}}, +     {?eh,cth,{empty_cth,pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[test]]}}, +     {?eh,cth,{empty_cth,post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[test]]}},       {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}},       {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, @@ -1051,12 +1140,12 @@ test_events(same_id_cth) ->        {?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}}},       {?eh,tc_start,{ct_cth_empty_SUITE,test_case}}, -     {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, +     {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}},       {negative, -      {?eh,cth,{'_',pre_init_per_testcase,[test_case,'$proplist',[]]}}, -      {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}}, +      {?eh,cth,{'_',pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}}, +      {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}}},       {negative, -      {?eh,cth,{'_',post_end_per_testcase,[test_case,'$proplist',ok,[]]}}, +      {?eh,cth,{'_',post_end_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',ok,[]]}},        {?eh,tc_done,{ct_cth_empty_SUITE,test_case,ok}}},       {?eh,tc_start,{ct_cth_empty_SUITE,end_per_suite}}, @@ -1094,11 +1183,13 @@ test_events(fail_n_skip_with_minimal_cth) ->  	 {?eh,tc_done,{ct_cth_fail_one_skip_one_SUITE,test_case2,{skipped,"skip it"}}},  	 {?eh,tc_start,{ct_cth_fail_one_skip_one_SUITE,test_case3}},  	 {?eh,tc_done,{ct_cth_fail_one_skip_one_SUITE,test_case3,{skipped,"skip it"}}}, -	 {?eh,cth,{empty_cth,on_tc_skip,[{test_case2,group2}, -					 {tc_user_skip,{skipped,"skip it"}}, +	 {?eh,cth,{empty_cth,on_tc_skip,[ct_cth_fail_one_skip_one_SUITE, +                                         {test_case2,group2}, +					 {tc_user_skip,"skip it"},  					 []]}}, -	 {?eh,cth,{empty_cth,on_tc_skip,[{test_case3,group2}, -					 {tc_user_skip,{skipped,"skip it"}}, +	 {?eh,cth,{empty_cth,on_tc_skip,[ct_cth_fail_one_skip_one_SUITE, +                                         {test_case3,group2}, +					 {tc_user_skip,"skip it"},  					 []]}},  	 {?eh,tc_start,{ct_cth_fail_one_skip_one_SUITE,{end_per_group,  							group2,[parallel]}}}, @@ -1115,13 +1206,24 @@ test_events(fail_n_skip_with_minimal_cth) ->      ];  test_events(prio_cth) -> -    GenPre = fun(Func,States) -> -		     [{?eh,cth,{'_',Func,['_','_',State]}} || State <- States] +    GenPre = fun(Func,States) when Func==pre_init_per_suite; +                                   Func==pre_end_per_suite -> +		     [{?eh,cth,{'_',Func,['_','_',State]}} || +                         State <- States]; +                (Func,States) -> +		     [{?eh,cth,{'_',Func,['_','_','_',State]}} || +			 State <- States]  	     end, -    GenPost = fun(Func,States) -> -		      [{?eh,cth,{'_',Func,['_','_','_',State]}} || State <- States] -	     end, +    GenPost = fun(Func,States) when Func==post_init_per_suite; +                                    Func==post_end_per_suite -> +		      [{?eh,cth,{'_',Func,['_','_','_',State]}} || +			  State <- States]; +                 (Func,States) -> +		      [{?eh,cth,{'_',Func,['_','_','_','_',State]}} || +			  State <- States] + +              end,      [{?eh,start_logging,{'DEF','RUNDIR'}},       {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}] ++ @@ -1197,30 +1299,30 @@ test_events(no_config) ->       {?eh,tc_done,{ct_framework,init_per_suite,ok}},       {?eh,tc_start,{ct_no_config_SUITE,test_case_1}},       {?eh,cth,{empty_cth,pre_init_per_testcase, -	       [test_case_1,'$proplist',[]]}}, +	       [ct_no_config_SUITE,test_case_1,'$proplist',[]]}},       {?eh,cth,{empty_cth,post_end_per_testcase, -	       [test_case_1,'$proplist',ok,[]]}}, +	       [ct_no_config_SUITE,test_case_1,'$proplist',ok,[]]}},       {?eh,tc_done,{ct_no_config_SUITE,test_case_1,ok}},       {?eh,test_stats,{1,0,{0,0}}},       [{?eh,tc_start,{ct_framework,{init_per_group,test_group,'$proplist'}}},        {?eh,cth,{empty_cth,pre_init_per_group, -		[test_group,'$proplist',[]]}}, +		[ct_no_config_SUITE,test_group,'$proplist',[]]}},        {?eh,cth,{empty_cth,post_init_per_group, -		[test_group,'$proplist','$proplist',[]]}}, +		[ct_no_config_SUITE,test_group,'$proplist','$proplist',[]]}},        {?eh,tc_done,{ct_framework,  		    {init_per_group,test_group,'$proplist'},ok}},        {?eh,tc_start,{ct_no_config_SUITE,test_case_2}},        {?eh,cth,{empty_cth,pre_init_per_testcase, -		[test_case_2,'$proplist',[]]}}, +		[ct_no_config_SUITE,test_case_2,'$proplist',[]]}},        {?eh,cth,{empty_cth,post_end_per_testcase, -		[test_case_2,'$proplist',ok,[]]}}, +		[ct_no_config_SUITE,test_case_2,'$proplist',ok,[]]}},        {?eh,tc_done,{ct_no_config_SUITE,test_case_2,ok}},        {?eh,test_stats,{2,0,{0,0}}},        {?eh,tc_start,{ct_framework,{end_per_group,test_group,'$proplist'}}},        {?eh,cth,{empty_cth,pre_end_per_group, -		[test_group,'$proplist',[]]}}, +		[ct_no_config_SUITE,test_group,'$proplist',[]]}},        {?eh,cth,{empty_cth,post_end_per_group, -		[test_group,'$proplist',ok,[]]}}, +		[ct_no_config_SUITE,test_group,'$proplist',ok,[]]}},        {?eh,tc_done,{ct_framework,{end_per_group,test_group,'$proplist'},ok}}],       {?eh,tc_start,{ct_framework,end_per_suite}},       {?eh,cth,{empty_cth,pre_end_per_suite, @@ -1233,6 +1335,166 @@ test_events(no_config) ->       {?eh,stop_logging,[]}      ]; +test_events(no_init_suite_config) -> +    [ +     {?eh,start_logging,{'DEF','RUNDIR'}}, +     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, +     {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, +     {?eh,start_info,{1,1,1}}, +     {?eh,tc_start,{ct_no_init_suite_config_SUITE,init_per_suite}}, +     {?eh,cth,{empty_cth,pre_init_per_suite, +	       [ct_no_init_suite_config_SUITE,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_init_per_suite, +	       [ct_no_init_suite_config_SUITE,'$proplist','_',[]]}}, +     {?eh,tc_done,{ct_no_init_suite_config_SUITE,init_per_suite, +                   {failed,{error,{undef,'_'}}}}}, +     {?eh,cth,{empty_cth,on_tc_fail,[ct_no_init_suite_config_SUITE, +                                     init_per_suite, +                                     {undef,'_'},[]]}}, +      {?eh,tc_auto_skip,{ct_no_init_suite_config_SUITE,test_case, +                         {failed,{ct_no_init_suite_config_SUITE,init_per_suite, +                                  {'EXIT',{undef,'_'}}}}}}, +     {?eh,cth,{empty_cth,on_tc_skip, +               [ct_no_init_suite_config_SUITE, +                test_case, +                {tc_auto_skip, +                 {failed,{ct_no_init_suite_config_SUITE,init_per_suite, +                          {'EXIT',{undef,'_'}}}}}, +                []]}}, +     {?eh,test_stats,{0,0,{0,1}}}, +     {?eh,tc_auto_skip,{ct_no_init_suite_config_SUITE,end_per_suite, +                        {failed,{ct_no_init_suite_config_SUITE,init_per_suite, +                                 {'EXIT',{undef,'_'}}}}}}, +     {?eh,cth,{empty_cth,on_tc_skip, +               [ct_no_init_suite_config_SUITE, +                end_per_suite, +                {tc_auto_skip, +                 {failed,{ct_no_init_suite_config_SUITE,init_per_suite, +                          {'EXIT',{undef,'_'}}}}}, +                []]}}, +     {?eh,test_done,{'DEF','STOP_TIME'}}, +     {?eh,cth,{empty_cth,terminate,[[]]}}, +     {?eh,stop_logging,[]} +    ]; + +test_events(no_init_config) -> +    [ +     {?eh,start_logging,{'DEF','RUNDIR'}}, +     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, +     {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, +     {?eh,start_info,{1,1,2}}, +     {?eh,tc_start,{ct_no_init_config_SUITE,init_per_suite}}, +     {?eh,cth,{empty_cth,pre_init_per_suite, +	       [ct_no_init_config_SUITE,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_init_per_suite, +	       [ct_no_init_config_SUITE,'$proplist','$proplist',[]]}}, +     {?eh,tc_done,{ct_no_init_config_SUITE,init_per_suite,ok}}, +     {?eh,tc_start,{ct_no_init_config_SUITE,test_case_1}}, +     {?eh,cth,{empty_cth,pre_init_per_testcase, +	       [ct_no_init_config_SUITE,test_case_1,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_end_per_testcase, +	       [ct_no_init_config_SUITE,test_case_1,'$proplist',ok,[]]}}, +     {?eh,tc_done,{ct_no_init_config_SUITE,test_case_1,ok}}, +     {?eh,test_stats,{1,0,{0,0}}}, +     [{?eh,tc_start,{ct_no_init_config_SUITE,{init_per_group,test_group,[]}}}, +      {?eh,cth,{empty_cth,pre_init_per_group, +		[ct_no_init_config_SUITE,test_group,'$proplist',[]]}}, +      {?eh,cth,{empty_cth,post_init_per_group, +		[ct_no_init_config_SUITE,test_group,'$proplist','_',[]]}}, +      {?eh,tc_done,{ct_no_init_config_SUITE,{init_per_group,test_group,[]}, +                    {failed,{error,{undef,'_'}}}}}, +      {?eh,cth,{empty_cth,on_tc_fail,[ct_no_init_config_SUITE, +                                      {init_per_group,test_group}, +                                      {undef,'_'},[]]}}, +      {?eh,tc_auto_skip,{ct_no_init_config_SUITE,{test_case_2,test_group}, +                         {failed,{ct_no_init_config_SUITE,init_per_group, +                                  {'EXIT',{undef,'_'}}}}}}, +      {?eh,cth,{empty_cth,on_tc_skip,[ct_no_init_config_SUITE, +                                      {test_case_2,test_group}, +                                      {tc_auto_skip, +                                       {failed, +                                        {ct_no_init_config_SUITE,init_per_group, +                                         {'EXIT',{undef,'_'}}}}}, +                                       []]}}, +      {?eh,test_stats,{1,0,{0,1}}}, +      {?eh,tc_auto_skip,{ct_no_init_config_SUITE,{end_per_group,test_group}, +                         {failed,{ct_no_init_config_SUITE,init_per_group, +                                  {'EXIT',{undef,'_'}}}}}}, +      {?eh,cth,{empty_cth,on_tc_skip,[ct_no_init_config_SUITE, +                                      {end_per_group,test_group}, +                                      {tc_auto_skip, +                                       {failed, +                                        {ct_no_init_config_SUITE,init_per_group, +                                         {'EXIT',{undef,'_'}}}}}, +                                       []]}}], +     {?eh,tc_start,{ct_no_init_config_SUITE,end_per_suite}}, +     {?eh,cth,{empty_cth,pre_end_per_suite, +	       [ct_no_init_config_SUITE,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_end_per_suite, +	       [ct_no_init_config_SUITE,'$proplist',ok,[]]}}, +     {?eh,tc_done,{ct_no_init_config_SUITE,end_per_suite,ok}}, +     {?eh,test_done,{'DEF','STOP_TIME'}}, +     {?eh,cth,{empty_cth,terminate,[[]]}}, +     {?eh,stop_logging,[]} +    ]; + +test_events(no_end_config) -> +    [ +     {?eh,start_logging,{'DEF','RUNDIR'}}, +     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, +     {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, +     {?eh,start_info,{1,1,2}}, +     {?eh,tc_start,{ct_no_end_config_SUITE,init_per_suite}}, +     {?eh,cth,{empty_cth,pre_init_per_suite, +	       [ct_no_end_config_SUITE,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_init_per_suite, +	       [ct_no_end_config_SUITE,'$proplist','$proplist',[]]}}, +     {?eh,tc_done,{ct_no_end_config_SUITE,init_per_suite,ok}}, +     {?eh,tc_start,{ct_no_end_config_SUITE,test_case_1}}, +     {?eh,cth,{empty_cth,pre_init_per_testcase, +	       [ct_no_end_config_SUITE,test_case_1,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_end_per_testcase, +	       [ct_no_end_config_SUITE,test_case_1,'$proplist',ok,[]]}}, +     {?eh,tc_done,{ct_no_end_config_SUITE,test_case_1,ok}}, +     {?eh,test_stats,{1,0,{0,0}}}, +     [{?eh,tc_start,{ct_no_end_config_SUITE, +                     {init_per_group,test_group,'$proplist'}}}, +      {?eh,cth,{empty_cth,pre_init_per_group, +		[ct_no_end_config_SUITE,test_group,'$proplist',[]]}}, +      {?eh,cth,{empty_cth,post_init_per_group, +		[ct_no_end_config_SUITE,test_group,'$proplist','$proplist',[]]}}, +      {?eh,tc_done,{ct_no_end_config_SUITE, +		    {init_per_group,test_group,'$proplist'},ok}}, +      {?eh,tc_start,{ct_no_end_config_SUITE,test_case_2}}, +      {?eh,cth,{empty_cth,pre_init_per_testcase, +		[ct_no_end_config_SUITE,test_case_2,'$proplist',[]]}}, +      {?eh,cth,{empty_cth,post_end_per_testcase, +		[ct_no_end_config_SUITE,test_case_2,'$proplist',ok,[]]}}, +      {?eh,tc_done,{ct_no_end_config_SUITE,test_case_2,ok}}, +      {?eh,test_stats,{2,0,{0,0}}}, +      {?eh,tc_start,{ct_no_end_config_SUITE, +                     {end_per_group,test_group,'$proplist'}}}, +      {?eh,cth,{empty_cth,pre_end_per_group, +		[ct_no_end_config_SUITE,test_group,'$proplist',[]]}}, +      {?eh,cth,{empty_cth,post_end_per_group, +		[ct_no_end_config_SUITE,test_group,'$proplist','_',[]]}}, +      {?eh,tc_done,{ct_no_end_config_SUITE,{end_per_group,test_group,[]}, +                    {failed,{error,{undef,'_'}}}}}, +      {?eh,cth,{empty_cth,on_tc_fail,[ct_no_end_config_SUITE, +                                      {end_per_group,test_group}, +                                      {undef,'_'},[]]}}], +     {?eh,tc_start,{ct_no_end_config_SUITE,end_per_suite}}, +     {?eh,cth,{empty_cth,pre_end_per_suite, +	       [ct_no_end_config_SUITE,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_end_per_suite, +	       [ct_no_end_config_SUITE,'$proplist','_',[]]}}, +     {?eh,tc_done,{ct_no_end_config_SUITE,end_per_suite, +                   {failed,{error,{undef,'_'}}}}}, +     {?eh,test_done,{'DEF','STOP_TIME'}}, +     {?eh,cth,{empty_cth,terminate,[[]]}}, +     {?eh,stop_logging,[]} +    ]; +  test_events(data_dir) ->      [       {?eh,start_logging,{'DEF','RUNDIR'}}, @@ -1247,30 +1509,30 @@ test_events(data_dir) ->       {?eh,tc_done,{ct_framework,init_per_suite,ok}},       {?eh,tc_start,{ct_data_dir_SUITE,test_case_1}},       {?eh,cth,{empty_cth,pre_init_per_testcase, -	       [test_case_1,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, +	       [ct_data_dir_SUITE,test_case_1,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},       {?eh,cth,{empty_cth,post_end_per_testcase, -	       [test_case_1,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, +	       [ct_data_dir_SUITE,test_case_1,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},       {?eh,tc_done,{ct_data_dir_SUITE,test_case_1,ok}},       {?eh,test_stats,{1,0,{0,0}}},       [{?eh,tc_start,{ct_framework,{init_per_group,test_group,'$proplist'}}},        {?eh,cth,{empty_cth,pre_init_per_group, -		[test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, +		[ct_data_dir_SUITE,test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},        {?eh,cth,{empty_cth,post_init_per_group, -		[test_group,'$proplist','$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, +		[ct_data_dir_SUITE,test_group,'$proplist','$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},        {?eh,tc_done,{ct_framework,  		    {init_per_group,test_group,'$proplist'},ok}},        {?eh,tc_start,{ct_data_dir_SUITE,test_case_2}},        {?eh,cth,{empty_cth,pre_init_per_testcase, -		[test_case_2,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, +		[ct_data_dir_SUITE,test_case_2,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},        {?eh,cth,{empty_cth,post_end_per_testcase, -		[test_case_2,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, +		[ct_data_dir_SUITE,test_case_2,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},        {?eh,tc_done,{ct_data_dir_SUITE,test_case_2,ok}},        {?eh,test_stats,{2,0,{0,0}}},        {?eh,tc_start,{ct_framework,{end_per_group,test_group,'$proplist'}}},        {?eh,cth,{empty_cth,pre_end_per_group, -		[test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, +		[ct_data_dir_SUITE,test_group,'$proplist',[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},        {?eh,cth,{empty_cth,post_end_per_group, -		[test_group,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}}, +		[ct_data_dir_SUITE,test_group,'$proplist',ok,[{data_dir_name,"ct_data_dir_SUITE_data"}]]}},        {?eh,tc_done,{ct_framework,{end_per_group,test_group,'$proplist'},ok}}],       {?eh,tc_start,{ct_framework,end_per_suite}},       {?eh,cth,{empty_cth,pre_end_per_suite, @@ -1303,6 +1565,645 @@ test_events(cth_log) ->       {?eh,stop_logging,[]}      ]; +test_events(fallback) -> +    [ +     {?eh,start_logging,{'DEF','RUNDIR'}}, +     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, +     {?eh,cth,{empty_cth,id,[[]]}}, +     {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, +     {?eh,tc_start,{all_hook_callbacks_SUITE,init_per_suite}}, +     {?eh,cth,{empty_cth,pre_init_per_suite, +	       [all_hook_callbacks_SUITE,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_init_per_suite, +	       [all_hook_callbacks_SUITE,'$proplist','$proplist',[]]}}, +     {?eh,tc_done,{all_hook_callbacks_SUITE,init_per_suite,ok}}, + +     [{?eh,tc_start,{ct_framework,{init_per_group,test_group,'$proplist'}}}, +      {?eh,cth,{empty_cth,pre_init_per_group, +		[fallback_nosuite,test_group,'$proplist',[]]}}, +      {?eh,cth,{empty_cth,post_init_per_group, +		[fallback_nosuite,test_group,'$proplist','$proplist',[]]}}, +      {?eh,tc_done,{ct_framework, +		    {init_per_group,test_group,'$proplist'},ok}}, +      {?eh,tc_start,{all_hook_callbacks_SUITE,test_case}}, +      {?eh,cth,{empty_cth,pre_init_per_testcase, +		[fallback_nosuite,test_case,'$proplist',[]]}}, +      {?eh,cth,{empty_cth,post_end_per_testcase, +		[fallback_nosuite,test_case,'$proplist',ok,[]]}}, +      {?eh,tc_done,{all_hook_callbacks_SUITE,test_case,ok}}, +      {?eh,test_stats,{1,0,{0,0}}}, +      {?eh,tc_start,{ct_framework,{end_per_group,test_group,'$proplist'}}}, +      {?eh,cth,{empty_cth,pre_end_per_group, +		[fallback_nosuite,test_group,'$proplist',[]]}}, +      {?eh,cth,{empty_cth,post_end_per_group, +		[fallback_nosuite,test_group,'$proplist',ok,[]]}}, +      {?eh,tc_done,{ct_framework,{end_per_group,test_group,'$proplist'},ok}}], +     {?eh,tc_start,{all_hook_callbacks_SUITE,test_case}}, +     {?eh,cth,{empty_cth,pre_init_per_testcase, +               [fallback_nosuite,test_case,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_init_per_testcase, +               [fallback_nosuite,test_case,'$proplist','_',[]]}}, +     {?eh,cth,{empty_cth,pre_end_per_testcase, +               [fallback_nosuite,test_case,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_end_per_testcase, +               [fallback_nosuite,test_case,'$proplist','_',[]]}}, +     {?eh,tc_done,{all_hook_callbacks_SUITE,test_case,ok}}, +     {?eh,test_stats,{2,0,{0,0}}}, +     {?eh,tc_start,{all_hook_callbacks_SUITE,skip_case}}, +     {?eh,cth,{empty_cth,pre_init_per_testcase, +               [fallback_nosuite,skip_case,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_init_per_testcase, +               [fallback_nosuite,skip_case,'$proplist', +                {skip,"Skipped in init_per_testcase/2"},[]]}}, +     {?eh,tc_done,{all_hook_callbacks_SUITE,skip_case, +                   {skipped,"Skipped in init_per_testcase/2"}}}, +     {?eh,cth,{empty_cth,on_tc_skip, +               [fallback_nosuite,skip_case, +                {tc_user_skip,"Skipped in init_per_testcase/2"}, +                []]}}, +     {?eh,test_stats,{2,0,{1,0}}}, +     {?eh,tc_start,{all_hook_callbacks_SUITE,end_per_suite}}, +     {?eh,cth,{empty_cth,pre_end_per_suite, +	       [all_hook_callbacks_SUITE,'$proplist',[]]}}, +     {?eh,cth,{empty_cth,post_end_per_suite, +               [all_hook_callbacks_SUITE,'$proplist','_',[]]}}, +     {?eh,tc_done,{all_hook_callbacks_SUITE,end_per_suite,ok}}, +     {?eh,test_done,{'DEF','STOP_TIME'}}, +     {?eh,cth,{empty_cth,terminate,[[]]}}, +     {?eh,stop_logging,[]} +    ]; + +test_events(callbacks_on_skip) -> +    %% skip_cth.erl will send a 'cth_error' event if a hook is +    %% erroneously called. Therefore, all Events are changed to +    %% {negative,{?eh,cth_error,'_'},Event} +    %% at the end of this function. +    Events = +        [ +         {?eh,start_logging,{'DEF','RUNDIR'}}, +         {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, +         {?eh,cth,{empty_cth,id,[[]]}}, +         {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, +         {?eh,start_info,{6,6,15}}, + +         %% all_hook_callbacks_SUITE is skipped in spec +         %% Only the on_tc_skip callback shall be called +         {?eh,tc_user_skip,{all_hook_callbacks_SUITE,all,"Skipped in spec"}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [all_hook_callbacks_SUITE,all, +                    {tc_user_skip,"Skipped in spec"}, +                    []]}}, +         {?eh,test_stats,{0,0,{1,0}}}, + +         %% skip_init_SUITE is skipped in its init_per_suite function +         %% No group- or testcase-functions shall be called. +         {?eh,tc_start,{skip_init_SUITE,init_per_suite}}, +         {?eh,cth,{empty_cth,pre_init_per_suite, +                   [skip_init_SUITE, +                    '$proplist', +                    []]}}, +         {?eh,cth,{empty_cth,post_init_per_suite, +                   [skip_init_SUITE, +                    '$proplist', +                    {skip,"Skipped in init_per_suite/1"}, +                    []]}}, +         {?eh,tc_done,{skip_init_SUITE,init_per_suite, +                       {skipped,"Skipped in init_per_suite/1"}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_init_SUITE,init_per_suite, +                    {tc_user_skip,"Skipped in init_per_suite/1"}, +                    []]}}, +         {?eh,tc_user_skip,{skip_init_SUITE,test_case,"Skipped in init_per_suite/1"}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_init_SUITE,test_case, +                    {tc_user_skip,"Skipped in init_per_suite/1"}, +                    []]}}, +         {?eh,test_stats,{0,0,{2,0}}}, +         {?eh,tc_user_skip,{skip_init_SUITE,end_per_suite, +                            "Skipped in init_per_suite/1"}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_init_SUITE,end_per_suite, +                    {tc_user_skip,"Skipped in init_per_suite/1"}, +                    []]}}, + +         %% skip_req_SUITE is auto-skipped since a 'require' statement +         %% returned by suite/0 is not fulfilled. +         %% No group- or testcase-functions shall be called. +         {?eh,tc_start,{skip_req_SUITE,init_per_suite}}, +         {?eh,tc_done,{skip_req_SUITE,init_per_suite, +                       {auto_skipped,{require_failed_in_suite0, +                                      {not_available,whatever}}}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_req_SUITE,init_per_suite, +                    {tc_auto_skip,{require_failed_in_suite0, +                                   {not_available,whatever}}}, +                    []]}}, +         {?eh,tc_auto_skip,{skip_req_SUITE,test_case,{require_failed_in_suite0, +                                                      {not_available,whatever}}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_req_SUITE,test_case, +                    {tc_auto_skip,{require_failed_in_suite0, +                                   {not_available,whatever}}}, +                    []]}}, +         {?eh,test_stats,{0,0,{2,1}}}, +         {?eh,tc_auto_skip,{skip_req_SUITE,end_per_suite, +                            {require_failed_in_suite0, +                             {not_available,whatever}}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_req_SUITE,end_per_suite, +                    {tc_auto_skip,{require_failed_in_suite0, +                                   {not_available,whatever}}}, +                    []]}}, + +         %% skip_fail_SUITE is auto-skipped since the suite/0 function +         %% retuns a faluty format. +         %% No group- or testcase-functions shall be called. +         {?eh,tc_start,{skip_fail_SUITE,init_per_suite}}, +         {?eh,tc_done,{skip_fail_SUITE,init_per_suite, +                       {failed,{error,{suite0_failed,bad_return_value}}}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_fail_SUITE,init_per_suite, +                    {tc_auto_skip, +                     {failed,{error,{suite0_failed,bad_return_value}}}}, +                    []]}}, +         {?eh,tc_auto_skip,{skip_fail_SUITE,test_case, +                            {failed,{error,{suite0_failed,bad_return_value}}}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_fail_SUITE,test_case, +                    {tc_auto_skip, +                     {failed,{error,{suite0_failed,bad_return_value}}}}, +                    []]}}, +         {?eh,test_stats,{0,0,{2,2}}}, +         {?eh,tc_auto_skip,{skip_fail_SUITE,end_per_suite, +                            {failed,{error,{suite0_failed,bad_return_value}}}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_fail_SUITE,end_per_suite, +                    {tc_auto_skip, +                     {failed,{error,{suite0_failed,bad_return_value}}}}, +                    []]}}, + +         %% skip_group_SUITE +         {?eh,tc_start,{skip_group_SUITE,init_per_suite}}, +         {?eh,cth,{empty_cth,pre_init_per_suite, +                   [skip_group_SUITE, +                    '$proplist', +                    []]}}, +         {?eh,cth,{empty_cth,post_init_per_suite, +                   [skip_group_SUITE, +                    '$proplist', +                    '_', +                    []]}}, +         {?eh,tc_done,{skip_group_SUITE,init_per_suite,ok}}, + +         %% test_group_1 - auto_skip due to require failed +         [{?eh,tc_start,{skip_group_SUITE,{init_per_group,test_group_1,[]}}}, +          {?eh,tc_done, +           {skip_group_SUITE,{init_per_group,test_group_1,[]}, +            {auto_skipped,{require_failed,{not_available,whatever}}}}}, +          {?eh,cth,{empty_cth,on_tc_skip, +                    [skip_group_SUITE, +                     {init_per_group,test_group_1}, +                     {tc_auto_skip,{require_failed,{not_available,whatever}}}, +                     []]}}, +          {?eh,tc_auto_skip,{skip_group_SUITE,{test_case,test_group_1}, +                             {require_failed,{not_available,whatever}}}}, +          {?eh,cth,{empty_cth,on_tc_skip, +                    [skip_group_SUITE, +                     {test_case,test_group_1}, +                     {tc_auto_skip,{require_failed,{not_available,whatever}}}, +                     []]}}, +          {?eh,test_stats,{0,0,{2,3}}}, +          {?eh,tc_auto_skip,{skip_group_SUITE,{end_per_group,test_group_1}, +                             {require_failed,{not_available,whatever}}}}], +         %% The following appears to be outside of the group, but +         %% that's only an implementation detail in +         %% ct_test_support.erl - it does not know about events from +         %% test suite specific hooks and regards the group ended with +         %% the above tc_auto_skip-event for end_per_group. +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_group_SUITE, +                    {end_per_group,test_group_1}, +                    {tc_auto_skip,{require_failed,{not_available,whatever}}}, +                    []]}}, + +         %% test_group_2 - auto_skip due to failed return from group/1 +         [{?eh,tc_start,{skip_group_SUITE,{init_per_group,test_group_2,[]}}}, +          {?eh,tc_done, +           {skip_group_SUITE,{init_per_group,test_group_2,[]}, +            {auto_skipped,{group0_failed,bad_return_value}}}}, +          {?eh,cth,{empty_cth,on_tc_skip, +                    [skip_group_SUITE, +                     {init_per_group,test_group_2}, +                     {tc_auto_skip,{group0_failed,bad_return_value}}, +                     []]}}, +          {?eh,tc_auto_skip,{skip_group_SUITE,{test_case,test_group_2}, +                             {group0_failed,bad_return_value}}}, +          {?eh,cth,{empty_cth,on_tc_skip, +                    [skip_group_SUITE, +                     {test_case,test_group_2}, +                     {tc_auto_skip,{group0_failed,bad_return_value}}, +                     []]}}, +          {?eh,test_stats,{0,0,{2,4}}}, +          {?eh,tc_auto_skip,{skip_group_SUITE,{end_per_group,test_group_2}, +                             {group0_failed,bad_return_value}}}], +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_group_SUITE, +                    {end_per_group,test_group_2}, +                    {tc_auto_skip,{group0_failed,bad_return_value}}, +                    []]}}, +         %% test_group_3 - user_skip in init_per_group/2 +         [{?eh,tc_start, +           {skip_group_SUITE,{init_per_group,test_group_3,[]}}}, +          {?eh,cth,{empty_cth,pre_init_per_group, +                    [skip_group_SUITE,test_group_3,'$proplist',[]]}}, +          {?eh,cth,{empty_cth,post_init_per_group, +                    [skip_group_SUITE,test_group_3,'$proplist', +                     {skip,"Skipped in init_per_group/2"}, +                     []]}}, +          {?eh,tc_done,{skip_group_SUITE, +                        {init_per_group,test_group_3,[]}, +                        {skipped,"Skipped in init_per_group/2"}}}, +          {?eh,cth,{empty_cth,on_tc_skip, +                    [skip_group_SUITE, +                     {init_per_group,test_group_3}, +                     {tc_user_skip,"Skipped in init_per_group/2"}, +                     []]}}, +          {?eh,tc_user_skip,{skip_group_SUITE, +                             {test_case,test_group_3}, +                             "Skipped in init_per_group/2"}}, +          {?eh,cth,{empty_cth,on_tc_skip, +                    [skip_group_SUITE, +                     {test_case,test_group_3}, +                     {tc_user_skip,"Skipped in init_per_group/2"}, +                     []]}}, +          {?eh,test_stats,{0,0,{3,4}}}, +          {?eh,tc_user_skip,{skip_group_SUITE, +                             {end_per_group,test_group_3}, +                             "Skipped in init_per_group/2"}}], +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_group_SUITE, +                    {end_per_group,test_group_3}, +                    {tc_user_skip,"Skipped in init_per_group/2"}, +                    []]}}, + +         {?eh,tc_start,{skip_group_SUITE,end_per_suite}}, +         {?eh,cth,{empty_cth,pre_end_per_suite, +                   [skip_group_SUITE, +                    '$proplist', +                    []]}}, +         {?eh,cth,{empty_cth,post_end_per_suite, +                   [skip_group_SUITE, +                    '$proplist', +                    ok,[]]}}, +         {?eh,tc_done,{skip_group_SUITE,end_per_suite,ok}}, + + +         %% skip_case_SUITE has 4 test cases which are all skipped in +         %% different ways +         {?eh,tc_start,{skip_case_SUITE,init_per_suite}}, +         {?eh,cth,{empty_cth,pre_init_per_suite, +                   [skip_case_SUITE, +                    '$proplist', +                    []]}}, +         {?eh,cth,{empty_cth,post_init_per_suite, +                   [skip_case_SUITE, +                    '$proplist', +                    '_', +                    []]}}, +         {?eh,tc_done,{skip_case_SUITE,init_per_suite,ok}}, + +         %% Skip in spec -> only on_tc_skip shall be called +         {?eh,tc_user_skip,{skip_case_SUITE,skip_in_spec,"Skipped in spec"}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_case_SUITE,skip_in_spec, +                    {tc_user_skip,"Skipped in spec"}, +                    []]}}, +         {?eh,test_stats,{0,0,{4,4}}}, + +         %% Skip in init_per_testcase -> pre/post_end_per_testcase +         %% shall not be called +         {?eh,tc_start,{skip_case_SUITE,skip_in_init}}, +         {?eh,cth,{empty_cth,pre_init_per_testcase, +                   [skip_case_SUITE,skip_in_init, +                    '$proplist', +                    []]}}, +         {?eh,cth,{empty_cth,post_init_per_testcase, +                   [skip_case_SUITE,skip_in_init, +                    '$proplist', +                    {skip,"Skipped in init_per_testcase/2"}, +                    []]}}, +         {?eh,tc_done,{skip_case_SUITE,skip_in_init, +                       {skipped,"Skipped in init_per_testcase/2"}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_case_SUITE,skip_in_init, +                    {tc_user_skip,"Skipped in init_per_testcase/2"}, +                    []]}}, +         {?eh,test_stats,{0,0,{5,4}}}, + +         %% Fail in init_per_testcase -> pre/post_end_per_testcase +         %% shall not be called +         {?eh,tc_start,{skip_case_SUITE,fail_in_init}}, +         {?eh,cth,{empty_cth,pre_init_per_testcase, +                   [skip_case_SUITE,fail_in_init, +                    '$proplist', +                    []]}}, +         {?eh,cth,{empty_cth,post_init_per_testcase, +                   [skip_case_SUITE,fail_in_init, +                    '$proplist', +                    {skip,{failed,'_'}}, +                    []]}}, +         {?eh,tc_done,{skip_case_SUITE,fail_in_init, +                       {auto_skipped,{failed,'_'}}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_case_SUITE,fail_in_init, +                    {tc_auto_skip,{failed,'_'}}, +                    []]}}, +         {?eh,test_stats,{0,0,{5,5}}}, + +         %% Exit in init_per_testcase -> pre/post_end_per_testcase +         %% shall not be called +         {?eh,tc_start,{skip_case_SUITE,exit_in_init}}, +         {?eh,cth,{empty_cth,pre_init_per_testcase, +                   [skip_case_SUITE,exit_in_init, +                    '$proplist', +                    []]}}, +         {?eh,cth,{empty_cth,post_init_per_testcase, +                   [skip_case_SUITE,exit_in_init, +                    '$proplist', +                    {skip,{failed,'_'}}, +                    []]}}, +         {?eh,tc_done,{skip_case_SUITE,exit_in_init, +                       {auto_skipped,{failed,'_'}}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_case_SUITE,exit_in_init, +                    {tc_auto_skip,{failed,'_'}}, +                    []]}}, +         {?eh,test_stats,{0,0,{5,6}}}, + +         %% Fail in end_per_testcase -> all hooks shall be called and +         %% test shall succeed. +         {?eh,tc_start,{skip_case_SUITE,fail_in_end}}, +         {?eh,cth,{empty_cth,pre_init_per_testcase, +                   [skip_case_SUITE,fail_in_end, +                    '$proplist', +                    []]}}, +         {?eh,cth,{empty_cth,post_init_per_testcase, +                   [skip_case_SUITE,fail_in_end, +                    '$proplist', +                    ok, +                    []]}}, +         {?eh,cth,{empty_cth,pre_end_per_testcase, +                   [skip_case_SUITE,fail_in_end, +                    '$proplist', +                    []]}}, +         {?eh,cth,{empty_cth,post_end_per_testcase, +                   [skip_case_SUITE,fail_in_end, +                    '$proplist', +                    {failed, +                     {skip_case_SUITE,end_per_testcase, +                      {'EXIT', +                       {test_case_failed,"Failed in end_per_testcase/2"}}}}, +                    []]}}, +         {?eh,tc_done,{skip_case_SUITE,fail_in_end, +                       {failed, +                        {skip_case_SUITE,end_per_testcase, +                         {'EXIT', +                          {test_case_failed,"Failed in end_per_testcase/2"}}}}}}, +         {?eh,test_stats,{1,0,{5,6}}}, + +         %% Exit in end_per_testcase -> all hooks shall be called and +         %% test shall succeed. +         {?eh,tc_start,{skip_case_SUITE,exit_in_end}}, +         {?eh,cth,{empty_cth,pre_init_per_testcase, +                   [skip_case_SUITE,exit_in_end, +                    '$proplist', +                    []]}}, +         {?eh,cth,{empty_cth,post_init_per_testcase, +                   [skip_case_SUITE,exit_in_end, +                    '$proplist', +                    ok, +                    []]}}, +         {?eh,cth,{empty_cth,pre_end_per_testcase, +                   [skip_case_SUITE,exit_in_end, +                    '$proplist', +                    []]}}, +         {?eh,cth,{empty_cth,post_end_per_testcase, +                   [skip_case_SUITE,exit_in_end, +                    '$proplist', +                    {failed, +                     {skip_case_SUITE,end_per_testcase, +                      {'EXIT',"Exit in end_per_testcase/2"}}}, +                    []]}}, +         {?eh,tc_done,{skip_case_SUITE,exit_in_end, +                       {failed, +                        {skip_case_SUITE,end_per_testcase, +                         {'EXIT',"Exit in end_per_testcase/2"}}}}}, +         {?eh,test_stats,{2,0,{5,6}}}, + +         %% Skip in testcase function -> all callbacks shall be called +         {?eh,tc_start,{skip_case_SUITE,skip_in_case}}, +         {?eh,cth,{empty_cth,pre_init_per_testcase, +                   [skip_case_SUITE,skip_in_case, +                    '$proplist', +                    []]}}, +         {?eh,cth,{empty_cth,post_init_per_testcase, +                   [skip_case_SUITE,skip_in_case, +                    '$proplist', +                    ok,[]]}}, +         {?eh,cth,{empty_cth,pre_end_per_testcase, +                   [skip_case_SUITE,skip_in_case, +                    '$proplist', +                    []]}}, +         {?eh,cth,{empty_cth,post_end_per_testcase, +                   [skip_case_SUITE,skip_in_case, +                    '$proplist', +                    {skip,"Skipped in test case function"}, +                    []]}}, +         {?eh,tc_done,{skip_case_SUITE,skip_in_case, +                       {skipped,"Skipped in test case function"}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_case_SUITE,skip_in_case, +                    {tc_user_skip,"Skipped in test case function"}, +                    []]}}, +         {?eh,test_stats,{2,0,{6,6}}}, + +         %% Auto skip due to failed 'require' -> only the on_tc_skip +         %% callback shall be called +         {?eh,tc_start,{skip_case_SUITE,req_auto_skip}}, +         {?eh,tc_done,{skip_case_SUITE,req_auto_skip, +                       {auto_skipped,{require_failed,{not_available,whatever}}}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_case_SUITE,req_auto_skip, +                    {tc_auto_skip,{require_failed,{not_available,whatever}}}, +                    []]}}, +         {?eh,test_stats,{2,0,{6,7}}}, + +         %% Auto skip due to failed testcase/0 function -> only the +         %% on_tc_skip callback shall be called +         {?eh,tc_start,{skip_case_SUITE,fail_auto_skip}}, +         {?eh,tc_done,{skip_case_SUITE,fail_auto_skip, +                       {auto_skipped,{testcase0_failed,bad_return_value}}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [skip_case_SUITE,fail_auto_skip, +                    {tc_auto_skip,{testcase0_failed,bad_return_value}}, +                    []]}}, +         {?eh,test_stats,{2,0,{6,8}}}, + +         {?eh,tc_start,{skip_case_SUITE,end_per_suite}}, +         {?eh,cth,{empty_cth,pre_end_per_suite, +                   [skip_case_SUITE, +                    '$proplist', +                    []]}}, +         {?eh,cth,{empty_cth,post_end_per_suite, +                   [skip_case_SUITE, +                    '$proplist', +                    ok,[]]}}, +         {?eh,tc_done,{skip_case_SUITE,end_per_suite,ok}}, +         {?eh,test_done,{'DEF','STOP_TIME'}}, +         {?eh,cth,{empty_cth,terminate,[[]]}}, +         {?eh,stop_logging,[]} +        ], +    %% Make sure no 'cth_error' events are received! +    [{negative,{?eh,cth_error,'_'},E} || E <- Events]; + +test_events(failed_sequence) -> +    %% skip_cth.erl will send a 'cth_error' event if a hook is +    %% erroneously called. Therefore, all Events are changed to +    %% {negative,{?eh,cth_error,'_'},Event} +    %% at the end of this function. +    Events = +        [ +         {?eh,start_logging,{'DEF','RUNDIR'}}, +         {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, +         {?eh,cth,{empty_cth,id,[[]]}}, +         {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, +         {?eh,start_info,{1,1,2}}, +         {?eh,tc_start,{ct_framework,init_per_suite}}, +         {?eh,cth,{empty_cth,pre_init_per_suite,[seq_SUITE,'$proplist',[]]}}, +         {?eh,cth,{empty_cth,post_init_per_suite, +                   [seq_SUITE,'$proplist','$proplist',[]]}}, +         {?eh,tc_done,{ct_framework,init_per_suite,ok}}, +         {?eh,tc_start,{seq_SUITE,test_case_1}}, +         {?eh,cth,{empty_cth,pre_init_per_testcase, +                   [seq_SUITE,test_case_1,'$proplist',[]]}}, +         {?eh,cth,{empty_cth,post_init_per_testcase, +                   [seq_SUITE,test_case_1,'$proplist',ok,[]]}}, +         {?eh,cth,{empty_cth,pre_end_per_testcase, +                   [seq_SUITE,test_case_1,'$proplist',[]]}}, +         {?eh,cth,{empty_cth,post_end_per_testcase, +                   [seq_SUITE,test_case_1,'$proplist', +                    {error,failed_on_purpose},[]]}}, +         {?eh,tc_done,{seq_SUITE,test_case_1,{failed,{error,failed_on_purpose}}}}, +         {?eh,cth,{empty_cth,on_tc_fail, +                   [seq_SUITE,test_case_1,failed_on_purpose,[]]}}, +         {?eh,test_stats,{0,1,{0,0}}}, +         {?eh,tc_start,{seq_SUITE,test_case_2}}, +         {?eh,tc_done,{seq_SUITE,test_case_2, +                       {auto_skipped,{sequence_failed,seq1,test_case_1}}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [seq_SUITE,test_case_2, +                    {tc_auto_skip,{sequence_failed,seq1,test_case_1}}, +                    []]}}, +         {?eh,test_stats,{0,1,{0,1}}}, +         {?eh,tc_start,{ct_framework,end_per_suite}}, +         {?eh,cth,{empty_cth,pre_end_per_suite,[seq_SUITE,'$proplist',[]]}}, +         {?eh,cth,{empty_cth,post_end_per_suite,[seq_SUITE,'$proplist',ok,[]]}}, +         {?eh,tc_done,{ct_framework,end_per_suite,ok}}, +         {?eh,test_done,{'DEF','STOP_TIME'}}, +         {?eh,cth,{empty_cth,terminate,[[]]}}, +         {?eh,stop_logging,[]} +        ], +    %% Make sure no 'cth_error' events are received! +    [{negative,{?eh,cth_error,'_'},E} || E <- Events]; + +test_events(repeat_force_stop) -> +    %% skip_cth.erl will send a 'cth_error' event if a hook is +    %% erroneously called. Therefore, all Events are changed to +    %% {negative,{?eh,cth_error,'_'},Event} +    %% at the end of this function. +    Events= +        [ +         {?eh,start_logging,{'DEF','RUNDIR'}}, +         {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, +         {?eh,cth,{empty_cth,id,[[]]}}, +         {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, +         {?eh,start_info,{1,1,2}}, +         {?eh,tc_start,{ct_framework,init_per_suite}}, +         {?eh,cth,{empty_cth,pre_init_per_suite,[repeat_SUITE,'$proplist',[]]}}, +         {?eh,cth,{empty_cth,post_init_per_suite, +                   [repeat_SUITE,'$proplist','$proplist',[]]}}, +         {?eh,tc_done,{ct_framework,init_per_suite,ok}}, +         {?eh,tc_start,{repeat_SUITE,test_case_1}}, +         {?eh,cth,{empty_cth,pre_init_per_testcase, +                   [repeat_SUITE,test_case_1,'$proplist',[]]}}, +         {?eh,cth,{empty_cth,post_init_per_testcase, +                   [repeat_SUITE,test_case_1,'$proplist',ok,[]]}}, +         {?eh,cth,{empty_cth,pre_end_per_testcase, +                   [repeat_SUITE,test_case_1,'$proplist',[]]}}, +         {?eh,cth,{empty_cth,post_end_per_testcase, +                   [repeat_SUITE,test_case_1,'$proplist',ok,[]]}}, +         {?eh,tc_done,{repeat_SUITE,test_case_1,ok}}, +         {?eh,test_stats,{1,0,{0,0}}}, +         {?eh,tc_start,{repeat_SUITE,test_case_2}}, +         {?eh,tc_done,{repeat_SUITE,test_case_2, +                       {auto_skipped, +                        "Repeated test stopped by force_stop option"}}}, +         {?eh,cth,{empty_cth,on_tc_skip, +                   [repeat_SUITE,test_case_2, +                    {tc_auto_skip,"Repeated test stopped by force_stop option"}, +                    []]}}, +         {?eh,test_stats,{1,0,{0,1}}}, +         {?eh,tc_start,{ct_framework,end_per_suite}}, +         {?eh,cth,{empty_cth,pre_end_per_suite,[repeat_SUITE,'$proplist',[]]}}, +         {?eh,cth,{empty_cth,post_end_per_suite, +                   [repeat_SUITE,'$proplist',ok,[]]}}, +         {?eh,tc_done,{ct_framework,end_per_suite,ok}}, +         {?eh,test_done,{'DEF','STOP_TIME'}}, +         {?eh,cth,{empty_cth,terminate,[[]]}}, +         {?eh,stop_logging,[]} +        ], +    %% Make sure no 'cth_error' events are received! +    [{negative,{?eh,cth_error,'_'},E} || E <- Events]; + +test_events(config_clash) -> +    %% skip_cth.erl will send a 'cth_error' event if a hook is +    %% erroneously called. Therefore, all Events are changed to +    %% {negative,{?eh,cth_error,'_'},Event} +    %% at the end of this function. +    Events = +        [ +         {?eh,start_logging,{'DEF','RUNDIR'}}, +         {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}}, +         {?eh,cth,{empty_cth,id,[[]]}}, +         {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}}, +         {?eh,start_info,{1,1,1}}, +         {?eh,tc_start,{ct_framework,init_per_suite}}, +         {?eh,cth,{empty_cth,pre_init_per_suite, +                   [config_clash_SUITE,'$proplist',[]]}}, +         {?eh,cth,{empty_cth,post_init_per_suite, +                   [config_clash_SUITE,'$proplist','$proplist',[]]}}, +         {?eh,tc_done,{ct_framework,init_per_suite,ok}}, +         {?eh,tc_start,{config_clash_SUITE,test_case_1}}, +         {?eh,tc_done,{config_clash_SUITE,test_case_1, +                       {failed,{error,{config_name_already_in_use,[aa]}}}}}, +         {?eh,cth,{empty_cth,on_tc_fail, +                   [config_clash_SUITE,test_case_1, +                    {config_name_already_in_use,[aa]}, +                    []]}}, +         {?eh,test_stats,{0,1,{0,0}}}, +         {?eh,tc_start,{ct_framework,end_per_suite}}, +         {?eh,cth,{empty_cth,pre_end_per_suite, +                   [config_clash_SUITE,'$proplist',[]]}}, +         {?eh,cth,{empty_cth,post_end_per_suite, +                   [config_clash_SUITE,'$proplist',ok,[]]}}, +         {?eh,tc_done,{ct_framework,end_per_suite,ok}}, +         {?eh,test_done,{'DEF','STOP_TIME'}}, +         {?eh,cth,{empty_cth,terminate,[[]]}}, +         {?eh,stop_logging,[]} +    ], +    %% Make sure no 'cth_error' events are received! +    [{negative,{?eh,cth_error,'_'},E} || E <- Events]; +  test_events(ok) ->      ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_hook_callbacks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_hook_callbacks_SUITE.erl new file mode 100644 index 0000000000..5b50548694 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_hook_callbacks_SUITE.erl @@ -0,0 +1,62 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(all_hook_callbacks_SUITE). + +-suite_defaults([{timetrap, {minutes, 10}}]). + +%% Note: This directive should only be used in test suites. +-compile(export_all). + +-include("ct.hrl"). + +%% Test server callback functions +init_per_suite(Config) -> +    Config. + +end_per_suite(_Config) -> +    ok. + +init_per_group(Config) -> +    Config. + +end_per_group(_Config) -> +    ok. + +init_per_testcase(skip_case, Config) -> +    {skip,"Skipped in init_per_testcase/2"}; +init_per_testcase(_TestCase, Config) -> +    Config. + +end_per_testcase(_TestCase, _Config) -> +    ok. + +all() -> +    [{group,test_group},test_case,skip_case]. + +groups() -> +    [{test_group,[test_case]}]. + +%% Test cases starts here. +test_case(Config) -> +    ok. + +skip_case(Config) -> +    ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl new file mode 100644 index 0000000000..f74c757cc1 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/config_clash_SUITE.erl @@ -0,0 +1,43 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(config_clash_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> +    [{require,aa,yy},{default_config,yy,"this is a default value"}]. + +init_per_testcase(_,Config) -> +    Config. + +end_per_testcase(_,_) -> +    ok. + +all() -> +    [test_case_1]. + +%% Test cases starts here. +test_case_1() -> +    [{require,aa,xx}]. +test_case_1(_Config) -> +    ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl new file mode 100644 index 0000000000..7cdaf2024b --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_end_config_SUITE.erl @@ -0,0 +1,51 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_no_end_config_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +%%% This suite is used to verify that all pre/post_end_per_* callbacks +%%% are called with correct SuiteName even if no end_per_* config +%%% function exist in the suite, and that the non-exported config +%%% functions fail with 'undef'. + +init_per_suite(Config) -> +    Config. + +init_per_group(_Group,Config) -> +    Config. + +init_per_testcase(_TC,Config) -> +    Config. + +all() -> +    [test_case_1, {group,test_group}]. + +groups() -> +    [{test_group,[],[test_case_2]}]. + +test_case_1(Config) -> +    ok. + +test_case_2(Config) -> +    ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl new file mode 100644 index 0000000000..43c062d66f --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_config_SUITE.erl @@ -0,0 +1,54 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_no_init_config_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +%%% This suite is used to verify that all +%%% pre/post_init_per_group/testcase callbacks are called with correct +%%% SuiteName even if no init_per_group/testcase function exist in the +%%% suite, and that the non-exported config functions fail with 'undef'. + +init_per_suite(Config) -> +    Config. + +end_per_suite(Config) -> +    Config. + +end_per_group(_Group,Config) -> +    Config. + +end_per_testcase(_TC,Config) -> +    Config. + +all() -> +    [test_case_1, {group,test_group}]. + +groups() -> +    [{test_group,[],[test_case_2]}]. + +test_case_1(Config) -> +    ok. + +test_case_2(Config) -> +    ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl new file mode 100644 index 0000000000..85dfe8ca4b --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_no_init_suite_config_SUITE.erl @@ -0,0 +1,39 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(ct_no_init_suite_config_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +%%% This suite is used to verify that pre/post_init_per_suite +%%% callbacks are called with correct SuiteName even if no +%%% init_per_suite function exist in the suite, and that the +%%% non-exported config function fails with 'undef'. + +end_per_suite(Config) -> +    Config. + +all() -> +    [test_case]. + +test_case(Config) -> +    ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl index c00eb5cf93..37742f0d20 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl @@ -44,18 +44,18 @@  -export([pre_end_per_suite/3]).  -export([post_end_per_suite/4]). --export([pre_init_per_group/3]). --export([post_init_per_group/4]). --export([pre_end_per_group/3]). --export([post_end_per_group/4]). +-export([pre_init_per_group/4]). +-export([post_init_per_group/5]). +-export([pre_end_per_group/4]). +-export([post_end_per_group/5]). --export([pre_init_per_testcase/3]). --export([post_init_per_testcase/4]). --export([pre_end_per_testcase/3]). --export([post_end_per_testcase/4]). +-export([pre_init_per_testcase/4]). +-export([post_init_per_testcase/5]). +-export([pre_end_per_testcase/4]). +-export([post_end_per_testcase/5]). --export([on_tc_fail/3]). --export([on_tc_skip/3]). +-export([on_tc_fail/4]). +-export([on_tc_skip/4]).  -export([terminate/1]). @@ -154,150 +154,160 @@ post_end_per_suite(Suite,Config,Return,State) ->  %% @doc Called before each init_per_group.  %% You can change the config in this function. --spec pre_init_per_group(Group :: atom(), -		     Config :: config(), -		     State :: #state{}) -> +-spec pre_init_per_group(Suite :: atom(), +                         Group :: atom(), +                         Config :: config(), +                         State :: #state{}) ->      {config() | skip_or_fail(), NewState :: #state{}}. -pre_init_per_group(Group,Config,State) -> +pre_init_per_group(Suite,Group,Config,State) ->      gen_event:notify(        ?CT_EVMGR_REF, #event{ name = cth, node = node(),  			     data = {?MODULE, pre_init_per_group, -				     [Group,Config,State]}}), -    ct:log("~w:pre_init_per_group(~w) called", [?MODULE,Group]), +				     [Suite,Group,Config,State]}}), +    ct:log("~w:pre_init_per_group(~w,~w) called", [?MODULE,Suite,Group]),      {Config, State}.  %% @doc Called after each init_per_group.  %% You can change the return value in this function. --spec post_init_per_group(Group :: atom(), +-spec post_init_per_group(Suite :: atom(), +                          Group :: atom(),  			  Config :: config(),  			  Return :: config() | skip_or_fail(),  			  State :: #state{}) ->      {config() | skip_or_fail(), NewState :: #state{}}. -post_init_per_group(Group,Config,Return,State) -> +post_init_per_group(Suite,Group,Config,Return,State) ->      gen_event:notify(        ?CT_EVMGR_REF, #event{ name = cth, node = node(),  			     data = {?MODULE, post_init_per_group, -				     [Group,Config,Return,State]}}), -    ct:log("~w:post_init_per_group(~w) called", [?MODULE,Group]), +				     [Suite,Group,Config,Return,State]}}), +    ct:log("~w:post_init_per_group(~w,~w) called", [?MODULE,Suite,Group]),      {Return, State}.  %% @doc Called after each end_per_group. The config/state can be changed here,  %% though it will only affect the *end_per_group functions. --spec pre_end_per_group(Group :: atom(), +-spec pre_end_per_group(Suite :: atom(), +                        Group :: atom(),  			Config :: config() | skip_or_fail(),  			State :: #state{}) ->      {ok | skip_or_fail(), NewState :: #state{}}. -pre_end_per_group(Group,Config,State) -> +pre_end_per_group(Suite,Group,Config,State) ->      gen_event:notify(        ?CT_EVMGR_REF, #event{ name = cth, node = node(),  			     data = {?MODULE, pre_end_per_group, -				     [Group,Config,State]}}), -    ct:log("~w:pre_end_per_group(~w) called", [?MODULE,Group]), +				     [Suite,Group,Config,State]}}), +    ct:log("~w:pre_end_per_group(~w~w) called", [?MODULE,Suite,Group]),      {Config, State}.  %% @doc Called after each end_per_group. Note that the config cannot be  %% changed here, only the status of the group. --spec post_end_per_group(Group :: atom(), +-spec post_end_per_group(Suite :: atom(), +                         Group :: atom(),  			 Config :: config(),  			 Return :: term(),  			 State :: #state{}) ->      {ok | skip_or_fail(), NewState :: #state{}}. -post_end_per_group(Group,Config,Return,State) -> +post_end_per_group(Suite,Group,Config,Return,State) ->      gen_event:notify(        ?CT_EVMGR_REF, #event{ name = cth, node = node(),  			     data = {?MODULE, post_end_per_group, -				     [Group,Config,Return,State]}}), -    ct:log("~w:post_end_per_group(~w) called", [?MODULE,Group]), +				     [Suite,Group,Config,Return,State]}}), +    ct:log("~w:post_end_per_group(~w,~w) called", [?MODULE,Suite,Group]),      {Return, State}.  %% @doc Called before init_per_testcase/2 for each test case.  %% You can change the config in this function. --spec pre_init_per_testcase(TC :: atom(), -		  Config :: config(), -		  State :: #state{}) -> +-spec pre_init_per_testcase(Suite :: atom(), +                            TC :: atom(), +                            Config :: config(), +                            State :: #state{}) ->      {config() | skip_or_fail(), NewState :: #state{}}. -pre_init_per_testcase(TC,Config,State) -> +pre_init_per_testcase(Suite,TC,Config,State) ->      gen_event:notify(        ?CT_EVMGR_REF, #event{ name = cth, node = node(),  			     data = {?MODULE, pre_init_per_testcase, -				     [TC,Config,State]}}), -    ct:log("~w:pre_init_per_testcase(~w) called", [?MODULE,TC]), +				     [Suite,TC,Config,State]}}), +    ct:log("~w:pre_init_per_testcase(~w,~w) called", [?MODULE,Suite,TC]),      {Config, State}.  %% @doc Called after init_per_testcase/2, and before the test case. --spec post_init_per_testcase(TC :: atom(), +-spec post_init_per_testcase(Suite :: atom(), +                             TC :: atom(),  			     Config :: config(),  			     Return :: config() | skip_or_fail(),  			     State :: #state{}) ->  				    {config() | skip_or_fail(), NewState :: #state{}}. -post_init_per_testcase(TC,Config,Return,State) -> +post_init_per_testcase(Suite,TC,Config,Return,State) ->      gen_event:notify(        ?CT_EVMGR_REF, #event{ name = cth, node = node(),  			     data = {?MODULE, post_init_per_testcase, -				     [TC,Config,Return,State]}}), -    ct:log("~w:post_init_per_testcase(~w) called", [?MODULE,TC]), +				     [Suite,TC,Config,Return,State]}}), +    ct:log("~w:post_init_per_testcase(~w,~w) called", [?MODULE,Suite,TC]),      {Return, State}.  %% @doc Called before end_per_testacse/2. No skip or fail allowed here,  %% only config additions. --spec pre_end_per_testcase(TC :: atom(), -		  Config :: config(), -		  State :: #state{}) -> +-spec pre_end_per_testcase(Suite :: atom(), +                           TC :: atom(), +                           Config :: config(), +                           State :: #state{}) ->      {config(), NewState :: #state{}}. -pre_end_per_testcase(TC,Config,State) -> +pre_end_per_testcase(Suite,TC,Config,State) ->      gen_event:notify(        ?CT_EVMGR_REF, #event{ name = cth, node = node(),  			     data = {?MODULE, pre_end_per_testcase, -				     [TC,Config,State]}}), -    ct:log("~w:pre_end_per_testcase(~w) called", [?MODULE,TC]), +				     [Suite,TC,Config,State]}}), +    ct:log("~w:pre_end_per_testcase(~w,~w) called", [?MODULE,Suite,TC]),      {Config, State}.  %% @doc Called after end_per_testcase/2 for each test case. Note that  %% the config cannot be changed here, only the status of the test case. --spec post_end_per_testcase(TC :: atom(), +-spec post_end_per_testcase(Suite :: atom(), +                            TC :: atom(),  			    Config :: config(),  			    Return :: term(),  			    State :: #state{}) ->      {ok | skip_or_fail(), NewState :: #state{}}. -post_end_per_testcase(TC,Config,Return,State) -> +post_end_per_testcase(Suite,TC,Config,Return,State) ->      gen_event:notify(        ?CT_EVMGR_REF, #event{ name = cth, node = node(),  			     data = {?MODULE, post_end_per_testcase, -				     [TC,Config,Return,State]}}), -    ct:log("~w:post_end_per_testcase(~w) called", [?MODULE,TC]), +				     [Suite,TC,Config,Return,State]}}), +    ct:log("~w:post_end_per_testcase(~w,~w) called", [?MODULE,Suite,TC]),      {Return, State}.  %% @doc Called after post_init_per_suite, post_end_per_suite, post_init_per_group,  %% post_end_per_group and post_end_per_tc if the suite, group or test case failed.  %% This function should be used for extra cleanup which might be needed.  %% It is not possible to modify the config or the status of the test run. --spec on_tc_fail(TC :: init_per_suite | end_per_suite | +-spec on_tc_fail(Suite :: atom(), +                 TC :: init_per_suite | end_per_suite |  		       init_per_group | end_per_group | atom() |  		       {Function :: atom(), GroupName :: atom()},  		 Reason :: term(), State :: #state{}) -> NewState :: #state{}. -on_tc_fail(TC, Reason, State) -> +on_tc_fail(Suite, TC, Reason, State) ->      gen_event:notify(        ?CT_EVMGR_REF, #event{ name = cth, node = node(),  			     data = {?MODULE, on_tc_fail, -				     [TC,Reason,State]}}), -    ct:log("~w:on_tc_fail(~w) called", [?MODULE,TC]), +				     [Suite,TC,Reason,State]}}), +    ct:log("~w:on_tc_fail(~w,~w) called", [?MODULE,Suite,TC]),      State.  %% @doc Called when a test case is skipped by either user action  %% or due to an init function failing. Test case can be  %% end_per_suite, init_per_group, end_per_group and the actual test cases.  --spec on_tc_skip(TC :: end_per_suite | +-spec on_tc_skip(Suite :: atom(), +                 TC :: end_per_suite |  		       init_per_group | end_per_group | atom() |  		       {Function :: atom(), GroupName :: atom()},  		 {tc_auto_skip, {failed, {Mod :: atom(), Function :: atom(), Reason :: term()}}} |  		 {tc_user_skip, {skipped, Reason :: term()}},  		 State :: #state{}) -> NewState :: #state{}. -on_tc_skip(TC, Reason, State) -> +on_tc_skip(Suite, TC, Reason, State) ->      gen_event:notify(        ?CT_EVMGR_REF, #event{ name = cth, node = node(),  			     data = {?MODULE, on_tc_skip, -				     [TC,Reason,State]}}), -    ct:log("~w:on_tc_skip(~w) called", [?MODULE,TC]), +				     [Suite,TC,Reason,State]}}), +    ct:log("~w:on_tc_skip(~w,~w) called", [?MODULE,Suite,TC]),      State.  %% @doc Called when the scope of the CTH is done, this depends on diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl index 559b22bc9f..141b933697 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_post_suite_cth.erl @@ -45,29 +45,29 @@ pre_end_per_suite(Suite,Config,State) ->  post_end_per_suite(Suite,Config,Return,State) ->
      empty_cth:post_end_per_suite(Suite,Config,Return,State).
 -pre_init_per_group(Group,Config,State) ->
 -    empty_cth:pre_init_per_group(Group,Config,State).
 +pre_init_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) ->
 -    empty_cth:post_init_per_group(Group,Config,Return,State).
 +post_init_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) ->
 -    empty_cth:pre_end_per_group(Group,Config,State).
 +pre_end_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) ->
 -    empty_cth:post_end_per_group(Group,Config,Return,State).
 +post_end_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) ->
 -    empty_cth:pre_init_per_testcase(TC,Config,State).
 +pre_init_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) ->
 -    empty_cth:post_end_per_testcase(TC,Config,Return,State).
 +post_end_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) ->
 -    empty_cth:on_tc_fail(TC,Reason,State).
 +on_tc_fail(Suite,TC, Reason, State) -> +    empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) ->
 -    empty_cth:on_tc_skip(TC,Reason,State).
 +on_tc_skip(Suite,TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State).  terminate(State) ->
      empty_cth:terminate(State).
 diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl index 51202443bf..07d7c84ed5 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fail_pre_suite_cth.erl @@ -45,35 +45,35 @@ pre_end_per_suite(Suite,Config,State) ->  post_end_per_suite(Suite,Config,Return,State) ->      empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> -    empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> -    empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> -    empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> -    empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> -    empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> -    empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> -    empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> +    empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> -    empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State).  terminate(State) ->      empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fallback_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fallback_cth.erl new file mode 100644 index 0000000000..59a3d5cbf9 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/fallback_cth.erl @@ -0,0 +1,81 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(fallback_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + + +%% CT Hooks +-compile(export_all). + +id(Opts) -> +    empty_cth:id(Opts). + +init(Id, Opts) -> +    empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> +    empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> +    empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> +    empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> +    empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Group,Config,State) -> +    empty_cth:pre_init_per_group(fallback_nosuite,Group,Config,State). + +post_init_per_group(Group,Config,Return,State) -> +    empty_cth:post_init_per_group(fallback_nosuite,Group,Config,Return,State). + +pre_end_per_group(Group,Config,State) -> +    empty_cth:pre_end_per_group(fallback_nosuite,Group,Config,State). + +post_end_per_group(Group,Config,Return,State) -> +    empty_cth:post_end_per_group(fallback_nosuite,Group,Config,Return,State). + +pre_init_per_testcase(TC,Config,State) -> +    empty_cth:pre_init_per_testcase(fallback_nosuite,TC,Config,State). + +post_init_per_testcase(TC,Config,Return,State) -> +    empty_cth:post_init_per_testcase(fallback_nosuite,TC,Config,Return,State). + +pre_end_per_testcase(TC,Config,State) -> +    empty_cth:pre_end_per_testcase(fallback_nosuite,TC,Config,State). + +post_end_per_testcase(TC,Config,Return,State) -> +    empty_cth:post_end_per_testcase(fallback_nosuite,TC,Config,Return,State). + +on_tc_fail(TC, Reason, State) -> +    empty_cth:on_tc_fail(fallback_nosuite,TC,Reason,State). + +on_tc_skip(TC, Reason, State) -> +    empty_cth:on_tc_skip(fallback_nosuite,TC,Reason,State). + +terminate(State) -> +    empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl index b49cbe7fb4..679f076f3a 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/minimal_terminate_cth.erl @@ -29,13 +29,13 @@  %% CT Hooks  -export([init/2]).  -export([terminate/1]). --export([on_tc_skip/3]). +-export([on_tc_skip/4]).  init(Id, Opts) ->      empty_cth:init(Id, Opts). -on_tc_skip(TC, Reason, State) -> -    empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite, TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State).  terminate(State) ->      empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl index a687743641..95bb76b4c1 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/prio_cth.erl @@ -47,35 +47,35 @@ pre_end_per_suite(Suite,Config,State) ->  post_end_per_suite(Suite,Config,Return,State) ->      empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> -    empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> -    empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> -    empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> -    empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> -    empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> -    empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> -    empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> +    empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> -    empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State).  terminate(State) ->      empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl index 4d9c60f1ca..3562d39967 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/recover_post_suite_cth.erl @@ -47,35 +47,35 @@ pre_end_per_suite(Suite,Config,State) ->  post_end_per_suite(Suite,Config,Return,State) ->      empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> -    empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> -    empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> -    empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> -    empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> -    empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> -    empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> -    empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> +    empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> -    empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State).  terminate(State) ->      empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl new file mode 100644 index 0000000000..fded4c02ab --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/repeat_SUITE.erl @@ -0,0 +1,42 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(repeat_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +init_per_testcase(_,Config) -> +    Config. + +end_per_testcase(_,_) -> +    ok. + +all() -> +    [test_case_1, test_case_2]. + +%% Test cases starts here. +test_case_1(_Config) -> +    timer:sleep(10000), +    ok. + +test_case_2(_Config) -> +    ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl index 494f398fc1..b9d9d4cec1 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/same_id_cth.erl @@ -48,35 +48,35 @@ pre_end_per_suite(Suite,Config,State) ->  post_end_per_suite(Suite,Config,Return,State) ->      empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> -    empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> -    empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> -    empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> -    empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> -    empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> -    empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> -    empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> +    empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> -    empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State).  terminate(State) ->      empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl new file mode 100644 index 0000000000..6d1302fd35 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/seq_SUITE.erl @@ -0,0 +1,45 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(seq_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +init_per_testcase(_,Config) -> +    Config. + +end_per_testcase(_,_) -> +    ok. + +all() -> +    [{sequence,seq1}]. + +sequences() -> +    [{seq1,[test_case_1,test_case_2]}]. + +%% Test cases starts here. +test_case_1(_Config) -> +    exit(failed_on_purpose). + +test_case_2(_Config) -> +    ct:fail("This test shall never be run since test_case_1 fails " +            "and they are run in sequence"). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec new file mode 100644 index 0000000000..a271c5e8b2 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip.spec @@ -0,0 +1,8 @@ +{suites,".",[all_hook_callbacks_SUITE, +             skip_init_SUITE, +             skip_req_SUITE, +             skip_fail_SUITE, +             skip_group_SUITE, +             skip_case_SUITE]}. +{skip_suites,".",all_hook_callbacks_SUITE,"Skipped in spec"}. +{skip_cases,".",skip_case_SUITE,skip_in_spec,"Skipped in spec"}. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl new file mode 100644 index 0000000000..dad80ae914 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_case_SUITE.erl @@ -0,0 +1,106 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_case_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> +    []. + +init_per_suite(Config) -> +    Config. + +end_per_suite(Config) -> +    ok. + +init_per_group(_,Config) -> +    Config. + +end_per_group(_,_) -> +    ok. + +init_per_testcase(skip_in_init,Config) -> +    {skip,"Skipped in init_per_testcase/2"}; +init_per_testcase(fail_in_init,Config) -> +    ct:fail("Failed in init_per_testcase/2"); +init_per_testcase(exit_in_init,Config) -> +    exit(self(),"Exit in init_per_testcase/2"); +init_per_testcase(_,Config) -> +    Config. + +end_per_testcase(fail_in_end,_) -> +    ct:fail("Failed in end_per_testcase/2"); +end_per_testcase(exit_in_end,_) -> +    exit(self(),"Exit in end_per_testcase/2"); +end_per_testcase(_,_) -> +    ok. + +all() -> +    [skip_in_spec, +     skip_in_init, +     fail_in_init, +     exit_in_init, +     fail_in_end, +     exit_in_end, +     skip_in_case, +     req_auto_skip, +     fail_auto_skip +    ]. + +%% Test cases starts here. +skip_in_spec(Config) -> +    ct:fail("This test shall never be run. " +            "It shall be skipped in the test spec."). + +skip_in_init(Config) -> +    ct:fail("This test shall never be run. " +            "It shall be skipped in init_per_testcase/2."). + +fail_in_init(Config) -> +    ct:fail("This test shall never be run. " +            "It shall fail in init_per_testcase/2."). + +exit_in_init(Config) -> +    ct:fail("This test shall never be run. " +            "It shall exit in init_per_testcase/2."). + +fail_in_end(Config) -> +    ok. + +exit_in_end(Config) -> +    ok. + +skip_in_case(Config) -> +    {skip,"Skipped in test case function"}. + +req_auto_skip() -> +    [{require,whatever}]. +req_auto_skip(Config) -> +    ct:fail("This test shall never be run due to " +            "failed require"). + +fail_auto_skip() -> +    faulty_return_value. +fail_auto_skip(Config) -> +    ct:fail("This test shall never be run due to " +            "faulty return from info function"). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl new file mode 100644 index 0000000000..16f015fe7a --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_cth.erl @@ -0,0 +1,182 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(skip_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +%% Send a cth_error event if a callback is called with unexpected arguments +-define(fail(Info), +        gen_event:notify( +          ?CT_EVMGR_REF,  +          #event{ name = cth_error, +                  node = node(), +                  data = {illegal_hook_callback,{?MODULE,?FUNCTION_NAME,Info}}})). + +%% CT Hooks +-compile(export_all). + +id(Opts) -> +    empty_cth:id(Opts). + +init(Id, Opts) -> +    empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> +    Suite==skip_init_SUITE +        orelse Suite==skip_group_SUITE +        orelse Suite==skip_case_SUITE +        orelse Suite==seq_SUITE +        orelse Suite==repeat_SUITE +        orelse Suite==config_clash_SUITE +        orelse ?fail(Suite), +    empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> +    Suite==skip_init_SUITE +        orelse Suite==skip_group_SUITE +        orelse Suite==skip_case_SUITE +        orelse Suite==seq_SUITE +        orelse Suite==repeat_SUITE +        orelse Suite==config_clash_SUITE +        orelse ?fail(Suite), +    empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> +    Suite==skip_case_SUITE +        orelse Suite==skip_group_SUITE +        orelse Suite==seq_SUITE +        orelse Suite==repeat_SUITE +        orelse Suite==config_clash_SUITE +        orelse ?fail(Suite), +    empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> +    Suite==skip_case_SUITE +        orelse Suite==skip_group_SUITE +        orelse Suite==seq_SUITE +        orelse Suite==repeat_SUITE +        orelse Suite==config_clash_SUITE +        orelse ?fail(Suite), +    empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Suite,Group,Config,State) -> +    (Suite==skip_group_SUITE andalso Group==test_group_3) +        orelse ?fail({Suite,Group}), +    empty_cth:pre_init_per_group(Suite,Group,Config,State). + +post_init_per_group(Suite,Group,Config,Return,State) -> +    (Suite==skip_group_SUITE andalso Group==test_group_3) +        orelse ?fail({Suite,Group}), +    empty_cth:post_init_per_group(Suite,Group,Config,Return,State). + +pre_end_per_group(Suite,Group,Config,State) -> +    ?fail({Suite,Group}), +    empty_cth:pre_end_per_group(Suite,Group,Config,State). + +post_end_per_group(Suite,Group,Config,Return,State) -> +    ?fail({Suite,Group}), +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State). + +pre_init_per_testcase(Suite,TC,Config,State) -> +    (Suite==skip_case_SUITE andalso (TC==skip_in_init +                                     orelse TC==fail_in_init +                                     orelse TC==exit_in_init +                                     orelse TC==fail_in_end +                                     orelse TC==exit_in_end +                                     orelse TC==skip_in_case)) +        orelse (Suite==seq_SUITE andalso TC==test_case_1) +        orelse (Suite==repeat_SUITE andalso TC==test_case_1) +        orelse ?fail({Suite,TC}), +    empty_cth:pre_init_per_testcase(Suite,TC,Config,State). + +post_init_per_testcase(Suite,TC,Config,Return,State) -> +    (Suite==skip_case_SUITE andalso (TC==skip_in_init +                                     orelse TC==fail_in_init +                                     orelse TC==exit_in_init +                                     orelse TC==fail_in_end +                                     orelse TC==exit_in_end +                                     orelse TC==skip_in_case)) +        orelse (Suite==seq_SUITE andalso TC==test_case_1) +        orelse (Suite==repeat_SUITE andalso TC==test_case_1) +        orelse ?fail({Suite,TC}), +    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). + +pre_end_per_testcase(Suite,TC,Config,State) -> +    (Suite==skip_case_SUITE andalso (TC==skip_in_case +                                     orelse TC==fail_in_end +                                     orelse TC==exit_in_end)) +        orelse (Suite==seq_SUITE andalso TC==test_case_1) +        orelse (Suite==repeat_SUITE andalso TC==test_case_1) +        orelse ?fail({Suite,TC}), +    empty_cth:pre_end_per_testcase(Suite,TC,Config,State). + +post_end_per_testcase(Suite,TC,Config,Return,State) -> +    (Suite==skip_case_SUITE andalso (TC==skip_in_case +                                     orelse TC==fail_in_end +                                     orelse TC==exit_in_end)) +        orelse (Suite==seq_SUITE andalso TC==test_case_1) +        orelse (Suite==repeat_SUITE andalso TC==test_case_1) +        orelse ?fail({Suite,TC}), +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). + +on_tc_fail(Suite,TC,Reason,State) -> +    (Suite==seq_SUITE andalso TC==test_case_1) +        orelse (Suite==config_clash_SUITE andalso TC==test_case_1) +        orelse ?fail({Suite,TC}), +    empty_cth:on_tc_fail(Suite,TC,Reason,State). + +on_tc_skip(all_hook_callbacks_SUITE=Suite,all=TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(Suite,TC,Reason,State) +  when (Suite==skip_init_SUITE +        orelse Suite==skip_req_SUITE +        orelse Suite==skip_fail_SUITE) +       andalso +       (TC==init_per_suite +        orelse TC==test_case +        orelse TC==end_per_suite) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(skip_group_SUITE=Suite,TC={C,G},Reason,State) +  when (C==init_per_group orelse C==test_case orelse C==end_per_group) andalso +       (G==test_group_1 orelse G==test_group_2 orelse G==test_group_3) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(skip_case_SUITE=Suite,TC,Reason,State) +  when TC==skip_in_spec; +       TC==skip_in_init; +       TC==fail_in_init; +       TC==exit_in_init; +       TC==skip_in_case; +       TC==req_auto_skip; +       TC==fail_auto_skip -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(Suite,TC,Reason,State) +  when (Suite==seq_SUITE andalso TC==test_case_2) +       orelse (Suite==repeat_SUITE andalso TC==test_case_2) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State); +on_tc_skip(Suite,TC,Reason,State) -> +    ?fail({Suite,TC}), +    empty_cth:on_tc_skip(Suite,TC,Reason,State). + +terminate(State) -> +    empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl new file mode 100644 index 0000000000..9f5dfee6b9 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_fail_SUITE.erl @@ -0,0 +1,53 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_fail_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> +    faulty_return_value. + +init_per_suite(Config) -> +    Config. + +end_per_suite(Config) -> +    ok. + +init_per_group(_,Config) -> +    Config. + +end_per_group(_,_) -> +    ok. + +init_per_testcase(_,Config) -> +    Config. + +end_per_testcase(_,_) -> +    ok. + +all() -> +    [test_case]. + +%% Test cases starts here. +test_case(Config) -> +    ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl new file mode 100644 index 0000000000..d3b848bfbd --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_group_SUITE.erl @@ -0,0 +1,64 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_group_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> +    []. + +init_per_suite(Config) -> +    Config. + +end_per_suite(Config) -> +    ok. + +group(test_group_1) -> +    [{require,whatever}]; +group(test_group_2) -> +    faulty_return_value; +group(_) -> +    []. + +init_per_group(test_group_3,Config) -> +    {skip,"Skipped in init_per_group/2"}; +init_per_group(_,Config) -> +    ct:fail("This shall never be run due to auto_skip from group/1"). + +end_per_group(_,_) -> +    ct:fail("This shall never be run"). + +all() -> +    [{group,test_group_1}, +     {group,test_group_2}, +     {group,test_group_3}]. + +groups() -> +    [{test_group_1,[test_case]}, +     {test_group_2,[test_case]}, +     {test_group_3,[test_case]}]. + +%% Test cases starts here. +test_case(_Config) ->     +    ct:fail("This test case shall never be run due to skip on group level"). + diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl new file mode 100644 index 0000000000..70305421ac --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_init_SUITE.erl @@ -0,0 +1,53 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_init_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> +    []. + +init_per_suite(Config) -> +    {skip,"Skipped in init_per_suite/1"}. + +end_per_suite(Config) -> +    ok. + +init_per_group(_,Config) -> +    Config. + +end_per_group(_,_) -> +    ok. + +init_per_testcase(_,Config) -> +    Config. + +end_per_testcase(_,_) -> +    ok. + +all() -> +    [test_case]. + +%% Test cases starts here. +test_case(Config) -> +    ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl index d5b347e723..48a2d70e22 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_post_suite_cth.erl @@ -45,35 +45,35 @@ pre_end_per_suite(Suite,Config,State) ->  post_end_per_suite(Suite,Config,Return,State) ->      empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> -    empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> -    empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> -    empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> -    empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> -    empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> -    empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> -    empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> +    empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> -    empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State).  terminate(State) ->      empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl index 36abac0bf8..d638954d3c 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_end_cth.erl @@ -46,36 +46,36 @@ pre_end_per_suite(Suite,Config,State) ->  post_end_per_suite(Suite,Config,Return,State) ->      empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> -    empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> -    empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> -    empty_cth:pre_end_per_group(Group,Config,State), +pre_end_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_end_per_group(Suite,Group,Config,State),      {{skip, "Test skip"}, State}. -post_end_per_group(Group,Config,Return,State) -> -    empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> -    empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> -    empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> -    empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> +    empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> -    empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State).  terminate(State) ->      empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl new file mode 100644 index 0000000000..e1d261d59a --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_init_tc_cth.erl @@ -0,0 +1,79 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + + +-module(skip_pre_init_tc_cth). + + +-include_lib("common_test/src/ct_util.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + + +%% CT Hooks +-compile(export_all). + +init(Id, Opts) -> +    empty_cth:init(Id, Opts). + +pre_init_per_suite(Suite, Config, State) -> +    empty_cth:pre_init_per_suite(Suite,Config,State). + +post_init_per_suite(Suite,Config,Return,State) -> +    empty_cth:post_init_per_suite(Suite,Config,Return,State). + +pre_end_per_suite(Suite,Config,State) -> +    empty_cth:pre_end_per_suite(Suite,Config,State). + +post_end_per_suite(Suite,Config,Return,State) -> +    empty_cth:post_end_per_suite(Suite,Config,Return,State). + +pre_init_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_init_per_group(Suite,Group,Config,State). + +post_init_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_init_per_group(Suite,Group,Config,Return,State). + +pre_end_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_end_per_group(Suite,Group,Config,State). + +post_end_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State). + +pre_init_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_init_per_testcase(Suite,TC,Config,State), +    {{skip, "Skipped in pre_init_per_testcase"}, State}. + +post_init_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). + +pre_end_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_end_per_testcase(Suite,TC,Config,State). + +post_end_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). + +on_tc_fail(Suite,TC, Reason, State) -> +    empty_cth:on_tc_fail(Suite,TC,Reason,State). + +on_tc_skip(Suite,TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State). + +terminate(State) -> +    empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl index fa510b2d54..d7b07ee33c 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_pre_suite_cth.erl @@ -46,35 +46,35 @@ pre_end_per_suite(Suite,Config,State) ->  post_end_per_suite(Suite,Config,Return,State) ->      empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> -    empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> -    empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> -    empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> -    empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> -    empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> -    empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> -    empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> +    empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> -    empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State).  terminate(State) ->      empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl new file mode 100644 index 0000000000..bc69dd5ea4 --- /dev/null +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/skip_req_SUITE.erl @@ -0,0 +1,53 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%%     http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(skip_req_SUITE). + +-compile(export_all). + +-include("ct.hrl"). + +suite() -> +    [{require,whatever}]. + +init_per_suite(Config) -> +    Config. + +end_per_suite(Config) -> +    ok. + +init_per_group(_,Config) -> +    Config. + +end_per_group(_,_) -> +    ok. + +init_per_testcase(_,Config) -> +    Config. + +end_per_testcase(_,_) -> +    ok. + +all() -> +    [test_case]. + +%% Test cases starts here. +test_case(Config) -> +    ok. diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl index 7ec0d458b6..c6e0419c50 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/state_update_cth.erl @@ -48,44 +48,44 @@ post_end_per_suite(Suite,Config,Return,State) ->      empty_cth:post_end_per_suite(Suite,Config,Return,State),      {Return, [post_end_per_suite|State]}. -pre_init_per_group(Group,Config,State) -> -    empty_cth:pre_init_per_group(Group,Config,State), +pre_init_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_init_per_group(Suite,Group,Config,State),      {Config, [pre_init_per_group|State]}. -post_init_per_group(Group,Config,Return,State) -> -    empty_cth:post_init_per_group(Group,Config,Return,State), +post_init_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_init_per_group(Suite,Group,Config,Return,State),      {Return, [post_init_per_group|State]}. -pre_end_per_group(Group,Config,State) -> -    empty_cth:pre_end_per_group(Group,Config,State), +pre_end_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_end_per_group(Suite,Group,Config,State),      {Config, [pre_end_per_group|State]}. -post_end_per_group(Group,Config,Return,State) -> -    empty_cth:post_end_per_group(Group,Config,Return,State), +post_end_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State),      {Return, [post_end_per_group|State]}. -pre_init_per_testcase(TC,Config,State) -> -    empty_cth:pre_init_per_testcase(TC,Config,State), +pre_init_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_init_per_testcase(Suite,TC,Config,State),      {Config, [pre_init_per_testcase|State]}. -post_init_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_init_per_testcase(TC,Config,Return,State), +post_init_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State),      {Return, [post_init_per_testcase|State]}. -pre_end_per_testcase(TC,Config,State) -> -    empty_cth:pre_end_per_testcase(TC,Config,State), +pre_end_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_end_per_testcase(Suite,TC,Config,State),      {Config, [pre_end_per_testcase|State]}. -post_end_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_end_per_testcase(TC,Config,Return,State), +post_end_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State),      {Return, [post_end_per_testcase|State]}. -on_tc_fail(TC, Reason, State) -> -    empty_cth:on_tc_fail(TC,Reason,State), +on_tc_fail(Suite,TC, Reason, State) -> +    empty_cth:on_tc_fail(Suite,TC,Reason,State),      [on_tc_fail|State]. -on_tc_skip(TC, Reason, State) -> -    empty_cth:on_tc_skip(TC,Reason,State), +on_tc_skip(Suite,TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State),      [on_tc_skip|State].  terminate(State) -> diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl index 2b9e726819..10a7047899 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/undef_cth.erl @@ -44,35 +44,35 @@ pre_end_per_suite(Suite,Config,State) ->  post_end_per_suite(Suite,Config,Return,State) ->      empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> -    empty_cth:pre_init_per_group(Group,Config,State). +pre_init_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> -    empty_cth:post_init_per_group(Group,Config,Return,State). +post_init_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> -    empty_cth:pre_end_per_group(Group,Config,State). +pre_end_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> -    empty_cth:post_end_per_group(Group,Config,Return,State). +post_end_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> -    empty_cth:pre_init_per_testcase(TC,Config,State). +pre_init_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> -    empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_end_per_testcase(TC,Config,Return,State). +post_end_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> -    empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> +    empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> -    empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State).  terminate(State) ->      empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl index d48981f667..f933c7702e 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/update_config_cth.erl @@ -50,43 +50,43 @@ post_end_per_suite(Suite,Config,Return,State) ->      NewConfig = [{post_end_per_suite,?now}|Config],      {NewConfig,NewConfig}. -pre_init_per_group(Group,Config,State) -> -    empty_cth:pre_init_per_group(Group,Config,State), +pre_init_per_group(Suite, Group,Config,State) -> +    empty_cth:pre_init_per_group(Suite,Group,Config,State),      {[{pre_init_per_group,?now}|Config],State}. -post_init_per_group(Group,Config,Return,State) -> -    empty_cth:post_init_per_group(Group,Config,Return,State), +post_init_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_init_per_group(Suite,Group,Config,Return,State),      {[{post_init_per_group,?now}|Return],State}. -pre_end_per_group(Group,Config,State) -> -    empty_cth:pre_end_per_group(Group,Config,State), +pre_end_per_group(Suite,Group,Config,State) -> +    empty_cth:pre_end_per_group(Suite,Group,Config,State),      {[{pre_end_per_group,?now}|Config],State}. -post_end_per_group(Group,Config,Return,State) -> -    empty_cth:post_end_per_group(Group,Config,Return,State), +post_end_per_group(Suite,Group,Config,Return,State) -> +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State),      {[{post_end_per_group,?now}|Config],State}. -pre_init_per_testcase(TC,Config,State) -> -    empty_cth:pre_init_per_testcase(TC,Config,State), +pre_init_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_init_per_testcase(Suite,TC,Config,State),      {[{pre_init_per_testcase,?now}|Config],State}. -post_init_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_init_per_testcase(TC,Config,Return,State), +post_init_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State),      {[{post_init_per_testcase,?now}|Config],State}. -pre_end_per_testcase(TC,Config,State) -> -    empty_cth:pre_end_per_testcase(TC,Config,State), +pre_end_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_end_per_testcase(Suite,TC,Config,State),      {[{pre_end_per_testcase,?now}|Config],State}. -post_end_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_end_per_testcase(TC,Config,Return,State), +post_end_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State),      {[{post_end_per_testcase,?now}|Config],State}. -on_tc_fail(TC, Reason, State) -> -    empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> +    empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> -    empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State).  terminate(State) ->      empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl index 71d84781e0..b29256a77e 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_config_cth.erl @@ -60,37 +60,37 @@ post_end_per_suite(Suite,Config,Return,State) ->      ct_no_config_SUITE = ct:get_config(suite_cfg),      empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> +pre_init_per_group(Suite,Group,Config,State) ->      true = ?val(post_init_per_suite, Config),      ct_no_config_SUITE = ct:get_config(suite_cfg),      test_group = ct:get_config(group_cfg), -    empty_cth:pre_init_per_group(Group, +    empty_cth:pre_init_per_group(Suite,Group,  				 [{pre_init_per_group,true} | Config],  				 State). -post_init_per_group(Group,Config,Return,State) -> +post_init_per_group(Suite,Group,Config,Return,State) ->      true = ?val(pre_init_per_group, Return),      test_group = ct:get_config(group_cfg), -    empty_cth:post_init_per_group(Group, +    empty_cth:post_init_per_group(Suite,Group,  				  Config,  				  [{post_init_per_group,true} | Return],  				  State). -pre_end_per_group(Group,Config,State) -> +pre_end_per_group(Suite,Group,Config,State) ->      true = ?val(post_init_per_group, Config),      ct_no_config_SUITE = ct:get_config(suite_cfg),      test_group = ct:get_config(group_cfg), -    empty_cth:pre_end_per_group(Group, +    empty_cth:pre_end_per_group(Suite,Group,  				[{pre_end_per_group,true} | Config],  				State). -post_end_per_group(Group,Config,Return,State) -> +post_end_per_group(Suite,Group,Config,Return,State) ->      true = ?val(pre_end_per_group, Config),      ct_no_config_SUITE = ct:get_config(suite_cfg),      test_group = ct:get_config(group_cfg), -    empty_cth:post_end_per_group(Group,Config,Return,State). +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> +pre_init_per_testcase(Suite,TC,Config,State) ->      true = ?val(post_init_per_suite, Config),      case ?val(name, ?val(tc_group_properties, Config)) of  	undefined ->  @@ -102,19 +102,19 @@ pre_init_per_testcase(TC,Config,State) ->      ct_no_config_SUITE = ct:get_config(suite_cfg),      CfgKey = list_to_atom(atom_to_list(TC) ++ "_cfg"),      TC = ct:get_config(CfgKey), -    empty_cth:pre_init_per_testcase(TC, +    empty_cth:pre_init_per_testcase(Suite,TC,  				    [{pre_init_per_testcase,true} | Config],  				    State).  %%! TODO: Verify Config also in post_init and pre_end! -post_init_per_testcase(TC,Config,Return,State) -> -    empty_cth:post_init_per_testcase(TC,Config,Return,State). +post_init_per_testcase(Suite,TC,Config,Return,State) -> +    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> -    empty_cth:pre_end_per_testcase(TC,Config,State). +pre_end_per_testcase(Suite,TC,Config,State) -> +    empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> +post_end_per_testcase(Suite,TC,Config,Return,State) ->      true = ?val(post_init_per_suite, Config),      true = ?val(pre_init_per_testcase, Config),      case ?val(name, ?val(tc_group_properties, Config)) of @@ -127,13 +127,13 @@ post_end_per_testcase(TC,Config,Return,State) ->      ct_no_config_SUITE = ct:get_config(suite_cfg),      CfgKey = list_to_atom(atom_to_list(TC) ++ "_cfg"),      TC = ct:get_config(CfgKey), -    empty_cth:post_end_per_testcase(TC,Config,Return,State). +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> -    empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> +    empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> -    empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State).  terminate(State) ->      empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl index 9abd2e5e83..42e086b96e 100644 --- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl +++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/verify_data_dir_cth.erl @@ -62,43 +62,43 @@ post_end_per_suite(Suite,Config,Return,State) ->      check_dirs(State,Config),      empty_cth:post_end_per_suite(Suite,Config,Return,State). -pre_init_per_group(Group,Config,State) -> +pre_init_per_group(Suite,Group,Config,State) ->      check_dirs(State,Config), -    empty_cth:pre_init_per_group(Group,Config,State). +    empty_cth:pre_init_per_group(Suite,Group,Config,State). -post_init_per_group(Group,Config,Return,State) -> +post_init_per_group(Suite,Group,Config,Return,State) ->      check_dirs(State,Return), -    empty_cth:post_init_per_group(Group,Config,Return,State). +    empty_cth:post_init_per_group(Suite,Group,Config,Return,State). -pre_end_per_group(Group,Config,State) -> +pre_end_per_group(Suite,Group,Config,State) ->      check_dirs(State,Config), -    empty_cth:pre_end_per_group(Group,Config,State). +    empty_cth:pre_end_per_group(Suite,Group,Config,State). -post_end_per_group(Group,Config,Return,State) -> +post_end_per_group(Suite,Group,Config,Return,State) ->      check_dirs(State,Config), -    empty_cth:post_end_per_group(Group,Config,Return,State). +    empty_cth:post_end_per_group(Suite,Group,Config,Return,State). -pre_init_per_testcase(TC,Config,State) -> +pre_init_per_testcase(Suite,TC,Config,State) ->      check_dirs(State,Config), -    empty_cth:pre_init_per_testcase(TC,Config,State). +    empty_cth:pre_init_per_testcase(Suite,TC,Config,State). -post_init_per_testcase(TC,Config,Return,State) -> +post_init_per_testcase(Suite,TC,Config,Return,State) ->      check_dirs(State,Config), -    empty_cth:post_init_per_testcase(TC,Config,Return,State). +    empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State). -pre_end_per_testcase(TC,Config,State) -> +pre_end_per_testcase(Suite,TC,Config,State) ->      check_dirs(State,Config), -    empty_cth:pre_end_per_testcase(TC,Config,State). +    empty_cth:pre_end_per_testcase(Suite,TC,Config,State). -post_end_per_testcase(TC,Config,Return,State) -> +post_end_per_testcase(Suite,TC,Config,Return,State) ->      check_dirs(State,Config), -    empty_cth:post_end_per_testcase(TC,Config,Return,State). +    empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State). -on_tc_fail(TC, Reason, State) -> -    empty_cth:on_tc_fail(TC,Reason,State). +on_tc_fail(Suite,TC, Reason, State) -> +    empty_cth:on_tc_fail(Suite,TC,Reason,State). -on_tc_skip(TC, Reason, State) -> -    empty_cth:on_tc_skip(TC,Reason,State). +on_tc_skip(Suite,TC, Reason, State) -> +    empty_cth:on_tc_skip(Suite,TC,Reason,State).  terminate(State) ->      empty_cth:terminate(State). diff --git a/lib/common_test/test/ct_repeat_testrun_SUITE.erl b/lib/common_test/test/ct_repeat_testrun_SUITE.erl index f8b6a379f6..76611a2db3 100644 --- a/lib/common_test/test/ct_repeat_testrun_SUITE.erl +++ b/lib/common_test/test/ct_repeat_testrun_SUITE.erl @@ -363,14 +363,17 @@ skip_first_tc1(Suite) ->       {?eh,tc_start,{Suite,tc1}},       {?eh,tc_done,{Suite,tc1,ok}},       {?eh,test_stats,{'_',0,{0,0}}}, +     {?eh,tc_start,{Suite,tc2}},       {?eh,tc_done,{Suite,tc2,?skipped}},       {?eh,test_stats,{'_',0,{0,1}}}, +     {?eh,tc_start,{Suite,{init_per_group,g,[]}}},       {?eh,tc_done,{Suite,{init_per_group,g,[]},?skipped}},       {?eh,tc_auto_skip,{Suite,{tc1,g},?skip_reason}},       {?eh,test_stats,{'_',0,{0,2}}},       {?eh,tc_auto_skip,{Suite,{tc2,g},?skip_reason}},       {?eh,test_stats,{'_',0,{0,3}}},       {?eh,tc_auto_skip,{Suite,{end_per_group,g},?skip_reason}}, +     {?eh,tc_start,{Suite,tc2}},       {?eh,tc_done,{Suite,tc2,?skipped}},       {?eh,test_stats,{'_',0,{0,4}}},       {?eh,tc_start,{Suite,end_per_suite}}, @@ -390,10 +393,12 @@ skip_tc1_in_group(Suite) ->        {?eh,tc_start,{Suite,tc1}},        {?eh,tc_done,{Suite,tc1,ok}},        {?eh,test_stats,{'_',0,{0,0}}}, +      {?eh,tc_start,{Suite,tc2}},        {?eh,tc_done,{Suite,tc2,?skipped}},        {?eh,test_stats,{'_',0,{0,1}}},        {?eh,tc_start,{Suite,{end_per_group,g,[]}}},        {?eh,tc_done,{Suite,{end_per_group,g,[]},ok}}], +     {?eh,tc_start,{Suite,tc2}},       {?eh,tc_done,{Suite,tc2,?skipped}},       {?eh,test_stats,{'_',0,{0,2}}},       {?eh,tc_start,{Suite,end_per_suite}}, diff --git a/lib/common_test/test/ct_surefire_SUITE.erl b/lib/common_test/test/ct_surefire_SUITE.erl index 42ec685c16..884217afc2 100644 --- a/lib/common_test/test/ct_surefire_SUITE.erl +++ b/lib/common_test/test/ct_surefire_SUITE.erl @@ -73,7 +73,9 @@ all() ->       relative_path,       url,       logdir, -     fail_pre_init_per_suite +     fail_pre_init_per_suite, +     skip_case_in_spec, +     skip_suite_in_spec      ].  %%-------------------------------------------------------------------- @@ -119,6 +121,18 @@ fail_pre_init_per_suite(Config) when is_list(Config) ->      run(fail_pre_init_per_suite,[fail_pre_init_per_suite,          {cth_surefire,[{path,Path}]}],Path,Config,[],Suites). +skip_case_in_spec(Config) -> +    DataDir = ?config(data_dir,Config), +    Spec = filename:join(DataDir,"skip_one_case.spec"), +    Path = "skip_case_in_spec.xml", +    run_spec(skip_case_in_spec,[{cth_surefire,[{path,Path}]}],Path,Config,Spec). + +skip_suite_in_spec(Config) -> +    DataDir = ?config(data_dir,Config), +    Spec = filename:join(DataDir,"skip_one_suite.spec"), +    Path = "skip_suite_in_spec.xml", +    run_spec(skip_suite_in_spec,[{cth_surefire,[{path,Path}]}],Path,Config,Spec). +  %%%-----------------------------------------------------------------  %%% HELP FUNCTIONS  %%%----------------------------------------------------------------- @@ -129,8 +143,15 @@ run(Case,CTHs,Report,Config,ExtraOpts) ->      Suite = filename:join(DataDir, "surefire_SUITE"),      run(Case,CTHs,Report,Config,ExtraOpts,Suite).  run(Case,CTHs,Report,Config,ExtraOpts,Suite) -> -    {Opts,ERPid} = setup([{suite,Suite},{ct_hooks,CTHs},{label,Case}|ExtraOpts], -			 Config), +    Test = [{suite,Suite},{ct_hooks,CTHs},{label,Case}|ExtraOpts], +    do_run(Case, Report, Test, Config). + +run_spec(Case,CTHs,Report,Config,Spec) -> +    Test = [{spec,Spec},{ct_hooks,CTHs},{label,Case}], +    do_run(Case, Report, Test, Config). + +do_run(Case, Report, Test, Config) -> +    {Opts,ERPid} = setup(Test, Config),      ok = execute(Case, Opts, ERPid, Config),      LogDir =  	case lists:keyfind(logdir,1,Opts) of @@ -201,7 +222,10 @@ test_suite_events(pass_SUITE) ->       {?eh,test_stats,{1,0,{0,0}}},       {?eh,tc_start,{ct_framework,end_per_suite}},       {?eh,tc_done,{ct_framework,end_per_suite,ok}}]; -test_suite_events(_) -> +test_suite_events(skip_all_surefire_SUITE) -> +    [{?eh,tc_user_skip,{skip_all_surefire_SUITE,all,"skipped in spec"}}, +     {?eh,test_stats,{0,0,{1,0}}}]; +test_suite_events(Test) ->      [{?eh,tc_start,{surefire_SUITE,init_per_suite}},       {?eh,tc_done,{surefire_SUITE,init_per_suite,ok}},       {?eh,tc_start,{surefire_SUITE,tc_ok}}, @@ -210,46 +234,55 @@ test_suite_events(_) ->       {?eh,tc_start,{surefire_SUITE,tc_fail}},       {?eh,tc_done,{surefire_SUITE,tc_fail,  		   {failed,{error,{test_case_failed,"this test should fail"}}}}}, -     {?eh,test_stats,{1,1,{0,0}}}, -     {?eh,tc_start,{surefire_SUITE,tc_skip}}, -     {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}, -     {?eh,test_stats,{1,1,{1,0}}}, -     {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, -     {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, -		   {auto_skipped,{require_failed,'_'}}}}, -     {?eh,test_stats,{1,1,{1,1}}}, -     [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}}, -      {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}}, -      {?eh,tc_start,{surefire_SUITE,tc_ok}}, -      {?eh,tc_done,{surefire_SUITE,tc_ok,ok}}, -      {?eh,test_stats,{2,1,{1,1}}}, -      {?eh,tc_start,{surefire_SUITE,tc_fail}}, -      {?eh,tc_done,{surefire_SUITE,tc_fail, -		    {failed,{error,{test_case_failed,"this test should fail"}}}}}, -      {?eh,test_stats,{2,2,{1,1}}}, -      {?eh,tc_start,{surefire_SUITE,tc_skip}}, -      {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}, -      {?eh,test_stats,{2,2,{2,1}}}, -      {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, -      {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, -		    {auto_skipped,{require_failed,'_'}}}}, -      {?eh,test_stats,{2,2,{2,2}}}, -      {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}}, -      {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}], -     [{?eh,tc_start,{surefire_SUITE,{init_per_group,g_fail,[]}}}, -      {?eh,tc_done,{surefire_SUITE,{init_per_group,g_fail,[]}, -		    {failed,{error,all_cases_should_be_skipped}}}}, -      {?eh,tc_auto_skip,{surefire_SUITE,{tc_ok,g_fail}, -			 {failed, -			  {surefire_SUITE,init_per_group, -			   {'EXIT',all_cases_should_be_skipped}}}}}, -      {?eh,test_stats,{2,2,{2,3}}}, -      {?eh,tc_auto_skip,{surefire_SUITE,{end_per_group,g_fail}, -			 {failed, -			  {surefire_SUITE,init_per_group, -			   {'EXIT',all_cases_should_be_skipped}}}}}], -     {?eh,tc_start,{surefire_SUITE,end_per_suite}}, -     {?eh,tc_done,{surefire_SUITE,end_per_suite,ok}}]. +     {?eh,test_stats,{1,1,{0,0}}}] ++ +        tc_skip_events(Test,undefined) ++ +        [{?eh,test_stats,{1,1,{1,0}}}, +         {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, +         {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, +                       {auto_skipped,{require_failed,'_'}}}}, +         {?eh,test_stats,{1,1,{1,1}}}, +         [{?eh,tc_start,{surefire_SUITE,{init_per_group,g,[]}}}, +          {?eh,tc_done,{surefire_SUITE,{init_per_group,g,[]},ok}}, +          {?eh,tc_start,{surefire_SUITE,tc_ok}}, +          {?eh,tc_done,{surefire_SUITE,tc_ok,ok}}, +          {?eh,test_stats,{2,1,{1,1}}}, +          {?eh,tc_start,{surefire_SUITE,tc_fail}}, +          {?eh,tc_done,{surefire_SUITE,tc_fail, +                        {failed,{error,{test_case_failed,"this test should fail"}}}}}, +          {?eh,test_stats,{2,2,{1,1}}}] ++ +             tc_skip_events(Test,g) ++ +             [{?eh,test_stats,{2,2,{2,1}}}, +              {?eh,tc_start,{surefire_SUITE,tc_autoskip_require}}, +              {?eh,tc_done,{surefire_SUITE,tc_autoskip_require, +                            {auto_skipped,{require_failed,'_'}}}}, +              {?eh,test_stats,{2,2,{2,2}}}, +              {?eh,tc_start,{surefire_SUITE,{end_per_group,g,[]}}}, +              {?eh,tc_done,{surefire_SUITE,{end_per_group,g,[]},ok}}], +         [{?eh,tc_start,{surefire_SUITE,{init_per_group,g_fail,[]}}}, +          {?eh,tc_done,{surefire_SUITE,{init_per_group,g_fail,[]}, +                        {failed,{error,all_cases_should_be_skipped}}}}, +          {?eh,tc_auto_skip,{surefire_SUITE,{tc_ok,g_fail}, +                             {failed, +                              {surefire_SUITE,init_per_group, +                               {'EXIT',all_cases_should_be_skipped}}}}}, +          {?eh,test_stats,{2,2,{2,3}}}, +          {?eh,tc_auto_skip,{surefire_SUITE,{end_per_group,g_fail}, +                             {failed, +                              {surefire_SUITE,init_per_group, +                               {'EXIT',all_cases_should_be_skipped}}}}}], +         {?eh,tc_start,{surefire_SUITE,end_per_suite}}, +         {?eh,tc_done,{surefire_SUITE,end_per_suite,ok}}]. + +tc_skip_events(skip_case_in_spec,Group) -> +    [{?eh,tc_user_skip,{surefire_SUITE,tc_skip_name(Group),"skipped in spec"}}]; +tc_skip_events(_Test,_Group) -> +    [{?eh,tc_start,{surefire_SUITE,tc_skip}}, +     {?eh,tc_done,{surefire_SUITE,tc_skip,{skipped,"this test is skipped"}}}]. + +tc_skip_name(undefined) -> +    tc_skip; +tc_skip_name(Group) -> +    {tc_skip,Group}.  test_events(fail_pre_init_per_suite) ->      [{?eh,start_logging,{'DEF','RUNDIR'}}, @@ -257,6 +290,10 @@ test_events(fail_pre_init_per_suite) ->       test_suite_events(pass_SUITE) ++       test_suite_events(fail_SUITE, {1,0,{0,1}}) ++       [{?eh,stop_logging,[]}]; +test_events(skip_suite_in_spec) -> +    [{?eh,start_logging,'_'},{?eh,start_info,{1,1,0}}] ++ +     test_suite_events(skip_all_surefire_SUITE) ++ +     [{?eh,stop_logging,[]}];  test_events(Test) ->      [{?eh,start_logging,'_'}, {?eh,start_info,{1,1,9}}] ++      test_suite_events(Test) ++ @@ -364,6 +401,8 @@ failed_or_skipped([]) ->  events_to_result(E) ->      events_to_result(E, []). +events_to_result([{?eh,tc_user_skip,{_Suite,all,_}}|E], Result) -> +    events_to_result(E, [[[s]]|Result]);  events_to_result([{?eh,tc_auto_skip,{_Suite,init_per_suite,_}}|E], Result) ->      {Suite,Rest} = events_to_result1(E),      events_to_result(Rest, [[[s]|Suite]|Result]); @@ -382,7 +421,7 @@ events_to_result1([{?eh,tc_done,{_Suite, end_per_suite,R}}|E]) ->  events_to_result1([{?eh,tc_done,{_Suite,_Case,R}}|E]) ->      {Suite,Rest} = events_to_result1(E),      {[result(R)|Suite],Rest}; -events_to_result1([{?eh,tc_auto_skip,_}|E]) -> +events_to_result1([{?eh,Skip,_}|E]) when Skip==tc_auto_skip; Skip==tc_user_skip ->      {Suite,Rest} = events_to_result1(E),      {[[s]|Suite],Rest};  events_to_result1([_|E]) -> diff --git a/lib/common_test/test/ct_surefire_SUITE_data/skip_one_case.spec b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_case.spec new file mode 100644 index 0000000000..42df8a7d1a --- /dev/null +++ b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_case.spec @@ -0,0 +1,2 @@ +{suites,".",surefire_SUITE}. +{skip_cases,".",surefire_SUITE,tc_skip,"skipped in spec"}. diff --git a/lib/common_test/test/ct_surefire_SUITE_data/skip_one_suite.spec b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_suite.spec new file mode 100644 index 0000000000..57966328ab --- /dev/null +++ b/lib/common_test/test/ct_surefire_SUITE_data/skip_one_suite.spec @@ -0,0 +1,2 @@ +{suites,".",[skip_all_surefire_SUITE]}. +{skip_suites,".",skip_all_surefire_SUITE,"skipped in spec"}. diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl index 228d900545..ea8a1a5662 100644 --- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl +++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl @@ -161,6 +161,7 @@ test_events(ts_if_1) ->       {?eh,tc_start,{ts_if_1_SUITE,tc4}},       {?eh,tc_done,{ts_if_1_SUITE,tc4,{failed,{error,failed_on_purpose}}}},       {?eh,test_stats,{1,2,{0,1}}}, +     {?eh,tc_start,{ts_if_1_SUITE,tc5}},       {?eh,tc_done,{ts_if_1_SUITE,tc5,{auto_skipped,{sequence_failed,seq1,tc4}}}},       {?eh,test_stats,{1,2,{0,2}}}, diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl index e926abd885..05a452b99d 100644 --- a/lib/common_test/test/ct_test_support.erl +++ b/lib/common_test/test/ct_test_support.erl @@ -765,23 +765,23 @@ locate({parallel,TEvs}, Node, Evs, Config) ->  				  {Done,RemEvs2,length(RemEvs2)}  			  end;  		     %% end_per_group auto- or user skipped -		     (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) +		     (TEv={TEH,AutoOrUserSkip,{M,{end_per_group,G},R}}, {Done,RemEvs,_RemSize})  			when AutoOrUserSkip == tc_auto_skip;  			     AutoOrUserSkip == tc_user_skip ->  			  RemEvs1 =   			      lists:dropwhile(  				fun({EH,#event{name=tc_auto_skip,  					       node=EvNode, -					       data={Mod,end_per_group,Reason}}}) when -				   EH == TEH, EvNode == Node, Mod == M -> +					       data={Mod,{end_per_group,EvGroupName},Reason}}}) when +				   EH == TEH, EvNode == Node, Mod == M, EvGroupName == G ->  					case match_data(R, Reason) of  					    match -> false;  					    _ -> true  					end;  				   ({EH,#event{name=tc_user_skip,  					       node=EvNode, -					       data={Mod,end_per_group,Reason}}}) when -				   EH == TEH, EvNode == Node, Mod == M -> +					       data={Mod,{end_per_group,EvGroupName},Reason}}}) when +				   EH == TEH, EvNode == Node, Mod == M, EvGroupName == G ->  					case match_data(R, Reason) of  					    match -> false;  					    _ -> true @@ -1008,20 +1008,20 @@ locate({shuffle,TEvs}, Node, Evs, Config) ->  				  {Done,RemEvs2,length(RemEvs2)}  			  end;  		     %% end_per_group auto-or user skipped -		     (TEv={TEH,AutoOrUserSkip,{M,end_per_group,R}}, {Done,RemEvs,_RemSize}) +		     (TEv={TEH,AutoOrUserSkip,{M,{end_per_group,G},R}}, {Done,RemEvs,_RemSize})  			when AutoOrUserSkip == tc_auto_skip;  			     AutoOrUserSkip == tc_user_skip ->  			  RemEvs1 =   			      lists:dropwhile(  				fun({EH,#event{name=tc_auto_skip,  					       node=EvNode, -					       data={Mod,end_per_group,Reason}}}) when -				   EH == TEH, EvNode == Node, Mod == M, Reason == R -> +					       data={Mod,{end_per_group,EvGroupName},Reason}}}) when +				   EH == TEH, EvNode == Node, Mod == M, EvGroupName == G, Reason == R ->  					false;  				   ({EH,#event{name=tc_user_skip,  					       node=EvNode, -					       data={Mod,end_per_group,Reason}}}) when -				   EH == TEH, EvNode == Node, Mod == M, Reason == R -> +					       data={Mod,{end_per_group,EvGroupName},Reason}}}) when +				   EH == TEH, EvNode == Node, Mod == M, EvGroupName == G, Reason == R ->  					false;  				   ({EH,#event{name=stop_logging,  					       node=EvNode,data=_}}) when @@ -1264,10 +1264,10 @@ log_events1([E={_EH,tc_done,{_M,{end_per_group,_GrName,Props},_R}} | Evs], Dev,  	    io:format(Dev, "~s~p]},~n", [Ind,E]),  	    log_events1(Evs, Dev, Ind--"  ")      end; -log_events1([E={_EH,tc_auto_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) -> +log_events1([E={_EH,tc_auto_skip,{_M,{end_per_group,_GrName},_Reason}} | Evs], Dev, Ind) ->      io:format(Dev, "~s~p],~n", [Ind,E]),      log_events1(Evs, Dev, Ind--" "); -log_events1([E={_EH,tc_user_skip,{_M,end_per_group,_Reason}} | Evs], Dev, Ind) -> +log_events1([E={_EH,tc_user_skip,{_M,{end_per_group,_GrName},_Reason}} | Evs], Dev, Ind) ->      io:format(Dev, "~s~p],~n", [Ind,E]),      log_events1(Evs, Dev, Ind--" ");  log_events1([E], Dev, Ind) -> | 
