aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorErlang/OTP <[email protected]>2019-04-16 16:37:34 +0200
committerErlang/OTP <[email protected]>2019-04-16 16:37:34 +0200
commit4351793954bc554c95a91d659e3f472b8b98d16c (patch)
tree69cdb803a96ac05ae19b970f07cb47bb3e86346f
parentcdcc5e2d74691d6a070f957afc7f6cb8b7141dd2 (diff)
parent54e599a8be99fe41b45c913b59277dd6e3189e27 (diff)
downloadotp-4351793954bc554c95a91d659e3f472b8b98d16c.tar.gz
otp-4351793954bc554c95a91d659e3f472b8b98d16c.tar.bz2
otp-4351793954bc554c95a91d659e3f472b8b98d16c.zip
Merge branch 'siri/ct/fuzzer_support/ERIERL-143/OTP-14746' into maint-20
* siri/ct/fuzzer_support/ERIERL-143/OTP-14746: [ct] Fix bug with faulty suite name in end_per_suite config [ct] Add {testcase,TC,RepeatProps} syntax for repeating test cases [ct] Add post_groups/2 and post_all/3 hook functions # Conflicts: # lib/common_test/test/Makefile
Diffstat
-rw-r--r--lib/common_test/doc/src/common_test_app.xml17
-rw-r--r--lib/common_test/doc/src/ct_hooks.xml125
-rw-r--r--lib/common_test/doc/src/write_test_chapter.xml18
-rw-r--r--lib/common_test/src/ct_framework.erl305
-rw-r--r--lib/common_test/src/ct_groups.erl83
-rw-r--r--lib/common_test/src/ct_hooks.erl76
-rw-r--r--lib/common_test/src/test_server_ctrl.erl133
-rw-r--r--lib/common_test/test/Makefile3
-rw-r--r--lib/common_test/test/ct_hooks_SUITE.erl345
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_SUITE.erl47
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_cth.erl100
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_match_state_cth.erl58
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_group_only_cth_SUITE.erl54
-rw-r--r--lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl28
-rw-r--r--lib/common_test/test/ct_tc_repeat_SUITE.erl438
-rw-r--r--lib/common_test/test/ct_tc_repeat_SUITE_data/tc_repeat_SUITE.erl85
16 files changed, 1765 insertions, 150 deletions
diff --git a/lib/common_test/doc/src/common_test_app.xml b/lib/common_test/doc/src/common_test_app.xml
index a3b3f927eb..5fa87901f6 100644
--- a/lib/common_test/doc/src/common_test_app.xml
+++ b/lib/common_test/doc/src/common_test_app.xml
@@ -72,14 +72,15 @@
<fsummary>Returns the list of all test case groups and test cases
in the module.</fsummary>
<type>
- <v>Tests = [TestCase | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}]</v>
+ <v>Tests = [TestCase | {testcase,TestCase,TCRepeatProps} | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}]</v>
<v>TestCase = atom()</v>
+ <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v>
<v>GroupName = atom()</v>
- <v>Properties = [parallel | sequence | Shuffle | {RepeatType,N}] | default</v>
+ <v>Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}] | default</v>
<v>SubGroups = [{GroupName,Properties} | {GroupName,Properties,SubGroups}]</v>
<v>Shuffle = shuffle | {shuffle,Seed}</v>
<v>Seed = {integer(),integer(),integer()}</v>
- <v>RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v>
+ <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v>
<v>N = integer() | forever</v>
<v>Reason = term()</v>
</type>
@@ -91,7 +92,8 @@
test suite module to be executed. This list also specifies the
order the cases and groups are executed by <c>Common Test</c>.
A test case is represented by an atom,
- the name of the test case function. A test case group is
+ the name of the test case function, or a <c>testcase</c> tuple
+ indicating that the test case shall be repeated. A test case group is
represented by a <c>group</c> tuple, where <c>GroupName</c>,
an atom, is the name of the group (defined in
<seealso marker="#Module:groups-0"><c>groups/0</c></seealso>).
@@ -121,12 +123,13 @@
<v>GroupDefs = [Group]</v>
<v>Group = {GroupName,Properties,GroupsAndTestCases}</v>
<v>GroupName = atom()</v>
- <v>Properties = [parallel | sequence | Shuffle | {RepeatType,N}]</v>
- <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase]</v>
+ <v>Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]</v>
+ <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase | {testcase,TestCase,TCRepeatProps}]</v>
<v>TestCase = atom()</v>
+ <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v>
<v>Shuffle = shuffle | {shuffle,Seed}</v>
<v>Seed = {integer(),integer(),integer()}</v>
- <v>RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v>
+ <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v>
<v>N = integer() | forever</v>
</type>
diff --git a/lib/common_test/doc/src/ct_hooks.xml b/lib/common_test/doc/src/ct_hooks.xml
index 954be0ffba..03dfceaa1f 100644
--- a/lib/common_test/doc/src/ct_hooks.xml
+++ b/lib/common_test/doc/src/ct_hooks.xml
@@ -109,6 +109,131 @@
</func>
<func>
+ <name>Module:post_groups(SuiteName, GroupDefs) -&gt; NewGroupDefs</name>
+ <fsummary>Called after groups/0.</fsummary>
+ <type>
+ <v>SuiteName = atom()</v>
+ <v>GroupDefs = NewGroupDefs = [Group]</v>
+ <v>Group = {GroupName,Properties,GroupsAndTestCases}</v>
+ <v>GroupName = atom()</v>
+ <v>Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]</v>
+ <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase | {testcase,TestCase,TCRepeatProps}]</v>
+ <v>TestCase = atom()</v>
+ <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v>
+ <v>Shuffle = shuffle | {shuffle,Seed}</v>
+ <v>Seed = {integer(),integer(),integer()}</v>
+ <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v>
+ <v>N = integer() | forever</v>
+ </type>
+ <desc>
+ <p>OPTIONAL</p>
+
+ <p>This function is called after
+ <seealso marker="common_test#Module:groups-0"><c>groups/0</c></seealso>.
+ It is used to modify the test group definitions, for
+ instance to add or remove groups or change group properties.</p>
+
+ <p><c>GroupDefs</c> is what
+ <seealso marker="common_test#Module:groups-0"><c>groups/0</c></seealso>
+ returned, that is, a list of group definitions.</p>
+
+ <p><c>NewGroupDefs</c> is the possibly modified version of this list.</p>
+
+ <p>This function is called only if the CTH is added before
+ <c>init_per_suite</c> is run. For details, see section
+ <seealso marker="ct_hooks_chapter#scope">CTH Scope</seealso>
+ in the User's Guide.</p>
+
+ <p>Notice that for CTHs that are installed by means of the
+ <seealso marker="common_test#Module:suite-0"><c>suite/0</c></seealso>
+ function, <c>post_groups/2</c> is called before
+ the <seealso marker="#Module:init-2"><c>init/2</c></seealso>
+ hook function. However, for CTHs that are installed by means
+ of the CT start flag,
+ the <seealso marker="#Module:init-2"><c>init/2</c></seealso>
+ function is called first.</p>
+
+ <note>
+ <p>Prior to each test execution, Common Test does a
+ simulated test run in order to count test suites, groups
+ and cases for logging purposes. This causes
+ the <c>post_groups/2</c> hook function to always be called
+ twice. For this reason, side effects are best avoided in
+ this callback.</p>
+ </note>
+ </desc>
+ </func>
+
+ <func>
+ <name>Module:post_all(SuiteName, Return, GroupDefs) -&gt; NewReturn</name>
+ <fsummary>Called after all/0.</fsummary>
+ <type>
+ <v>SuiteName = atom()</v>
+ <v>Return = NewReturn = Tests | {skip,Reason}</v>
+ <v>Tests = [TestCase | {testcase,TestCase,TCRepeatProps} | {group,GroupName} | {group,GroupName,Properties} | {group,GroupName,Properties,SubGroups}]</v>
+ <v>TestCase = atom()</v>
+ <v>TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</v>
+ <v>GroupName = atom()</v>
+ <v>Properties = GroupProperties | default</v>
+ <v>SubGroups = [{GroupName,Properties} | {GroupName,Properties,SubGroups}]</v>
+ <v>Shuffle = shuffle | {shuffle,Seed}</v>
+ <v>Seed = {integer(),integer(),integer()}</v>
+ <v>GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail | repeat_until_any_ok | repeat_until_any_fail</v>
+ <v>N = integer() | forever</v>
+ <v>GroupDefs = NewGroupDefs = [Group]</v>
+ <v>Group = {GroupName,GroupProperties,GroupsAndTestCases}</v>
+ <v>GroupProperties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]</v>
+ <v>GroupsAndTestCases = [Group | {group,GroupName} | TestCase]</v>
+ <v>Reason = term()</v>
+ </type>
+ <desc>
+ <p>OPTIONAL</p>
+
+ <p>This function is called after
+ <seealso marker="common_test#Module:all-0"><c>all/0</c></seealso>.
+ It is used to modify the set of test cases and test group to
+ be executed, for instance to add or remove test cases and
+ groups, change group properties, or even skip all tests in
+ the suite.</p>
+
+ <p><c>Return</c> is what
+ <seealso marker="common_test#Module:all-0"><c>all/0</c></seealso>
+ returned, that is, a list of test cases and groups to be
+ executed, or a tuple <c>{skip,Reason}</c>.</p>
+
+ <p><c>GroupDefs</c> is what
+ <seealso marker="common_test#Module:groups-0"><c>groups/0</c></seealso>
+ or the <c>post_groups/2</c> hook returned, that is, a list
+ of group definitions.</p>
+
+ <p><c>NewReturn</c> is the possibly modified version of <c>Return</c>.</p>
+
+ <p>This function is called only if the CTH is added before
+ <c>init_per_suite</c> is run. For details, see section
+ <seealso marker="ct_hooks_chapter#scope">CTH Scope</seealso>
+ in the User's Guide.</p>
+
+ <p>Notice that for CTHs that are installed by means of the
+ <seealso marker="common_test#Module:suite-0"><c>suite/0</c></seealso>
+ function, <c>post_all/2</c> is called before
+ the <seealso marker="#Module:init-2"><c>init/2</c></seealso>
+ hook function. However, for CTHs that are installed by means
+ of the CT start flag,
+ the <seealso marker="#Module:init-2"><c>init/2</c></seealso>
+ function is called first.</p>
+
+ <note>
+ <p>Prior to each test execution, Common Test does a
+ simulated test run in order to count test suites, groups
+ and cases for logging purposes. This causes
+ the <c>post_all/3</c> hook function to always be called
+ twice. For this reason, side effects are best avoided in
+ this callback.</p>
+ </note>
+ </desc>
+ </func>
+
+ <func>
<name>Module:pre_init_per_suite(SuiteName, InitData, CTHState) -&gt; Result</name>
<fsummary>Called before init_per_suite.</fsummary>
<type>
diff --git a/lib/common_test/doc/src/write_test_chapter.xml b/lib/common_test/doc/src/write_test_chapter.xml
index 82dc06834f..5eed748b08 100644
--- a/lib/common_test/doc/src/write_test_chapter.xml
+++ b/lib/common_test/doc/src/write_test_chapter.xml
@@ -455,8 +455,10 @@
GroupDefs = [GroupDef]
GroupDef = {GroupName,Properties,GroupsAndTestCases}
GroupName = atom()
- GroupsAndTestCases = [GroupDef | {group,GroupName} | TestCase]
- TestCase = atom()</pre>
+ GroupsAndTestCases = [GroupDef | {group,GroupName} | TestCase |
+ {testcase,TestCase,TCRepeatProps}]
+ TestCase = atom()
+ TCRepeatProps = [{repeat,N} | {repeat_until_ok,N} | {repeat_until_fail,N}]</pre>
<p><c>GroupName</c> is the name of the group and must be unique within
the test suite module. Groups can be nested, by including a group definition
@@ -464,11 +466,11 @@
<c>Properties</c> is the list of execution
properties for the group. The possible values are as follows:</p>
<pre>
- Properties = [parallel | sequence | Shuffle | {RepeatType,N}]
+ Properties = [parallel | sequence | Shuffle | {GroupRepeatType,N}]
Shuffle = shuffle | {shuffle,Seed}
Seed = {integer(),integer(),integer()}
- RepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
- repeat_until_any_ok | repeat_until_any_fail
+ GroupRepeatType = repeat | repeat_until_all_ok | repeat_until_all_fail |
+ repeat_until_any_ok | repeat_until_any_fail
N = integer() | forever</pre>
<p><em>Explanations:</em></p>
@@ -481,8 +483,8 @@
Dependencies Between Test Cases and Suites.</p></item>
<tag><c>shuffle</c></tag>
<item><p>The cases in the group are executed in random order.</p></item>
- <tag><c>repeat</c></tag>
- <item><p>Orders <c>Common Test</c> to repeat execution of the cases in the
+ <tag><c>repeat, repeat_until_*</c></tag>
+ <item><p>Orders <c>Common Test</c> to repeat execution of all the cases in the
group a given number of times, or until any, or all, cases fail or succeed.</p></item>
</taglist>
@@ -496,7 +498,7 @@
<c>{group,GroupName}</c> to the <c>all/0</c> list.</p>
<p><em>Example:</em></p>
<pre>
- all() -> [testcase1, {group,group1}, testcase2, {group,group2}].</pre>
+ all() -> [testcase1, {group,group1}, {testcase,testcase2,[{repeat,10}]}, {group,group2}].</pre>
<p>Execution properties with a group tuple in
<c>all/0</c>: <c>{group,GroupName,Properties}</c> can also be specified.
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index de72344611..0437d6786f 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -1066,21 +1066,41 @@ group_or_func(Func, _Config) ->
%%% should be returned.
get_suite(Mod, all) ->
- case catch apply(Mod, groups, []) of
- {'EXIT',_} ->
- get_all(Mod, []);
- GroupDefs when is_list(GroupDefs) ->
- case catch ct_groups:find_groups(Mod, all, all, GroupDefs) of
- {error,_} = Error ->
- %% this makes test_server call error_in_suite as first
- %% (and only) test case so we can report Error properly
- [{?MODULE,error_in_suite,[[Error]]}];
- ConfTests ->
- get_all(Mod, ConfTests)
- end;
- _ ->
+ case safe_apply_groups_0(Mod,{ok,[]}) of
+ {ok,GroupDefs} ->
+ try ct_groups:find_groups(Mod, all, all, GroupDefs) of
+ ConfTests when is_list(ConfTests) ->
+ get_all(Mod, ConfTests)
+ catch
+ throw:{error,Error} ->
+ [{?MODULE,error_in_suite,[[{error,Error}]]}];
+ _:Error ->
+ S = erlang:get_stacktrace(),
+ [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
+ end;
+ {error,{bad_return,_Bad}} ->
E = "Bad return value from "++atom_to_list(Mod)++":groups/0",
- [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}]
+ [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}];
+ {error,{bad_hook_return,Bad}} ->
+ E = "Bad return value from post_groups/2 hook function",
+ [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}];
+ {error,{failed,ExitReason}} ->
+ case ct_util:get_testdata({error_in_suite,Mod}) of
+ undefined ->
+ ErrStr = io_lib:format("~n*** ERROR *** "
+ "~w:groups/0 failed: ~p~n",
+ [Mod,ExitReason]),
+ io:format(?def_gl, ErrStr, []),
+ %% save the error info so it doesn't get printed twice
+ ct_util:set_testdata_async({{error_in_suite,Mod},
+ ExitReason});
+ _ExitReason ->
+ ct_util:delete_testdata({error_in_suite,Mod})
+ end,
+ Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"),
+ [{?MODULE,error_in_suite,[[{error,Reason}]]}];
+ {error,What} ->
+ [{?MODULE,error_in_suite,[[{error,What}]]}]
end;
%%!============================================================
@@ -1090,54 +1110,75 @@ get_suite(Mod, all) ->
%% group
get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
- Name = ?val(name, Props),
- case catch apply(Mod, groups, []) of
- {'EXIT',_} ->
- [Group];
- GroupDefs when is_list(GroupDefs) ->
- case catch ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of
- {error,_} = Error ->
- %% this makes test_server call error_in_suite as first
- %% (and only) test case so we can report Error properly
- [{?MODULE,error_in_suite,[[Error]]}];
- [] ->
- [];
- ConfTests ->
- case lists:member(skipped, Props) of
- true ->
- %% a *subgroup* specified *only* as skipped (and not
- %% as an explicit test) should not be returned, or
- %% init/end functions for top groups will be executed
- case catch ?val(name, element(2, hd(ConfTests))) of
- Name -> % top group
- ct_groups:delete_subs(ConfTests, ConfTests);
- _ ->
- []
- end;
- false ->
- ConfTests1 = ct_groups:delete_subs(ConfTests,
- ConfTests),
- case ?val(override, Props) of
- undefined ->
- ConfTests1;
- [] ->
- ConfTests1;
- ORSpec ->
- ORSpec1 = if is_tuple(ORSpec) -> [ORSpec];
- true -> ORSpec end,
- ct_groups:search_and_override(ConfTests1,
- ORSpec1, Mod)
- end
- end
- end;
- _ ->
+ case safe_apply_groups_0(Mod,{ok,[Group]}) of
+ {ok,GroupDefs} ->
+ Name = ?val(name, Props),
+ try ct_groups:find_groups(Mod, Name, TCs, GroupDefs) of
+ [] ->
+ [];
+ ConfTests when is_list(ConfTests) ->
+ case lists:member(skipped, Props) of
+ true ->
+ %% a *subgroup* specified *only* as skipped (and not
+ %% as an explicit test) should not be returned, or
+ %% init/end functions for top groups will be executed
+ try ?val(name, element(2, hd(ConfTests))) of
+ Name -> % top group
+ ct_groups:delete_subs(ConfTests, ConfTests);
+ _ -> []
+ catch
+ _:_ -> []
+ end;
+ false ->
+ ConfTests1 = ct_groups:delete_subs(ConfTests,
+ ConfTests),
+ case ?val(override, Props) of
+ undefined ->
+ ConfTests1;
+ [] ->
+ ConfTests1;
+ ORSpec ->
+ ORSpec1 = if is_tuple(ORSpec) -> [ORSpec];
+ true -> ORSpec end,
+ ct_groups:search_and_override(ConfTests1,
+ ORSpec1, Mod)
+ end
+ end
+ catch
+ throw:{error,Error} ->
+ [{?MODULE,error_in_suite,[[{error,Error}]]}];
+ _:Error ->
+ S = erlang:get_stacktrace(),
+ [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
+ end;
+ {error,{bad_return,_Bad}} ->
E = "Bad return value from "++atom_to_list(Mod)++":groups/0",
- [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}]
+ [{?MODULE,error_in_suite,[[{error,list_to_atom(E)}]]}];
+ {error,{bad_hook_return,Bad}} ->
+ E = "Bad return value from post_groups/2 hook function",
+ [{?MODULE,error_in_suite,[[{error,{list_to_atom(E),Bad}}]]}];
+ {error,{failed,ExitReason}} ->
+ case ct_util:get_testdata({error_in_suite,Mod}) of
+ undefined ->
+ ErrStr = io_lib:format("~n*** ERROR *** "
+ "~w:groups/0 failed: ~p~n",
+ [Mod,ExitReason]),
+ io:format(?def_gl, ErrStr, []),
+ %% save the error info so it doesn't get printed twice
+ ct_util:set_testdata_async({{error_in_suite,Mod},
+ ExitReason});
+ _ExitReason ->
+ ct_util:delete_testdata({error_in_suite,Mod})
+ end,
+ Reason = list_to_atom(atom_to_list(Mod)++":groups/0 failed"),
+ [{?MODULE,error_in_suite,[[{error,Reason}]]}];
+ {error,What} ->
+ [{?MODULE,error_in_suite,[[{error,What}]]}]
end;
%% testcase
get_suite(Mod, Name) ->
- get_seq(Mod, Name).
+ get_seq(Mod, Name).
%%%-----------------------------------------------------------------
@@ -1171,21 +1212,49 @@ get_all_cases1(_, []) ->
%%%-----------------------------------------------------------------
-get_all(Mod, ConfTests) ->
- case catch apply(Mod, all, []) of
- {'EXIT',{undef,[{Mod,all,[],_} | _]}} ->
+get_all(Mod, ConfTests) ->
+ case safe_apply_all_0(Mod) of
+ {ok,AllTCs} ->
+ %% expand group references using ConfTests
+ try ct_groups:expand_groups(AllTCs, ConfTests, Mod) of
+ {error,_} = Error ->
+ [{?MODULE,error_in_suite,[[Error]]}];
+ Tests0 ->
+ Tests = ct_groups:delete_subs(Tests0, Tests0),
+ expand_tests(Mod, Tests)
+ catch
+ throw:{error,Error} ->
+ [{?MODULE,error_in_suite,[[{error,Error}]]}];
+ _:Error ->
+ S = erlang:get_stacktrace(),
+ [{?MODULE,error_in_suite,[[{error,{Error,S}}]]}]
+ end;
+ Skip = {skip,_Reason} ->
+ Skip;
+ {error,undef} ->
+ Reason =
+ case code:which(Mod) of
+ non_existing ->
+ list_to_atom(
+ atom_to_list(Mod)++
+ " cannot be compiled or loaded");
+ _ ->
+ list_to_atom(
+ atom_to_list(Mod)++":all/0 is missing")
+ end,
+ %% this makes test_server call error_in_suite as first
+ %% (and only) test case so we can report Reason properly
+ [{?MODULE,error_in_suite,[[{error,Reason}]]}];
+ {error,{bad_return,_Bad}} ->
Reason =
- case code:which(Mod) of
- non_existing ->
- list_to_atom(atom_to_list(Mod)++
- " can not be compiled or loaded");
- _ ->
- list_to_atom(atom_to_list(Mod)++":all/0 is missing")
- end,
- %% this makes test_server call error_in_suite as first
- %% (and only) test case so we can report Reason properly
+ list_to_atom("Bad return value from "++
+ atom_to_list(Mod)++":all/0"),
[{?MODULE,error_in_suite,[[{error,Reason}]]}];
- {'EXIT',ExitReason} ->
+ {error,{bad_hook_return,Bad}} ->
+ Reason =
+ list_to_atom("Bad return value from post_all/3 hook function"),
+ [{?MODULE,error_in_suite,[[{error,{Reason,Bad}}]]}];
+ {error,{failed,ExitReason}} ->
case ct_util:get_testdata({error_in_suite,Mod}) of
undefined ->
ErrStr = io_lib:format("~n*** ERROR *** "
@@ -1202,28 +1271,8 @@ get_all(Mod, ConfTests) ->
%% this makes test_server call error_in_suite as first
%% (and only) test case so we can report Reason properly
[{?MODULE,error_in_suite,[[{error,Reason}]]}];
- AllTCs when is_list(AllTCs) ->
- case catch save_seqs(Mod,AllTCs) of
- {error,What} ->
- [{?MODULE,error_in_suite,[[{error,What}]]}];
- SeqsAndTCs ->
- %% expand group references in all() using ConfTests
- case catch ct_groups:expand_groups(SeqsAndTCs,
- ConfTests,
- Mod) of
- {error,_} = Error ->
- [{?MODULE,error_in_suite,[[Error]]}];
- Tests ->
- ct_groups:delete_subs(Tests, Tests)
- end
- end;
- Skip = {skip,_Reason} ->
- Skip;
- _ ->
- Reason =
- list_to_atom("Bad return value from "++
- atom_to_list(Mod)++":all/0"),
- [{?MODULE,error_in_suite,[[{error,Reason}]]}]
+ {error,What} ->
+ [{?MODULE,error_in_suite,[[{error,What}]]}]
end.
%%!============================================================
@@ -1581,3 +1630,75 @@ get_html_wrapper(TestName, PrintLabel, Cwd, TableCols, Encoding) ->
%%% @spec get_log_dir() -> {ok,LogDir}
get_log_dir() ->
ct_logs:get_log_dir(true).
+
+%%%-----------------------------------------------------------------
+%%% Call all and group callbacks and post_* hooks with error handling
+safe_apply_all_0(Mod) ->
+ try apply(Mod, all, []) of
+ AllTCs0 when is_list(AllTCs0) ->
+ try save_seqs(Mod,AllTCs0) of
+ SeqsAndTCs when is_list(SeqsAndTCs) ->
+ all_hook(Mod,SeqsAndTCs)
+ catch throw:{error,What} ->
+ {error,What}
+ end;
+ {skip,_}=Skip ->
+ all_hook(Mod,Skip);
+ Bad ->
+ {error,{bad_return,Bad}}
+ catch
+ _:Reason ->
+ handle_callback_crash(Reason,erlang:get_stacktrace(),Mod,all,{error,undef})
+ end.
+
+all_hook(Mod, All) ->
+ case ct_hooks:all(Mod, All) of
+ AllTCs when is_list(AllTCs) ->
+ {ok,AllTCs};
+ {skip,_}=Skip ->
+ Skip;
+ {fail,Reason} ->
+ {error,Reason};
+ Bad ->
+ {error,{bad_hook_return,Bad}}
+ end.
+
+safe_apply_groups_0(Mod,Default) ->
+ try apply(Mod, groups, []) of
+ GroupDefs when is_list(GroupDefs) ->
+ case ct_hooks:groups(Mod, GroupDefs) of
+ GroupDefs1 when is_list(GroupDefs1) ->
+ {ok,GroupDefs1};
+ {fail,Reason} ->
+ {error,Reason};
+ Bad ->
+ {error,{bad_hook_return,Bad}}
+ end;
+ Bad ->
+ {error,{bad_return,Bad}}
+ catch
+ _:Reason ->
+ handle_callback_crash(Reason,erlang:get_stacktrace(),
+ Mod,groups,Default)
+ end.
+
+handle_callback_crash(undef,[{Mod,Func,[],_}|_],Mod,Func,Default) ->
+ case ct_hooks:Func(Mod, []) of
+ [] ->
+ Default;
+ List when is_list(List) ->
+ {ok,List};
+ {fail,Reason} ->
+ {error,Reason};
+ Bad ->
+ {error,{bad_hook_return,Bad}}
+ end;
+handle_callback_crash(Reason,Stacktrace,_Mod,_Func,_Default) ->
+ {error,{failed,{Reason,Stacktrace}}}.
+
+expand_tests(Mod, [{testcase,Case,[Prop]}|Tests]) ->
+ [{repeat,{Mod,Case},Prop}|expand_tests(Mod,Tests)];
+expand_tests(Mod,[Test|Tests]) ->
+ [Test|expand_tests(Mod,Tests)];
+expand_tests(_Mod,[]) ->
+ [].
diff --git a/lib/common_test/src/ct_groups.erl b/lib/common_test/src/ct_groups.erl
index 2235365a0e..b734455f15 100644
--- a/lib/common_test/src/ct_groups.erl
+++ b/lib/common_test/src/ct_groups.erl
@@ -103,23 +103,34 @@ find(Mod, [], TCs, Tests, _Known, _Defs, false) ->
[{Mod,TC}];
({group,_}) ->
[];
+ ({testcase,TC,[Prop]}) when is_atom(TC), TC ==all ->
+ [{repeat,{Mod,TC},Prop}];
({_,_}=TC) when TCs == all ->
[TC];
- (TC) ->
- if is_atom(TC) ->
- Tuple = {Mod,TC},
- case lists:member(Tuple, TCs) of
- true ->
- [Tuple];
- false ->
- case lists:member(TC, TCs) of
- true -> [{Mod,TC}];
- false -> []
- end
- end;
- true ->
- []
- end
+ (TC) when is_atom(TC) ->
+ Tuple = {Mod,TC},
+ case lists:member(Tuple, TCs) of
+ true ->
+ [Tuple];
+ false ->
+ case lists:member(TC, TCs) of
+ true -> [Tuple];
+ false -> []
+ end
+ end;
+ ({testcase,TC,[Prop]}) when is_atom(TC) ->
+ Tuple = {Mod,TC},
+ case lists:member(Tuple, TCs) of
+ true ->
+ [{repeat,Tuple,Prop}];
+ false ->
+ case lists:member(TC, TCs) of
+ true -> [{repeat,Tuple,Prop}];
+ false -> []
+ end
+ end;
+ (_) ->
+ []
end, Tests),
if Cases == [] -> ['NOMATCH'];
true -> Cases
@@ -174,12 +185,19 @@ find(Mod, GrNames, all, [{M,TC} | Gs], Known,
Defs, FindAll) when is_atom(M), M /= group, is_atom(TC) ->
[{M,TC} | find(Mod, GrNames, all, Gs, Known, Defs, FindAll)];
+%% Save test case
+find(Mod, GrNames, all, [{testcase,TC,[Prop]} | Gs], Known,
+ Defs, FindAll) when is_atom(TC) ->
+ [{repeat,{Mod,TC},Prop} | find(Mod, GrNames, all, Gs, Known, Defs, FindAll)];
+
%% Check if test case should be saved
-find(Mod, GrNames, TCs, [TC | Gs], Known,
- Defs, FindAll) when is_atom(TC) orelse
- ((size(TC) == 2) and (element(1,TC) /= group)) ->
+find(Mod, GrNames, TCs, [TC | Gs], Known, Defs, FindAll)
+ when is_atom(TC) orelse
+ ((size(TC) == 3) andalso (element(1,TC) == testcase)) orelse
+ ((size(TC) == 2) and (element(1,TC) /= group)) ->
Case =
- if is_atom(TC) ->
+ case TC of
+ _ when is_atom(TC) ->
Tuple = {Mod,TC},
case lists:member(Tuple, TCs) of
true ->
@@ -190,7 +208,18 @@ find(Mod, GrNames, TCs, [TC | Gs], Known,
false -> []
end
end;
- true ->
+ {testcase,TC0,[Prop]} when is_atom(TC0) ->
+ Tuple = {Mod,TC0},
+ case lists:member(Tuple, TCs) of
+ true ->
+ {repeat,Tuple,Prop};
+ false ->
+ case lists:member(TC0, TCs) of
+ true -> {repeat,{Mod,TC0},Prop};
+ false -> []
+ end
+ end;
+ _ ->
case lists:member(TC, TCs) of
true -> {Mod,TC};
false -> []
@@ -291,12 +320,22 @@ modify_tc_list(GrSpecTs, TSCs, []) ->
modify_tc_list1(GrSpecTs, TSCs);
modify_tc_list(GrSpecTs, _TSCs, _) ->
- [Test || Test <- GrSpecTs, not is_atom(Test)].
+ [Test || Test <- GrSpecTs, not is_atom(Test), element(1,Test)=/=testcase].
modify_tc_list1(GrSpecTs, TSCs) ->
%% remove all cases in group tc list that should not be executed
GrSpecTs1 =
- lists:flatmap(fun(Test) when is_tuple(Test),
+ lists:flatmap(fun(Test={testcase,TC,_}) ->
+ case lists:keysearch(TC, 2, TSCs) of
+ {value,_} ->
+ [Test];
+ _ ->
+ case lists:member(TC, TSCs) of
+ true -> [Test];
+ false -> []
+ end
+ end;
+ (Test) when is_tuple(Test),
(size(Test) > 2) ->
[Test];
(Test={group,_}) ->
diff --git a/lib/common_test/src/ct_hooks.erl b/lib/common_test/src/ct_hooks.erl
index f0592a40be..903710963c 100644
--- a/lib/common_test/src/ct_hooks.erl
+++ b/lib/common_test/src/ct_hooks.erl
@@ -26,6 +26,8 @@
%% API Exports
-export([init/1]).
+-export([groups/2]).
+-export([all/2]).
-export([init_tc/3]).
-export([end_tc/5]).
-export([terminate/1]).
@@ -41,7 +43,8 @@
opts = [],
prio = ctfirst }]).
--record(ct_hook_config, {id, module, prio, scope, opts = [], state = []}).
+-record(ct_hook_config, {id, module, prio, scope, opts = [],
+ state = [], groups = []}).
%% -------------------------------------------------------------------------
%% API Functions
@@ -54,6 +57,47 @@ init(Opts) ->
call(get_builtin_hooks(Opts) ++ get_new_hooks(Opts, undefined),
ok, init, []).
+%% Call the post_groups/2 hook callback
+groups(Mod, Groups) ->
+ Info = try proplists:get_value(ct_hooks, Mod:suite(), []) of
+ CTHooks when is_list(CTHooks) ->
+ [{?config_name,CTHooks}];
+ CTHook when is_atom(CTHook) ->
+ [{?config_name,[CTHook]}]
+ catch _:_ ->
+ %% since this might be the first time Mod:suite()
+ %% is called, and it might just fail or return
+ %% something bad, we allow any failure here - it
+ %% will be catched later if there is something
+ %% really wrong.
+ [{?config_name,[]}]
+ end,
+ case call(fun call_generic/3, Info ++ [{'$ct_groups',Groups}], [post_groups, Mod]) of
+ [{'$ct_groups',NewGroups}] ->
+ NewGroups;
+ Other ->
+ Other
+ end.
+
+%% Call the post_all/3 hook callback
+all(Mod, Tests) ->
+ Info = try proplists:get_value(ct_hooks, Mod:suite(), []) of
+ CTHooks when is_list(CTHooks) ->
+ [{?config_name,CTHooks}];
+ CTHook when is_atom(CTHook) ->
+ [{?config_name,[CTHook]}]
+ catch _:_ ->
+ %% just allow any failure here - it will be catched
+ %% later if there is something really wrong.
+ [{?config_name,[]}]
+ end,
+ case call(fun call_generic/3, Info ++ [{'$ct_all',Tests}], [post_all, Mod]) of
+ [{'$ct_all',NewTests}] ->
+ NewTests;
+ Other ->
+ Other
+ end.
+
%% @doc Called after all suites are done.
-spec terminate(Hooks :: term()) ->
ok.
@@ -88,6 +132,7 @@ init_tc(Mod, init_per_suite, Config) ->
[{?config_name,[]}]
end,
call(fun call_generic/3, Config ++ Info, [pre_init_per_suite, Mod]);
+
init_tc(Mod, end_per_suite, Config) ->
call(fun call_generic/3, Config, [pre_end_per_suite, Mod]);
init_tc(Mod, {init_per_group, GroupName, Properties}, Config) ->
@@ -163,7 +208,7 @@ call_id(#ct_hook_config{ module = Mod, opts = Opts} = Hook, Config, Scope) ->
{Config, Hook#ct_hook_config{ id = Id, scope = scope(Scope)}}.
call_init(#ct_hook_config{ module = Mod, opts = Opts, id = Id, prio = P} = Hook,
- Config,_Meta) ->
+ Config, _Meta) ->
case Mod:init(Id, Opts) of
{ok, NewState} when P =:= undefined ->
{Config, Hook#ct_hook_config{ state = NewState, prio = 0 } };
@@ -194,6 +239,18 @@ call_generic(Hook, Value, Meta) ->
call_generic_fallback(Hook, Value, Meta) ->
do_call_generic(Hook, Value, Meta, true).
+do_call_generic(#ct_hook_config{ module = Mod} = Hook,
+ [{'$ct_groups',Groups}], [post_groups | Args], Fallback) ->
+ NewGroups = catch_apply(Mod, post_groups, Args ++ [Groups],
+ Groups, Fallback),
+ {[{'$ct_groups',NewGroups}], Hook#ct_hook_config{ groups = NewGroups } };
+
+do_call_generic(#ct_hook_config{ module = Mod, groups = Groups} = Hook,
+ [{'$ct_all',Tests}], [post_all | Args], Fallback) ->
+ NewTests = catch_apply(Mod, post_all, Args ++ [Tests, Groups],
+ Tests, Fallback),
+ {[{'$ct_all',NewTests}], Hook};
+
do_call_generic(#ct_hook_config{ module = Mod, state = State} = Hook,
Value, [Function | Args], Fallback) ->
{NewValue, NewState} = catch_apply(Mod, Function, Args ++ [Value, State],
@@ -228,6 +285,12 @@ call([{Hook, call_id, NextFun} | Rest], Config, Meta, Hooks) ->
Rest ++ [{NewId, call_init}]};
ExistingHook when is_tuple(ExistingHook) ->
{Hooks, Rest};
+ _ when hd(Meta)=:=post_groups; hd(Meta)=:=post_all ->
+ %% If CTH is started because of a call from
+ %% groups/2 or all/2, CTH:init/1 must not be
+ %% called (the suite scope should be used).
+ {Hooks ++ [NewHook],
+ Rest ++ [{NewId,NextFun}]};
_ ->
{Hooks ++ [NewHook],
Rest ++ [{NewId, call_init}, {NewId,NextFun}]}
@@ -237,8 +300,8 @@ call([{Hook, call_id, NextFun} | Rest], Config, Meta, Hooks) ->
Trace = erlang:get_stacktrace(),
ct_logs:log("Suite Hook","Failed to start a CTH: ~tp:~tp",
[Error,{Reason,Trace}]),
- call([], {fail,"Failed to start CTH"
- ", see the CT Log for details"}, Meta, Hooks)
+ call([], {fail,"Failed to start CTH, "
+ "see the CT Log for details"}, Meta, Hooks)
end;
call([{HookId, call_init} | Rest], Config, Meta, Hooks) ->
call([{HookId, fun call_init/3} | Rest], Config, Meta, Hooks);
@@ -278,6 +341,10 @@ scope([pre_init_per_suite, SuiteName|_]) ->
[post_end_per_suite, SuiteName];
scope([post_init_per_suite, SuiteName|_]) ->
[post_end_per_suite, SuiteName];
+scope([post_groups, SuiteName|_]) ->
+ [post_groups, SuiteName];
+scope([post_all, SuiteName|_]) ->
+ [post_all, SuiteName];
scope(init) ->
none.
@@ -364,6 +431,7 @@ resort(Calls,Hooks,[F|_R]) when F == pre_end_per_testcase;
F == pre_end_per_suite;
F == post_end_per_suite ->
lists:reverse(resort(Calls,Hooks));
+
resort(Calls,Hooks,_Meta) ->
resort(Calls,Hooks).
diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl
index 69669bbeef..3bb7b6263c 100644
--- a/lib/common_test/src/test_server_ctrl.erl
+++ b/lib/common_test/src/test_server_ctrl.erl
@@ -1443,6 +1443,8 @@ remove_conf([C={Mod,error_in_suite,_}|Cases], NoConf, Repeats) ->
true ->
remove_conf(Cases, [C|NoConf], Repeats)
end;
+remove_conf([C={repeat,_,_}|Cases], NoConf, _Repeats) ->
+ remove_conf(Cases, [C|NoConf], true);
remove_conf([C|Cases], NoConf, Repeats) ->
remove_conf(Cases, [C|NoConf], Repeats);
remove_conf([], NoConf, true) ->
@@ -2060,6 +2062,14 @@ add_init_and_end_per_suite([SkipCase|Cases], LastMod, LastRef, FwMod)
[SkipCase|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)];
add_init_and_end_per_suite([{conf,_,_,_}=Case|Cases], LastMod, LastRef, FwMod) ->
[Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)];
+add_init_and_end_per_suite([{repeat,{Mod,_},_}=Case|Cases], LastMod, LastRef, FwMod)
+ when Mod =/= LastMod, Mod =/= FwMod ->
+ {PreCases, NextMod, NextRef} =
+ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod),
+ PreCases ++ [Case|add_init_and_end_per_suite(Cases, NextMod,
+ NextRef, FwMod)];
+add_init_and_end_per_suite([{repeat,_,_}=Case|Cases], LastMod, LastRef, FwMod) ->
+ [Case|add_init_and_end_per_suite(Cases, LastMod, LastRef, FwMod)];
add_init_and_end_per_suite([{Mod,_}=Case|Cases], LastMod, LastRef, FwMod)
when Mod =/= LastMod, Mod =/= FwMod ->
{PreCases, NextMod, NextRef} =
@@ -2137,7 +2147,7 @@ do_add_init_and_end_per_suite(LastMod, LastRef, Mod, FwMod) ->
%% let's call a "fake" end_per_suite if it exists
case erlang:function_exported(FwMod, end_per_suite, 1) of
true ->
- [{conf,LastRef,[{suite,Mod}],
+ [{conf,LastRef,[{suite,LastMod}],
{FwMod,end_per_suite}}|Init];
false ->
[{conf,LastRef,[],{LastMod,end_per_suite}}|Init]
@@ -2925,6 +2935,29 @@ run_test_cases_loop([{conf,_Ref,_Props,_X}=Conf|_Cases0],
Config, _TimetrapData, _Mode, _Status) ->
erlang:error(badarg, [Conf,Config]);
+run_test_cases_loop([{repeat,Case,{RepeatType,N}}|Cases0], Config,
+ TimeTrapData, Mode, Status) ->
+ Ref = make_ref(),
+ Parallel = check_prop(parallel, Mode) =/= false,
+ Sequence = check_prop(sequence, Mode) =/= false,
+ RepeatStop = RepeatType=:=repeat_until_fail
+ orelse RepeatType=:=repeat_until_ok,
+
+ if Parallel andalso RepeatStop ->
+ %% Cannot check results of test case during parallal
+ %% execution, so only RepeatType=:=repeat is allowed in
+ %% combination with parallel groups.
+ erlang:error({illegal_combination,{parallel,RepeatType}});
+ Sequence andalso RepeatStop ->
+ %% Sequence is stop on fail + skip rest, so only
+ %% RepeatType=:=repeat makes sense inside a sequence.
+ erlang:error({illegal_combination,{sequence,RepeatType}});
+ true ->
+ Mode1 = [{Ref,[{repeat,{RepeatType,1,N}}],?now}|Mode],
+ run_test_cases_loop([Case | Cases0], Config, TimeTrapData,
+ Mode1, Status)
+ end;
+
run_test_cases_loop([{Mod,Case}|Cases], Config, TimetrapData, Mode, Status) ->
ActualCfg =
case get(test_server_create_priv_dir) of
@@ -2937,7 +2970,7 @@ run_test_cases_loop([{Mod,Case}|Cases], Config, TimetrapData, Mode, Status) ->
run_test_cases_loop([{Mod,Case,[ActualCfg]}|Cases], Config,
TimetrapData, Mode, Status);
-run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status) ->
+run_test_cases_loop([{Mod,Func,Args}=Case|Cases], Config, TimetrapData, Mode0, Status) ->
{Num,RunInit} =
case FwMod = get_fw_mod(?MODULE) of
Mod when Func == error_in_suite ->
@@ -2947,6 +2980,14 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status)
run_init}
end,
+ Mode =
+ case Mode0 of
+ [{_,[{repeat,{_,_,_}}],_}|RestMode] ->
+ RestMode;
+ _ ->
+ Mode0
+ end,
+
%% check the current execution mode and save info about the case if
%% detected that printouts to common log files is handled later
@@ -2974,36 +3015,42 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status)
if is_tuple(RetVal) -> element(1,RetVal);
true -> undefined
end,
- {Failed,Status1} =
+ {Result,Failed,Status1} =
case RetTag of
Skip when Skip==skip; Skip==skipped ->
- {false,update_status(skipped, Mod, Func, Status)};
+ {skipped,false,update_status(skipped, Mod, Func, Status)};
Fail when Fail=='EXIT'; Fail==failed ->
- {true,update_status(failed, Mod, Func, Status)};
+ {failed,true,update_status(failed, Mod, Func, Status)};
_ when Time==died, RetVal=/=ok ->
- {true,update_status(failed, Mod, Func, Status)};
+ {failed,true,update_status(failed, Mod, Func, Status)};
_ ->
- {false,update_status(ok, Mod, Func, Status)}
+ {ok,false,update_status(ok, Mod, Func, Status)}
end,
case check_prop(sequence, Mode) of
false ->
+ {Cases1,Mode1} =
+ check_repeat_testcase(Case,Result,Cases,Mode0),
stop_minor_log_file(),
- run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status1);
+ run_test_cases_loop(Cases1, Config, TimetrapData, Mode1, Status1);
Ref ->
%% the case is in a sequence; we must check the result and
%% determine if the following cases should run or be skipped
if not Failed -> % proceed with next case
+ {Cases1,Mode1} =
+ check_repeat_testcase(Case,Result,Cases,Mode0),
stop_minor_log_file(),
- run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status1);
+ run_test_cases_loop(Cases1, Config, TimetrapData, Mode1, Status1);
true -> % skip rest of cases in sequence
print(minor, "~n*** ~tw failed.~n"
" Skipping all other cases in sequence.",
[Func]),
+ {Cases1,Mode1} =
+ check_repeat_testcase(Case,Result,Cases,Mode0),
Reason = {failed,{Mod,Func}},
- Cases2 = skip_cases_upto(Ref, Cases, Reason, tc,
+ Cases2 = skip_cases_upto(Ref, Cases1, Reason, tc,
Mode, auto_skip_case),
stop_minor_log_file(),
- run_test_cases_loop(Cases2, Config, TimetrapData, Mode, Status1)
+ run_test_cases_loop(Cases2, Config, TimetrapData, Mode1, Status1)
end
end;
%% the test case is being executed in parallel with the main process (and
@@ -3012,7 +3059,8 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status)
%% io from Pid will be buffered by the test_server_io process and
%% handled later, so we have to save info about the case
queue_test_case_io(undefined, Pid, Num+1, Mod, Func),
- run_test_cases_loop(Cases, Config, TimetrapData, Mode, Status)
+ {Cases1,Mode1} = check_repeat_testcase(Case,ok,Cases,Mode0),
+ run_test_cases_loop(Cases1, Config, TimetrapData, Mode1, Status)
end;
%% TestSpec processing finished
@@ -3450,9 +3498,19 @@ modify_cases_upto1(Ref, {skip,Reason,FType,Mode,SkipType},
T, Orig, Alt)
end;
-%% next is some other case, ignore or copy
-modify_cases_upto1(Ref, {skip,_,_,_,_}=Op, [_Other|T], Orig, Alt) ->
+%% next is a repeated test case
+modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}=Op,
+ [{repeat,{_M,_F}=MF,_Repeat}|T], Orig, Alt) ->
+ modify_cases_upto1(Ref, Op, T, Orig, [{SkipType,{MF,Reason},Mode}|Alt]);
+
+%% next is an already skipped case, ignore or copy
+modify_cases_upto1(Ref, {skip,_,_,_,_}=Op, [{SkipType,_,_}|T], Orig, Alt)
+ when SkipType=:=skip_case; SkipType=:=auto_skip_case ->
modify_cases_upto1(Ref, Op, T, Orig, Alt);
+
+%% next is some other case, mark as skipped or copy
+modify_cases_upto1(Ref, {skip,Reason,_,Mode,SkipType}=Op, [Other|T], Orig, Alt) ->
+ modify_cases_upto1(Ref, Op, T, Orig, [{SkipType,{Other,Reason},Mode}|Alt]);
modify_cases_upto1(Ref, CopyOp, [C|T], Orig, Alt) ->
modify_cases_upto1(Ref, CopyOp, T, [C|Orig], [C|Alt]).
@@ -4799,6 +4857,14 @@ collect_cases({make,InitMFA,CaseList,FinMFA}, St0, Mode) ->
{error,_Reason} = Error -> Error
end;
+collect_cases({repeat,{Module, Case}, Repeat}, St, Mode) ->
+ case catch collect_case([Case], St#cc{mod=Module}, [], Mode) of
+ {ok, [{Module,Case}], _} ->
+ {ok, [{repeat,{Module, Case}, Repeat}], St};
+ Other ->
+ {error,Other}
+ end;
+
collect_cases({Module, Cases}, St, Mode) when is_list(Cases) ->
case (catch collect_case(Cases, St#cc{mod=Module}, [], Mode)) of
Result = {ok,_,_} ->
@@ -5762,3 +5828,42 @@ encoding(File) ->
E ->
E
end.
+
+check_repeat_testcase(Case,Result,Cases,
+ [{Ref,[{repeat,RepeatData0}],StartTime}|Mode0]) ->
+ case do_update_repeat_data(Result,RepeatData0) of
+ false ->
+ {Cases,Mode0};
+ RepeatData ->
+ {[Case|Cases],[{Ref,[{repeat,RepeatData}],StartTime}|Mode0]}
+ end;
+check_repeat_testcase(_,_,Cases,Mode) ->
+ {Cases,Mode}.
+
+do_update_repeat_data(_,{RT,N,N}) when is_integer(N) ->
+ report_repeat_testcase(N,N),
+ report_stop_repeat_testcase(done,{RT,N}),
+ false;
+do_update_repeat_data(ok,{repeat_until_ok=RT,M,N}) ->
+ report_repeat_testcase(M,N),
+ report_stop_repeat_testcase(RT,{RT,N}),
+ false;
+do_update_repeat_data(failed,{repeat_until_fail=RT,M,N}) ->
+ report_repeat_testcase(M,N),
+ report_stop_repeat_testcase(RT,{RT,N}),
+ false;
+do_update_repeat_data(_,{RT,M,N}) when is_integer(M) ->
+ report_repeat_testcase(M,N),
+ {RT,M+1,N};
+do_update_repeat_data(_,{_,M,N}=RepeatData) ->
+ report_repeat_testcase(M,N),
+ RepeatData.
+
+report_stop_repeat_testcase(Reason,RepVal) ->
+ print(minor, "~n*** Stopping test case repeat operation: ~w", [Reason]),
+ print(1, "Stopping test case repeat operation: ~w", [RepVal]).
+
+report_repeat_testcase(M,forever) ->
+ print(minor, "~n=== Repeated test case: ~w of infinity", [M]);
+report_repeat_testcase(M,N) ->
+ print(minor, "~n=== Repeated test case: ~w of ~w", [M,N]).
diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index bb3e7a1eca..e510b74d6a 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -75,7 +75,8 @@ MODULES= \
ct_keep_logs_SUITE \
ct_unicode_SUITE \
ct_auto_clean_SUITE \
- ct_util_SUITE
+ ct_util_SUITE \
+ ct_tc_repeat_SUITE
ERL_FILES= $(MODULES:%=%.erl)
HRL_FILES= test_server_test_lib.hrl
diff --git a/lib/common_test/test/ct_hooks_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE.erl
index 7900fcd5bc..edcfec4fb5 100644
--- a/lib/common_test/test/ct_hooks_SUITE.erl
+++ b/lib/common_test/test/ct_hooks_SUITE.erl
@@ -73,11 +73,15 @@ all() ->
all(suite) ->
lists:reverse(
[
+ crash_groups, crash_all, bad_return_groups, bad_return_all,
+ illegal_values_groups, illegal_values_all, alter_groups, alter_all,
+ alter_all_to_skip, alter_all_from_skip,
one_cth, two_cth, faulty_cth_no_init, faulty_cth_id_no_init,
faulty_cth_exit_in_init, faulty_cth_exit_in_id,
faulty_cth_exit_in_init_scope_suite, minimal_cth,
minimal_and_maximal_cth, faulty_cth_undef,
scope_per_suite_cth, scope_per_group_cth, scope_suite_cth,
+ scope_suite_group_only_cth,
scope_per_suite_state_cth, scope_per_group_state_cth,
scope_suite_state_cth,
fail_pre_suite_cth, double_fail_pre_suite_cth,
@@ -152,6 +156,11 @@ scope_suite_cth(Config) when is_list(Config) ->
do_test(scope_suite_cth, "ct_scope_suite_cth_SUITE.erl",
[],Config).
+scope_suite_group_only_cth(Config) when is_list(Config) ->
+ do_test(scope_suite_group_only_cth,
+ "ct_scope_suite_group_only_cth_SUITE.erl",
+ [],Config,ok,2,[{group,g1}]).
+
scope_per_group_cth(Config) when is_list(Config) ->
do_test(scope_per_group_cth, "ct_scope_per_group_cth_SUITE.erl",
[],Config).
@@ -299,10 +308,74 @@ repeat_force_stop(Config) ->
[{force_stop,skip_rest},{duration,"000009"}]).
%% Test that expected callbacks, and only those, are called when a test
-%% are fails due to clash in config alias names
+%% fails due to clash in config alias names
config_clash(Config) ->
do_test(config_clash, "config_clash_SUITE.erl", [skip_cth], Config).
+%% Test post_groups and post_all hook callbacks, introduced by OTP-14746
+alter_groups(Config) ->
+ CfgFile = gen_config(?FUNCTION_NAME,
+ [{post_groups_return,[{new_group,[tc1,tc2]}]},
+ {post_all_return,[{group,new_group}]}],Config),
+ do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth],
+ Config, ok, 2, [{config,CfgFile}]).
+
+alter_all(Config) ->
+ CfgFile = gen_config(?FUNCTION_NAME,[{post_all_return,[tc2]}],Config),
+ do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth],
+ Config, ok, 2, [{config,CfgFile}]).
+
+alter_all_from_skip(Config) ->
+ CfgFile = gen_config(?FUNCTION_NAME,[{all_return,{skip,"skipped by all/0"}},
+ {post_all_return,[tc2]}],Config),
+ do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth],
+ Config, ok, 2, [{config,CfgFile}]).
+
+alter_all_to_skip(Config) ->
+ CfgFile = gen_config(?FUNCTION_NAME,
+ [{post_all_return,{skip,"skipped by post_all/3"}}],
+ Config),
+ do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth],
+ Config, ok, 2, [{config,CfgFile}]).
+
+bad_return_groups(Config) ->
+ CfgFile = gen_config(?FUNCTION_NAME,[{post_groups_return,not_a_list}],
+ Config),
+ do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth],
+ Config, ok, 2, [{config,CfgFile}]).
+
+bad_return_all(Config) ->
+ CfgFile = gen_config(?FUNCTION_NAME,[{post_all_return,not_a_list}],
+ Config),
+ do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth],
+ Config, ok, 2, [{config,CfgFile}]).
+
+illegal_values_groups(Config) ->
+ CfgFile = gen_config(?FUNCTION_NAME,
+ [{post_groups_return,[{new_group,[this_test_does_not_exist]},
+ this_is_not_a_group_def]}],
+ Config),
+ do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth],
+ Config, ok, 2, [{config,CfgFile}]).
+
+illegal_values_all(Config) ->
+ CfgFile = gen_config(?FUNCTION_NAME,
+ [{post_all_return,[{group,this_group_does_not_exist},
+ {this_is_not_a_valid_term}]}],
+ Config),
+ do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth],
+ Config, ok, 2, [{config,CfgFile}]).
+
+crash_groups(Config) ->
+ CfgFile = gen_config(?FUNCTION_NAME,[{post_groups_return,crash}],Config),
+ do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth],
+ Config, ok, 2, [{config,CfgFile}]).
+
+crash_all(Config) ->
+ CfgFile = gen_config(?FUNCTION_NAME,[{post_all_return,crash}],Config),
+ do_test(?FUNCTION_NAME, "all_and_groups_SUITE.erl", [all_and_groups_cth],
+ Config, ok, 2, [{config,CfgFile}]).
+
%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
%%%-----------------------------------------------------------------
@@ -322,6 +395,7 @@ do_test(Tag, {WhatTag,Wildcard}, CTHs, Config, Res, EC, ExtraOpts) ->
filename:join([DataDir,"cth/tests",Wildcard])),
{Opts,ERPid} =
setup([{WhatTag,Files},{ct_hooks,CTHs},{label,Tag}|ExtraOpts], Config),
+
Res = ct_test_support:run(Opts, Config),
Events = ct_test_support:get_events(ERPid, Config),
@@ -347,6 +421,13 @@ reformat(Events, EH) ->
%reformat(Events, _EH) ->
% Events.
+gen_config(Name,KeyVals,Config) ->
+ PrivDir = ?config(priv_dir,Config),
+ File = filename:join(PrivDir,atom_to_list(Name)++".cfg"),
+ ok = file:write_file(File,[io_lib:format("~p.~n",[{Key,Value}])
+ || {Key,Value} <- KeyVals]),
+ File.
+
%%%-----------------------------------------------------------------
%%% TEST EVENTS
%%%-----------------------------------------------------------------
@@ -365,13 +446,16 @@ test_events(one_empty_cth) ->
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
{?eh,cth,{empty_cth,id,[[]]}},
{?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ %% check that post_groups and post_all comes after init when hook
+ %% is installed with start flag/option.
+ {?eh,cth,{empty_cth,post_groups,[ct_cth_empty_SUITE,[]]}},
+ {?eh,cth,{empty_cth,post_all,[ct_cth_empty_SUITE,[test_case],[]]}},
{?eh,tc_start,{ct_cth_empty_SUITE,init_per_suite}},
{?eh,cth,{empty_cth,pre_init_per_suite,
[ct_cth_empty_SUITE,'$proplist',[]]}},
{?eh,cth,{empty_cth,post_init_per_suite,
[ct_cth_empty_SUITE,'$proplist','$proplist',[]]}},
{?eh,tc_done,{ct_cth_empty_SUITE,init_per_suite,ok}},
-
{?eh,tc_start,{ct_cth_empty_SUITE,test_case}},
{?eh,cth,{empty_cth,pre_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist',[]]}},
{?eh,cth,{empty_cth,post_init_per_testcase,[ct_cth_empty_SUITE,test_case,'$proplist','_',[]]}},
@@ -580,6 +664,10 @@ test_events(scope_suite_cth) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ %% check that post_groups and post_all comes before init when hook
+ %% is installed in suite/0
+ {?eh,cth,{'_',post_groups,['_',[]]}},
+ {?eh,cth,{'_',post_all,['_','_',[]]}},
{?eh,tc_start,{ct_scope_suite_cth_SUITE,init_per_suite}},
{?eh,cth,{'_',id,[[]]}},
{?eh,cth,{'_',init,['_',[]]}},
@@ -601,6 +689,34 @@ test_events(scope_suite_cth) ->
{?eh,stop_logging,[]}
];
+test_events(scope_suite_group_only_cth) ->
+ Suite = ct_scope_suite_group_only_cth_SUITE,
+ CTH = empty_cth,
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,1}},
+ %% check that post_groups and post_all comes before init when hook
+ %% is installed in suite/0
+ {?eh,cth,{CTH,post_groups,['_',['_']]}},
+ {negative,
+ {?eh,cth,{CTH,post_all,['_','_','_']}},
+ {?eh,tc_start,{Suite,init_per_suite}}},
+ {?eh,cth,{CTH,id,[[]]}},
+ {?eh,cth,{CTH,init,['_',[]]}},
+ {?eh,cth,{CTH,pre_init_per_suite,[Suite,'$proplist',mystate]}},
+ {?eh,cth,{CTH,post_init_per_suite,[Suite,'$proplist','$proplist',mystate]}},
+ {?eh,tc_done,{Suite,init_per_suite,ok}},
+
+ {?eh,tc_start,{Suite,end_per_suite}},
+ {?eh,cth,{CTH,pre_end_per_suite,[Suite,'$proplist',mystate]}},
+ {?eh,cth,{CTH,post_end_per_suite,[Suite,'$proplist','_',mystate]}},
+ {?eh,cth,{CTH,terminate,[mystate]}},
+ {?eh,tc_done,{Suite,end_per_suite,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ];
+
test_events(scope_per_group_cth) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
@@ -660,6 +776,8 @@ test_events(scope_suite_state_cth) ->
[
{?eh,start_logging,{'DEF','RUNDIR'}},
{?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{'_',post_groups,['_',[]]}},
+ {?eh,cth,{'_',post_all,['_','_',[]]}},
{?eh,tc_start,{ct_scope_suite_state_cth_SUITE,init_per_suite}},
{?eh,cth,{'_',id,[[test]]}},
{?eh,cth,{'_',init,['_',[test]]}},
@@ -2308,6 +2426,229 @@ test_events(config_clash) ->
%% Make sure no 'cth_error' events are received!
[{negative,{?eh,cth_error,'_'},E} || E <- Events];
+test_events(alter_groups) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,
+ [{new_group,[tc1,tc2]}]]}},
+ {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[{group,new_group}],
+ [{new_group,[tc1,tc2]}]]}},
+ {?eh,start_info,{1,1,2}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,
+ [{new_group,[tc1,tc2]}]]}},
+ {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[{group,new_group}],
+ [{new_group,[tc1,tc2]}]]}},
+ {?eh,tc_start,{all_and_groups_SUITE,{init_per_group,new_group,[]}}},
+ {?eh,tc_done,{all_and_groups_SUITE,
+ {init_per_group,new_group,'$proplist'},ok}},
+ {?eh,tc_start,{all_and_groups_SUITE,tc1}},
+ {?eh,tc_done,{all_and_groups_SUITE,tc1,ok}},
+ {?eh,tc_start,{all_and_groups_SUITE,tc2}},
+ {?eh,tc_done,{all_and_groups_SUITE,tc2,ok}},
+ {?eh,tc_start,{all_and_groups_SUITE,{end_per_group,new_group,[]}}},
+ {?eh,tc_done,{all_and_groups_SUITE,
+ {end_per_group,new_group,'$proplist'},ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(alter_all) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,
+ [{test_group,[tc1]}]]}},
+ {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2],
+ [{test_group,[tc1]}]]}},
+ {?eh,start_info,{1,1,1}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}},
+ {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2],'_']}},
+ {?eh,tc_start,{all_and_groups_SUITE,tc2}},
+ {?eh,tc_done,{all_and_groups_SUITE,tc2,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(alter_all_from_skip) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,
+ [{test_group,[tc1]}]]}},
+ {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2],
+ [{test_group,[tc1]}]]}},
+ {?eh,start_info,{1,1,1}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}},
+ {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,[tc2],'_']}},
+ {?eh,tc_start,{all_and_groups_SUITE,tc2}},
+ {?eh,tc_done,{all_and_groups_SUITE,tc2,ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(alter_all_to_skip) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,
+ [{test_group,[tc1]}]]}},
+ {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,
+ {skip,"skipped by post_all/3"},
+ [{test_group,[tc1]}]]}},
+ {?eh,start_info,{1,1,0}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}},
+ {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,
+ {skip,"skipped by post_all/3"},
+ '_']}},
+ {?eh,tc_user_skip,{all_and_groups_SUITE,all,"skipped by post_all/3"}},
+ {?eh,cth,{'_',on_tc_skip,[all_and_groups_SUITE,all,
+ {tc_user_skip,"skipped by post_all/3"},
+ []]}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(illegal_values_groups) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,cth,{empty_cth,post_groups,
+ [all_and_groups_SUITE,
+ [{new_group,[this_test_does_not_exist]},
+ this_is_not_a_group_def]]}},
+ {?eh,start_info,{1,0,0}},
+ {?eh,cth,{empty_cth,post_groups,
+ [all_and_groups_SUITE,
+ [{new_group,[this_test_does_not_exist]},
+ this_is_not_a_group_def]]}},
+ {?eh,tc_start,{ct_framework,error_in_suite}},
+ {?eh,tc_done,{ct_framework,error_in_suite,{failed,{error,'_'}}}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(illegal_values_all) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}},
+ {?eh,cth,{empty_cth,post_all,
+ [all_and_groups_SUITE,
+ [{group,this_group_does_not_exist},
+ {this_is_not_a_valid_term}],'_']}},
+ {?eh,start_info,{1,0,0}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}},
+ {?eh,cth,{empty_cth,post_all,
+ [all_and_groups_SUITE,
+ [{group,this_group_does_not_exist},
+ {this_is_not_a_valid_term}],'_']}},
+ {?eh,tc_start,{ct_framework,error_in_suite}},
+ {?eh,tc_done,
+ {ct_framework,error_in_suite,
+ {failed,
+ {error,'Invalid reference to group this_group_does_not_exist in all_and_groups_SUITE:all/0'}}}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(bad_return_groups) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,not_a_list]}},
+ {?eh,start_info,{1,0,0}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,not_a_list]}},
+ {?eh,tc_start,{ct_framework,error_in_suite}},
+ {?eh,tc_done,
+ {ct_framework,error_in_suite,
+ {failed,
+ {error,
+ {'Bad return value from post_groups/2 hook function',not_a_list}}}}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(bad_return_all) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}},
+ {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,not_a_list,'_']}},
+ {?eh,start_info,{1,0,0}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}},
+ {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,not_a_list,'_']}},
+ {?eh,tc_start,{ct_framework,error_in_suite}},
+ {?eh,tc_done,
+ {ct_framework,error_in_suite,
+ {failed,
+ {error,{'Bad return value from post_all/3 hook function',not_a_list}}}}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(crash_groups) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,crash]}},
+ {?eh,start_info,{1,0,0}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,crash]}},
+ {?eh,tc_start,{ct_framework,error_in_suite}},
+ {?eh,tc_done,{ct_framework,error_in_suite,
+ {failed,
+ {error,"all_and_groups_cth:post_groups/2 CTH call failed"}}}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(crash_all) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,cth,{empty_cth,id,[[]]}},
+ {?eh,cth,{empty_cth,init,[{'_','_','_'},[]]}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}},
+ {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,crash,'_']}},
+ {?eh,start_info,{1,0,0}},
+ {?eh,cth,{empty_cth,post_groups,[all_and_groups_SUITE,'_']}},
+ {?eh,cth,{empty_cth,post_all,[all_and_groups_SUITE,crash,'_']}},
+ {?eh,tc_start,{ct_framework,error_in_suite}},
+ {?eh,tc_done,{ct_framework,error_in_suite,
+ {failed,
+ {error,"all_and_groups_cth:post_all/3 CTH call failed"}}}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,cth,{empty_cth,terminate,[[]]}},
+ {?eh,stop_logging,[]}
+ ];
+
test_events(ok) ->
ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_SUITE.erl
new file mode 100644
index 0000000000..adc86005f9
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_SUITE.erl
@@ -0,0 +1,47 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(all_and_groups_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+init_per_group(_Group,Config) ->
+ Config.
+
+end_per_group(_Group,Config) ->
+ ok.
+
+all() ->
+ ct:get_config(all_return,[{group,test_group}]).
+
+groups() ->
+ [{test_group,[tc1]}].
+
+%% Test cases starts here.
+tc1(Config) ->
+ ok.
+
+tc2(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_cth.erl
new file mode 100644
index 0000000000..9ebc00e9de
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/all_and_groups_cth.erl
@@ -0,0 +1,100 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(all_and_groups_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+%% Send a cth_error event if a callback is called with unexpected arguments
+-define(fail(Info),
+ gen_event:notify(
+ ?CT_EVMGR_REF,
+ #event{ name = cth_error,
+ node = node(),
+ data = {illegal_hook_callback,{?MODULE,?FUNCTION_NAME,Info}}})).
+
+%% CT Hooks
+-compile(export_all).
+
+id(Opts) ->
+ empty_cth:id(Opts).
+
+post_groups(Suite,Groups) ->
+ case empty_cth:post_groups(Suite,ct:get_config(post_groups_return,Groups)) of
+ crash -> error(crash_in_post_groups);
+ R -> R
+ end.
+
+post_all(Suite,Tests,Groups) ->
+ case empty_cth:post_all(Suite,ct:get_config(post_all_return,Tests),Groups) of
+ crash -> error(crash_in_post_all);
+ R -> R
+ end.
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts).
+
+pre_init_per_suite(Suite, Config, State) ->
+ empty_cth:pre_init_per_suite(Suite,Config,State).
+
+post_init_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,State).
+
+pre_end_per_suite(Suite,Config,State) ->
+ empty_cth:pre_end_per_suite(Suite,Config,State).
+
+post_end_per_suite(Suite,Config,Return,State) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,State).
+
+pre_init_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_init_per_group(Suite,Group,Config,State).
+
+post_init_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_init_per_group(Suite,Group,Config,Return,State).
+
+pre_end_per_group(Suite,Group,Config,State) ->
+ empty_cth:pre_end_per_group(Suite,Group,Config,State).
+
+post_end_per_group(Suite,Group,Config,Return,State) ->
+ empty_cth:post_end_per_group(Suite,Group,Config,Return,State).
+
+pre_init_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_init_per_testcase(Suite,TC,Config,State).
+
+post_init_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_init_per_testcase(Suite,TC,Config,Return,State).
+
+pre_end_per_testcase(Suite,TC,Config,State) ->
+ empty_cth:pre_end_per_testcase(Suite,TC,Config,State).
+
+post_end_per_testcase(Suite,TC,Config,Return,State) ->
+ empty_cth:post_end_per_testcase(Suite,TC,Config,Return,State).
+
+on_tc_fail(Suite,TC,Reason,State) ->
+ empty_cth:on_tc_fail(Suite,TC,Reason,State).
+
+on_tc_skip(Suite,TC,Reason,State) ->
+ empty_cth:on_tc_skip(Suite,TC,Reason,State).
+
+terminate(State) ->
+ empty_cth:terminate(State).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_match_state_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_match_state_cth.erl
new file mode 100644
index 0000000000..38c9da903d
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_match_state_cth.erl
@@ -0,0 +1,58 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+
+-module(ct_match_state_cth).
+
+
+-include_lib("common_test/src/ct_util.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-compile(export_all).
+
+id(Opts) ->
+ empty_cth:id(Opts).
+
+post_groups(Suite, Groups) ->
+ empty_cth:post_groups(Suite, Groups).
+
+post_all(Suite, Tests, Groups) ->
+ empty_cth:post_all(Suite, Tests, Groups).
+
+init(Id, Opts) ->
+ empty_cth:init(Id, Opts),
+ {ok,mystate}.
+
+%% In the following, always match against the state value, to ensure
+%% that init has indeed been called before the rest of the hooks.
+pre_init_per_suite(Suite,Config,mystate) ->
+ empty_cth:pre_init_per_suite(Suite,Config,mystate).
+
+post_init_per_suite(Suite,Config,Return,mystate) ->
+ empty_cth:post_init_per_suite(Suite,Config,Return,mystate).
+
+pre_end_per_suite(Suite,Config,mystate) ->
+ empty_cth:pre_end_per_suite(Suite,Config,mystate).
+
+post_end_per_suite(Suite,Config,Return,mystate) ->
+ empty_cth:post_end_per_suite(Suite,Config,Return,mystate).
+
+terminate(mystate) ->
+ empty_cth:terminate(mystate).
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_group_only_cth_SUITE.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_group_only_cth_SUITE.erl
new file mode 100644
index 0000000000..537c97d3f0
--- /dev/null
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/ct_scope_suite_group_only_cth_SUITE.erl
@@ -0,0 +1,54 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2016. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(ct_scope_suite_group_only_cth_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+%% Test server callback functions
+suite() ->
+ [{ct_hooks,[ct_match_state_cth]}].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(_Config) ->
+ ok.
+
+init_per_testcase(_TestCase, Config) ->
+ Config.
+
+end_per_testcase(_TestCase, _Config) ->
+ ok.
+
+all() ->
+ [test_case].
+
+groups() ->
+ [{g1,[test_case]}].
+
+%% Test cases starts here.
+test_case(Config) when is_list(Config) ->
+ ok.
diff --git a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl
index 961ea68d2d..324f1dc80a 100644
--- a/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl
+++ b/lib/common_test/test/ct_hooks_SUITE_data/cth/tests/empty_cth.erl
@@ -39,6 +39,9 @@
-export([id/1]).
-export([init/2]).
+-export([post_all/3]).
+-export([post_groups/2]).
+
-export([pre_init_per_suite/3]).
-export([post_init_per_suite/4]).
-export([pre_end_per_suite/3]).
@@ -71,6 +74,31 @@
-record(state, { id = ?MODULE :: term()}).
+%% @doc Called after groups/0.
+%% you can change the return value in this function.
+-spec post_groups(Suite :: atom(), Groups :: list()) -> list().
+post_groups(Suite,Groups) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_groups,
+ [Suite,Groups]}}),
+ ct:log("~w:post_groups(~w) called", [?MODULE,Suite]),
+ Groups.
+
+%% @doc Called after all/0.
+%% you can change the return value in this function.
+-spec post_all(Suite :: atom(),
+ Tests :: list(),
+ Groups :: term()) ->
+ list().
+post_all(Suite,Tests,Groups) ->
+ gen_event:notify(
+ ?CT_EVMGR_REF, #event{ name = cth, node = node(),
+ data = {?MODULE, post_all,
+ [Suite,Tests,Groups]}}),
+ ct:log("~w:post_all(~w) called", [?MODULE,Suite]),
+ Tests.
+
%% @doc Always called before any other callback function. Use this to initiate
%% any common state. It should return an state for this CTH.
-spec init(Id :: term(), Opts :: proplists:proplist()) ->
diff --git a/lib/common_test/test/ct_tc_repeat_SUITE.erl b/lib/common_test/test/ct_tc_repeat_SUITE.erl
new file mode 100644
index 0000000000..433b5456fe
--- /dev/null
+++ b/lib/common_test/test/ct_tc_repeat_SUITE.erl
@@ -0,0 +1,438 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(ct_tc_repeat_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ DataDir = ?config(data_dir, Config),
+ ct_test_support:init_per_suite([{path_dirs,[DataDir]} | Config]).
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+
+suite() ->
+ [{timetrap,{minutes,1}}].
+
+all() ->
+ all(suite).
+
+all(suite) ->
+ [
+ repeat,
+ repeat_parallel_until_ok,
+ repeat_parallel_until_fail,
+ repeat_sequence_until_ok,
+ repeat_sequence_until_fail,
+ pick_one_test_from_group,
+ pick_one_test_from_subgroup
+ ].
+
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+%%%-----------------------------------------------------------------
+%%%
+%% Test post_groups and post_all hook callbacks, introduced by OTP-14746
+repeat(Config) ->
+ ok = do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [], [], Config).
+
+repeat_parallel_until_ok(Config) ->
+ {error,{{illegal_combination,{parallel,repeat_until_ok}},_}} =
+ do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_parallel_until_ok}],
+ [], Config, 1, []).
+
+repeat_parallel_until_fail(Config) ->
+ {error,{{illegal_combination,{parallel,repeat_until_fail}},_}} =
+ do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_parallel_until_fail}],
+ [], Config, 1, []).
+
+repeat_sequence_until_ok(Config) ->
+ {error,{{illegal_combination,{sequence,repeat_until_ok}},_}} =
+ do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_sequence_until_ok}],
+ [], Config, 1, []).
+
+repeat_sequence_until_fail(Config) ->
+ {error,{{illegal_combination,{sequence,repeat_until_fail}},_}} =
+ do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_sequence_until_fail}],
+ [], Config, 1, []).
+
+pick_one_test_from_group(Config) ->
+ do_test(?FUNCTION_NAME, "tc_repeat_SUITE", [{group,g_mixed},{testcase,tc2}],
+ [], Config, 1, []).
+
+pick_one_test_from_subgroup(Config) ->
+ do_test(?FUNCTION_NAME, "tc_repeat_SUITE",
+ [{group,[[g_mixed,subgroup]]},{testcase,tc2}],
+ [], Config, 1, []).
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+
+do_test(Tag, Suite, WTT, CTHs, Config) ->
+ do_test(Tag, Suite, WTT, CTHs, Config, 2, []).
+
+do_test(Tag, Suite0, WTT, CTHs, Config, EC, ExtraOpts) ->
+ DataDir = ?config(data_dir, Config),
+ Suite = filename:join([DataDir,Suite0]),
+ {Opts,ERPid} =
+ setup([{suite,Suite}|WTT]++[{ct_hooks,CTHs},{label,Tag}|ExtraOpts],
+ Config),
+ Res = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+ %% io:format("~p~n",[Events]),
+
+ ct_test_support:log_events(Tag,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+
+ TestEvents = events_to_check(Tag, EC),
+ ok = ct_test_support:verify_events(TestEvents, Events, Config),
+ Res.
+
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+
+gen_config(Name,KeyVals,Config) ->
+ PrivDir = ?config(priv_dir,Config),
+ File = filename:join(PrivDir,atom_to_list(Name)++".cfg"),
+ ok = file:write_file(File,[io_lib:format("~p.~n",[{Key,Value}])
+ || {Key,Value} <- KeyVals]),
+ File.
+
+%%%-----------------------------------------------------------------
+%%% TEST EVENTS
+%%%-----------------------------------------------------------------
+events_to_check(Test) ->
+ %% 2 tests (ct:run_test + script_start) is default
+ events_to_check(Test, 2).
+
+events_to_check(_, 0) ->
+ [];
+events_to_check(Test, N) ->
+ test_events(Test) ++ events_to_check(Test, N-1).
+
+test_events(repeat) ->
+ S = tc_repeat_SUITE,
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,start_info,{1,1,unknown}},
+
+ %% tc1, {repeat,2}
+ {?eh,tc_start,{S,tc1}},
+ {?eh,tc_done,{S,tc1,ok}},
+ {?eh,test_stats,{1,0,{0,0}}},
+ {?eh,tc_start,{S,tc1}},
+ {?eh,tc_done,{S,tc1,ok}},
+ {?eh,test_stats,{2,0,{0,0}}},
+ %% tc2, {repeat_until_ok,3}
+ {?eh,tc_start,{S,tc2}},
+ {?eh,tc_done,{S,tc2,ok}},
+ {?eh,test_stats,{3,0,{0,0}}},
+ %% tc3, {repeat_until_ok,3}
+ {?eh,tc_start,{S,tc3}},
+ {?eh,tc_done,{tc_repeat_SUITE,tc3,
+ {failed,{error,{test_case_failed,always_fail}}}}},
+ {?eh,test_stats,{3,1,{0,0}}},
+ {?eh,tc_start,{S,tc3}},
+ {?eh,tc_done,{S,tc3,{failed,{error,{test_case_failed,always_fail}}}}},
+ {?eh,test_stats,{3,2,{0,0}}},
+ {?eh,tc_start,{S,tc3}},
+ {?eh,tc_done,{S,tc3,{failed,{error,{test_case_failed,always_fail}}}}},
+ {?eh,test_stats,{3,3,{0,0}}},
+ %% tc4, {repeat_until_fail,3}
+ {?eh,tc_start,{S,tc4}},
+ {?eh,tc_done,{S,tc4,ok}},
+ {?eh,test_stats,{4,3,{0,0}}},
+ {?eh,tc_start,{S,tc4}},
+ {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}},
+ {?eh,test_stats,{4,4,{0,0}}},
+ %% g, tc1, {repeat,2}
+ {?eh,tc_start,{S,{init_per_group,g,[]}}},
+ {?eh,tc_done,{S,{init_per_group,g,[]},ok}},
+ {?eh,tc_start,{S,tc1}},
+ {?eh,tc_done,{S,tc1,ok}},
+ {?eh,test_stats,{5,4,{0,0}}},
+ {?eh,tc_start,{S,tc1}},
+ {?eh,tc_done,{S,tc1,ok}},
+ {?eh,test_stats,{6,4,{0,0}}},
+ {?eh,tc_start,{S,{end_per_group,g,[]}}},
+ {?eh,tc_done,{S,{end_per_group,g,[]},ok}},
+ %% g_until_ok, tc2, {repeat_until_ok,3}
+ {?eh,tc_start,{S,{init_per_group,g_until_ok,[]}}},
+ {?eh,tc_done,{S,{init_per_group,g_until_ok,[]},ok}},
+ {?eh,tc_start,{S,tc2}},
+ {?eh,tc_done,{S,tc2,ok}},
+ {?eh,test_stats,{7,4,{0,0}}},
+ {?eh,tc_start,{S,{end_per_group,g_until_ok,[]}}},
+ {?eh,tc_done,{S,{end_per_group,g_until_ok,[]},ok}},
+ %% g_until_fail, tc4, {repeat_until_fail,3}
+ {?eh,tc_start,{S,{init_per_group,g_until_fail,[]}}},
+ {?eh,tc_done,{S,{init_per_group,g_until_fail,[]},ok}},
+ {?eh,tc_start,{S,tc4}},
+ {?eh,tc_done,{S,tc4,ok}},
+ {?eh,test_stats,{8,4,{0,0}}},
+ {?eh,tc_start,{S,tc4}},
+ {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}},
+ {?eh,test_stats,{8,5,{0,0}}},
+ {?eh,tc_start,{S,{end_per_group,g_until_fail,[]}}},
+ {?eh,tc_done,{S,{end_per_group,g_until_fail,[]},ok}},
+ %% g, parallel, tc1, {repeat,2}
+ {parallel,
+ [{?eh,tc_start,{S,{init_per_group,g,[parallel]}}},
+ {?eh,tc_done,{S,{init_per_group,g,[parallel]},ok}},
+ {?eh,tc_start,{S,tc1}},
+ {?eh,tc_done,{S,tc1,ok}},
+ {?eh,test_stats,{9,5,{0,0}}},
+ {?eh,tc_start,{S,tc1}},
+ {?eh,tc_done,{S,tc1,ok}},
+ {?eh,test_stats,{10,5,{0,0}}},
+ {?eh,tc_start,{S,{end_per_group,g,[parallel]}}},
+ {?eh,tc_done,{S,{end_per_group,g,[parallel]},ok}}]},
+ %% g, sequence, tc1, {repeat,2}
+ {?eh,tc_start,{S,{init_per_group,g,[sequence]}}},
+ {?eh,tc_done,{S,{init_per_group,g,[sequence]},ok}},
+ {?eh,tc_start,{S,tc1}},
+ {?eh,tc_done,{S,tc1,ok}},
+ {?eh,test_stats,{11,5,{0,0}}},
+ {?eh,tc_start,{S,tc1}},
+ {?eh,tc_done,{S,tc1,ok}},
+ {?eh,test_stats,{12,5,{0,0}}},
+ {?eh,tc_start,{S,{end_per_group,g,[sequence]}}},
+ {?eh,tc_done,{S,{end_per_group,g,[sequence]},ok}},
+ %% g_sequence_skip_rest,
+ {?eh,tc_start,{S,{init_per_group,g_mixed,[sequence]}}},
+ {?eh,tc_done,{S,{init_per_group,g_mixed,[sequence]},ok}},
+ {?eh,tc_start,{S,tc1}},
+ {?eh,tc_done,{S,tc1,ok}},
+ {?eh,test_stats,{13,5,{0,0}}},
+ {?eh,tc_start,{S,tc1}},
+ {?eh,tc_done,{S,tc1,ok}},
+ {?eh,test_stats,{14,5,{0,0}}},
+ {?eh,tc_start,{S,tc4}},
+ {?eh,tc_done,{S,tc4,ok}},
+ {?eh,test_stats,{15,5,{0,0}}},
+ {?eh,tc_start,{S,tc4}},
+ {?eh,tc_done,{S,tc4,{failed,{error,{test_case_failed,second_time_fail}}}}},
+ {?eh,test_stats,{15,6,{0,0}}},
+ %% ----> fail in sequence, so skip rest
+ {?eh,tc_auto_skip,{S,{tc4,g_mixed}, % last of current repeat tc4
+ {failed,{tc_repeat_SUITE,tc4}}}},
+ {?eh,test_stats,{15,6,{0,1}}},
+ {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % single tc1
+ {failed,{tc_repeat_SUITE,tc4}}}},
+ {?eh,test_stats,{15,6,{0,2}}},
+ {?eh,tc_auto_skip,{S,{tc1,g}, % group g, tc1, {repeat,2}
+ {failed,{tc_repeat_SUITE,tc4}}}},
+ {?eh,test_stats,{15,6,{0,3}}},
+ {?eh,tc_auto_skip,{S,{tc1,subgroup}, % subgroup, single tc1
+ {failed,{tc_repeat_SUITE,tc4}}}},
+ {?eh,test_stats,{15,6,{0,4}}},
+ {?eh,tc_auto_skip,{S,{tc2,subgroup}, % subgroup, tc2, {repeat,2}
+ {failed,{tc_repeat_SUITE,tc4}}}},
+ {?eh,test_stats,{15,6,{0,5}}},
+ {?eh,tc_auto_skip,{S,{tc2,g_mixed}, % tc2, {repeat,2}
+ {failed,{tc_repeat_SUITE,tc4}}}},
+ {?eh,test_stats,{15,6,{0,6}}},
+ {?eh,tc_auto_skip,{S,{tc2,g_mixed}, % single tc2
+ {failed,{tc_repeat_SUITE,tc4}}}},
+ {?eh,test_stats,{15,6,{0,7}}},
+ {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % tc1, {repeat,2}
+ {failed,{tc_repeat_SUITE,tc4}}}},
+ {?eh,test_stats,{15,6,{0,8}}},
+ {?eh,tc_auto_skip,{S,{tc1,g_mixed}, % single tc1
+ {failed,{tc_repeat_SUITE,tc4}}}},
+ {?eh,test_stats,{15,6,{0,9}}},
+ {?eh,tc_start,{S,{end_per_group,g_mixed,'_'}}},
+ {?eh,tc_done,{S,{end_per_group,g_mixed,'_'},ok}},
+ %% done
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(repeat_parallel_until_ok) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,tc_start,{'_',{init_per_group,g_parallel_until_ok,[parallel]}}},
+ {?eh,tc_done,{'_',{init_per_group,g_parallel_until_ok,[parallel]},ok}},
+ {?eh,severe_error,{{illegal_combination,{parallel,repeat_until_ok}},'_'}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(repeat_parallel_until_fail) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,tc_start,{'_',{init_per_group,g_parallel_until_fail,[parallel]}}},
+ {?eh,tc_done,{'_',{init_per_group,g_parallel_until_fail,[parallel]},ok}},
+ {?eh,severe_error,{{illegal_combination,{parallel,repeat_until_fail}},'_'}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(repeat_sequence_until_ok) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,tc_start,{'_',{init_per_group,g_sequence_until_ok,[sequence]}}},
+ {?eh,tc_done,{'_',{init_per_group,g_sequence_until_ok,[sequence]},ok}},
+ {?eh,severe_error,{{illegal_combination,{sequence,repeat_until_ok}},'_'}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(repeat_sequence_until_fail) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,tc_start,{'_',{init_per_group,g_sequence_until_fail,[sequence]}}},
+ {?eh,tc_done,{'_',{init_per_group,g_sequence_until_fail,[sequence]},ok}},
+ {?eh,severe_error,{{illegal_combination,{sequence,repeat_until_fail}},'_'}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(pick_one_test_from_group) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,tc_start,{'_',{init_per_group,g_mixed,[]}}},
+ {?eh,tc_done,{'_',{init_per_group,g_mixed,[]},ok}},
+ {negative,
+ {?eh,tc_start,{'_',tc1}},
+ {?eh,tc_start,{'_',tc2}}}, % single tc2
+ {?eh,tc_done,{'_',tc2,ok}},
+ {?eh,tc_start,{'_',tc2}}, % tc2, {repeat,2}
+ {?eh,tc_done,{'_',tc2,ok}},
+ {?eh,tc_start,{'_',tc2}},
+ {?eh,tc_done,{'_',tc2,ok}},
+ {negative,
+ {?eh,tc_start,{'_',tc1}},
+ {?eh,tc_start,{'_',{end_per_group,g_mixed,[]}}}},
+ {?eh,tc_done,{'_',{end_per_group,g_mixed,[]},ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(pick_one_test_from_subgroup) ->
+ [
+ {?eh,start_logging,{'DEF','RUNDIR'}},
+ {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+ {?eh,tc_start,{'_',{init_per_group,g_mixed,[]}}},
+ {?eh,tc_done,{'_',{init_per_group,g_mixed,[]},ok}},
+ {negative,
+ {?eh,tc_start,{'_',tc2}},
+ {?eh,tc_start,{'_',{init_per_group,subgroup,[]}}}},
+ {?eh,tc_done,{'_',{init_per_group,subgroup,[]},ok}},
+ {negative,
+ {?eh,tc_start,{'_',tc1}},
+ {?eh,tc_start,{'_',tc2}}}, % tc2, {repeat,2}
+ {?eh,tc_done,{'_',tc2,ok}},
+ {?eh,tc_start,{'_',tc2}},
+ {?eh,tc_done,{'_',tc2,ok}},
+ {negative,
+ {?eh,tc_start,{'_',tc1}},
+ {?eh,tc_start,{'_',{end_per_group,subgroup,[]}}}},
+ {?eh,tc_done,{'_',{end_per_group,subgroup,[]},ok}},
+ {negative,
+ {?eh,tc_start,{'_',tc2}},
+ {?eh,tc_start,{'_',{end_per_group,g_mixed,[]}}}},
+ {?eh,tc_done,{'_',{end_per_group,g_mixed,[]},ok}},
+ {?eh,test_done,{'DEF','STOP_TIME'}},
+ {?eh,stop_logging,[]}
+ ];
+
+test_events(ok) ->
+ ok.
+
+%% test events help functions
+contains(List) ->
+ fun(Proplist) when is_list(Proplist) ->
+ contains(List,Proplist)
+ end.
+
+contains([{not_in_order,List}|T],Rest) ->
+ contains_parallel(List,Rest),
+ contains(T,Rest);
+contains([{Ele,Pos}|T] = L,[H|T2]) ->
+ case element(Pos,H) of
+ Ele ->
+ contains(T,T2);
+ _ ->
+ contains(L,T2)
+ end;
+contains([Ele|T],[{Ele,_}|T2])->
+ contains(T,T2);
+contains([Ele|T],[Ele|T2])->
+ contains(T,T2);
+contains(List,[_|T]) ->
+ contains(List,T);
+contains([],_) ->
+ match.
+
+contains_parallel([Key | T], Elems) ->
+ contains([Key],Elems),
+ contains_parallel(T,Elems);
+contains_parallel([],_Elems) ->
+ match.
+
+not_contains(List) ->
+ fun(Proplist) when is_list(Proplist) ->
+ [] = [Ele || {Ele,_} <- Proplist,
+ Test <- List,
+ Test =:= Ele]
+ end.
diff --git a/lib/common_test/test/ct_tc_repeat_SUITE_data/tc_repeat_SUITE.erl b/lib/common_test/test/ct_tc_repeat_SUITE_data/tc_repeat_SUITE.erl
new file mode 100644
index 0000000000..f5d960d12f
--- /dev/null
+++ b/lib/common_test/test/ct_tc_repeat_SUITE_data/tc_repeat_SUITE.erl
@@ -0,0 +1,85 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2017. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-module(tc_repeat_SUITE).
+
+-suite_defaults([{timetrap, {minutes, 10}}]).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include("ct.hrl").
+
+init_per_group(_Group,Config) ->
+ Config.
+
+end_per_group(_Group,Config) ->
+ ok.
+
+all() ->
+ [{testcase,tc1,[{repeat,2}]},
+ {testcase,tc2,[{repeat_until_ok,3}]},
+ {testcase,tc3,[{repeat_until_ok,3}]},
+ {testcase,tc4,[{repeat_until_fail,3}]},
+ {group,g},
+ {group,g_until_ok},
+ {group,g_until_fail},
+ {group,g,[parallel]},
+ {group,g,[sequence]},
+ {group,g_mixed,[sequence]}].
+
+groups() ->
+ [{g,[{testcase,tc1,[{repeat,2}]}]},
+ {g_until_ok,[{testcase,tc2,[{repeat_until_ok,3}]}]},
+ {g_until_fail,[{testcase,tc4,[{repeat_until_fail,3}]}]},
+ {g_parallel_until_ok,[parallel],[{testcase,tc2,[{repeat_until_ok,3}]}]},
+ {g_parallel_until_fail,[parallel],[{testcase,tc1,[{repeat_until_fail,2}]}]},
+ {g_sequence_until_ok,[sequence],[{testcase,tc2,[{repeat_until_ok,3}]}]},
+ {g_sequence_until_fail,[sequence],[{testcase,tc1,[{repeat_until_fail,2}]}]},
+ {g_mixed,[{testcase,tc1,[{repeat,2}]},
+ {testcase,tc4,[{repeat,3}]},
+ tc1,
+ {group,g},
+ {subgroup,[tc1,{testcase,tc2,[{repeat,2}]}]},
+ {testcase,tc2,[{repeat,2}]},
+ tc2,
+ {testcase,tc1,[{repeat,2}]},
+ tc1]}].
+
+%% Test cases starts here.
+tc1(_Config) ->
+ ok.
+
+tc2(_Config) ->
+ ok.
+
+tc3(_Config) ->
+ ct:fail(always_fail).
+
+tc4(Config) ->
+ case ?config(saved_config,Config) of
+ {tc4,_} ->
+ ct:fail(second_time_fail);
+ undefined ->
+ {save_config,Config}
+ end.
+
+tc5(_Config) ->
+ {skip,"just skip this"}.
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-21 18:12:40 +0000