aboutsummaryrefslogtreecommitdiffstats
path: root/lib/common_test
diff options
context:
space:
mode:
authorLukas Larsson <lukas@erix.ericsson.se>2011-01-20 17:38:30 +0100
committerLukas Larsson <lukas@erix.ericsson.se>2011-02-15 13:25:51 +0100
commit05a18c6877dde2e7d0d64edb384607098cf8af90 (patch)
tree54e35e26637889070ca21d62a6e8e5e906fddb27 /lib/common_test
parent18ca34b08868d5c045a854b022a4d9fb3fa727a0 (diff)
downloadotp-05a18c6877dde2e7d0d64edb384607098cf8af90.tar.gz
otp-05a18c6877dde2e7d0d64edb384607098cf8af90.tar.bz2
otp-05a18c6877dde2e7d0d64edb384607098cf8af90.zip
Update test spec parser to not group suites, groups and cases which are hierarchically related if merge_tests config i set to false
Diffstat (limited to 'lib/common_test')
-rw-r--r--lib/common_test/doc/src/run_test_chapter.xml10
-rw-r--r--lib/common_test/src/ct_testspec.erl133
-rw-r--r--lib/common_test/src/ct_util.hrl3
-rw-r--r--lib/common_test/test/ct_testspec_1_SUITE.erl342
4 files changed, 408 insertions, 80 deletions
diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml
index 94fcf6bf01..1662923a69 100644
--- a/lib/common_test/doc/src/run_test_chapter.xml
+++ b/lib/common_test/doc/src/run_test_chapter.xml
@@ -365,15 +365,17 @@
either one or more suites, one or more test case groups, or one
or more test cases in a group or suite.</p>
<p>An arbitrary number of test terms may be declared in sequence.
- Common Test will compile the terms into one or more tests to be
- performed in one resulting test run. Note that a term that
+ Common Test will by default compile the terms into one or more tests
+ to be performed in one resulting test run. Note that a term that
specifies a set of test cases will "swallow" one that only
specifies a subset of these cases. E.g. the result of merging
one term that specifies that all cases in suite S should be
executed, with another term specifying only test case X and Y in
S, is a test of all cases in S. However, if a term specifying
test case X and Y in S is merged with a term specifying case Z
- in S, the result is a test of X, Y and Z in S.</p>
+ in S, the result is a test of X, Y and Z in S. To disable this
+ behaviour, it is possible in test specification to set the
+ <c>merge_tests</c> term to <c>false</c>.</p>
<p>A test term can also specify one or more test suites, groups,
or test cases to be skipped. Skipped suites, groups and cases
are not executed and show up in the HTML test log files as
@@ -432,6 +434,8 @@
{userconfig, NodeRefs, {CallbackModule, ConfigStrings}}.
{alias, DirAlias, Dir}.
+
+ {merge_tests, Bool}.
{logdir, LogDir}.
{logdir, NodeRefs, LogDir}.
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index f5069427a2..bf9282900b 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -68,7 +68,8 @@ prepare_tests(TestSpec) when is_record(TestSpec,testspec) ->
%% Create initial list of {Node,{Run,Skip}} tuples
NodeList = lists:map(fun(N) -> {N,{[],[]}} end, list_nodes(TestSpec)),
%% Get all Run tests sorted per node basis.
- NodeList1 = run_per_node(Run,NodeList),
+ NodeList1 = run_per_node(Run,NodeList,
+ TestSpec#testspec.merge_tests),
%% Get all Skip entries sorted per node basis.
NodeList2 = skip_per_node(Skip,NodeList1),
%% Change representation.
@@ -89,11 +90,17 @@ prepare_tests(TestSpec) when is_record(TestSpec,testspec) ->
%% run_per_node/2 takes the Run list as input and returns a list
%% of {Node,RunPerNode,[]} tuples where the tests have been sorted
%% on a per node basis.
-run_per_node([{{Node,Dir},Test}|Ts],Result) ->
+run_per_node([{{Node,Dir},Test}|Ts],Result, MergeTests) ->
{value,{Node,{Run,Skip}}} = lists:keysearch(Node,1,Result),
- Run1 = merge_tests(Dir,Test,Run),
- run_per_node(Ts,insert_in_order({Node,{Run1,Skip}},Result));
-run_per_node([],Result) ->
+ Run1 = case MergeTests of
+ false ->
+ append({Dir, Test}, Run);
+ true ->
+ merge_tests(Dir,Test,Run)
+ end,
+ run_per_node(Ts,insert_in_order({Node,{Run1,Skip}},Result),
+ MergeTests);
+run_per_node([],Result,_) ->
Result.
merge_tests(Dir,Test={all,_},TestDirs) ->
@@ -281,6 +288,8 @@ collect_tests(Terms,TestSpec,Relaxed) ->
{Terms2, TestSpec3} = filter_init_terms(Terms, [], TestSpec2),
add_tests(Terms2,TestSpec3).
+get_global([{merge_tests, Bool} | Ts], Spec) ->
+ get_global(Ts,Spec#testspec{ merge_tests = Bool });
get_global([{alias,Ref,Dir}|Ts],Spec=#testspec{alias=Refs}) ->
get_global(Ts,Spec#testspec{alias=[{Ref,get_absdir(Dir,Spec)}|Refs]});
get_global([{node,Ref,Node}|Ts],Spec=#testspec{nodes=Refs}) ->
@@ -656,7 +665,7 @@ add_tests([{suites,Node,Dir,Ss}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = insert_suites(ref2node(Node,Spec#testspec.nodes),
ref2dir(Dir,Spec#testspec.alias),
- Ss,Tests),
+ Ss,Tests, Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
%% --- groups ---
@@ -682,13 +691,15 @@ add_tests([{groups,Node,Dir,Suite,Gs}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = insert_groups(ref2node(Node,Spec#testspec.nodes),
ref2dir(Dir,Spec#testspec.alias),
- Suite,Gs,all,Tests),
+ Suite,Gs,all,Tests,
+ Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
add_tests([{groups,Node,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = insert_groups(ref2node(Node,Spec#testspec.nodes),
ref2dir(Dir,Spec#testspec.alias),
- Suite,Gs,TCs,Tests),
+ Suite,Gs,TCs,Tests,
+ Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
%% --- cases ---
@@ -703,7 +714,7 @@ add_tests([{cases,Node,Dir,Suite,Cs}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = insert_cases(ref2node(Node,Spec#testspec.nodes),
ref2dir(Dir,Spec#testspec.alias),
- Suite,Cs,Tests),
+ Suite,Cs,Tests, Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
%% --- skip_suites ---
@@ -718,7 +729,8 @@ add_tests([{skip_suites,Node,Dir,Ss,Cmt}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = skip_suites(ref2node(Node,Spec#testspec.nodes),
ref2dir(Dir,Spec#testspec.alias),
- Ss,Cmt,Tests),
+ Ss,Cmt,Tests,
+ Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
%% --- skip_groups ---
@@ -740,13 +752,15 @@ add_tests([{skip_groups,Node,Dir,Suite,Gs,Cmt}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = skip_groups(ref2node(Node,Spec#testspec.nodes),
ref2dir(Dir,Spec#testspec.alias),
- Suite,Gs,all,Cmt,Tests),
+ Suite,Gs,all,Cmt,Tests,
+ Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
add_tests([{skip_groups,Node,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = skip_groups(ref2node(Node,Spec#testspec.nodes),
ref2dir(Dir,Spec#testspec.alias),
- Suite,Gs,TCs,Cmt,Tests),
+ Suite,Gs,TCs,Cmt,Tests,
+ Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
%% --- skip_cases ---
@@ -761,7 +775,7 @@ add_tests([{skip_cases,Node,Dir,Suite,Cs,Cmt}|Ts],Spec) ->
Tests = Spec#testspec.tests,
Tests1 = skip_cases(ref2node(Node,Spec#testspec.nodes),
ref2dir(Dir,Spec#testspec.alias),
- Suite,Cs,Cmt,Tests),
+ Suite,Cs,Cmt,Tests,Spec#testspec.merge_tests),
add_tests(Ts,Spec#testspec{tests=Tests1});
%% --- handled/errors ---
@@ -771,6 +785,9 @@ add_tests([{alias,_,_}|Ts],Spec) -> % handled
add_tests([{node,_,_}|Ts],Spec) -> % handled
add_tests(Ts,Spec);
+add_tests([{merge_tests, _} | Ts], Spec) -> % handled
+ add_tests(Ts,Spec);
+
%% check if it's a CT term that has bad format or if the user seems to
%% have added something of his/her own, which we'll let pass if relaxed
%% mode is enabled.
@@ -823,17 +840,22 @@ separate([],_,_,_) ->
%% {Suite2,[{GrOrCase21,{skip,Cmt}},GrOrCase22,...]},...]}
%% GrOrCase = {GroupName,[Case1,Case2,...]} | Case
-insert_suites(Node,Dir,[S|Ss],Tests) ->
- Tests1 = insert_cases(Node,Dir,S,all,Tests),
- insert_suites(Node,Dir,Ss,Tests1);
-insert_suites(_Node,_Dir,[],Tests) ->
+insert_suites(Node,Dir,[S|Ss],Tests, MergeTests) ->
+ Tests1 = insert_cases(Node,Dir,S,all,Tests,MergeTests),
+ insert_suites(Node,Dir,Ss,Tests1,MergeTests);
+insert_suites(_Node,_Dir,[],Tests,_MergeTests) ->
Tests;
-insert_suites(Node,Dir,S,Tests) ->
- insert_suites(Node,Dir,[S],Tests).
+insert_suites(Node,Dir,S,Tests,MergeTests) ->
+ insert_suites(Node,Dir,[S],Tests,MergeTests).
-insert_groups(Node,Dir,Suite,Group,Cases,Tests) when is_atom(Group) ->
- insert_groups(Node,Dir,Suite,[Group],Cases,Tests);
-insert_groups(Node,Dir,Suite,Groups,Cases,Tests) when
+insert_groups(Node,Dir,Suite,Group,Cases,Tests,MergeTests)
+ when is_atom(Group) ->
+ insert_groups(Node,Dir,Suite,[Group],Cases,Tests,MergeTests);
+insert_groups(Node,Dir,Suite,Groups,Cases,Tests,false) when
+ ((Cases == all) or is_list(Cases)) and is_list(Groups) ->
+ Groups1 = [{Gr,Cases} || Gr <- Groups],
+ append({{Node,Dir},[{Suite,Groups1}]},Tests);
+insert_groups(Node,Dir,Suite,Groups,Cases,Tests,true) when
((Cases == all) or is_list(Cases)) and is_list(Groups) ->
case lists:keysearch({Node,Dir},1,Tests) of
{value,{{Node,Dir},[{all,_}]}} ->
@@ -847,9 +869,10 @@ insert_groups(Node,Dir,Suite,Groups,Cases,Tests) when
Groups1 = [{Gr,Cases} || Gr <- Groups],
insert_in_order({{Node,Dir},[{Suite,Groups1}]},Tests)
end;
-insert_groups(Node,Dir,Suite,Groups,Case,Tests) when is_atom(Case) ->
+insert_groups(Node,Dir,Suite,Groups,Case,Tests, MergeTests)
+ when is_atom(Case) ->
Cases = if Case == all -> all; true -> [Case] end,
- insert_groups(Node,Dir,Suite,Groups,Cases,Tests).
+ insert_groups(Node,Dir,Suite,Groups,Cases,Tests, MergeTests).
insert_groups1(_Suite,_Groups,all) ->
all;
@@ -879,7 +902,9 @@ insert_groups2([Group={GrName,Cases}|Groups],GrAndCases) ->
insert_groups2([],GrAndCases) ->
GrAndCases.
-insert_cases(Node,Dir,Suite,Cases,Tests) when is_list(Cases) ->
+insert_cases(Node,Dir,Suite,Cases,Tests,false) when is_list(Cases) ->
+ append({{Node,Dir},[{Suite,Cases}]},Tests);
+insert_cases(Node,Dir,Suite,Cases,Tests,true) when is_list(Cases) ->
case lists:keysearch({Node,Dir},1,Tests) of
{value,{{Node,Dir},[{all,_}]}} ->
Tests;
@@ -889,8 +914,8 @@ insert_cases(Node,Dir,Suite,Cases,Tests) when is_list(Cases) ->
false ->
insert_in_order({{Node,Dir},[{Suite,Cases}]},Tests)
end;
-insert_cases(Node,Dir,Suite,Case,Tests) when is_atom(Case) ->
- insert_cases(Node,Dir,Suite,[Case],Tests).
+insert_cases(Node,Dir,Suite,Case,Tests,MergeTests) when is_atom(Case) ->
+ insert_cases(Node,Dir,Suite,[Case],Tests,MergeTests).
insert_cases1(_Suite,_Cases,all) ->
all;
@@ -905,22 +930,28 @@ insert_cases1(Suite,Cases,Suites0) ->
insert_in_order({Suite,Cases},Suites0)
end.
-skip_suites(Node,Dir,[S|Ss],Cmt,Tests) ->
- Tests1 = skip_cases(Node,Dir,S,all,Cmt,Tests),
- skip_suites(Node,Dir,Ss,Cmt,Tests1);
-skip_suites(_Node,_Dir,[],_Cmt,Tests) ->
+skip_suites(Node,Dir,[S|Ss],Cmt,Tests,MergeTests) ->
+ Tests1 = skip_cases(Node,Dir,S,all,Cmt,Tests,MergeTests),
+ skip_suites(Node,Dir,Ss,Cmt,Tests1,MergeTests);
+skip_suites(_Node,_Dir,[],_Cmt,Tests,_MergeTests) ->
Tests;
-skip_suites(Node,Dir,S,Cmt,Tests) ->
- skip_suites(Node,Dir,[S],Cmt,Tests).
-
-skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests) when is_atom(Group) ->
- skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests);
-skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests) when is_atom(Group) ->
- skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests);
-skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case),
- Case =/= all ->
- skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests);
-skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when
+skip_suites(Node,Dir,S,Cmt,Tests,MergeTests) ->
+ skip_suites(Node,Dir,[S],Cmt,Tests,MergeTests).
+
+skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests,MergeTests)
+ when is_atom(Group) ->
+ skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests,MergeTests);
+skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests,MergeTests)
+ when is_atom(Group) ->
+ skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests,MergeTests);
+skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests,MergeTests)
+ when is_atom(Case),Case =/= all ->
+ skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests,MergeTests);
+skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,false) when
+ ((Cases == all) or is_list(Cases)) and is_list(Groups) ->
+ Suites1 = skip_groups1(Suite,[{Gr,Cases} || Gr <- Groups],Cmt,[]),
+ append({{Node,Dir},Suites1},Tests);
+skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,true) when
((Cases == all) or is_list(Cases)) and is_list(Groups) ->
Suites =
case lists:keysearch({Node,Dir},1,Tests) of
@@ -931,9 +962,10 @@ skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when
end,
Suites1 = skip_groups1(Suite,[{Gr,Cases} || Gr <- Groups],Cmt,Suites),
insert_in_order({{Node,Dir},Suites1},Tests);
-skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case) ->
+skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests,MergeTests)
+ when is_atom(Case) ->
Cases = if Case == all -> all; true -> [Case] end,
- skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests).
+ skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,MergeTests).
skip_groups1(Suite,Groups,Cmt,Suites0) ->
SkipGroups = lists:map(fun(Group) ->
@@ -947,7 +979,10 @@ skip_groups1(Suite,Groups,Cmt,Suites0) ->
insert_in_order({Suite,SkipGroups},Suites0)
end.
-skip_cases(Node,Dir,Suite,Cases,Cmt,Tests) when is_list(Cases) ->
+skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,false) when is_list(Cases) ->
+ Suites1 = skip_cases1(Suite,Cases,Cmt,[]),
+ append({{Node,Dir},Suites1},Tests);
+skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,true) when is_list(Cases) ->
Suites =
case lists:keysearch({Node,Dir},1,Tests) of
{value,{{Node,Dir},Suites0}} ->
@@ -957,8 +992,8 @@ skip_cases(Node,Dir,Suite,Cases,Cmt,Tests) when is_list(Cases) ->
end,
Suites1 = skip_cases1(Suite,Cases,Cmt,Suites),
insert_in_order({{Node,Dir},Suites1},Tests);
-skip_cases(Node,Dir,Suite,Case,Cmt,Tests) when is_atom(Case) ->
- skip_cases(Node,Dir,Suite,[Case],Cmt,Tests).
+skip_cases(Node,Dir,Suite,Case,Cmt,Tests,MergeTests) when is_atom(Case) ->
+ skip_cases(Node,Dir,Suite,[Case],Cmt,Tests,MergeTests).
skip_cases1(Suite,Cases,Cmt,Suites0) ->
SkipCases = lists:map(fun(C) ->
@@ -972,6 +1007,9 @@ skip_cases1(Suite,Cases,Cmt,Suites0) ->
insert_in_order({Suite,SkipCases},Suites0)
end.
+append(Elem, List) ->
+ List ++ [Elem].
+
insert_in_order([E|Es],List) ->
List1 = insert_elem(E,List,[]),
insert_in_order(Es,List1);
@@ -1044,6 +1082,7 @@ valid_terms() ->
{userconfig,2},
{userconfig,3},
{alias,3},
+ {merge_tests,1},
{logdir,2},
{logdir,3},
{label,2},
diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl
index ee973f6220..66021ae99b 100644
--- a/lib/common_test/src/ct_util.hrl
+++ b/lib/common_test/src/ct_util.hrl
@@ -40,7 +40,8 @@
multiply_timetraps=[],
scale_timetraps=[],
alias=[],
- tests=[]}).
+ tests=[],
+ merge_tests = true }).
-record(cover, {app=none,
level=details,
diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl
index f592d697bd..7b09310e2e 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE.erl
@@ -60,25 +60,35 @@ all(doc) ->
["Run smoke tests of Common Test."];
all(suite) ->
- [all_suites, skip_all_suites,
- suite, skip_suite,
- all_testcases, skip_all_testcases,
- testcase, skip_testcase,
- all_groups, skip_all_groups,
- group, skip_group,
- group_all_testcases, skip_group_all_testcases,
- group_testcase, skip_group_testcase,
- topgroup,
- subgroup, skip_subgroup,
- subgroup_all_testcases, skip_subgroup_all_testcases,
- subgroup_testcase, skip_subgroup_testcase,
- sub_skipped_by_top,
- testcase_in_multiple_groups,
- order_of_tests_in_multiple_dirs,
- order_of_tests_in_multiple_suites,
- order_of_suites_in_multiple_dirs,
- order_of_groups_in_multiple_dirs,
- order_of_groups_in_multiple_suites
+ [
+ %% all_suites, skip_all_suites,
+ %% suite, skip_suite,
+ %% all_testcases, skip_all_testcases,
+ %% testcase, skip_testcase,
+ %% all_groups, skip_all_groups,
+ %% group, skip_group,
+ %% group_all_testcases, skip_group_all_testcases,
+ %% group_testcase, skip_group_testcase,
+ %% topgroup,
+ %% subgroup, skip_subgroup,
+ %% subgroup_all_testcases, skip_subgroup_all_testcases,
+ %% subgroup_testcase, skip_subgroup_testcase,
+ %% sub_skipped_by_top,
+ %% testcase_in_multiple_groups,
+ %% order_of_tests_in_multiple_dirs_no_merge_tests,
+ %% order_of_tests_in_multiple_suites_no_merge_tests,
+ %% order_of_suites_in_multiple_dirs_no_merge_tests,
+ %% order_of_groups_in_multiple_dirs_no_merge_tests,
+ %% order_of_groups_in_multiple_suites_no_merge_tests,
+ %% order_of_tests_in_multiple_dirs,
+ %% order_of_tests_in_multiple_suites,
+ %% order_of_suites_in_multiple_dirs,
+ %% order_of_groups_in_multiple_dirs,
+ %% order_of_groups_in_multiple_suites,
+ order_of_tests_in_multiple_suites_with_skip_no_merge_tests,
+ order_of_tests_in_multiple_suites_with_skip,
+ all_plus_one_tc_no_merge_tests,
+ all_plus_one_tc
].
%%--------------------------------------------------------------------
@@ -374,6 +384,105 @@ testcase_in_multiple_groups(Config) when is_list(Config) ->
%%%-----------------------------------------------------------------
%%%
+order_of_tests_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+
+ TestDir1 = filename:join(DataDir, "groups_1"),
+ TestDir2 = filename:join(DataDir, "groups_2"),
+ TestSpec = [{merge_tests, false},
+ {cases,TestDir1,groups_12_SUITE,[testcase_1a]},
+ {cases,TestDir2,groups_22_SUITE,[testcase_1]},
+ {cases,TestDir1,groups_12_SUITE,[testcase_1b]}],
+
+ setup_and_execute(order_of_tests_in_multiple_dirs_no_merge_tests,
+ TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_tests_in_multiple_suites_no_merge_tests(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+
+ TestDir1 = filename:join(DataDir, "groups_1"),
+ TestSpec = [{merge_tests, false},
+ {cases,TestDir1,groups_12_SUITE,[testcase_1a]},
+ {cases,TestDir1,groups_11_SUITE,[testcase_1]},
+ {cases,TestDir1,groups_12_SUITE,[testcase_1b]}],
+
+ setup_and_execute(order_of_tests_in_multiple_suites_no_merge_tests,
+ TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_suites_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+
+ TestDir1 = filename:join(DataDir, "groups_1"),
+ TestDir2 = filename:join(DataDir, "groups_2"),
+ TestSpec = [{merge_tests, false},
+ {suites,TestDir1,groups_12_SUITE},
+ {suites,TestDir2,groups_22_SUITE},
+ {suites,TestDir1,groups_11_SUITE}],
+
+ setup_and_execute(order_of_suites_in_multiple_dirs_no_merge_tests,
+ TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_groups_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+
+ TestDir1 = filename:join(DataDir, "groups_1"),
+ TestDir2 = filename:join(DataDir, "groups_2"),
+ TestSpec = [{merge_tests, false},
+ {groups,TestDir1,groups_12_SUITE,test_group_1a},
+ {groups,TestDir2,groups_22_SUITE,test_group_1a},
+ {groups,TestDir1,groups_12_SUITE,test_group_1b}],
+
+ setup_and_execute(order_of_groups_in_multiple_dirs_no_merge_tests,
+ TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_groups_in_multiple_suites_no_merge_tests(Config)
+ when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+
+ TestDir1 = filename:join(DataDir, "groups_1"),
+ TestSpec = [{merge_tests, false},
+ {groups,TestDir1,groups_12_SUITE,test_group_1a},
+ {groups,TestDir1,groups_11_SUITE,test_group_1a},
+ {groups,TestDir1,groups_12_SUITE,test_group_1b}],
+
+ setup_and_execute(order_of_groups_in_multiple_suites_no_merge_tests,
+ TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_tests_in_multiple_suites_with_skip_no_merge_tests(Config)
+ when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+
+ TestDir1 = filename:join(DataDir, "groups_1"),
+ TestSpec = [{merge_tests, false},
+ {cases,TestDir1,groups_12_SUITE,[testcase_1a]},
+ {cases,TestDir1,groups_11_SUITE,[testcase_1]},
+ {cases,TestDir1,groups_12_SUITE,[testcase_1b]},
+ {cases,TestDir1,groups_11_SUITE,[testcase_2]},
+ {skip_cases,TestDir1,groups_12_SUITE,[testcase_1b],"Skip it"}],
+
+ setup_and_execute(
+ order_of_tests_in_multiple_suites_with_skip_no_merge_tests,
+ TestSpec, Config).
+
+
+%%%-----------------------------------------------------------------
+%%%
+
order_of_tests_in_multiple_dirs(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
@@ -383,7 +492,8 @@ order_of_tests_in_multiple_dirs(Config) when is_list(Config) ->
{cases,TestDir2,groups_22_SUITE,[testcase_1]},
{cases,TestDir1,groups_12_SUITE,[testcase_1b]}],
- setup_and_execute(order_of_tests_in_multiple_dirs, TestSpec, Config).
+ setup_and_execute(order_of_tests_in_multiple_dirs,
+ TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
@@ -396,7 +506,8 @@ order_of_tests_in_multiple_suites(Config) when is_list(Config) ->
{cases,TestDir1,groups_11_SUITE,[testcase_1]},
{cases,TestDir1,groups_12_SUITE,[testcase_1b]}],
- setup_and_execute(order_of_tests_in_multiple_suites, TestSpec, Config).
+ setup_and_execute(order_of_tests_in_multiple_suites,
+ TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
@@ -410,7 +521,8 @@ order_of_suites_in_multiple_dirs(Config) when is_list(Config) ->
{suites,TestDir2,groups_22_SUITE},
{suites,TestDir1,groups_11_SUITE}],
- setup_and_execute(order_of_suites_in_multiple_dirs, TestSpec, Config).
+ setup_and_execute(order_of_suites_in_multiple_dirs,
+ TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
@@ -424,7 +536,8 @@ order_of_groups_in_multiple_dirs(Config) when is_list(Config) ->
{groups,TestDir2,groups_22_SUITE,test_group_1a},
{groups,TestDir1,groups_12_SUITE,test_group_1b}],
- setup_and_execute(order_of_groups_in_multiple_dirs, TestSpec, Config).
+ setup_and_execute(order_of_groups_in_multiple_dirs,
+ TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
@@ -437,7 +550,53 @@ order_of_groups_in_multiple_suites(Config) when is_list(Config) ->
{groups,TestDir1,groups_11_SUITE,test_group_1a},
{groups,TestDir1,groups_12_SUITE,test_group_1b}],
- setup_and_execute(order_of_groups_in_multiple_suites, TestSpec, Config).
+ setup_and_execute(order_of_groups_in_multiple_suites,
+ TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_tests_in_multiple_suites_with_skip(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+
+ TestDir1 = filename:join(DataDir, "groups_1"),
+ TestSpec = [{cases,TestDir1,groups_12_SUITE,[testcase_1a]},
+ {cases,TestDir1,groups_11_SUITE,[testcase_1]},
+ {cases,TestDir1,groups_12_SUITE,[testcase_1b]},
+ {cases,TestDir1,groups_11_SUITE,[testcase_2]},
+ {skip_cases,TestDir1,groups_12_SUITE,[testcase_1b],"Skip it!"}],
+
+ setup_and_execute(order_of_tests_in_multiple_suites_with_skip,
+ TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+all_plus_one_tc_no_merge_tests(Config) when is_list(Config) ->
+
+ DataDir = ?config(data_dir, Config),
+
+ TestDir1 = filename:join(DataDir, "groups_1"),
+ TestSpec = [{merge_tests,false},
+ {suites,TestDir1,groups_12_SUITE},
+ {cases,TestDir1,groups_12_SUITE,[testcase_1a]}],
+
+ setup_and_execute(all_plus_one_tc_no_merge_tests,
+ TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+all_plus_one_tc(Config) when is_list(Config) ->
+
+ DataDir = ?config(data_dir, Config),
+
+ TestDir1 = filename:join(DataDir, "groups_1"),
+ TestSpec = [{suites,TestDir1,groups_12_SUITE},
+ {cases,TestDir1,groups_12_SUITE,[testcase_1a]}],
+
+ setup_and_execute(all_plus_one_tc,
+ TestSpec, Config).
%%%-----------------------------------------------------------------
%%% HELP FUNCTIONS
@@ -503,7 +662,7 @@ events_to_check(Test, N) ->
test_events(Test) ++ events_to_check(Test, N-1).
-test_events(order_of_tests_in_multiple_dirs) ->
+test_events(order_of_tests_in_multiple_dirs_no_merge_tests) ->
[{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
{ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}},
{ct_test_support_eh,tc_done,
@@ -517,7 +676,7 @@ test_events(order_of_tests_in_multiple_dirs) ->
{failed,{error,{test_case_failed,no_group_data}}}}},
{ct_test_support_eh,stop_logging,[]}
];
-test_events(order_of_tests_in_multiple_suites) ->
+test_events(order_of_tests_in_multiple_suites_no_merge_tests) ->
[{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
{ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}},
{ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}},
@@ -527,7 +686,7 @@ test_events(order_of_tests_in_multiple_suites) ->
{ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}},
{ct_test_support_eh,stop_logging,[]}
];
-test_events(order_of_suites_in_multiple_dirs) ->
+test_events(order_of_suites_in_multiple_dirs_no_merge_tests) ->
[{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
{ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}},
{ct_test_support_eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}},
@@ -542,7 +701,7 @@ test_events(order_of_suites_in_multiple_dirs) ->
{ct_test_support_eh,tc_start,{groups_11_SUITE,end_per_suite}},
{ct_test_support_eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
{ct_test_support_eh,stop_logging,[]}];
-test_events(order_of_groups_in_multiple_dirs) ->
+test_events(order_of_groups_in_multiple_dirs_no_merge_tests) ->
[{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
{ct_test_support_eh,tc_start,
@@ -561,7 +720,7 @@ test_events(order_of_groups_in_multiple_dirs) ->
{groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}},
{ct_test_support_eh,stop_logging,[]}];
-test_events(order_of_groups_in_multiple_suites) ->
+test_events(order_of_groups_in_multiple_suites_no_merge_tests) ->
[{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
{ct_test_support_eh,tc_start,
@@ -580,6 +739,131 @@ test_events(order_of_groups_in_multiple_suites) ->
{groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}},
{ct_test_support_eh,stop_logging,[]}];
+test_events(order_of_tests_in_multiple_suites_with_skip_no_merge_tests) ->
+ [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}},
+ {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}},
+ {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_1}},
+ {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_1,ok}},
+ {ct_test_support_eh,tc_user_skip,{groups_12_SUITE,testcase_1b,'_'}},
+ {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_2}},
+ {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_2,ok}},
+ {ct_test_support_eh,stop_logging,[]}
+ ];
+
+test_events(order_of_tests_in_multiple_dirs) ->
+ [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}},
+ {ct_test_support_eh,tc_done,
+ {groups_12_SUITE,testcase_1a,
+ {failed,{error,{test_case_failed,no_group_data}}}}},
+ {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1b}},
+ {ct_test_support_eh,tc_done,
+ {groups_12_SUITE,testcase_1b,
+ {failed,{error,{test_case_failed,no_group_data}}}}},
+ {ct_test_support_eh,tc_start,{groups_22_SUITE,testcase_1}},
+ {ct_test_support_eh,tc_done,{groups_22_SUITE,testcase_1,ok}},
+ {ct_test_support_eh,stop_logging,[]}
+ ];
+test_events(order_of_tests_in_multiple_suites) ->
+ [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}},
+ {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}},
+
+ {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1b}},
+ {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}},
+
+ {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_1}},
+ {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_1,ok}},
+ {ct_test_support_eh,stop_logging,[]}
+ ];
+test_events(order_of_suites_in_multiple_dirs) ->
+ [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}},
+ {ct_test_support_eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}},
+ {ct_test_support_eh,tc_start,{groups_12_SUITE,end_per_suite}},
+ {ct_test_support_eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+
+ {ct_test_support_eh,tc_start,{groups_11_SUITE,init_per_suite}},
+ {ct_test_support_eh,tc_done,{groups_11_SUITE,init_per_suite,'_'}},
+ {ct_test_support_eh,tc_start,{groups_11_SUITE,end_per_suite}},
+ {ct_test_support_eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+
+ {ct_test_support_eh,tc_start,{groups_22_SUITE,init_per_suite}},
+ {ct_test_support_eh,tc_done,{groups_22_SUITE,init_per_suite,'_'}},
+ {ct_test_support_eh,tc_start,{groups_22_SUITE,end_per_suite}},
+ {ct_test_support_eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}},
+ {ct_test_support_eh,stop_logging,[]}];
+test_events(order_of_groups_in_multiple_dirs) ->
+ [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+
+ {ct_test_support_eh,tc_start,
+ {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}},
+ {ct_test_support_eh,tc_done,
+ {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}},
+
+ {ct_test_support_eh,tc_start,
+ {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}},
+ {ct_test_support_eh,tc_done,
+ {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}},
+
+ {ct_test_support_eh,tc_start,
+ {groups_22_SUITE,{init_per_group,test_group_1a,'_'}}},
+ {ct_test_support_eh,tc_done,
+ {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}},
+
+ {ct_test_support_eh,stop_logging,[]}];
+test_events(order_of_groups_in_multiple_suites) ->
+ [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+
+ {ct_test_support_eh,tc_start,
+ {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}},
+ {ct_test_support_eh,tc_done,
+ {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}},
+
+ {ct_test_support_eh,tc_start,
+ {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}},
+ {ct_test_support_eh,tc_done,
+ {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}},
+
+ {ct_test_support_eh,tc_start,
+ {groups_11_SUITE,{init_per_group,test_group_1a,'_'}}},
+ {ct_test_support_eh,tc_done,
+ {groups_11_SUITE,{end_per_group,test_group_1a,'_'},'_'}},
+
+ {ct_test_support_eh,stop_logging,[]}];
+
+test_events(order_of_tests_in_multiple_suites_with_skip) ->
+ [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,tc_start,{groups_12_SUITE,testcase_1a}},
+ {ct_test_support_eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}},
+ {ct_test_support_eh,tc_user_skip,{groups_12_SUITE,testcase_1b,'_'}},
+ {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_1}},
+ {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_1,ok}},
+ {ct_test_support_eh,tc_start,{groups_11_SUITE,testcase_2}},
+ {ct_test_support_eh,tc_done,{groups_11_SUITE,testcase_2,ok}},
+ {ct_test_support_eh,stop_logging,[]}
+ ];
+
+test_events(all_plus_one_tc_no_merge_tests) ->
+
+ [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}},
+ {ct_test_support_eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+ {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}},
+ {ct_test_support_eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+ {ct_test_support_eh,stop_logging,[]}
+ ];
+
+test_events(all_plus_one_tc) ->
+
+ [{ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
+ {ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}},
+ {ct_test_support_eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+ {negative,{ct_test_support_eh,tc_start,{groups_12_SUITE,init_per_suite}},
+ {ct_test_support_eh,stop_logging,[]}}
+ ];
+
test_events(_) ->
[
].