From 6b51c530111e478c9605cb88bb90178682ffe669 Mon Sep 17 00:00:00 2001
From: Peter Andersson <peppe@erlang.org>
Date: Thu, 1 Jul 2010 20:41:33 +0200
Subject: Improve handling of test case group specifications
Still a few things missing:
* Documentation
* More tests
* Fix old broken test cases
---
lib/common_test/src/ct_framework.erl | 17 +++++++-
lib/common_test/src/ct_run.erl | 22 +++++++---
lib/common_test/src/ct_testspec.erl | 7 ++--
lib/common_test/test/ct_testspec_1_SUITE.erl | 48 +++++++++++++---------
.../groups_1/groups_12_SUITE.erl | 40 +++++++++++++++---
lib/test_server/src/test_server_ctrl.erl | 10 +++--
6 files changed, 105 insertions(+), 39 deletions(-)
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 17215e5eb5..4104115484 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -687,7 +687,20 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
[] ->
{error,{invalid_group_spec,Name}};
ConfTests ->
- ConfTests
+ case lists:member(skipped, Props) of
+ true ->
+ %% a *subgroup* specified *only* as skipped (and not
+ %% as an explicit test) should not be returned, or
+ %% init/end functions for top groups will be executed
+ case catch proplists:get_value(name, element(2, hd(ConfTests))) of
+ Name -> % top group
+ ConfTests;
+ _ ->
+ []
+ end;
+ false ->
+ ConfTests
+ end
end;
_ ->
E = "Bad return value from "++atom_to_list(Mod)++":groups/0",
@@ -874,7 +887,7 @@ make_all_conf(Mod) ->
[] ->
{error,{invalid_group_spec,Mod}};
ConfTests ->
- ConfTests
+ [{conf,Props,Init,all,End} || {conf,Props,Init,_,End} <- ConfTests]
end
end.
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index 2591890898..c5bfd01642 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -1378,15 +1378,17 @@ suite_tuples([]) ->
final_tests(Tests, Skip, Bad) ->
-%%! --- Thu Jun 24 15:47:27 2010 --- peppe was here!
-io:format(user, "FINAL0 = ~p~nSKIP0 = ~p~n", [Tests, Skip]),
+ %%! --- Thu Jun 24 15:47:27 2010 --- peppe was here!
+ %%! io:format(user, "FINAL0 = ~p~nSKIP0 = ~p~n", [Tests, Skip]),
{Tests1,Skip1} = final_tests1(Tests, [], Skip, Bad),
+ Skip2 = final_skip(Skip1, []),
-%%! --- Thu Jun 24 15:47:27 2010 --- peppe was here!
-io:format(user, "FINAL1 = ~p~nSKIP1 = ~p~n", [Tests1, Skip1]),
- {Tests1,final_skip(Skip1, [])}.
+ %%! --- Thu Jun 24 15:47:27 2010 --- peppe was here!
+ %%! io:format(user, "FINAL1 = ~p~nSKIP1 = ~p~n", [Tests1, Skip2]),
+
+ {Tests1,Skip2}.
final_tests1([{TestDir,Suites,_}|Tests], Final, Skip, Bad) when
is_list(Suites), is_atom(hd(Suites)) ->
@@ -1440,7 +1442,10 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when
fun({all,all}) ->
ct_framework:make_all_conf(TestDir,
Suite, []);
- ({Group,TCs}) ->
+ ({skipped,Group,TCs}) ->
+ [ct_framework:make_conf(TestDir, Suite,
+ Group, [skipped], TCs)];
+ ({Group,TCs}) ->
[ct_framework:make_conf(TestDir, Suite,
Group, [], TCs)];
(TC) ->
@@ -1453,6 +1458,11 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when
final_tests1([], Final, Skip, _Bad) ->
{lists:reverse(Final),Skip}.
+final_skip([{TestDir,Suite,{all,all},Reason}|Skips], Final) ->
+ SkipConf = ct_framework:make_conf(TestDir, Suite, all, [], all),
+ Skip = {TestDir,Suite,SkipConf,Reason},
+ final_skip(Skips, [Skip|Final]);
+
final_skip([{TestDir,Suite,{Group,TCs},Reason}|Skips], Final) ->
Conf = ct_framework:make_conf(TestDir, Suite, Group, [], TCs),
Skip = {TestDir,Suite,Conf,Reason},
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 1ca60dc92c..f5069427a2 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -185,13 +185,13 @@ prepare_cases(Node,Dir,Suite,Cases) ->
{[{{Node,Dir},{Suite,all}}],SkipAll};
Skipped ->
%% note: this adds a test even if only skip is specified
- PrepC = lists:foldr(fun({{G,_}=Group,{skip,_Cmt}}, Acc) when
+ PrepC = lists:foldr(fun({{G,Cs},{skip,_Cmt}}, Acc) when
is_atom(G) ->
case lists:keymember(G, 1, Cases) of
true ->
Acc;
false ->
- [Group|Acc]
+ [{skipped,G,Cs}|Acc]
end;
({C,{skip,_Cmt}},Acc) ->
case lists:member(C,Cases) of
@@ -917,7 +917,8 @@ skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests) when is_atom(Group) ->
skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests);
skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests) when is_atom(Group) ->
skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests);
-skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case) ->
+skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case),
+ Case =/= all ->
skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests);
skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when
((Cases == all) or is_list(Cases)) and is_list(Groups) ->
diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl
index cc66c99ea5..dc399bfb4c 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE.erl
@@ -72,6 +72,7 @@ all(suite) ->
subgroup, skip_subgroup,
subgroup_all_testcases, skip_subgroup_all_testcases,
subgroup_testcase, skip_subgroup_testcase,
+ sub_skipped_by_top,
testcase_in_multiple_groups].
%%--------------------------------------------------------------------
@@ -174,9 +175,6 @@ all_groups(Config) when is_list(Config) ->
setup_and_execute(all_groups, TestSpec, Config).
-
-%%! --- Wed Jun 30 00:12:31 2010 --- peppe was here!
-%%! NOT WORKING
skip_all_groups(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
@@ -189,8 +187,6 @@ skip_all_groups(Config) when is_list(Config) ->
%%%-----------------------------------------------------------------
%%%
-%%! --- Wed Jun 30 00:12:31 2010 --- peppe was here!
-%%! NOT WORKING
group(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
@@ -206,7 +202,7 @@ skip_group(Config) when is_list(Config) ->
TestSpec = [{groups,TestDir,groups_11_SUITE,[test_group_1a,
test_group_1b]},
{skip_groups,TestDir,groups_11_SUITE,
- [test_group_1b,test_group_2],"SKIPPED!"}],
+ [test_group_1b,test_group_2,test_group_7],"SKIPPED!"}],
setup_and_execute(skip_group, TestSpec, Config).
@@ -214,8 +210,6 @@ skip_group(Config) when is_list(Config) ->
%%%-----------------------------------------------------------------
%%%
-%%! --- Wed Jun 30 00:18:11 2010 --- peppe was here!
-%%! NOT WORKING
group_all_testcases(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
@@ -255,7 +249,7 @@ skip_group_testcase(Config) when is_list(Config) ->
TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a,
{cases,[testcase_1a,testcase_1b]}},
{groups,TestDir,groups_11_SUITE,test_group_1b,
- {cases,[testcase_1a,testcase_1b]}},
+ {cases,[testcase_1b,testcase_1a]}},
{skip_groups,TestDir,groups_11_SUITE,
test_group_1a,{cases,testcase_1b},"SKIPPED!"},
{skip_groups,TestDir,groups_11_SUITE,
@@ -266,21 +260,18 @@ skip_group_testcase(Config) when is_list(Config) ->
%%%-----------------------------------------------------------------
%%%
-%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here!
-%%! NOT WORKING! Group props disappear from get_suite?
topgroup(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
TestDir = filename:join(DataDir, "groups_1"),
- TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_2}],
+ TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_2},
+ {groups,TestDir,groups_12_SUITE,test_group_4}],
setup_and_execute(topgroup, TestSpec, Config).
%%%-----------------------------------------------------------------
%%%
-%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here!
-%%! NOT WORKING!
subgroup(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
@@ -289,15 +280,13 @@ subgroup(Config) when is_list(Config) ->
setup_and_execute(subgroup, TestSpec, Config).
-%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here!
-%%! NOT WORKING!
skip_subgroup(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
TestDir = filename:join(DataDir, "groups_1"),
- TestSpec = [{groups,TestDir,groups_12_SUITE,[test_group_2]},
+ TestSpec = [{groups,TestDir,groups_12_SUITE,[test_group_4]},
{skip_groups,TestDir,groups_12_SUITE,
- test_group_3,"SKIPPED!"}],
+ test_group_8,"SKIPPED!"}],
setup_and_execute(skip_subgroup, TestSpec, Config).
@@ -309,7 +298,9 @@ subgroup_all_testcases(Config) when is_list(Config) ->
TestDir = filename:join(DataDir, "groups_1"),
TestSpec = [{groups,TestDir,groups_12_SUITE,
- test_group_5,{cases,all}}],
+ test_group_5,{cases,all}},
+ {groups,TestDir,groups_12_SUITE,
+ test_group_3,{cases,all}}],
setup_and_execute(subgroup_all_testcases, TestSpec, Config).
@@ -331,7 +322,9 @@ subgroup_testcase(Config) when is_list(Config) ->
TestDir = filename:join(DataDir, "groups_1"),
TestSpec = [{groups,TestDir,groups_12_SUITE,
- test_group_7,{cases,testcase_7a}}],
+ test_group_7,{cases,testcase_7a}},
+ {groups,TestDir,groups_12_SUITE,
+ test_group_3,{cases,testcase_3b}}],
setup_and_execute(subgroup_testcase, TestSpec, Config).
@@ -348,6 +341,21 @@ skip_subgroup_testcase(Config) when is_list(Config) ->
%%%-----------------------------------------------------------------
%%%
+%%!
+%%! Somewhat weird result from this one:
+%%!
+sub_skipped_by_top(Config) when is_list(Config) ->
+ DataDir = ?config(data_dir, Config),
+
+ TestDir = filename:join(DataDir, "groups_1"),
+ TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_5},
+ {skip_groups,TestDir,groups_12_SUITE,test_group_4,"SKIPPED!"}],
+
+ setup_and_execute(sub_skipped_by_top, TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
testcase_in_multiple_groups(Config) when is_list(Config) ->
DataDir = ?config(data_dir, Config),
diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl
index dbf1576c76..69c06f9b83 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl
@@ -48,9 +48,12 @@ groups() ->
testcase_5b]}]},
- {test_group_6, [parallel], [{group, test_group_7}]},
+ {test_group_6, [parallel], [{group, test_group_7},
+ {group, test_group_8}]},
- {test_group_7, [sequence], [testcase_7a,testcase_7b]}
+ {test_group_7, [sequence], [testcase_7a,testcase_7b]},
+
+ {test_group_8, [shuffle,sequence], [testcase_8a,testcase_8b]}
].
all() ->
@@ -68,7 +71,8 @@ grs_and_tcs() ->
test_group_1a, test_group_1b,
test_group_2, test_group_3,
test_group_4, test_group_5,
- test_group_6, test_group_7
+ test_group_6, test_group_7,
+ test_group_8
],
[
testcase_1a, testcase_1b, testcase_1c,
@@ -78,7 +82,8 @@ grs_and_tcs() ->
testcase_3a, testcase_3b,
testcase_3,
testcase_5a, testcase_5b,
- testcase_7a, testcase_7b
+ testcase_7a, testcase_7b,
+ testcase_8a, testcase_8b
]}.
%%--------------------------------------------------------------------
@@ -107,7 +112,9 @@ init_per_group(Group, Config) ->
{test_group_4,[{name,test_group_4}]} -> ok;
{test_group_5,[{name,test_group_5},parallel]} -> "parallel";
{test_group_6,[{name,test_group_6},parallel]} -> "parallel";
- {test_group_7,[{name,test_group_7},sequence]} -> "sequence"
+ {test_group_7,[{name,test_group_7},sequence]} -> "sequence";
+ {test_group_8,[{shuffle,_},{name,test_group_8},sequence]} ->
+ "shuffle & sequence"
end,
{Grs,_} = grs_and_tcs(),
case lists:member(Group, Grs) of
@@ -312,3 +319,26 @@ testcase_7b(Config) ->
undefined = ?config(testcase_7a,Config),
testcase_7b = ?config(testcase_7b,Config),
ok.
+
+testcase_8a() ->
+ [].
+testcase_8a(Config) ->
+ init = ?config(suite,Config),
+ undefined = ?config(test_group_3,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ test_group_6 = ?config(test_group_6,Config),
+ test_group_8 = ?config(test_group_8,Config),
+ testcase_8a = ?config(testcase_8a,Config),
+ ok.
+testcase_8b() ->
+ [].
+testcase_8b(Config) ->
+ init = ?config(suite,Config),
+ test_group_4 = ?config(test_group_4,Config),
+ test_group_5 = ?config(test_group_5,Config),
+ test_group_6 = ?config(test_group_6,Config),
+ test_group_8 = ?config(test_group_8,Config),
+ undefined = ?config(testcase_8a,Config),
+ testcase_8b = ?config(testcase_8b,Config),
+ ok.
diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl
index 38a0b6d94d..72f274d63a 100644
--- a/lib/test_server/src/test_server_ctrl.erl
+++ b/lib/test_server/src/test_server_ctrl.erl
@@ -1671,6 +1671,7 @@ do_test_cases(TopCases, SkipCases,
put(test_server_case_num, 0),
TestSpec =
add_init_and_end_per_suite(TestSpec0, undefined, undefined),
+
TI = get_target_info(),
print(1, "Starting test~s", [print_if_known(N, {", ~w test cases",[N]},
{" (with repeated test cases)",[]})]),
@@ -4436,7 +4437,7 @@ collect_all_cases(Top, Skip) when is_list(Skip) ->
Result =
case collect_cases(Top, #cc{mod=[],skip=Skip}) of
{ok,Cases,_St} -> Cases;
- Other -> Other
+ Other -> Other
end,
Result.
@@ -4555,8 +4556,8 @@ collect_cases(Case, St) when is_atom(Case), is_atom(St#cc.mod) ->
collect_cases(Other, St) ->
{error,{bad_subtest_spec,St#cc.mod,Other}}.
-collect_case({_Mod,{conf,_,_,_,_}=Conf}, St) ->
- collect_cases(Conf, St);
+collect_case({Mod,{conf,_,_,_,_}=Conf}, St) ->
+ collect_case_invoke(Mod, Conf, [], St);
collect_case(MFA, St) ->
case in_skip_list(MFA, St#cc.skip) of
@@ -4593,6 +4594,7 @@ collect_case_invoke(Mod, Case, MFA, St) ->
collect_subcases(Mod, Case, MFA, St, Suite) ->
case Suite of
[] when Case == all -> {ok,[],St};
+ [] when element(1, Case) == conf -> {ok,[],St};
[] -> {ok,[MFA],St};
%%%! --- START Kept for backwards compatibilty ---
%%%! Requirements are not used
@@ -4680,6 +4682,8 @@ in_skip_list({Mod,{conf,Props,InitMF,_CaseList,_FinMF}}, SkipList) ->
fun({M,{conf,SProps,_,SCaseList,_},Cmt}) when
M == Mod ->
case proplists:get_value(name, SProps) of
+ all ->
+ [{M,all,Cmt}];
Name ->
case SCaseList of
all ->
--
cgit v1.2.3