diff options
author | Peter Andersson <peppe@erlang.org> | 2010-07-01 20:41:33 +0200 |
---|---|---|
committer | Peter Andersson <peppe@erlang.org> | 2010-07-06 02:44:27 +0200 |
commit | 6b51c530111e478c9605cb88bb90178682ffe669 (patch) | |
tree | f18cba35d1870500008701d13e9469826520c001 | |
parent | afbd6fa341f89211697a3c3f59f888b8247aa7f4 (diff) | |
download | otp-6b51c530111e478c9605cb88bb90178682ffe669.tar.gz otp-6b51c530111e478c9605cb88bb90178682ffe669.tar.bz2 otp-6b51c530111e478c9605cb88bb90178682ffe669.zip |
Improve handling of test case group specifications
Still a few things missing:
* Documentation
* More tests
* Fix old broken test cases
-rw-r--r-- | lib/common_test/src/ct_framework.erl | 17 | ||||
-rw-r--r-- | lib/common_test/src/ct_run.erl | 22 | ||||
-rw-r--r-- | lib/common_test/src/ct_testspec.erl | 7 | ||||
-rw-r--r-- | lib/common_test/test/ct_testspec_1_SUITE.erl | 48 | ||||
-rw-r--r-- | lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl | 40 | ||||
-rw-r--r-- | lib/test_server/src/test_server_ctrl.erl | 10 |
6 files changed, 105 insertions, 39 deletions
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 17215e5eb5..4104115484 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -687,7 +687,20 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) -> [] -> {error,{invalid_group_spec,Name}}; ConfTests -> - ConfTests + case lists:member(skipped, Props) of + true -> + %% a *subgroup* specified *only* as skipped (and not + %% as an explicit test) should not be returned, or + %% init/end functions for top groups will be executed + case catch proplists:get_value(name, element(2, hd(ConfTests))) of + Name -> % top group + ConfTests; + _ -> + [] + end; + false -> + ConfTests + end end; _ -> E = "Bad return value from "++atom_to_list(Mod)++":groups/0", @@ -874,7 +887,7 @@ make_all_conf(Mod) -> [] -> {error,{invalid_group_spec,Mod}}; ConfTests -> - ConfTests + [{conf,Props,Init,all,End} || {conf,Props,Init,_,End} <- ConfTests] end end. diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 2591890898..c5bfd01642 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -1378,15 +1378,17 @@ suite_tuples([]) -> final_tests(Tests, Skip, Bad) -> -%%! --- Thu Jun 24 15:47:27 2010 --- peppe was here! -io:format(user, "FINAL0 = ~p~nSKIP0 = ~p~n", [Tests, Skip]), + %%! --- Thu Jun 24 15:47:27 2010 --- peppe was here! + %%! io:format(user, "FINAL0 = ~p~nSKIP0 = ~p~n", [Tests, Skip]), {Tests1,Skip1} = final_tests1(Tests, [], Skip, Bad), + Skip2 = final_skip(Skip1, []), -%%! --- Thu Jun 24 15:47:27 2010 --- peppe was here! -io:format(user, "FINAL1 = ~p~nSKIP1 = ~p~n", [Tests1, Skip1]), - {Tests1,final_skip(Skip1, [])}. + %%! --- Thu Jun 24 15:47:27 2010 --- peppe was here! + %%! io:format(user, "FINAL1 = ~p~nSKIP1 = ~p~n", [Tests1, Skip2]), + + {Tests1,Skip2}. final_tests1([{TestDir,Suites,_}|Tests], Final, Skip, Bad) when is_list(Suites), is_atom(hd(Suites)) -> @@ -1440,7 +1442,10 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when fun({all,all}) -> ct_framework:make_all_conf(TestDir, Suite, []); - ({Group,TCs}) -> + ({skipped,Group,TCs}) -> + [ct_framework:make_conf(TestDir, Suite, + Group, [skipped], TCs)]; + ({Group,TCs}) -> [ct_framework:make_conf(TestDir, Suite, Group, [], TCs)]; (TC) -> @@ -1453,6 +1458,11 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when final_tests1([], Final, Skip, _Bad) -> {lists:reverse(Final),Skip}. +final_skip([{TestDir,Suite,{all,all},Reason}|Skips], Final) -> + SkipConf = ct_framework:make_conf(TestDir, Suite, all, [], all), + Skip = {TestDir,Suite,SkipConf,Reason}, + final_skip(Skips, [Skip|Final]); + final_skip([{TestDir,Suite,{Group,TCs},Reason}|Skips], Final) -> Conf = ct_framework:make_conf(TestDir, Suite, Group, [], TCs), Skip = {TestDir,Suite,Conf,Reason}, diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl index 1ca60dc92c..f5069427a2 100644 --- a/lib/common_test/src/ct_testspec.erl +++ b/lib/common_test/src/ct_testspec.erl @@ -185,13 +185,13 @@ prepare_cases(Node,Dir,Suite,Cases) -> {[{{Node,Dir},{Suite,all}}],SkipAll}; Skipped -> %% note: this adds a test even if only skip is specified - PrepC = lists:foldr(fun({{G,_}=Group,{skip,_Cmt}}, Acc) when + PrepC = lists:foldr(fun({{G,Cs},{skip,_Cmt}}, Acc) when is_atom(G) -> case lists:keymember(G, 1, Cases) of true -> Acc; false -> - [Group|Acc] + [{skipped,G,Cs}|Acc] end; ({C,{skip,_Cmt}},Acc) -> case lists:member(C,Cases) of @@ -917,7 +917,8 @@ skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests) when is_atom(Group) -> skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests); skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests) when is_atom(Group) -> skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests); -skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case) -> +skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case), + Case =/= all -> skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests); skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when ((Cases == all) or is_list(Cases)) and is_list(Groups) -> diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl index cc66c99ea5..dc399bfb4c 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE.erl @@ -72,6 +72,7 @@ all(suite) -> subgroup, skip_subgroup, subgroup_all_testcases, skip_subgroup_all_testcases, subgroup_testcase, skip_subgroup_testcase, + sub_skipped_by_top, testcase_in_multiple_groups]. %%-------------------------------------------------------------------- @@ -174,9 +175,6 @@ all_groups(Config) when is_list(Config) -> setup_and_execute(all_groups, TestSpec, Config). - -%%! --- Wed Jun 30 00:12:31 2010 --- peppe was here! -%%! NOT WORKING skip_all_groups(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -189,8 +187,6 @@ skip_all_groups(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% -%%! --- Wed Jun 30 00:12:31 2010 --- peppe was here! -%%! NOT WORKING group(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -206,7 +202,7 @@ skip_group(Config) when is_list(Config) -> TestSpec = [{groups,TestDir,groups_11_SUITE,[test_group_1a, test_group_1b]}, {skip_groups,TestDir,groups_11_SUITE, - [test_group_1b,test_group_2],"SKIPPED!"}], + [test_group_1b,test_group_2,test_group_7],"SKIPPED!"}], setup_and_execute(skip_group, TestSpec, Config). @@ -214,8 +210,6 @@ skip_group(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% -%%! --- Wed Jun 30 00:18:11 2010 --- peppe was here! -%%! NOT WORKING group_all_testcases(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -255,7 +249,7 @@ skip_group_testcase(Config) when is_list(Config) -> TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a, {cases,[testcase_1a,testcase_1b]}}, {groups,TestDir,groups_11_SUITE,test_group_1b, - {cases,[testcase_1a,testcase_1b]}}, + {cases,[testcase_1b,testcase_1a]}}, {skip_groups,TestDir,groups_11_SUITE, test_group_1a,{cases,testcase_1b},"SKIPPED!"}, {skip_groups,TestDir,groups_11_SUITE, @@ -266,21 +260,18 @@ skip_group_testcase(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% -%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here! -%%! NOT WORKING! Group props disappear from get_suite? topgroup(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir = filename:join(DataDir, "groups_1"), - TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_2}], + TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_2}, + {groups,TestDir,groups_12_SUITE,test_group_4}], setup_and_execute(topgroup, TestSpec, Config). %%%----------------------------------------------------------------- %%% -%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here! -%%! NOT WORKING! subgroup(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), @@ -289,15 +280,13 @@ subgroup(Config) when is_list(Config) -> setup_and_execute(subgroup, TestSpec, Config). -%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here! -%%! NOT WORKING! skip_subgroup(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), TestDir = filename:join(DataDir, "groups_1"), - TestSpec = [{groups,TestDir,groups_12_SUITE,[test_group_2]}, + TestSpec = [{groups,TestDir,groups_12_SUITE,[test_group_4]}, {skip_groups,TestDir,groups_12_SUITE, - test_group_3,"SKIPPED!"}], + test_group_8,"SKIPPED!"}], setup_and_execute(skip_subgroup, TestSpec, Config). @@ -309,7 +298,9 @@ subgroup_all_testcases(Config) when is_list(Config) -> TestDir = filename:join(DataDir, "groups_1"), TestSpec = [{groups,TestDir,groups_12_SUITE, - test_group_5,{cases,all}}], + test_group_5,{cases,all}}, + {groups,TestDir,groups_12_SUITE, + test_group_3,{cases,all}}], setup_and_execute(subgroup_all_testcases, TestSpec, Config). @@ -331,7 +322,9 @@ subgroup_testcase(Config) when is_list(Config) -> TestDir = filename:join(DataDir, "groups_1"), TestSpec = [{groups,TestDir,groups_12_SUITE, - test_group_7,{cases,testcase_7a}}], + test_group_7,{cases,testcase_7a}}, + {groups,TestDir,groups_12_SUITE, + test_group_3,{cases,testcase_3b}}], setup_and_execute(subgroup_testcase, TestSpec, Config). @@ -348,6 +341,21 @@ skip_subgroup_testcase(Config) when is_list(Config) -> %%%----------------------------------------------------------------- %%% +%%! +%%! Somewhat weird result from this one: +%%! +sub_skipped_by_top(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + + TestDir = filename:join(DataDir, "groups_1"), + TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_5}, + {skip_groups,TestDir,groups_12_SUITE,test_group_4,"SKIPPED!"}], + + setup_and_execute(sub_skipped_by_top, TestSpec, Config). + +%%%----------------------------------------------------------------- +%%% + testcase_in_multiple_groups(Config) when is_list(Config) -> DataDir = ?config(data_dir, Config), diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl index dbf1576c76..69c06f9b83 100644 --- a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl +++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl @@ -48,9 +48,12 @@ groups() -> testcase_5b]}]}, - {test_group_6, [parallel], [{group, test_group_7}]}, + {test_group_6, [parallel], [{group, test_group_7}, + {group, test_group_8}]}, - {test_group_7, [sequence], [testcase_7a,testcase_7b]} + {test_group_7, [sequence], [testcase_7a,testcase_7b]}, + + {test_group_8, [shuffle,sequence], [testcase_8a,testcase_8b]} ]. all() -> @@ -68,7 +71,8 @@ grs_and_tcs() -> test_group_1a, test_group_1b, test_group_2, test_group_3, test_group_4, test_group_5, - test_group_6, test_group_7 + test_group_6, test_group_7, + test_group_8 ], [ testcase_1a, testcase_1b, testcase_1c, @@ -78,7 +82,8 @@ grs_and_tcs() -> testcase_3a, testcase_3b, testcase_3, testcase_5a, testcase_5b, - testcase_7a, testcase_7b + testcase_7a, testcase_7b, + testcase_8a, testcase_8b ]}. %%-------------------------------------------------------------------- @@ -107,7 +112,9 @@ init_per_group(Group, Config) -> {test_group_4,[{name,test_group_4}]} -> ok; {test_group_5,[{name,test_group_5},parallel]} -> "parallel"; {test_group_6,[{name,test_group_6},parallel]} -> "parallel"; - {test_group_7,[{name,test_group_7},sequence]} -> "sequence" + {test_group_7,[{name,test_group_7},sequence]} -> "sequence"; + {test_group_8,[{shuffle,_},{name,test_group_8},sequence]} -> + "shuffle & sequence" end, {Grs,_} = grs_and_tcs(), case lists:member(Group, Grs) of @@ -312,3 +319,26 @@ testcase_7b(Config) -> undefined = ?config(testcase_7a,Config), testcase_7b = ?config(testcase_7b,Config), ok. + +testcase_8a() -> + []. +testcase_8a(Config) -> + init = ?config(suite,Config), + undefined = ?config(test_group_3,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_8 = ?config(test_group_8,Config), + testcase_8a = ?config(testcase_8a,Config), + ok. +testcase_8b() -> + []. +testcase_8b(Config) -> + init = ?config(suite,Config), + test_group_4 = ?config(test_group_4,Config), + test_group_5 = ?config(test_group_5,Config), + test_group_6 = ?config(test_group_6,Config), + test_group_8 = ?config(test_group_8,Config), + undefined = ?config(testcase_8a,Config), + testcase_8b = ?config(testcase_8b,Config), + ok. diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 38a0b6d94d..72f274d63a 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -1671,6 +1671,7 @@ do_test_cases(TopCases, SkipCases, put(test_server_case_num, 0), TestSpec = add_init_and_end_per_suite(TestSpec0, undefined, undefined), + TI = get_target_info(), print(1, "Starting test~s", [print_if_known(N, {", ~w test cases",[N]}, {" (with repeated test cases)",[]})]), @@ -4436,7 +4437,7 @@ collect_all_cases(Top, Skip) when is_list(Skip) -> Result = case collect_cases(Top, #cc{mod=[],skip=Skip}) of {ok,Cases,_St} -> Cases; - Other -> Other + Other -> Other end, Result. @@ -4555,8 +4556,8 @@ collect_cases(Case, St) when is_atom(Case), is_atom(St#cc.mod) -> collect_cases(Other, St) -> {error,{bad_subtest_spec,St#cc.mod,Other}}. -collect_case({_Mod,{conf,_,_,_,_}=Conf}, St) -> - collect_cases(Conf, St); +collect_case({Mod,{conf,_,_,_,_}=Conf}, St) -> + collect_case_invoke(Mod, Conf, [], St); collect_case(MFA, St) -> case in_skip_list(MFA, St#cc.skip) of @@ -4593,6 +4594,7 @@ collect_case_invoke(Mod, Case, MFA, St) -> collect_subcases(Mod, Case, MFA, St, Suite) -> case Suite of [] when Case == all -> {ok,[],St}; + [] when element(1, Case) == conf -> {ok,[],St}; [] -> {ok,[MFA],St}; %%%! --- START Kept for backwards compatibilty --- %%%! Requirements are not used @@ -4680,6 +4682,8 @@ in_skip_list({Mod,{conf,Props,InitMF,_CaseList,_FinMF}}, SkipList) -> fun({M,{conf,SProps,_,SCaseList,_},Cmt}) when M == Mod -> case proplists:get_value(name, SProps) of + all -> + [{M,all,Cmt}]; Name -> case SCaseList of all -> |