Improve handling of test case group specifications

Still a few things missing:

 * Documentation
 * More tests
 * Fix old broken test cases

6 files changed, 105 insertions, 39 deletions

diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 17215e5eb5..4104115484 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -687,7 +687,20 @@ get_suite(Mod, Group={conf,Props,_Init,TCs,_End}) ->
 		[] ->
 		    {error,{invalid_group_spec,Name}};
 		ConfTests ->
-		    ConfTests
+		    case lists:member(skipped, Props) of
+			true ->
+			    %% a *subgroup* specified *only* as skipped (and not
+			    %% as an explicit test) should not be returned, or
+			    %% init/end functions for top groups will be executed
+			    case catch proplists:get_value(name, element(2, hd(ConfTests))) of
+				Name ->		% top group
+				    ConfTests;
+				_ ->
+				    []
+			    end;
+			false ->
+			    ConfTests
+		    end
 	    end;
 	_ ->
 	    E = "Bad return value from "++atom_to_list(Mod)++":groups/0",
@@ -874,7 +887,7 @@ make_all_conf(Mod) ->
 		[] ->
 		    {error,{invalid_group_spec,Mod}};
 		ConfTests ->
-		    ConfTests
+		    [{conf,Props,Init,all,End} || {conf,Props,Init,_,End} <- ConfTests]
 	    end
     end.
 
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index 2591890898..c5bfd01642 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -1378,15 +1378,17 @@ suite_tuples([]) ->
 
 final_tests(Tests, Skip, Bad) ->
 
-%%! --- Thu Jun 24 15:47:27 2010 --- peppe was here!
-io:format(user, "FINAL0 = ~p~nSKIP0 = ~p~n", [Tests, Skip]),
+    %%! --- Thu Jun 24 15:47:27 2010 --- peppe was here!
+    %%! io:format(user, "FINAL0 = ~p~nSKIP0 = ~p~n", [Tests, Skip]),
 
     {Tests1,Skip1} = final_tests1(Tests, [], Skip, Bad),
+    Skip2 = final_skip(Skip1, []),
 
-%%! --- Thu Jun 24 15:47:27 2010 --- peppe was here!
-io:format(user, "FINAL1 = ~p~nSKIP1 = ~p~n", [Tests1, Skip1]),
 
-    {Tests1,final_skip(Skip1, [])}.
+    %%! --- Thu Jun 24 15:47:27 2010 --- peppe was here!
+    %%! io:format(user, "FINAL1 = ~p~nSKIP1 = ~p~n", [Tests1, Skip2]),
+
+    {Tests1,Skip2}.
 
 final_tests1([{TestDir,Suites,_}|Tests], Final, Skip, Bad) when
       is_list(Suites), is_atom(hd(Suites)) ->
@@ -1440,7 +1442,10 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when
 		  fun({all,all}) ->
 			  ct_framework:make_all_conf(TestDir,
 						      Suite, []);
-		      ({Group,TCs}) ->
+		     ({skipped,Group,TCs}) ->
+			  [ct_framework:make_conf(TestDir, Suite,
+						  Group, [skipped], TCs)];
+		     ({Group,TCs}) ->
 			  [ct_framework:make_conf(TestDir, Suite,
 						  Group, [], TCs)];
 		     (TC) ->
@@ -1453,6 +1458,11 @@ final_tests1([{TestDir,Suite,GrsOrCs}|Tests], Final, Skip, Bad) when
 final_tests1([], Final, Skip, _Bad) ->
     {lists:reverse(Final),Skip}.
 
+final_skip([{TestDir,Suite,{all,all},Reason}|Skips], Final) ->
+    SkipConf =  ct_framework:make_conf(TestDir, Suite, all, [], all),
+    Skip = {TestDir,Suite,SkipConf,Reason},
+    final_skip(Skips, [Skip|Final]);
+
 final_skip([{TestDir,Suite,{Group,TCs},Reason}|Skips], Final) ->
     Conf =  ct_framework:make_conf(TestDir, Suite, Group, [], TCs),
     Skip = {TestDir,Suite,Conf,Reason},
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 1ca60dc92c..f5069427a2 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -185,13 +185,13 @@ prepare_cases(Node,Dir,Suite,Cases) ->
 	    {[{{Node,Dir},{Suite,all}}],SkipAll};
 	Skipped ->
 	    %% note: this adds a test even if only skip is specified
-	    PrepC = lists:foldr(fun({{G,_}=Group,{skip,_Cmt}}, Acc) when
+	    PrepC = lists:foldr(fun({{G,Cs},{skip,_Cmt}}, Acc) when
 					  is_atom(G) ->
 					case lists:keymember(G, 1, Cases) of
 					    true ->
 						Acc;
 					    false ->
-						[Group|Acc]
+						[{skipped,G,Cs}|Acc]
 					end;
 				   ({C,{skip,_Cmt}},Acc) ->
 					case lists:member(C,Cases) of
@@ -917,7 +917,8 @@ skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests) when is_atom(Group) ->
     skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests);
 skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests) when is_atom(Group) ->
     skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests);
-skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case) ->
+skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case),
+						       Case =/= all ->
     skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests);
 skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when
       ((Cases == all) or is_list(Cases)) and is_list(Groups) ->
diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl
index cc66c99ea5..dc399bfb4c 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE.erl
@@ -72,6 +72,7 @@ all(suite) ->
      subgroup, skip_subgroup,
      subgroup_all_testcases, skip_subgroup_all_testcases,
      subgroup_testcase, skip_subgroup_testcase,
+     sub_skipped_by_top,
      testcase_in_multiple_groups].
 
 %%--------------------------------------------------------------------
@@ -174,9 +175,6 @@ all_groups(Config) when is_list(Config) ->
 
     setup_and_execute(all_groups, TestSpec, Config).
 
-
-%%! --- Wed Jun 30 00:12:31 2010 --- peppe was here!
-%%! NOT WORKING
 skip_all_groups(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
@@ -189,8 +187,6 @@ skip_all_groups(Config) when is_list(Config) ->
 %%%-----------------------------------------------------------------
 %%%
 
-%%! --- Wed Jun 30 00:12:31 2010 --- peppe was here!
-%%! NOT WORKING
 group(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
@@ -206,7 +202,7 @@ skip_group(Config) when is_list(Config) ->
     TestSpec = [{groups,TestDir,groups_11_SUITE,[test_group_1a,
 						 test_group_1b]},
 		{skip_groups,TestDir,groups_11_SUITE,
-		 [test_group_1b,test_group_2],"SKIPPED!"}],
+		 [test_group_1b,test_group_2,test_group_7],"SKIPPED!"}],
 
     setup_and_execute(skip_group, TestSpec, Config).
 
@@ -214,8 +210,6 @@ skip_group(Config) when is_list(Config) ->
 %%%-----------------------------------------------------------------
 %%%
 
-%%! --- Wed Jun 30 00:18:11 2010 --- peppe was here!
-%%! NOT WORKING
 group_all_testcases(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
@@ -255,7 +249,7 @@ skip_group_testcase(Config) when is_list(Config) ->
     TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a,
 		 {cases,[testcase_1a,testcase_1b]}},
 		{groups,TestDir,groups_11_SUITE,test_group_1b,
-		 {cases,[testcase_1a,testcase_1b]}},
+		 {cases,[testcase_1b,testcase_1a]}},
 		{skip_groups,TestDir,groups_11_SUITE,
 		 test_group_1a,{cases,testcase_1b},"SKIPPED!"},
 		{skip_groups,TestDir,groups_11_SUITE,
@@ -266,21 +260,18 @@ skip_group_testcase(Config) when is_list(Config) ->
 %%%-----------------------------------------------------------------
 %%%
 
-%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here!
-%%! NOT WORKING! Group props disappear from get_suite?
 topgroup(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
     TestDir = filename:join(DataDir, "groups_1"),
-    TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_2}],
+    TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_2},
+		{groups,TestDir,groups_12_SUITE,test_group_4}],
 
     setup_and_execute(topgroup, TestSpec, Config).
 
 %%%-----------------------------------------------------------------
 %%%
 
-%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here!
-%%! NOT WORKING!
 subgroup(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
@@ -289,15 +280,13 @@ subgroup(Config) when is_list(Config) ->
 
     setup_and_execute(subgroup, TestSpec, Config).
 
-%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here!
-%%! NOT WORKING!
 skip_subgroup(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
     TestDir = filename:join(DataDir, "groups_1"),
-    TestSpec = [{groups,TestDir,groups_12_SUITE,[test_group_2]},
+    TestSpec = [{groups,TestDir,groups_12_SUITE,[test_group_4]},
 		{skip_groups,TestDir,groups_12_SUITE,
-		 test_group_3,"SKIPPED!"}],
+		 test_group_8,"SKIPPED!"}],
 
     setup_and_execute(skip_subgroup, TestSpec, Config).
 
@@ -309,7 +298,9 @@ subgroup_all_testcases(Config) when is_list(Config) ->
 
     TestDir = filename:join(DataDir, "groups_1"),
     TestSpec = [{groups,TestDir,groups_12_SUITE,
-		 test_group_5,{cases,all}}],
+		 test_group_5,{cases,all}},
+		{groups,TestDir,groups_12_SUITE,
+		 test_group_3,{cases,all}}],
 
     setup_and_execute(subgroup_all_testcases, TestSpec, Config).
 
@@ -331,7 +322,9 @@ subgroup_testcase(Config) when is_list(Config) ->
 
     TestDir = filename:join(DataDir, "groups_1"),
     TestSpec = [{groups,TestDir,groups_12_SUITE,
-		 test_group_7,{cases,testcase_7a}}],
+		 test_group_7,{cases,testcase_7a}},
+		{groups,TestDir,groups_12_SUITE,
+		 test_group_3,{cases,testcase_3b}}],
 
     setup_and_execute(subgroup_testcase, TestSpec, Config).
 
@@ -348,6 +341,21 @@ skip_subgroup_testcase(Config) when is_list(Config) ->
 %%%-----------------------------------------------------------------
 %%%
 
+%%!
+%%! Somewhat weird result from this one:
+%%!
+sub_skipped_by_top(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_5},
+		{skip_groups,TestDir,groups_12_SUITE,test_group_4,"SKIPPED!"}],
+
+    setup_and_execute(sub_skipped_by_top, TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
 testcase_in_multiple_groups(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl
index dbf1576c76..69c06f9b83 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl
@@ -48,9 +48,12 @@ groups() ->
 
 						  testcase_5b]}]},
 
-      {test_group_6, [parallel], [{group, test_group_7}]},
+      {test_group_6, [parallel], [{group, test_group_7},
+				  {group, test_group_8}]},
 
-      {test_group_7, [sequence], [testcase_7a,testcase_7b]}
+      {test_group_7, [sequence], [testcase_7a,testcase_7b]},
+
+      {test_group_8, [shuffle,sequence], [testcase_8a,testcase_8b]}
      ].
 
 all() ->
@@ -68,7 +71,8 @@ grs_and_tcs() ->
       test_group_1a, test_group_1b,
       test_group_2, test_group_3,
       test_group_4, test_group_5,
-      test_group_6, test_group_7
+      test_group_6, test_group_7,
+      test_group_8
      ],
      [
       testcase_1a, testcase_1b, testcase_1c,
@@ -78,7 +82,8 @@ grs_and_tcs() ->
       testcase_3a, testcase_3b,
       testcase_3,
       testcase_5a, testcase_5b,
-      testcase_7a, testcase_7b
+      testcase_7a, testcase_7b,
+      testcase_8a, testcase_8b
      ]}.
 
 %%--------------------------------------------------------------------
@@ -107,7 +112,9 @@ init_per_group(Group, Config) ->
 	    {test_group_4,[{name,test_group_4}]} -> ok;
 	    {test_group_5,[{name,test_group_5},parallel]} -> "parallel";
 	    {test_group_6,[{name,test_group_6},parallel]} -> "parallel";
-	    {test_group_7,[{name,test_group_7},sequence]} -> "sequence"
+	    {test_group_7,[{name,test_group_7},sequence]} -> "sequence";
+	    {test_group_8,[{shuffle,_},{name,test_group_8},sequence]} ->
+		"shuffle & sequence"
 	end,
     {Grs,_} = grs_and_tcs(),
     case lists:member(Group, Grs) of
@@ -312,3 +319,26 @@ testcase_7b(Config) ->
     undefined = ?config(testcase_7a,Config),
     testcase_7b = ?config(testcase_7b,Config),
     ok.
+
+testcase_8a() ->
+    [].
+testcase_8a(Config) ->
+    init = ?config(suite,Config),
+    undefined = ?config(test_group_3,Config),
+    test_group_4 = ?config(test_group_4,Config),
+    test_group_5 = ?config(test_group_5,Config),
+    test_group_6 = ?config(test_group_6,Config),
+    test_group_8 = ?config(test_group_8,Config),
+    testcase_8a = ?config(testcase_8a,Config),
+    ok.
+testcase_8b() ->
+    [].
+testcase_8b(Config) ->
+    init = ?config(suite,Config),
+    test_group_4 = ?config(test_group_4,Config),
+    test_group_5 = ?config(test_group_5,Config),
+    test_group_6 = ?config(test_group_6,Config),
+    test_group_8 = ?config(test_group_8,Config),
+    undefined = ?config(testcase_8a,Config),
+    testcase_8b = ?config(testcase_8b,Config),
+    ok.
diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl
index 38a0b6d94d..72f274d63a 100644
--- a/lib/test_server/src/test_server_ctrl.erl
+++ b/lib/test_server/src/test_server_ctrl.erl
@@ -1671,6 +1671,7 @@ do_test_cases(TopCases, SkipCases,
 	    put(test_server_case_num, 0),
 	    TestSpec =
 		add_init_and_end_per_suite(TestSpec0, undefined, undefined),
+
 	    TI = get_target_info(),
 	    print(1, "Starting test~s", [print_if_known(N, {", ~w test cases",[N]},
 							{" (with repeated test cases)",[]})]),
@@ -4436,7 +4437,7 @@ collect_all_cases(Top, Skip) when is_list(Skip) ->
     Result =
 	case collect_cases(Top, #cc{mod=[],skip=Skip}) of
 	    {ok,Cases,_St} -> Cases;
-	    Other -> Other
+	    Other          -> Other
 	end,
     Result.
 
@@ -4555,8 +4556,8 @@ collect_cases(Case, St) when is_atom(Case), is_atom(St#cc.mod) ->
 collect_cases(Other, St) ->
     {error,{bad_subtest_spec,St#cc.mod,Other}}.
 
-collect_case({_Mod,{conf,_,_,_,_}=Conf}, St) ->
-    collect_cases(Conf, St);
+collect_case({Mod,{conf,_,_,_,_}=Conf}, St) ->
+    collect_case_invoke(Mod, Conf, [], St);
 
 collect_case(MFA, St) ->
     case in_skip_list(MFA, St#cc.skip) of
@@ -4593,6 +4594,7 @@ collect_case_invoke(Mod, Case, MFA, St) ->
 collect_subcases(Mod, Case, MFA, St, Suite) ->
     case Suite of
 	[] when Case == all -> {ok,[],St};
+	[] when element(1, Case) == conf -> {ok,[],St};
 	[] -> {ok,[MFA],St};
 %%%! --- START Kept for backwards compatibilty ---
 %%%! Requirements are not used
@@ -4680,6 +4682,8 @@ in_skip_list({Mod,{conf,Props,InitMF,_CaseList,_FinMF}}, SkipList) ->
 			  fun({M,{conf,SProps,_,SCaseList,_},Cmt}) when
 				    M == Mod ->
 				  case proplists:get_value(name, SProps) of
+				      all ->
+					  [{M,all,Cmt}];
 				      Name ->
 					  case SCaseList of
 					      all ->