Improve handling of test case group specifications

Still a few things missing:

 * Documentation
 * More tests
 * Fix old broken test cases

2 files changed, 63 insertions, 25 deletions

diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl
index cc66c99ea5..dc399bfb4c 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE.erl
@@ -72,6 +72,7 @@ all(suite) ->
      subgroup, skip_subgroup,
      subgroup_all_testcases, skip_subgroup_all_testcases,
      subgroup_testcase, skip_subgroup_testcase,
+     sub_skipped_by_top,
      testcase_in_multiple_groups].
 
 %%--------------------------------------------------------------------
@@ -174,9 +175,6 @@ all_groups(Config) when is_list(Config) ->
 
     setup_and_execute(all_groups, TestSpec, Config).
 
-
-%%! --- Wed Jun 30 00:12:31 2010 --- peppe was here!
-%%! NOT WORKING
 skip_all_groups(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
@@ -189,8 +187,6 @@ skip_all_groups(Config) when is_list(Config) ->
 %%%-----------------------------------------------------------------
 %%%
 
-%%! --- Wed Jun 30 00:12:31 2010 --- peppe was here!
-%%! NOT WORKING
 group(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
@@ -206,7 +202,7 @@ skip_group(Config) when is_list(Config) ->
     TestSpec = [{groups,TestDir,groups_11_SUITE,[test_group_1a,
 						 test_group_1b]},
 		{skip_groups,TestDir,groups_11_SUITE,
-		 [test_group_1b,test_group_2],"SKIPPED!"}],
+		 [test_group_1b,test_group_2,test_group_7],"SKIPPED!"}],
 
     setup_and_execute(skip_group, TestSpec, Config).
 
@@ -214,8 +210,6 @@ skip_group(Config) when is_list(Config) ->
 %%%-----------------------------------------------------------------
 %%%
 
-%%! --- Wed Jun 30 00:18:11 2010 --- peppe was here!
-%%! NOT WORKING
 group_all_testcases(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
@@ -255,7 +249,7 @@ skip_group_testcase(Config) when is_list(Config) ->
     TestSpec = [{groups,TestDir,groups_11_SUITE,test_group_1a,
 		 {cases,[testcase_1a,testcase_1b]}},
 		{groups,TestDir,groups_11_SUITE,test_group_1b,
-		 {cases,[testcase_1a,testcase_1b]}},
+		 {cases,[testcase_1b,testcase_1a]}},
 		{skip_groups,TestDir,groups_11_SUITE,
 		 test_group_1a,{cases,testcase_1b},"SKIPPED!"},
 		{skip_groups,TestDir,groups_11_SUITE,
@@ -266,21 +260,18 @@ skip_group_testcase(Config) when is_list(Config) ->
 %%%-----------------------------------------------------------------
 %%%
 
-%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here!
-%%! NOT WORKING! Group props disappear from get_suite?
 topgroup(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
     TestDir = filename:join(DataDir, "groups_1"),
-    TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_2}],
+    TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_2},
+		{groups,TestDir,groups_12_SUITE,test_group_4}],
 
     setup_and_execute(topgroup, TestSpec, Config).
 
 %%%-----------------------------------------------------------------
 %%%
 
-%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here!
-%%! NOT WORKING!
 subgroup(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
@@ -289,15 +280,13 @@ subgroup(Config) when is_list(Config) ->
 
     setup_and_execute(subgroup, TestSpec, Config).
 
-%%! --- Wed Jun 30 00:25:59 2010 --- peppe was here!
-%%! NOT WORKING!
 skip_subgroup(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
     TestDir = filename:join(DataDir, "groups_1"),
-    TestSpec = [{groups,TestDir,groups_12_SUITE,[test_group_2]},
+    TestSpec = [{groups,TestDir,groups_12_SUITE,[test_group_4]},
 		{skip_groups,TestDir,groups_12_SUITE,
-		 test_group_3,"SKIPPED!"}],
+		 test_group_8,"SKIPPED!"}],
 
     setup_and_execute(skip_subgroup, TestSpec, Config).
 
@@ -309,7 +298,9 @@ subgroup_all_testcases(Config) when is_list(Config) ->
 
     TestDir = filename:join(DataDir, "groups_1"),
     TestSpec = [{groups,TestDir,groups_12_SUITE,
-		 test_group_5,{cases,all}}],
+		 test_group_5,{cases,all}},
+		{groups,TestDir,groups_12_SUITE,
+		 test_group_3,{cases,all}}],
 
     setup_and_execute(subgroup_all_testcases, TestSpec, Config).
 
@@ -331,7 +322,9 @@ subgroup_testcase(Config) when is_list(Config) ->
 
     TestDir = filename:join(DataDir, "groups_1"),
     TestSpec = [{groups,TestDir,groups_12_SUITE,
-		 test_group_7,{cases,testcase_7a}}],
+		 test_group_7,{cases,testcase_7a}},
+		{groups,TestDir,groups_12_SUITE,
+		 test_group_3,{cases,testcase_3b}}],
 
     setup_and_execute(subgroup_testcase, TestSpec, Config).
 
@@ -348,6 +341,21 @@ skip_subgroup_testcase(Config) when is_list(Config) ->
 %%%-----------------------------------------------------------------
 %%%
 
+%%!
+%%! Somewhat weird result from this one:
+%%!
+sub_skipped_by_top(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir,groups_12_SUITE,test_group_5},
+		{skip_groups,TestDir,groups_12_SUITE,test_group_4,"SKIPPED!"}],
+
+    setup_and_execute(sub_skipped_by_top, TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
 testcase_in_multiple_groups(Config) when is_list(Config) ->
     DataDir = ?config(data_dir, Config),
 
diff --git a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl
index dbf1576c76..69c06f9b83 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE_data/groups_1/groups_12_SUITE.erl
@@ -48,9 +48,12 @@ groups() ->
 
 						  testcase_5b]}]},
 
-      {test_group_6, [parallel], [{group, test_group_7}]},
+      {test_group_6, [parallel], [{group, test_group_7},
+				  {group, test_group_8}]},
 
-      {test_group_7, [sequence], [testcase_7a,testcase_7b]}
+      {test_group_7, [sequence], [testcase_7a,testcase_7b]},
+
+      {test_group_8, [shuffle,sequence], [testcase_8a,testcase_8b]}
      ].
 
 all() ->
@@ -68,7 +71,8 @@ grs_and_tcs() ->
       test_group_1a, test_group_1b,
       test_group_2, test_group_3,
       test_group_4, test_group_5,
-      test_group_6, test_group_7
+      test_group_6, test_group_7,
+      test_group_8
      ],
      [
       testcase_1a, testcase_1b, testcase_1c,
@@ -78,7 +82,8 @@ grs_and_tcs() ->
       testcase_3a, testcase_3b,
       testcase_3,
       testcase_5a, testcase_5b,
-      testcase_7a, testcase_7b
+      testcase_7a, testcase_7b,
+      testcase_8a, testcase_8b
      ]}.
 
 %%--------------------------------------------------------------------
@@ -107,7 +112,9 @@ init_per_group(Group, Config) ->
 	    {test_group_4,[{name,test_group_4}]} -> ok;
 	    {test_group_5,[{name,test_group_5},parallel]} -> "parallel";
 	    {test_group_6,[{name,test_group_6},parallel]} -> "parallel";
-	    {test_group_7,[{name,test_group_7},sequence]} -> "sequence"
+	    {test_group_7,[{name,test_group_7},sequence]} -> "sequence";
+	    {test_group_8,[{shuffle,_},{name,test_group_8},sequence]} ->
+		"shuffle & sequence"
 	end,
     {Grs,_} = grs_and_tcs(),
     case lists:member(Group, Grs) of
@@ -312,3 +319,26 @@ testcase_7b(Config) ->
     undefined = ?config(testcase_7a,Config),
     testcase_7b = ?config(testcase_7b,Config),
     ok.
+
+testcase_8a() ->
+    [].
+testcase_8a(Config) ->
+    init = ?config(suite,Config),
+    undefined = ?config(test_group_3,Config),
+    test_group_4 = ?config(test_group_4,Config),
+    test_group_5 = ?config(test_group_5,Config),
+    test_group_6 = ?config(test_group_6,Config),
+    test_group_8 = ?config(test_group_8,Config),
+    testcase_8a = ?config(testcase_8a,Config),
+    ok.
+testcase_8b() ->
+    [].
+testcase_8b(Config) ->
+    init = ?config(suite,Config),
+    test_group_4 = ?config(test_group_4,Config),
+    test_group_5 = ?config(test_group_5,Config),
+    test_group_6 = ?config(test_group_6,Config),
+    test_group_8 = ?config(test_group_8,Config),
+    undefined = ?config(testcase_8a,Config),
+    testcase_8b = ?config(testcase_8b,Config),
+    ok.