Add new tests for test case groups and test specifications

6 files changed, 680 insertions, 2 deletions

diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 1ab9242e4b..dc017921cc 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -624,6 +624,25 @@ add_tests([{suites,Node,Dir,Ss}|Ts],Spec) ->
 			   Ss,Tests),
     add_tests(Ts,Spec#testspec{tests=Tests1});
 
+%% --- groups ---
+%% Later make it possible to specify group execution properties
+%% that will override thse in the suite. Also make it possible
+%% create dynamic groups in specification, i.e. to group test cases
+%% by means of groups defined only in the test specification.
+add_tests([{groups,all_nodes,Dir,Suite,Gs}|Ts],Spec) ->
+    add_tests([{groups,list_nodes(Spec),Dir,Suite,Gs}|Ts],Spec);
+add_tests([{groups,Dir,Suite,Gs}|Ts],Spec) ->
+    add_tests([{groups,all_nodes,Dir,Suite,Gs}|Ts],Spec);
+add_tests([{groups,Nodes,Dir,Suite,Gs}|Ts],Spec) when is_list(Nodes) ->
+    Ts1 = separate(Nodes,groups,[Dir,Suite,Gs],Ts,Spec#testspec.nodes),
+    add_tests(Ts1,Spec);
+add_tests([{groups,Node,Dir,Suite,Gs}|Ts],Spec) ->
+    Tests = Spec#testspec.tests,
+    Tests1 = insert_cases(ref2node(Node,Spec#testspec.nodes),
+			   ref2dir(Dir,Spec#testspec.alias),
+			   Suite,Gs,Tests),
+    add_tests(Ts,Spec#testspec{tests=Tests1});
+
 %% --- cases ---
 add_tests([{cases,all_nodes,Dir,Suite,Cs}|Ts],Spec) ->
     add_tests([{cases,list_nodes(Spec),Dir,Suite,Cs}|Ts],Spec);
@@ -654,6 +673,21 @@ add_tests([{skip_suites,Node,Dir,Ss,Cmt}|Ts],Spec) ->
 			 Ss,Cmt,Tests),
     add_tests(Ts,Spec#testspec{tests=Tests1});
 
+%% --- skip_groups ---
+add_tests([{skip_groups,all_nodes,Dir,Suite,Gs,Cmt}|Ts],Spec) ->
+    add_tests([{skip_groups,list_nodes(Spec),Dir,Suite,Gs,Cmt}|Ts],Spec);
+add_tests([{skip_groups,Dir,Suite,Gs,Cmt}|Ts],Spec) ->
+    add_tests([{skip_groups,all_nodes,Dir,Suite,Gs,Cmt}|Ts],Spec);
+add_tests([{skip_groups,Nodes,Dir,Suite,Gs,Cmt}|Ts],Spec) when is_list(Nodes) ->
+    Ts1 = separate(Nodes,skip_groups,[Dir,Suite,Gs,Cmt],Ts,Spec#testspec.nodes),
+    add_tests(Ts1,Spec);
+add_tests([{skip_groups,Node,Dir,Suite,Gs,Cmt}|Ts],Spec) ->
+    Tests = Spec#testspec.tests,
+    Tests1 = skip_cases(ref2node(Node,Spec#testspec.nodes),
+			ref2dir(Dir,Spec#testspec.alias),
+			Suite,Gs,Cmt,Tests),
+    add_tests(Ts,Spec#testspec{tests=Tests1});
+
 %% --- skip_cases ---
 add_tests([{skip_cases,all_nodes,Dir,Suite,Cs,Cmt}|Ts],Spec) ->
     add_tests([{skip_cases,list_nodes(Spec),Dir,Suite,Cs,Cmt}|Ts],Spec);
diff --git a/lib/common_test/test/ct_groups_test_2_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE.erl
index 75348e2f4a..1fbeff0126 100644
--- a/lib/common_test/test/ct_groups_test_2_SUITE.erl
+++ b/lib/common_test/test/ct_groups_test_2_SUITE.erl
@@ -60,7 +60,7 @@ all(doc) ->
     ["Run smoke tests of Common Test."];
 
 all(suite) -> 
-    [missing_conf].
+    [missing_conf, testspec_1].
 
 %%--------------------------------------------------------------------
 %% TEST CASES
@@ -84,6 +84,25 @@ missing_conf(Config) when is_list(Config) ->
 
     TestEvents = events_to_check(missing_conf),
     ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+testspec_1(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestSpec = filename:join(DataDir, "specs/groups_2.1.spec"),
+
+    {Opts,ERPid} = setup({spec,TestSpec}, Config),
+    ok = ct_test_support:run(Opts, Config),
+    Events = ct_test_support:get_events(ERPid, Config),
+
+    ct_test_support:log_events(testspec_1,
+			       reformat(Events, ?eh),
+			       ?config(priv_dir, Config)),
+
+    TestEvents = events_to_check(testspec_1),
+    ok = ct_test_support:verify_events(TestEvents, Events, Config).
     
 %%%-----------------------------------------------------------------
 %%% HELP FUNCTIONS
@@ -133,4 +152,7 @@ test_events(missing_conf) ->
      {?eh,test_stats,{4,0,{0,0}}},
      {?eh,test_done,{'DEF','STOP_TIME'}},
      {?eh,stop_logging,[]}
-    ].
+    ];
+
+test_events(testspec_1) ->
+    [].
diff --git a/lib/common_test/test/ct_groups_test_2_SUITE_data/cfgs/groups_2.1.cfg b/lib/common_test/test/ct_groups_test_2_SUITE_data/cfgs/groups_2.1.cfg
new file mode 100644
index 0000000000..4928505157
--- /dev/null
+++ b/lib/common_test/test/ct_groups_test_2_SUITE_data/cfgs/groups_2.1.cfg
@@ -0,0 +1 @@
+{dummy_key, "dummy_data"}.
diff --git a/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_2/groups_21_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_2/groups_21_SUITE.erl
new file mode 100644
index 0000000000..a63aa31a7f
--- /dev/null
+++ b/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_2/groups_21_SUITE.erl
@@ -0,0 +1,281 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(groups_21_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%====================================================================
+%% COMMON TEST CALLBACK FUNCTIONS
+%%====================================================================
+
+suite() ->
+    [{timetrap,{minutes,1}}].
+
+groups() ->
+    [
+      {test_group_1a, [testcase_1a,testcase_1b]},
+
+      {test_group_1b, [], [testcase_1a,testcase_1b]},
+
+      {test_group_2, [], [testcase_2a,
+
+			  {test_group_3, [], [testcase_3a,
+					      testcase_3b]},
+			  testcase_2b]},
+
+      {test_group_4, [{test_group_5, [], [testcase_5a,
+
+					  {group, test_group_6},
+
+					  testcase_5b]}]},
+
+      {test_group_6, [{group, test_group_7}]},
+
+      {test_group_7, [testcase_7a,testcase_7b]}
+     ].
+
+all() ->
+    [testcase_1,
+     {group, test_group_1a},
+     {group, test_group_1b},
+     testcase_2,
+     {group, test_group_2},
+     testcase_3,
+     {group, test_group_4}].
+
+%% this func only for internal test purposes
+grs_and_tcs() ->
+    {[
+      test_group_1a, test_group_1b,
+      test_group_2, test_group_3,
+      test_group_4, test_group_5,
+      test_group_6, test_group_7
+     ],
+     [
+      testcase_1,
+      testcase_1a, testcase_1b,
+      testcase_2,
+      testcase_2a, testcase_2b,
+      testcase_3a, testcase_3b,
+      testcase_3,
+      testcase_5a, testcase_5b,
+      testcase_7a, testcase_7b
+     ]}.
+
+%%--------------------------------------------------------------------
+%% Suite Configuration
+%%--------------------------------------------------------------------
+
+init_per_suite(Config) ->
+    [{suite,init}|Config].
+
+end_per_suite(Config) ->
+    init = ?config(suite,Config).
+
+%%--------------------------------------------------------------------
+%% Group Configuration
+%%--------------------------------------------------------------------
+
+init_per_group(Group, Config) ->
+    [{name,Group}] = ?config(tc_group_properties,Config),
+    {Grs,_} = grs_and_tcs(),
+    case lists:member(Group, Grs) of
+	true ->
+	    ct:comment(io_lib:format("~w", [Group])),
+	    init = ?config(suite,Config),
+	    [{Group,Group} | Config];
+	false ->
+	    ct:fail({bad_group,Group})
+    end.
+
+end_per_group(Group, Config) ->
+    {Grs,_} = grs_and_tcs(),
+    case lists:member(Group, Grs) of
+	true ->
+	    init = ?config(suite,Config),
+	    Group = ?config(Group,Config),
+	    ok;
+	false ->
+	    ct:fail({bad_group,Group})
+    end.
+
+%%--------------------------------------------------------------------
+%% Testcase Configuration
+%%--------------------------------------------------------------------
+
+init_per_testcase(TestCase, Config) ->
+    {_,TCs} = grs_and_tcs(),
+    case lists:member(TestCase, TCs) of
+	true ->
+	    init = ?config(suite,Config),
+	    [{TestCase,TestCase} | Config];
+	false ->
+	    ct:fail({unknown_testcase,TestCase})
+    end.
+
+end_per_testcase(TestCase, Config) ->
+    {_,TCs} = grs_and_tcs(),
+    case lists:member(TestCase, TCs) of
+	true ->
+	    init = ?config(suite,Config),
+	    TestCase = ?config(TestCase,Config),
+	    ok;
+	false ->
+	    ct:fail({unknown_testcase,TestCase})
+    end.
+
+
+%%--------------------------------------------------------------------
+%% Testcases
+%%--------------------------------------------------------------------
+
+testcase_1() ->
+    [].
+testcase_1(Config) ->
+    init = ?config(suite,Config),
+    testcase_1 = ?config(testcase_1,Config),
+    ok.
+
+testcase_1a() ->
+    [].
+testcase_1a(Config) ->
+    init = ?config(suite,Config),
+    case ?config(test_group_1a,Config) of
+	test_group_1a -> ok;
+	_ ->
+	    case ?config(test_group_1b,Config) of
+		test_group_1b -> ok;
+		_ -> ct:fail(no_group_data)
+	    end
+    end,
+    testcase_1a = ?config(testcase_1a,Config),
+    ok.
+testcase_1b() ->
+    [].
+testcase_1b(Config) ->
+    init = ?config(suite,Config),
+    case ?config(test_group_1a,Config) of
+	test_group_1a -> ok;
+	_ ->
+	    case ?config(test_group_1b,Config) of
+		test_group_1b -> ok;
+		_ -> ct:fail(no_group_data)
+	    end
+    end,
+    undefined = ?config(testcase_1a,Config),
+    testcase_1b = ?config(testcase_1b,Config),
+    ok.
+
+testcase_2() ->
+    [].
+testcase_2(Config) ->
+    init = ?config(suite,Config),
+    undefined = ?config(test_group_1a,Config),
+    undefined = ?config(test_group_1b,Config),
+    testcase_2 = ?config(testcase_2,Config),
+    ok.
+
+testcase_2a() ->
+    [].
+testcase_2a(Config) ->
+    init = ?config(suite,Config),
+    test_group_2 = ?config(test_group_2,Config),
+    testcase_2a = ?config(testcase_2a,Config),
+    ok.
+testcase_2b() ->
+    [].
+testcase_2b(Config) ->
+    init = ?config(suite,Config),
+    test_group_2 = ?config(test_group_2,Config),
+    undefined = ?config(testcase_2a,Config),
+    testcase_2b = ?config(testcase_2b,Config),
+    ok.
+
+testcase_3a() ->
+    [].
+testcase_3a(Config) ->
+    init = ?config(suite,Config),
+    test_group_2 = ?config(test_group_2,Config),
+    test_group_3 = ?config(test_group_3,Config),
+    undefined = ?config(testcase_2b,Config),
+    testcase_3a = ?config(testcase_3a,Config),
+    ok.
+testcase_3b() ->
+    [].
+testcase_3b(Config) ->
+    init = ?config(suite,Config),
+    test_group_2 = ?config(test_group_2,Config),
+    test_group_3 = ?config(test_group_3,Config),
+    undefined = ?config(testcase_3a,Config),
+    testcase_3b = ?config(testcase_3b,Config),
+    ok.
+
+testcase_3() ->
+    [].
+testcase_3(Config) ->
+    init = ?config(suite,Config),
+    undefined = ?config(test_group_2,Config),
+    undefined = ?config(test_group_3,Config),
+    testcase_3 = ?config(testcase_3,Config),
+    ok.
+
+testcase_5a() ->
+    [].
+testcase_5a(Config) ->
+    init = ?config(suite,Config),
+    undefined = ?config(test_group_3,Config),
+    test_group_4 = ?config(test_group_4,Config),
+    test_group_5 = ?config(test_group_5,Config),
+    undefined = ?config(testcase_3,Config),
+    testcase_5a = ?config(testcase_5a,Config),
+    ok.
+testcase_5b() ->
+    [].
+testcase_5b(Config) ->
+    init = ?config(suite,Config),
+    test_group_4 = ?config(test_group_4,Config),
+    test_group_5 = ?config(test_group_5,Config),
+    undefined = ?config(testcase_5a,Config),
+    testcase_5b = ?config(testcase_5b,Config),
+    ok.
+
+testcase_7a() ->
+    [].
+testcase_7a(Config) ->
+    init = ?config(suite,Config),
+    undefined = ?config(test_group_3,Config),
+    test_group_4 = ?config(test_group_4,Config),
+    test_group_5 = ?config(test_group_5,Config),
+    test_group_6 = ?config(test_group_6,Config),
+    test_group_7 = ?config(test_group_7,Config),
+    testcase_7a = ?config(testcase_7a,Config),
+    ok.
+testcase_7b() ->
+    [].
+testcase_7b(Config) ->
+    init = ?config(suite,Config),
+    test_group_4 = ?config(test_group_4,Config),
+    test_group_5 = ?config(test_group_5,Config),
+    test_group_6 = ?config(test_group_6,Config),
+    test_group_7 = ?config(test_group_7,Config),
+    undefined = ?config(testcase_7a,Config),
+    testcase_7b = ?config(testcase_7b,Config),
+    ok.
diff --git a/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_2/groups_22_SUITE.erl b/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_2/groups_22_SUITE.erl
new file mode 100644
index 0000000000..308df616d5
--- /dev/null
+++ b/lib/common_test/test/ct_groups_test_2_SUITE_data/groups_2/groups_22_SUITE.erl
@@ -0,0 +1,314 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2009-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(groups_22_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%====================================================================
+%% COMMON TEST CALLBACK FUNCTIONS
+%%====================================================================
+
+suite() ->
+    [{timetrap,{minutes,1}}].
+
+groups() ->
+    [
+      {test_group_1a, [shuffle], [testcase_1a,testcase_1b,testcase_1c]},
+
+      {test_group_1b, [parallel], [testcase_1a,testcase_1b]},
+
+      {test_group_2, [parallel], [testcase_2a,
+
+				  {test_group_3, [{repeat,1}],
+				   [testcase_3a, testcase_3b]},
+
+				  testcase_2b]},
+
+      {test_group_4, [{test_group_5, [parallel], [testcase_5a,
+
+						  {group, test_group_6},
+
+						  testcase_5b]}]},
+
+      {test_group_6, [parallel], [{group, test_group_7}]},
+
+      {test_group_7, [sequence], [testcase_7a,testcase_7b]}
+     ].
+
+all() ->
+    [{group, test_group_1a},
+     {group, test_group_1b},
+     testcase_1,
+     testcase_2,
+     {group, test_group_2},
+     testcase_3,
+     {group, test_group_4}].
+
+%% this func only for internal test purposes
+grs_and_tcs() ->
+    {[
+      test_group_1a, test_group_1b,
+      test_group_2, test_group_3,
+      test_group_4, test_group_5,
+      test_group_6, test_group_7
+     ],
+     [
+      testcase_1a, testcase_1b, testcase_1c,
+      testcase_1,
+      testcase_2,
+      testcase_2a, testcase_2b,
+      testcase_3a, testcase_3b,
+      testcase_3,
+      testcase_5a, testcase_5b,
+      testcase_7a, testcase_7b
+     ]}.
+
+%%--------------------------------------------------------------------
+%% Suite Configuration
+%%--------------------------------------------------------------------
+
+init_per_suite(Config) ->
+    [{suite,init}|Config].
+
+end_per_suite(Config) ->
+    init = ?config(suite,Config).
+
+%%--------------------------------------------------------------------
+%% Group Configuration
+%%--------------------------------------------------------------------
+
+init_per_group(Group, Config) ->
+    Cmt =
+	case {Group,?config(tc_group_properties,Config)} of
+	    {test_group_1a,[{shuffle,S},{name,test_group_1a}]} ->
+		io_lib:format("shuffled, ~w", [S]);
+	    {test_group_1b,[{name,test_group_1b},parallel]} -> "parallel";
+	    {test_group_2,[{name,test_group_2},parallel]} -> "parallel";
+	    {test_group_3,[{name,test_group_3},{repeat,1}]} -> "repeat 1";
+	    {test_group_3,[{name,test_group_3}]} -> "repeat 0";
+	    {test_group_4,[{name,test_group_4}]} -> ok;
+	    {test_group_5,[{name,test_group_5},parallel]} -> "parallel";
+	    {test_group_6,[{name,test_group_6},parallel]} -> "parallel";
+	    {test_group_7,[{name,test_group_7},sequence]} -> "sequence"
+	end,
+    {Grs,_} = grs_and_tcs(),
+    case lists:member(Group, Grs) of
+	true ->
+	    init = ?config(suite,Config),
+	    ct:comment(io_lib:format("~w, ~s", [Group,Cmt])),
+	    [{Group,Group} | Config];
+	false ->
+	    ct:fail({bad_group,Group})
+    end.
+
+end_per_group(Group, Config) ->
+    {Grs,_} = grs_and_tcs(),
+    case lists:member(Group, Grs) of
+	true ->
+	    init = ?config(suite,Config),
+	    Group = ?config(Group,Config),
+	    ok;
+	false ->
+	    ct:fail({bad_group,Group})
+    end.
+
+%%--------------------------------------------------------------------
+%% Testcase Configuration
+%%--------------------------------------------------------------------
+
+init_per_testcase(TestCase, Config) ->
+    {_,TCs} = grs_and_tcs(),
+    case lists:member(TestCase, TCs) of
+	true ->
+	    init = ?config(suite,Config),
+	    [{TestCase,TestCase} | Config];
+	false ->
+	    ct:fail({unknown_testcase,TestCase})
+    end.
+
+end_per_testcase(TestCase, Config) ->
+    {_,TCs} = grs_and_tcs(),
+    case lists:member(TestCase, TCs) of
+	true ->
+	    init = ?config(suite,Config),
+	    TestCase = ?config(TestCase,Config),
+	    ok;
+	false ->
+	    ct:fail({unknown_testcase,TestCase})
+    end.
+
+
+%%--------------------------------------------------------------------
+%% Testcases
+%%--------------------------------------------------------------------
+
+testcase_1a() ->
+    [].
+testcase_1a(Config) ->
+    init = ?config(suite,Config),
+    case ?config(test_group_1a,Config) of
+	test_group_1a -> ok;
+	_ ->
+	    case ?config(test_group_1b,Config) of
+		test_group_1b -> ok;
+		_ -> ct:fail(no_group_data)
+	    end
+    end,
+    testcase_1a = ?config(testcase_1a,Config),
+    ok.
+testcase_1b() ->
+    [].
+testcase_1b(Config) ->
+    init = ?config(suite,Config),
+    case ?config(test_group_1a,Config) of
+	test_group_1a -> ok;
+	_ ->
+	    case ?config(test_group_1b,Config) of
+		test_group_1b -> ok;
+		_ -> ct:fail(no_group_data)
+	    end
+    end,
+    undefined = ?config(testcase_1a,Config),
+    testcase_1b = ?config(testcase_1b,Config),
+    ok.
+
+testcase_1c() ->
+    [].
+testcase_1c(Config) ->
+    init = ?config(suite,Config),
+    case ?config(test_group_1a,Config) of
+	test_group_1a -> ok;
+	_ ->
+	    case ?config(test_group_1b,Config) of
+		test_group_1b -> ok;
+		_ -> ct:fail(no_group_data)
+	    end
+    end,
+    undefined = ?config(testcase_1b,Config),
+    testcase_1c = ?config(testcase_1c,Config),
+    ok.
+
+testcase_1() ->
+    [].
+testcase_1(Config) ->
+    init = ?config(suite,Config),
+    testcase_1 = ?config(testcase_1,Config),
+    ok.
+
+testcase_2() ->
+    [].
+testcase_2(Config) ->
+    init = ?config(suite,Config),
+    undefined = ?config(test_group_1a,Config),
+    undefined = ?config(test_group_1b,Config),
+    testcase_2 = ?config(testcase_2,Config),
+    ok.
+
+testcase_2a() ->
+    [].
+testcase_2a(Config) ->
+    init = ?config(suite,Config),
+    test_group_2 = ?config(test_group_2,Config),
+    testcase_2a = ?config(testcase_2a,Config),
+    ok.
+testcase_2b() ->
+    [].
+testcase_2b(Config) ->
+    init = ?config(suite,Config),
+    test_group_2 = ?config(test_group_2,Config),
+    undefined = ?config(testcase_2a,Config),
+    testcase_2b = ?config(testcase_2b,Config),
+    ok.
+
+testcase_3a() ->
+    [].
+testcase_3a(Config) ->
+    init = ?config(suite,Config),
+    test_group_2 = ?config(test_group_2,Config),
+    test_group_3 = ?config(test_group_3,Config),
+    undefined = ?config(testcase_2b,Config),
+    testcase_3a = ?config(testcase_3a,Config),
+    ok.
+testcase_3b() ->
+    [].
+testcase_3b(Config) ->
+    init = ?config(suite,Config),
+    test_group_2 = ?config(test_group_2,Config),
+    test_group_3 = ?config(test_group_3,Config),
+    undefined = ?config(testcase_3a,Config),
+    testcase_3b = ?config(testcase_3b,Config),
+    ok.
+
+testcase_3() ->
+    [].
+testcase_3(Config) ->
+    init = ?config(suite,Config),
+    undefined = ?config(test_group_2,Config),
+    undefined = ?config(test_group_3,Config),
+    testcase_3 = ?config(testcase_3,Config),
+    ok.
+
+testcase_5a() ->
+    [].
+testcase_5a(Config) ->
+    init = ?config(suite,Config),
+    undefined = ?config(test_group_3,Config),
+    test_group_4 = ?config(test_group_4,Config),
+    test_group_5 = ?config(test_group_5,Config),
+    undefined = ?config(testcase_3,Config),
+    testcase_5a = ?config(testcase_5a,Config),
+    %% increase chance the done event will come
+    %% during execution of subgroup (could be
+    %% tricky to handle)
+    timer:sleep(3),
+    ok.
+testcase_5b() ->
+    [].
+testcase_5b(Config) ->
+    init = ?config(suite,Config),
+    test_group_4 = ?config(test_group_4,Config),
+    test_group_5 = ?config(test_group_5,Config),
+    undefined = ?config(testcase_5a,Config),
+    testcase_5b = ?config(testcase_5b,Config),
+    ok.
+
+testcase_7a() ->
+    [].
+testcase_7a(Config) ->
+    init = ?config(suite,Config),
+    undefined = ?config(test_group_3,Config),
+    test_group_4 = ?config(test_group_4,Config),
+    test_group_5 = ?config(test_group_5,Config),
+    test_group_6 = ?config(test_group_6,Config),
+    test_group_7 = ?config(test_group_7,Config),
+    testcase_7a = ?config(testcase_7a,Config),
+    ok.
+testcase_7b() ->
+    [].
+testcase_7b(Config) ->
+    init = ?config(suite,Config),
+    test_group_4 = ?config(test_group_4,Config),
+    test_group_5 = ?config(test_group_5,Config),
+    test_group_6 = ?config(test_group_6,Config),
+    test_group_7 = ?config(test_group_7,Config),
+    undefined = ?config(testcase_7a,Config),
+    testcase_7b = ?config(testcase_7b,Config),
+    ok.
diff --git a/lib/common_test/test/ct_groups_test_2_SUITE_data/specs/groups_2.1.spec b/lib/common_test/test/ct_groups_test_2_SUITE_data/specs/groups_2.1.spec
new file mode 100644
index 0000000000..396b8ac645
--- /dev/null
+++ b/lib/common_test/test/ct_groups_test_2_SUITE_data/specs/groups_2.1.spec
@@ -0,0 +1,26 @@
+
+{config, "../cfgs/groups_2.1.cfg"}.
+{alias, groups_2, "../groups_2"}.
+
+{suites, groups_2, groups_21_SUITE}.
+{skip_groups, groups_2, groups_21_SUITE,
+    [test_group_1b, test_group_7], "Skip tg_1b & tg_7"}.
+{skip_cases, groups_2, groups_21_SUITE,
+    [testcase_1b, testcase_3a], "Skip tc_1b & tc_3a"}.
+
+{groups, groups_2, groups_22_SUITE,
+    test_group_1a}.
+{skip_cases, groups_2, groups_22_SUITE,
+    testcase_1a, "Skip tc_1a"}.
+
+{groups, groups_2, groups_22_SUITE,
+    test_group_1b}.
+{skip_cases, groups_2, groups_22_SUITE,
+    testcase_1b, "Skip tc_1b"}.
+{skip_groups, groups_2, groups_21_SUITE,
+    [test_group_3], "Skip tg_3"}.
+
+{groups, groups_2, groups_22_SUITE,
+    test_group_5}.
+{skip_cases, groups_2, groups_22_SUITE,
+    [testcase_7a, testcase_7b], "Skip tc_7a & tc_7b"}.