Merge branch 'lukas/common_test/test_order_wrong/OTP-9026' into dev

* lukas/common_test/test_order_wrong/OTP-9026:
  Add test event info for the old tests
  Update test spec parser to not group suites, groups and cases which are hierarchically related if merge_tests config i set to false
  Add test cases for test spec ordering of testcases in spec

Conflicts:
	lib/common_test/test/ct_testspec_1_SUITE.erl

4 files changed, 1040 insertions, 53 deletions

diff --git a/lib/common_test/doc/src/run_test_chapter.xml b/lib/common_test/doc/src/run_test_chapter.xml
index 7b485bdc35..f052adf25e 100644
--- a/lib/common_test/doc/src/run_test_chapter.xml
+++ b/lib/common_test/doc/src/run_test_chapter.xml
@@ -368,15 +368,17 @@
       either one or more suites, one or more test case groups, or one
       or more test cases in a group or suite.</p>
     <p>An arbitrary number of test terms may be declared in sequence.
-      Common Test will compile the terms into one or more tests to be
-      performed in one resulting test run. Note that a term that
+      Common Test will by default compile the terms into one or more tests 
+      to be performed in one resulting test run. Note that a term that
       specifies a set of test cases will "swallow" one that only
       specifies a subset of these cases. E.g. the result of merging
       one term that specifies that all cases in suite S should be
       executed, with another term specifying only test case X and Y in
       S, is a test of all cases in S. However, if a term specifying
       test case X and Y in S is merged with a term specifying case Z
-      in S, the result is a test of X, Y and Z in S.</p>
+      in S, the result is a test of X, Y and Z in S. To disable this
+      behaviour, it is possible in test specification to set the 
+      <c>merge_tests</c> term to <c>false</c>.</p>
     <p>A test term can also specify one or more test suites, groups,
       or test cases to be skipped. Skipped suites, groups and cases
       are not executed and show up in the HTML test log files as
@@ -435,6 +437,8 @@
       {userconfig, NodeRefs, {CallbackModule, ConfigStrings}}.
       
       {alias, DirAlias, Dir}.
+
+      {merge_tests, Bool}.
       
       {logdir, LogDir}.                                        
       {logdir, NodeRefs, LogDir}.
diff --git a/lib/common_test/src/ct_testspec.erl b/lib/common_test/src/ct_testspec.erl
index 2b6abefb72..2c7cd3577c 100644
--- a/lib/common_test/src/ct_testspec.erl
+++ b/lib/common_test/src/ct_testspec.erl
@@ -68,7 +68,8 @@ prepare_tests(TestSpec) when is_record(TestSpec,testspec) ->
     %% Create initial list of {Node,{Run,Skip}} tuples
     NodeList = lists:map(fun(N) -> {N,{[],[]}} end, list_nodes(TestSpec)),
     %% Get all Run tests sorted per node basis.
-    NodeList1 = run_per_node(Run,NodeList),    
+    NodeList1 = run_per_node(Run,NodeList, 
+			     TestSpec#testspec.merge_tests),    
     %% Get all Skip entries sorted per node basis.
     NodeList2 = skip_per_node(Skip,NodeList1),
     %% Change representation.
@@ -89,11 +90,17 @@ prepare_tests(TestSpec) when is_record(TestSpec,testspec) ->
 %% run_per_node/2 takes the Run list as input and returns a list
 %% of {Node,RunPerNode,[]} tuples where the tests have been sorted
 %% on a per node basis.
-run_per_node([{{Node,Dir},Test}|Ts],Result) ->
+run_per_node([{{Node,Dir},Test}|Ts],Result, MergeTests) ->
     {value,{Node,{Run,Skip}}} = lists:keysearch(Node,1,Result),
-    Run1 = merge_tests(Dir,Test,Run),
-    run_per_node(Ts,insert_in_order({Node,{Run1,Skip}},Result));
-run_per_node([],Result) ->
+    Run1 = case MergeTests of
+	       false ->
+		   append({Dir, Test}, Run);
+	       true ->
+		   merge_tests(Dir,Test,Run)
+	   end,
+    run_per_node(Ts,insert_in_order({Node,{Run1,Skip}},Result), 
+		 MergeTests);
+run_per_node([],Result,_) ->
     Result.
 
 merge_tests(Dir,Test={all,_},TestDirs) ->
@@ -281,6 +288,8 @@ collect_tests(Terms,TestSpec,Relaxed) ->
     {Terms2, TestSpec3} = filter_init_terms(Terms, [], TestSpec2),
     add_tests(Terms2,TestSpec3).
 
+get_global([{merge_tests, Bool} | Ts], Spec) ->
+    get_global(Ts,Spec#testspec{ merge_tests = Bool });
 get_global([{alias,Ref,Dir}|Ts],Spec=#testspec{alias=Refs}) ->
     get_global(Ts,Spec#testspec{alias=[{Ref,get_absdir(Dir,Spec)}|Refs]});
 get_global([{node,Ref,Node}|Ts],Spec=#testspec{nodes=Refs}) ->
@@ -668,7 +677,7 @@ add_tests([{suites,Node,Dir,Ss}|Ts],Spec) ->
     Tests = Spec#testspec.tests,
     Tests1 = insert_suites(ref2node(Node,Spec#testspec.nodes),
 			   ref2dir(Dir,Spec#testspec.alias),
-			   Ss,Tests),
+			   Ss,Tests, Spec#testspec.merge_tests),
     add_tests(Ts,Spec#testspec{tests=Tests1});
 
 %% --- groups ---
@@ -694,13 +703,15 @@ add_tests([{groups,Node,Dir,Suite,Gs}|Ts],Spec) ->
     Tests = Spec#testspec.tests,
     Tests1 = insert_groups(ref2node(Node,Spec#testspec.nodes),
 			   ref2dir(Dir,Spec#testspec.alias),
-			   Suite,Gs,all,Tests),
+			   Suite,Gs,all,Tests,
+			   Spec#testspec.merge_tests),
     add_tests(Ts,Spec#testspec{tests=Tests1});
 add_tests([{groups,Node,Dir,Suite,Gs,{cases,TCs}}|Ts],Spec) ->
     Tests = Spec#testspec.tests,
     Tests1 = insert_groups(ref2node(Node,Spec#testspec.nodes),
 			   ref2dir(Dir,Spec#testspec.alias),
-			   Suite,Gs,TCs,Tests),
+			   Suite,Gs,TCs,Tests,
+			   Spec#testspec.merge_tests),
     add_tests(Ts,Spec#testspec{tests=Tests1});
 
 %% --- cases ---
@@ -715,7 +726,7 @@ add_tests([{cases,Node,Dir,Suite,Cs}|Ts],Spec) ->
     Tests = Spec#testspec.tests,
     Tests1 = insert_cases(ref2node(Node,Spec#testspec.nodes),
 			  ref2dir(Dir,Spec#testspec.alias),
-			  Suite,Cs,Tests),
+			  Suite,Cs,Tests, Spec#testspec.merge_tests),
     add_tests(Ts,Spec#testspec{tests=Tests1});
 
 %% --- skip_suites ---
@@ -730,7 +741,8 @@ add_tests([{skip_suites,Node,Dir,Ss,Cmt}|Ts],Spec) ->
     Tests = Spec#testspec.tests,
     Tests1 = skip_suites(ref2node(Node,Spec#testspec.nodes),
 			 ref2dir(Dir,Spec#testspec.alias),
-			 Ss,Cmt,Tests),
+			 Ss,Cmt,Tests,
+			 Spec#testspec.merge_tests),
     add_tests(Ts,Spec#testspec{tests=Tests1});
 
 %% --- skip_groups ---
@@ -752,13 +764,15 @@ add_tests([{skip_groups,Node,Dir,Suite,Gs,Cmt}|Ts],Spec) ->
     Tests = Spec#testspec.tests,
     Tests1 = skip_groups(ref2node(Node,Spec#testspec.nodes),
 			 ref2dir(Dir,Spec#testspec.alias),
-			 Suite,Gs,all,Cmt,Tests),
+			 Suite,Gs,all,Cmt,Tests,
+			 Spec#testspec.merge_tests),
     add_tests(Ts,Spec#testspec{tests=Tests1});
 add_tests([{skip_groups,Node,Dir,Suite,Gs,{cases,TCs},Cmt}|Ts],Spec) ->
     Tests = Spec#testspec.tests,
     Tests1 = skip_groups(ref2node(Node,Spec#testspec.nodes),
 			 ref2dir(Dir,Spec#testspec.alias),
-			 Suite,Gs,TCs,Cmt,Tests),
+			 Suite,Gs,TCs,Cmt,Tests,
+			 Spec#testspec.merge_tests),
     add_tests(Ts,Spec#testspec{tests=Tests1});
 
 %% --- skip_cases ---
@@ -773,7 +787,7 @@ add_tests([{skip_cases,Node,Dir,Suite,Cs,Cmt}|Ts],Spec) ->
     Tests = Spec#testspec.tests,
     Tests1 = skip_cases(ref2node(Node,Spec#testspec.nodes),
 			ref2dir(Dir,Spec#testspec.alias),
-			Suite,Cs,Cmt,Tests),
+			Suite,Cs,Cmt,Tests,Spec#testspec.merge_tests),
     add_tests(Ts,Spec#testspec{tests=Tests1});
 
 %% --- handled/errors ---
@@ -783,6 +797,9 @@ add_tests([{alias,_,_}|Ts],Spec) ->		% handled
 add_tests([{node,_,_}|Ts],Spec) ->		% handled
     add_tests(Ts,Spec);
 
+add_tests([{merge_tests, _} | Ts], Spec) -> % handled
+    add_tests(Ts,Spec);
+
 %% check if it's a CT term that has bad format or if the user seems to
 %% have added something of his/her own, which we'll let pass if relaxed
 %% mode is enabled.
@@ -835,17 +852,22 @@ separate([],_,_,_) ->
 %%              {Suite2,[{GrOrCase21,{skip,Cmt}},GrOrCase22,...]},...]}
 %% GrOrCase = {GroupName,[Case1,Case2,...]} | Case
 
-insert_suites(Node,Dir,[S|Ss],Tests) ->
-    Tests1 = insert_cases(Node,Dir,S,all,Tests),
-    insert_suites(Node,Dir,Ss,Tests1);
-insert_suites(_Node,_Dir,[],Tests) ->
+insert_suites(Node,Dir,[S|Ss],Tests, MergeTests) ->
+    Tests1 = insert_cases(Node,Dir,S,all,Tests,MergeTests),
+    insert_suites(Node,Dir,Ss,Tests1,MergeTests);
+insert_suites(_Node,_Dir,[],Tests,_MergeTests) ->
     Tests;
-insert_suites(Node,Dir,S,Tests) ->
-    insert_suites(Node,Dir,[S],Tests).
+insert_suites(Node,Dir,S,Tests,MergeTests) ->
+    insert_suites(Node,Dir,[S],Tests,MergeTests).
 
-insert_groups(Node,Dir,Suite,Group,Cases,Tests) when is_atom(Group) ->
-    insert_groups(Node,Dir,Suite,[Group],Cases,Tests);
-insert_groups(Node,Dir,Suite,Groups,Cases,Tests) when
+insert_groups(Node,Dir,Suite,Group,Cases,Tests,MergeTests) 
+  when is_atom(Group) ->
+    insert_groups(Node,Dir,Suite,[Group],Cases,Tests,MergeTests);
+insert_groups(Node,Dir,Suite,Groups,Cases,Tests,false) when
+      ((Cases == all) or is_list(Cases)) and is_list(Groups) ->
+    Groups1 = [{Gr,Cases} || Gr <- Groups],
+    append({{Node,Dir},[{Suite,Groups1}]},Tests);
+insert_groups(Node,Dir,Suite,Groups,Cases,Tests,true) when
       ((Cases == all) or is_list(Cases)) and is_list(Groups) ->
     case lists:keysearch({Node,Dir},1,Tests) of
 	{value,{{Node,Dir},[{all,_}]}} ->
@@ -859,9 +881,10 @@ insert_groups(Node,Dir,Suite,Groups,Cases,Tests) when
 	    Groups1 = [{Gr,Cases} || Gr <- Groups],
 	    insert_in_order({{Node,Dir},[{Suite,Groups1}]},Tests)
     end;
-insert_groups(Node,Dir,Suite,Groups,Case,Tests) when is_atom(Case) ->
+insert_groups(Node,Dir,Suite,Groups,Case,Tests, MergeTests) 
+  when is_atom(Case) ->
     Cases = if Case == all -> all; true -> [Case] end,
-    insert_groups(Node,Dir,Suite,Groups,Cases,Tests).
+    insert_groups(Node,Dir,Suite,Groups,Cases,Tests, MergeTests).
 
 insert_groups1(_Suite,_Groups,all) ->
     all;
@@ -891,7 +914,9 @@ insert_groups2([Group={GrName,Cases}|Groups],GrAndCases) ->
 insert_groups2([],GrAndCases) ->
     GrAndCases.
 
-insert_cases(Node,Dir,Suite,Cases,Tests) when is_list(Cases) ->
+insert_cases(Node,Dir,Suite,Cases,Tests,false) when is_list(Cases) ->
+    append({{Node,Dir},[{Suite,Cases}]},Tests);
+insert_cases(Node,Dir,Suite,Cases,Tests,true) when is_list(Cases) ->
     case lists:keysearch({Node,Dir},1,Tests) of
 	{value,{{Node,Dir},[{all,_}]}} ->
 	    Tests;
@@ -901,8 +926,8 @@ insert_cases(Node,Dir,Suite,Cases,Tests) when is_list(Cases) ->
 	false ->
 	    insert_in_order({{Node,Dir},[{Suite,Cases}]},Tests)
     end;
-insert_cases(Node,Dir,Suite,Case,Tests) when is_atom(Case) ->
-    insert_cases(Node,Dir,Suite,[Case],Tests).
+insert_cases(Node,Dir,Suite,Case,Tests,MergeTests) when is_atom(Case) ->
+    insert_cases(Node,Dir,Suite,[Case],Tests,MergeTests).
 
 insert_cases1(_Suite,_Cases,all) ->
     all;
@@ -917,22 +942,28 @@ insert_cases1(Suite,Cases,Suites0) ->
 	    insert_in_order({Suite,Cases},Suites0)
     end.
 
-skip_suites(Node,Dir,[S|Ss],Cmt,Tests) ->
-    Tests1 = skip_cases(Node,Dir,S,all,Cmt,Tests),
-    skip_suites(Node,Dir,Ss,Cmt,Tests1);
-skip_suites(_Node,_Dir,[],_Cmt,Tests) ->
+skip_suites(Node,Dir,[S|Ss],Cmt,Tests,MergeTests) ->
+    Tests1 = skip_cases(Node,Dir,S,all,Cmt,Tests,MergeTests),
+    skip_suites(Node,Dir,Ss,Cmt,Tests1,MergeTests);
+skip_suites(_Node,_Dir,[],_Cmt,Tests,_MergeTests) ->
     Tests;
-skip_suites(Node,Dir,S,Cmt,Tests) ->
-    skip_suites(Node,Dir,[S],Cmt,Tests).
-
-skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests) when is_atom(Group) ->
-    skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests);
-skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests) when is_atom(Group) ->
-    skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests);
-skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case),
-						       Case =/= all ->
-    skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests);
-skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when
+skip_suites(Node,Dir,S,Cmt,Tests,MergeTests) ->
+    skip_suites(Node,Dir,[S],Cmt,Tests,MergeTests).
+
+skip_groups(Node,Dir,Suite,Group,all,Cmt,Tests,MergeTests) 
+  when is_atom(Group) ->
+    skip_groups(Node,Dir,Suite,[Group],all,Cmt,Tests,MergeTests);
+skip_groups(Node,Dir,Suite,Group,Cases,Cmt,Tests,MergeTests) 
+  when is_atom(Group) ->
+    skip_groups(Node,Dir,Suite,[Group],Cases,Cmt,Tests,MergeTests);
+skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests,MergeTests) 
+  when is_atom(Case),Case =/= all ->
+    skip_groups(Node,Dir,Suite,Groups,[Case],Cmt,Tests,MergeTests);
+skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,false) when
+      ((Cases == all) or is_list(Cases)) and is_list(Groups) ->
+    Suites1 = skip_groups1(Suite,[{Gr,Cases} || Gr <- Groups],Cmt,[]),
+    append({{Node,Dir},Suites1},Tests);
+skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,true) when
       ((Cases == all) or is_list(Cases)) and is_list(Groups) ->
     Suites =
 	case lists:keysearch({Node,Dir},1,Tests) of
@@ -943,9 +974,10 @@ skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests) when
 	end,
     Suites1 = skip_groups1(Suite,[{Gr,Cases} || Gr <- Groups],Cmt,Suites),
     insert_in_order({{Node,Dir},Suites1},Tests);
-skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests) when is_atom(Case) ->
+skip_groups(Node,Dir,Suite,Groups,Case,Cmt,Tests,MergeTests) 
+  when is_atom(Case) ->
     Cases = if Case == all -> all; true -> [Case] end,
-    skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests).
+    skip_groups(Node,Dir,Suite,Groups,Cases,Cmt,Tests,MergeTests).
 
 skip_groups1(Suite,Groups,Cmt,Suites0) ->
     SkipGroups = lists:map(fun(Group) ->
@@ -959,7 +991,10 @@ skip_groups1(Suite,Groups,Cmt,Suites0) ->
 	    insert_in_order({Suite,SkipGroups},Suites0)
     end.
 
-skip_cases(Node,Dir,Suite,Cases,Cmt,Tests) when is_list(Cases) ->
+skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,false) when is_list(Cases) ->
+    Suites1 = skip_cases1(Suite,Cases,Cmt,[]),
+    append({{Node,Dir},Suites1},Tests);
+skip_cases(Node,Dir,Suite,Cases,Cmt,Tests,true) when is_list(Cases) ->
     Suites =
 	case lists:keysearch({Node,Dir},1,Tests) of
 	    {value,{{Node,Dir},Suites0}} ->
@@ -969,8 +1004,8 @@ skip_cases(Node,Dir,Suite,Cases,Cmt,Tests) when is_list(Cases) ->
 	end,
     Suites1 = skip_cases1(Suite,Cases,Cmt,Suites),
     insert_in_order({{Node,Dir},Suites1},Tests);
-skip_cases(Node,Dir,Suite,Case,Cmt,Tests) when is_atom(Case) ->
-    skip_cases(Node,Dir,Suite,[Case],Cmt,Tests).
+skip_cases(Node,Dir,Suite,Case,Cmt,Tests,MergeTests) when is_atom(Case) ->
+    skip_cases(Node,Dir,Suite,[Case],Cmt,Tests,MergeTests).
 
 skip_cases1(Suite,Cases,Cmt,Suites0) ->
     SkipCases = lists:map(fun(C) ->
@@ -984,6 +1019,9 @@ skip_cases1(Suite,Cases,Cmt,Suites0) ->
 	    insert_in_order({Suite,SkipCases},Suites0)
     end.
 
+append(Elem, List) ->
+    List ++ [Elem].
+
 insert_in_order([E|Es],List) ->
     List1 = insert_elem(E,List,[]),
     insert_in_order(Es,List1);
@@ -1056,6 +1094,7 @@ valid_terms() ->
      {userconfig,2},
      {userconfig,3},
      {alias,3},
+     {merge_tests,1},
      {logdir,2},
      {logdir,3},
      {label,2},
diff --git a/lib/common_test/src/ct_util.hrl b/lib/common_test/src/ct_util.hrl
index 4ed29bdcd1..f0255c533c 100644
--- a/lib/common_test/src/ct_util.hrl
+++ b/lib/common_test/src/ct_util.hrl
@@ -41,7 +41,8 @@
 		   multiply_timetraps=[],
 		   scale_timetraps=[],
 		   alias=[],
-		   tests=[]}).
+		   tests=[],
+		   merge_tests = true }).
 
 -record(cover, {app=none,
 		level=details,
diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl
index c2a7bd8fd7..e080e074bf 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE.erl
@@ -67,7 +67,21 @@ all() ->
      skip_group_testcase, topgroup, subgroup, skip_subgroup,
      subgroup_all_testcases, skip_subgroup_all_testcases,
      subgroup_testcase, skip_subgroup_testcase,
-     sub_skipped_by_top, testcase_in_multiple_groups].
+     sub_skipped_by_top, testcase_in_multiple_groups,
+     order_of_tests_in_multiple_dirs_no_merge_tests,
+     order_of_tests_in_multiple_suites_no_merge_tests,
+     order_of_suites_in_multiple_dirs_no_merge_tests,
+     order_of_groups_in_multiple_dirs_no_merge_tests,
+     order_of_groups_in_multiple_suites_no_merge_tests,
+     order_of_tests_in_multiple_dirs,
+     order_of_tests_in_multiple_suites,
+     order_of_suites_in_multiple_dirs,
+     order_of_groups_in_multiple_dirs,
+     order_of_groups_in_multiple_suites,
+     order_of_tests_in_multiple_suites_with_skip_no_merge_tests,
+     order_of_tests_in_multiple_suites_with_skip,
+     all_plus_one_tc_no_merge_tests,
+     all_plus_one_tc].
 
 groups() -> 
     [].
@@ -78,7 +92,6 @@ init_per_group(_GroupName, Config) ->
 end_per_group(_GroupName, Config) ->
 	Config.
 
-
 %%--------------------------------------------------------------------
 %% TEST CASES
 %%--------------------------------------------------------------------
@@ -370,6 +383,223 @@ testcase_in_multiple_groups(Config) when is_list(Config) ->
     setup_and_execute(testcase_in_multiple_groups, TestSpec, Config).
 
 %%%-----------------------------------------------------------------
+%%%
+
+order_of_tests_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir1 = filename:join(DataDir, "groups_1"),
+    TestDir2 = filename:join(DataDir, "groups_2"),
+    TestSpec = [{merge_tests, false},
+		{cases,TestDir1,groups_12_SUITE,[testcase_1a]},
+		{cases,TestDir2,groups_22_SUITE,[testcase_1]},
+		{cases,TestDir1,groups_12_SUITE,[testcase_1b]}],
+
+    setup_and_execute(order_of_tests_in_multiple_dirs_no_merge_tests, 
+		      TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_tests_in_multiple_suites_no_merge_tests(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir1 = filename:join(DataDir, "groups_1"),
+    TestSpec = [{merge_tests, false},
+		{cases,TestDir1,groups_12_SUITE,[testcase_1a]},
+		{cases,TestDir1,groups_11_SUITE,[testcase_1]},
+		{cases,TestDir1,groups_12_SUITE,[testcase_1b]}],
+
+    setup_and_execute(order_of_tests_in_multiple_suites_no_merge_tests, 
+		      TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_suites_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir1 = filename:join(DataDir, "groups_1"),
+    TestDir2 = filename:join(DataDir, "groups_2"),
+    TestSpec = [{merge_tests, false},
+		{suites,TestDir1,groups_12_SUITE},
+		{suites,TestDir2,groups_22_SUITE},
+		{suites,TestDir1,groups_11_SUITE}],
+
+    setup_and_execute(order_of_suites_in_multiple_dirs_no_merge_tests, 
+		      TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_groups_in_multiple_dirs_no_merge_tests(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir1 = filename:join(DataDir, "groups_1"),
+    TestDir2 = filename:join(DataDir, "groups_2"),
+    TestSpec = [{merge_tests, false},
+		{groups,TestDir1,groups_12_SUITE,test_group_1a},
+		{groups,TestDir2,groups_22_SUITE,test_group_1a},
+		{groups,TestDir1,groups_12_SUITE,test_group_1b}],
+
+    setup_and_execute(order_of_groups_in_multiple_dirs_no_merge_tests, 
+		      TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_groups_in_multiple_suites_no_merge_tests(Config) 
+  when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir1 = filename:join(DataDir, "groups_1"),
+    TestSpec = [{merge_tests, false},
+		{groups,TestDir1,groups_12_SUITE,test_group_1a},
+		{groups,TestDir1,groups_11_SUITE,test_group_1a},
+		{groups,TestDir1,groups_12_SUITE,test_group_1b}],
+
+    setup_and_execute(order_of_groups_in_multiple_suites_no_merge_tests, 
+		      TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_tests_in_multiple_suites_with_skip_no_merge_tests(Config) 
+  when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir1 = filename:join(DataDir, "groups_1"),
+    TestSpec = [{merge_tests, false},
+		{cases,TestDir1,groups_12_SUITE,[testcase_1a]},
+		{cases,TestDir1,groups_11_SUITE,[testcase_1]},
+		{cases,TestDir1,groups_12_SUITE,[testcase_1b]},
+		{cases,TestDir1,groups_11_SUITE,[testcase_2]},
+		{skip_cases,TestDir1,groups_12_SUITE,[testcase_1b],"Skip it"}],
+
+    setup_and_execute(
+      order_of_tests_in_multiple_suites_with_skip_no_merge_tests,
+      TestSpec, Config).
+     
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_tests_in_multiple_dirs(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir1 = filename:join(DataDir, "groups_1"),
+    TestDir2 = filename:join(DataDir, "groups_2"),
+    TestSpec = [{cases,TestDir1,groups_12_SUITE,[testcase_1a]},
+		{cases,TestDir2,groups_22_SUITE,[testcase_1]},
+		{cases,TestDir1,groups_12_SUITE,[testcase_1b]}],
+
+    setup_and_execute(order_of_tests_in_multiple_dirs, 
+		      TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_tests_in_multiple_suites(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir1 = filename:join(DataDir, "groups_1"),
+    TestSpec = [{cases,TestDir1,groups_12_SUITE,[testcase_1a]},
+		{cases,TestDir1,groups_11_SUITE,[testcase_1]},
+		{cases,TestDir1,groups_12_SUITE,[testcase_1b]}],
+
+    setup_and_execute(order_of_tests_in_multiple_suites, 
+		      TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_suites_in_multiple_dirs(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir1 = filename:join(DataDir, "groups_1"),
+    TestDir2 = filename:join(DataDir, "groups_2"),
+    TestSpec = [{suites,TestDir1,groups_12_SUITE},
+		{suites,TestDir2,groups_22_SUITE},
+		{suites,TestDir1,groups_11_SUITE}],
+
+    setup_and_execute(order_of_suites_in_multiple_dirs, 
+		      TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_groups_in_multiple_dirs(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir1 = filename:join(DataDir, "groups_1"),
+    TestDir2 = filename:join(DataDir, "groups_2"),
+    TestSpec = [{groups,TestDir1,groups_12_SUITE,test_group_1a},
+		{groups,TestDir2,groups_22_SUITE,test_group_1a},
+		{groups,TestDir1,groups_12_SUITE,test_group_1b}],
+
+    setup_and_execute(order_of_groups_in_multiple_dirs, 
+		      TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_groups_in_multiple_suites(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir1 = filename:join(DataDir, "groups_1"),
+    TestSpec = [{groups,TestDir1,groups_12_SUITE,test_group_1a},
+		{groups,TestDir1,groups_11_SUITE,test_group_1a},
+		{groups,TestDir1,groups_12_SUITE,test_group_1b}],
+
+    setup_and_execute(order_of_groups_in_multiple_suites, 
+		      TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+order_of_tests_in_multiple_suites_with_skip(Config) when is_list(Config) ->
+    DataDir = ?config(data_dir, Config),
+
+    TestDir1 = filename:join(DataDir, "groups_1"),
+    TestSpec = [{cases,TestDir1,groups_12_SUITE,[testcase_1a]},
+		{cases,TestDir1,groups_11_SUITE,[testcase_1]},
+		{cases,TestDir1,groups_12_SUITE,[testcase_1b]},
+		{cases,TestDir1,groups_11_SUITE,[testcase_2]},
+		{skip_cases,TestDir1,groups_12_SUITE,[testcase_1b],"Skip it!"}],
+
+    setup_and_execute(order_of_tests_in_multiple_suites_with_skip, 
+		      TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+all_plus_one_tc_no_merge_tests(Config) when is_list(Config) ->
+    
+    DataDir = ?config(data_dir, Config),
+
+    TestDir1 = filename:join(DataDir, "groups_1"),
+    TestSpec = [{merge_tests,false},
+		{suites,TestDir1,groups_12_SUITE},
+		{cases,TestDir1,groups_12_SUITE,[testcase_1a]}],
+
+    setup_and_execute(all_plus_one_tc_no_merge_tests, 
+		      TestSpec, Config).
+
+%%%-----------------------------------------------------------------
+%%%
+
+all_plus_one_tc(Config) when is_list(Config) ->
+    
+    DataDir = ?config(data_dir, Config),
+
+    TestDir1 = filename:join(DataDir, "groups_1"),
+    TestSpec = [{suites,TestDir1,groups_12_SUITE},
+		{cases,TestDir1,groups_12_SUITE,[testcase_1a]}],
+
+    setup_and_execute(all_plus_one_tc, 
+		      TestSpec, Config).
+
+%%%-----------------------------------------------------------------
 %%% HELP FUNCTIONS
 %%%-----------------------------------------------------------------
 
@@ -432,6 +662,719 @@ events_to_check(_, 0) ->
 events_to_check(Test, N) ->
     test_events(Test) ++ events_to_check(Test, N-1).
 
+test_events(all_suites) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{simple_1_SUITE,init_per_suite}},
+     {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}},
+     {?eh,tc_start,{simple_2_SUITE,init_per_suite}},
+     {?eh,test_stats,{4,0,{0,0}}},
+     {?eh,tc_done,{simple_2_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(skip_all_suites) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_user_skip,{simple_1_SUITE,all,"SKIPPED!"}},
+     {?eh,tc_user_skip,{simple_2_SUITE,all,"SKIPPED!"}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(suite) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{simple_1_SUITE,init_per_suite}},
+     {?eh,test_stats,{2,0,{0,0}}},
+     {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+ 
+test_events(skip_suite) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_user_skip,{simple_1_SUITE,all,"SKIPPED!"}},
+     {?eh,tc_done,{simple_2_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(all_testcases) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{simple_1_SUITE,init_per_suite}},
+     {?eh,test_stats,{2,0,{0,0}}},
+     {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+ 
+test_events(skip_all_testcases) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_user_skip,{simple_1_SUITE,all,"SKIPPED!"}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(testcase) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{simple_1_SUITE,init_per_suite}},
+     {?eh,test_stats,{1,0,{0,0}}},
+     {negative,{?eh,test_stats,{2,0,{0,0}}},
+      {?eh,tc_done,{simple_1_SUITE,end_per_suite,'_'}}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+ 
+test_events(skip_testcase) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{simple_1_SUITE,init_per_suite}},
+     {?eh,tc_user_skip,{simple_1_SUITE,tc1,"SKIPPED!"}},
+     {?eh,tc_start,{simple_1_SUITE,tc2}},
+     {?eh,tc_start,{simple_1_SUITE,end_per_suite}},
+
+     {?eh,tc_start,{simple_2_SUITE,init_per_suite}},
+     {?eh,tc_user_skip,{simple_2_SUITE,tc2,"SKIPPED!"}},
+     {?eh,tc_start,{simple_2_SUITE,tc1}},
+     {?eh,test_stats,{2,0,{2,0}}},
+     {?eh,tc_start,{simple_2_SUITE,end_per_suite}},
+     
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(all_groups) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     {?eh,test_stats,{12,0,{0,0}}},
+     {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+ 
+test_events(skip_all_groups) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}},
+     {?eh,test_stats,{0,0,{1,0}}},
+     {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}},
+     {?eh,test_stats,{0,0,{2,0}}},
+     {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_2},"SKIPPED!"}},
+     {?eh,test_stats,{0,0,{3,0}}},
+     {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_4},"SKIPPED!"}},
+     {?eh,test_stats,{0,0,{4,0}}},
+     {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(group) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_1a}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_1b}},
+     {?eh,test_stats,{2,0,{0,0}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}},
+     {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+ 
+test_events(skip_group) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_1a}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_1b}},
+     {?eh,test_stats,{2,0,{0,0}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}},
+     
+     {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}},
+     {?eh,test_stats,{2,0,{1,0}}},
+     {negative,{?eh,tc_user_skip,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(group_all_testcases) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_1a}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_1b}},
+     {?eh,test_stats,{2,0,{0,0}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}},
+     {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+ 
+test_events(skip_group_all_testcases) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1a},"SKIPPED!"}},
+     {?eh,tc_user_skip, {groups_11_SUITE,{group,test_group_1b},"SKIPPED!"}},
+     {?eh,test_stats,{0,0,{2,0}}},
+     {?eh,tc_start,{groups_11_SUITE,end_per_suite}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(group_testcase) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_1a}},
+     {?eh,test_stats,{1,0,{0,0}}},
+     {negative,{?eh,test_stats,{2,0,{0,0}}},
+      {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}}},
+
+     {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+ 
+test_events(skip_group_testcase) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1a,[]}}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_1a}},
+     {?eh,tc_user_skip,{groups_11_SUITE,testcase_1b,"SKIPPED!"}},
+     {?eh,test_stats,{1,0,{1,0}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1a,[]},'_'}},
+     
+     {?eh,tc_start,{groups_11_SUITE,{init_per_group,test_group_1b,[]}}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_1b}},
+     {?eh,tc_user_skip,{groups_11_SUITE,testcase_1a,"SKIPPED!"}},
+     {?eh,test_stats,{2,0,{2,0}}},
+     {?eh,tc_done,{groups_11_SUITE,{end_per_group,test_group_1b,[]},'_'}},
+
+     {negative,{?eh,tc_user_skip,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(topgroup) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_12_SUITE,init_per_suite}},
+     
+     {parallel, 
+      [{?eh,tc_start,
+	{groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}},
+       {?eh,tc_done,
+	{groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}},
+       [{?eh,tc_start,
+	 {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}},
+	{?eh,tc_start,
+	 {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}}
+       ],
+       [{?eh,tc_start,
+	 {groups_12_SUITE,{init_per_group,test_group_3,[]}}},
+	{?eh,tc_start,
+	 {groups_12_SUITE,{end_per_group,test_group_3,[]}}}
+       ],
+       {?eh,test_stats,{6,0,{0,0}}},
+       {?eh,tc_start,
+	{groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}},
+       {?eh,tc_done,
+	{groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]},
+
+     [{?eh,tc_start,
+       {groups_12_SUITE,{init_per_group,test_group_4,[]}}},
+      {parallel,
+       [{?eh,tc_start,
+	 {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}},
+	{?eh,tc_done,
+	 {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}},
+	{parallel,
+	 [{?eh,tc_start,
+	   {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}},
+	  {?eh,tc_done,
+	   {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}},
+	  [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}},
+	   {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}],
+	  {shuffle,
+	   [{?eh,tc_start,
+	     {groups_12_SUITE,{init_per_group,test_group_8,
+			       [{shuffle,'_'},sequence]}}},
+	    {?eh,tc_done,
+	     {groups_12_SUITE,{init_per_group,test_group_8,
+			       [{shuffle,'_'},sequence]},ok}},
+	    {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_8,
+					    [shuffle,sequence]}}},
+	    {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_8,
+					    [shuffle,sequence]},ok}}
+	   ]},
+	  {?eh,tc_start,
+	   {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}},
+	  {?eh,tc_done,
+	   {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}}
+	 ]},
+	{?eh,test_stats,{12,0,{0,0}}},
+	{?eh,tc_start,
+	 {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}},
+	{?eh,tc_done,
+	 {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]},
+      {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}],
+
+     {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(subgroup) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_12_SUITE,init_per_suite}},
+
+     {parallel, 
+      [{?eh,tc_start,
+	{groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}},
+       {?eh,tc_done,
+	{groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}},
+       [{?eh,tc_start,
+	 {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}},
+	{?eh,tc_start,
+	 {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}}
+       ],
+       [{?eh,tc_start,
+	 {groups_12_SUITE,{init_per_group,test_group_3,[]}}},
+	{?eh,tc_start,
+	 {groups_12_SUITE,{end_per_group,test_group_3,[]}}}
+       ],
+       {?eh,test_stats,{4,0,{0,0}}},
+       {?eh,tc_start,
+	{groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}},
+       {?eh,tc_done,
+	{groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]},
+     {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+ 
+test_events(skip_subgroup) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_12_SUITE,init_per_suite}},
+
+     [{?eh,tc_start,
+       {groups_12_SUITE,{init_per_group,test_group_4,[]}}},
+      {parallel,
+       [{?eh,tc_start,
+	 {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}},
+	{?eh,tc_done,
+	 {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}},
+	{parallel,
+	 [{?eh,tc_start,
+	   {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}},
+	  {?eh,tc_done,
+	   {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}},
+	  [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}},
+	   {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}],
+	  {?eh,tc_user_skip,
+	   {groups_12_SUITE,{group,test_group_8},"SKIPPED!"}},
+	  {?eh,tc_start,
+	   {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}},
+	  {?eh,tc_done,
+	   {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}}
+	 ]},
+	{?eh,tc_start,
+	 {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}},
+	{?eh,tc_done,
+	 {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]},
+      {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}],
+
+     {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(subgroup_all_testcases) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_12_SUITE,init_per_suite}},
+
+     [{?eh,tc_start,
+       {groups_12_SUITE,{init_per_group,test_group_4,[]}}},
+      {parallel,
+       [{?eh,tc_start,
+	 {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}},
+	{?eh,tc_done,
+	 {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}},
+	{parallel,
+	 [{?eh,tc_start,
+	   {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}},
+	  {?eh,tc_done,
+	   {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}},
+	  [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}},
+	   {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}],
+	  {shuffle,
+	   [{?eh,tc_start,
+	     {groups_12_SUITE,{init_per_group,test_group_8,
+			       [{shuffle,'_'},sequence]}}},
+	    {?eh,tc_done,
+	     {groups_12_SUITE,{init_per_group,test_group_8,
+			       [{shuffle,'_'},sequence]},ok}},
+	    {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_8,
+					    [shuffle,sequence]}}},
+	    {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_8,
+					    [shuffle,sequence]},ok}}
+	   ]},
+	  {?eh,tc_start,
+	   {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}},
+	  {?eh,tc_done,
+	   {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}}
+	 ]},
+	{?eh,test_stats,{6,0,{0,0}}},
+	{?eh,tc_start,
+	 {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}},
+	{?eh,tc_done,
+	 {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]},
+      {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}],
+
+     {parallel, 
+      [{?eh,tc_start,
+	{groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}},
+       {?eh,tc_done,
+	{groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}},
+       [{?eh,tc_start,
+	 {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}},
+	{?eh,tc_start,
+	 {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}}
+       ],
+       [{?eh,tc_start,
+	 {groups_12_SUITE,{init_per_group,test_group_3,[]}}},
+	{?eh,test_stats,{10,0,{0,0}}},
+	{?eh,tc_start,
+	 {groups_12_SUITE,{end_per_group,test_group_3,[]}}}
+       ],
+       {?eh,tc_start,
+	{groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}},
+       {?eh,tc_done,
+	{groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]},
+
+     {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+ 
+test_events(skip_subgroup_all_testcases) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_12_SUITE,init_per_suite}},
+
+     [{?eh,tc_start,
+       {groups_12_SUITE,{init_per_group,test_group_4,[]}}},
+      {?eh,tc_done,
+       {groups_12_SUITE,{init_per_group,test_group_4,[]},ok}},
+      {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_5},"SKIPPED!"}},
+      {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}},
+      {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_4,[]},ok}}
+     ],
+
+     {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(subgroup_testcase) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_12_SUITE,init_per_suite}},
+
+     [{?eh,tc_start,
+       {groups_12_SUITE,{init_per_group,test_group_4,[]}}},
+      {parallel,
+       [{?eh,tc_start,
+	 {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}},
+	{?eh,tc_done,
+	 {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}},
+	{parallel,
+	 [{?eh,tc_start,
+	   {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}},
+	  {?eh,tc_done,
+	   {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}},
+	  [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}},
+	   {?eh,test_stats,{1,0,{0,0}}},
+	   {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}],
+	  {?eh,tc_start,
+	   {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}},
+	  {?eh,tc_done,
+	   {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}}
+	 ]},
+	{?eh,tc_start,
+	 {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}},
+	{?eh,tc_done,
+	 {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]},
+      {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}],
+
+     {parallel, 
+      [{?eh,tc_start,
+	{groups_12_SUITE,{init_per_group,test_group_2,[parallel]}}},
+       {?eh,tc_done,
+	{groups_12_SUITE,{init_per_group,test_group_2,[parallel]},ok}},
+       [{?eh,tc_start,
+	 {groups_12_SUITE,{init_per_group,test_group_3,[{repeat,2}]}}},
+	{?eh,test_stats,{2,0,{0,0}}},
+	{?eh,tc_start,
+	 {groups_12_SUITE,{end_per_group,test_group_3,[{repeat,2}]}}}
+       ],
+       [{?eh,tc_start,
+	 {groups_12_SUITE,{init_per_group,test_group_3,[]}}},
+	{?eh,test_stats,{3,0,{0,0}}},
+	{?eh,tc_start,
+	 {groups_12_SUITE,{end_per_group,test_group_3,[]}}}
+       ],
+       {?eh,tc_start,
+	{groups_12_SUITE,{end_per_group,test_group_2,[parallel]}}},
+       {?eh,tc_done,
+	{groups_12_SUITE,{end_per_group,test_group_2,[parallel]},ok}}]},
+
+     {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+ 
+test_events(skip_subgroup_testcase) ->
+    [
+
+    {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_12_SUITE,init_per_suite}},
+
+     [{?eh,tc_start,
+       {groups_12_SUITE,{init_per_group,test_group_4,[]}}},
+      {parallel,
+       [{?eh,tc_start,
+	 {groups_12_SUITE,{init_per_group,test_group_5,[parallel]}}},
+	{?eh,tc_done,
+	 {groups_12_SUITE,{init_per_group,test_group_5,[parallel]},ok}},
+	{parallel,
+	 [{?eh,tc_start,
+	   {groups_12_SUITE,{init_per_group,test_group_6,[parallel]}}},
+	  {?eh,tc_done,
+	   {groups_12_SUITE,{init_per_group,test_group_6,[parallel]},ok}},
+	  [{?eh,tc_start,{groups_12_SUITE,{init_per_group,test_group_7,'_'}}},
+	   {?eh,tc_user_skip, {groups_12_SUITE,testcase_7a,"SKIPPED!"}},
+           {?eh,test_stats,{1,0,{1,0}}},
+           {?eh,tc_user_skip, {groups_12_SUITE,testcase_7b,"SKIPPED!"}},
+           {?eh,test_stats,{1,0,{2,0}}},
+	   {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_7,'_'}}}],
+	  {shuffle,
+	   [{?eh,tc_start,
+	     {groups_12_SUITE,{init_per_group,test_group_8,
+			       [{shuffle,'_'},sequence]}}},
+	    {?eh,tc_done,
+	     {groups_12_SUITE,{init_per_group,test_group_8,
+			       [{shuffle,'_'},sequence]},ok}},
+	    {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_8,
+					    [shuffle,sequence]}}},
+	    {?eh,tc_done,{groups_12_SUITE,{end_per_group,test_group_8,
+					    [shuffle,sequence]},ok}}
+	   ]},
+	  {?eh,tc_start,
+	   {groups_12_SUITE,{end_per_group,test_group_6,[parallel]}}},
+	  {?eh,tc_done,
+	   {groups_12_SUITE,{end_per_group,test_group_6,[parallel]},ok}}
+	 ]},
+	{?eh,test_stats,{4,0,{2,0}}},
+	{?eh,tc_start,
+	 {groups_12_SUITE,{end_per_group,test_group_5,[parallel]}}},
+	{?eh,tc_done,
+	 {groups_12_SUITE,{end_per_group,test_group_5,[parallel]},ok}}]},
+      {?eh,tc_start,{groups_12_SUITE,{end_per_group,test_group_4,[]}}}],
+
+     {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+     
+    ];
+
+test_events(sub_skipped_by_top) ->
+    [
+     {?eh,start_logging,'_'},
+     {?eh,tc_start,{groups_12_SUITE,init_per_suite}},
+
+     {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}},
+
+     {negative,
+      {?eh,tc_user_skip,{groups_12_SUITE,{group,test_group_4},"SKIPPED!"}},
+      {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}}},
+
+     {negative,{?eh,tc_start,'_'},{?eh,stop_logging,'_'}}
+    ];
+
+test_events(testcase_in_multiple_groups) ->
+    [];
+
+test_events(order_of_tests_in_multiple_dirs_no_merge_tests) ->
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,tc_start,{groups_12_SUITE,testcase_1a}},
+     {?eh,tc_done, {groups_12_SUITE,testcase_1a,
+		    {failed,{error,{test_case_failed,no_group_data}}}}},
+     {?eh,tc_start,{groups_22_SUITE,testcase_1}},
+     {?eh,tc_done,{groups_22_SUITE,testcase_1,ok}},
+     {?eh,tc_start,{groups_12_SUITE,testcase_1b}},
+     {?eh,tc_done, {groups_12_SUITE,testcase_1b,
+		    {failed,{error,{test_case_failed,no_group_data}}}}},
+     {?eh,stop_logging,[]}
+    ];
+test_events(order_of_tests_in_multiple_suites_no_merge_tests) ->
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,tc_start,{groups_12_SUITE,testcase_1a}},
+     {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_1}},
+     {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}},
+     {?eh,tc_start,{groups_12_SUITE,testcase_1b}},
+     {?eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}},
+     {?eh,stop_logging,[]}
+    ];
+test_events(order_of_suites_in_multiple_dirs_no_merge_tests) ->
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,tc_start,{groups_12_SUITE,init_per_suite}},
+     {?eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}},
+     {?eh,tc_start,{groups_12_SUITE,end_per_suite}},
+     {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+     {?eh,tc_start,{groups_22_SUITE,init_per_suite}},
+     {?eh,tc_done,{groups_22_SUITE,init_per_suite,'_'}},
+     {?eh,tc_start,{groups_22_SUITE,end_per_suite}},
+     {?eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}},
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     {?eh,tc_done,{groups_11_SUITE,init_per_suite,'_'}},
+     {?eh,tc_start,{groups_11_SUITE,end_per_suite}},
+     {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+     {?eh,stop_logging,[]}];
+test_events(order_of_groups_in_multiple_dirs_no_merge_tests) ->
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     
+     {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}},
+     {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}},
+
+     {?eh,tc_start, {groups_22_SUITE,{init_per_group,test_group_1a,'_'}}},
+     {?eh,tc_done, {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}},
+      
+     {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}},
+     {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}},
+
+     {?eh,stop_logging,[]}];
+test_events(order_of_groups_in_multiple_suites_no_merge_tests) ->
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     
+     {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}},
+     {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}},
+
+     {?eh,tc_start, {groups_11_SUITE,{init_per_group,test_group_1a,'_'}}},
+     {?eh,tc_done, {groups_11_SUITE,{end_per_group,test_group_1a,'_'},'_'}},
+      
+     {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}},
+     {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}},
+
+     {?eh,stop_logging,[]}];
+test_events(order_of_tests_in_multiple_suites_with_skip_no_merge_tests) ->
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,tc_start,{groups_12_SUITE,testcase_1a}},
+     {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_1}},
+     {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}},
+     {?eh,tc_user_skip,{groups_12_SUITE,testcase_1b,'_'}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_2}},
+     {?eh,tc_done,{groups_11_SUITE,testcase_2,ok}},
+     {?eh,stop_logging,[]}
+    ];
+
+test_events(order_of_tests_in_multiple_dirs) ->
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,tc_start,{groups_12_SUITE,testcase_1a}},
+     {?eh,tc_done,
+      {groups_12_SUITE,testcase_1a,
+       {failed,{error,{test_case_failed,no_group_data}}}}},
+     {?eh,tc_start,{groups_12_SUITE,testcase_1b}},
+     {?eh,tc_done,
+      {groups_12_SUITE,testcase_1b,
+       {failed,{error,{test_case_failed,no_group_data}}}}},
+     {?eh,tc_start,{groups_22_SUITE,testcase_1}},
+     {?eh,tc_done,{groups_22_SUITE,testcase_1,ok}},
+     {?eh,stop_logging,[]}
+    ];
+test_events(order_of_tests_in_multiple_suites) ->
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,tc_start,{groups_12_SUITE,testcase_1a}},
+     {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}},
+
+     {?eh,tc_start,{groups_12_SUITE,testcase_1b}},
+     {?eh,tc_done,{groups_12_SUITE,testcase_1b,'_'}},
+
+     {?eh,tc_start,{groups_11_SUITE,testcase_1}},
+     {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}},
+     {?eh,stop_logging,[]}
+    ];
+test_events(order_of_suites_in_multiple_dirs) ->
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,tc_start,{groups_12_SUITE,init_per_suite}},
+     {?eh,tc_done,{groups_12_SUITE,init_per_suite,'_'}},
+     {?eh,tc_start,{groups_12_SUITE,end_per_suite}},
+     {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+
+     {?eh,tc_start,{groups_11_SUITE,init_per_suite}},
+     {?eh,tc_done,{groups_11_SUITE,init_per_suite,'_'}},
+     {?eh,tc_start,{groups_11_SUITE,end_per_suite}},
+     {?eh,tc_done,{groups_11_SUITE,end_per_suite,'_'}},
+
+     {?eh,tc_start,{groups_22_SUITE,init_per_suite}},
+     {?eh,tc_done,{groups_22_SUITE,init_per_suite,'_'}},
+     {?eh,tc_start,{groups_22_SUITE,end_per_suite}},
+     {?eh,tc_done,{groups_22_SUITE,end_per_suite,'_'}},
+     {?eh,stop_logging,[]}];
+test_events(order_of_groups_in_multiple_dirs) ->
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     
+     {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}},
+     {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}},
+
+     {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}},
+     {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}},
+
+     {?eh,tc_start, {groups_22_SUITE,{init_per_group,test_group_1a,'_'}}},
+     {?eh,tc_done, {groups_22_SUITE,{end_per_group,test_group_1a,'_'},'_'}},
+
+     {?eh,stop_logging,[]}];
+test_events(order_of_groups_in_multiple_suites) ->
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     
+     {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1a,'_'}}},
+     {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1a,'_'},'_'}},
+
+     {?eh,tc_start, {groups_12_SUITE,{init_per_group,test_group_1b,'_'}}},
+     {?eh,tc_done, {groups_12_SUITE,{end_per_group,test_group_1b,'_'},'_'}},
+
+     {?eh,tc_start, {groups_11_SUITE,{init_per_group,test_group_1a,'_'}}},
+     {?eh,tc_done, {groups_11_SUITE,{end_per_group,test_group_1a,'_'},'_'}},
+
+     {?eh,stop_logging,[]}];
+
+test_events(order_of_tests_in_multiple_suites_with_skip) ->
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,tc_start,{groups_12_SUITE,testcase_1a}},
+     {?eh,tc_done,{groups_12_SUITE,testcase_1a,'_'}},
+     {?eh,tc_user_skip,{groups_12_SUITE,testcase_1b,'_'}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_1}},
+     {?eh,tc_done,{groups_11_SUITE,testcase_1,ok}},
+     {?eh,tc_start,{groups_11_SUITE,testcase_2}},
+     {?eh,tc_done,{groups_11_SUITE,testcase_2,ok}},
+     {?eh,stop_logging,[]}
+    ];
+
+test_events(all_plus_one_tc_no_merge_tests) ->
+
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,tc_start,{groups_12_SUITE,init_per_suite}},
+     {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+     {?eh,tc_start,{groups_12_SUITE,init_per_suite}},
+     {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+     {?eh,stop_logging,[]}
+    ];
+
+test_events(all_plus_one_tc) ->
+
+    [{?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,tc_start,{groups_12_SUITE,init_per_suite}},
+     {?eh,tc_done,{groups_12_SUITE,end_per_suite,'_'}},
+     {negative,{?eh,tc_start,{groups_12_SUITE,init_per_suite}},
+      {?eh,stop_logging,[]}}
+    ];
+
 test_events(_) ->
     [
     ].