Merge remote branch 'origin/peppe/common_test/change_skip_behaviour' into maint

* origin/peppe/common_test/change_skip_behaviour:
  Modify the auto_skip report for group config funcs to include group name
  Fix problems with info functions and add more tests
  Add test cases for new and modified functionality
  Fix problem with suites and groups skipped from test specification
  Change report tag for failed init_per_testcase from skipped to auto_skipped
  Add tests for skipping parallel groups
  Correct tests cases that fail because of modified events
  Correct various bugs related to auto_skip and groups
  Change status from skip to auto_skip for config func that fails due to require
  Fix problem with handling Config and FW reports correctly
  Change skip vs auto_skip behaviour

OTP-11305

1 files changed, 356 insertions, 77 deletions

diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl
index b8be55f43a..b0a6c839a2 100644
--- a/lib/common_test/test/ct_skip_SUITE.erl
+++ b/lib/common_test/test/ct_skip_SUITE.erl
@@ -59,7 +59,7 @@ end_per_testcase(TestCase, Config) ->
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
 all() -> 
-    [auto_skip, user_skip].
+    [auto_skip, user_skip, testspec_skip].
 
 groups() -> 
     [].
@@ -91,7 +91,8 @@ auto_skip(Config) when is_list(Config) ->
 	      Join(DataDir, "auto_skip_8_SUITE"),
 	      Join(DataDir, "auto_skip_9_SUITE"),
 	      Join(DataDir, "auto_skip_10_SUITE"),
-	      Join(DataDir, "auto_skip_11_SUITE")
+	      Join(DataDir, "auto_skip_11_SUITE"),
+	      Join(DataDir, "auto_skip_12_SUITE")
 	     ],
 
     {Opts,ERPid} = setup({suite,Suites}, Config),
@@ -116,7 +117,8 @@ user_skip(Config) when is_list(Config) ->
 	      Join(DataDir, "user_skip_2_SUITE"),	      
 	      Join(DataDir, "user_skip_3_SUITE"),
 	      Join(DataDir, "user_skip_4_SUITE"),
-	      Join(DataDir, "user_skip_5_SUITE")],
+	      Join(DataDir, "user_skip_5_SUITE"),
+	      Join(DataDir, "user_skip_6_SUITE")],
 
     {Opts,ERPid} = setup({suite,Suites}, Config),
     ok = ct_test_support:run(Opts, Config),
@@ -131,8 +133,56 @@ user_skip(Config) when is_list(Config) ->
     ok = ct_test_support:verify_events(TestEvents, Events, Config).
 
 %%%-----------------------------------------------------------------
+%%% 
+testspec_skip(Config) when is_list(Config) ->
+    TestDir = filename:join(?config(data_dir, Config),
+			    filename:join("skip", "test")),
+    TestSpec1 = [{suites, TestDir, user_skip_7_SUITE},
+		 {skip_cases, TestDir, user_skip_7_SUITE, [tc1,tc3], "SKIPPED"}],
+
+    TestSpec2 = [{suites, TestDir, user_skip_7_SUITE},
+		 {skip_groups, TestDir, user_skip_7_SUITE, ptop1, "SKIPPED"}],
+
+    TestSpec3 = [{suites, TestDir, user_skip_7_SUITE},
+		 {skip_groups, TestDir, user_skip_7_SUITE, psub1, "SKIPPED"}],
+
+    TestSpec4 = [{suites, TestDir, user_skip_7_SUITE},
+		 {skip_suites, TestDir, user_skip_7_SUITE, "SKIPPED"}],
+
+    TestSpec5 = [{groups, TestDir, user_skip_6_SUITE, ptop1},
+		 {skip_groups, TestDir, user_skip_6_SUITE, psub1, "SKIPPED"}],
+
+    {Opts,ERPid} = setup_testspec([{ts1,TestSpec1},
+				   {ts2,TestSpec2},
+				   {ts3,TestSpec3},
+				   {ts4,TestSpec4},
+				   {ts5,TestSpec5}], Config),
+
+    ok = ct_test_support:run(Opts, Config),
+
+    Events = ct_test_support:get_events(ERPid, Config),
+
+    ct_test_support:log_events(testspec_skip, 
+			       reformat(Events, ?eh),
+			       ?config(priv_dir, Config),
+			       Opts),
+
+    TestEvents = events_to_check(testspec_skip),
+    ok = ct_test_support:verify_events(TestEvents, Events, Config).
+
+%%%-----------------------------------------------------------------
 %%% HELP FUNCTIONS
 %%%-----------------------------------------------------------------
+setup_testspec(TestSpecs, Config) ->
+    SpecFiles =
+	[begin SpecFile = filename:join(?config(priv_dir, Config),
+					atom_to_list(SpecName)++".spec"),
+	       {ok,Dev} = file:open(SpecFile, [write]),
+	       [io:format(Dev, "~p.~n", [Term]) || Term <- TestSpec],
+	       file:close(Dev),
+	       SpecFile
+	 end || {SpecName,TestSpec} <- TestSpecs],
+    setup({spec,SpecFiles}, Config).
 
 setup(Test, Config) ->
     Opts0 = ct_test_support:get_opts(Config),
@@ -163,7 +213,7 @@ test_events(auto_skip) ->
     [
      {?eh,start_logging,{'DEF','RUNDIR'}},
      {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
-     {?eh,start_info,{11,11,34}},
+     {?eh,start_info,{12,12,43}},
 
      {?eh,tc_start,{auto_skip_1_SUITE,init_per_suite}},
      {?eh,tc_done,
@@ -196,8 +246,8 @@ test_events(auto_skip) ->
      {?eh,tc_start,{auto_skip_3_SUITE,tc1}},
      {?eh,tc_done,
       {auto_skip_3_SUITE,tc1,
-       {skipped,{failed,{auto_skip_3_SUITE,init_per_testcase,
-			 {{init_per_testcase,tc1,failed},'_'}}}}}},
+       {auto_skipped,{failed,{auto_skip_3_SUITE,init_per_testcase,
+			      {'init_per_testcase for tc1 failed','_'}}}}}},
      {?eh,test_stats,{0,0,{0,4}}},
      {?eh,tc_start,{auto_skip_3_SUITE,tc2}},
      {?eh,tc_done,{auto_skip_3_SUITE,tc2,ok}},
@@ -209,7 +259,7 @@ test_events(auto_skip) ->
      {?eh,tc_done,{auto_skip_4_SUITE,init_per_suite,ok}},
      {?eh,tc_start,{auto_skip_4_SUITE,tc1}},
      {?eh,tc_done,{auto_skip_4_SUITE,tc1,
-		   {skipped,{failed,{auto_skip_4_SUITE,init_per_testcase,
+		   {auto_skipped,{failed,{auto_skip_4_SUITE,init_per_testcase,
 				     {timetrap_timeout,1000}}}}}},
      {?eh,test_stats,{1,0,{0,5}}},     
      {?eh,tc_start,{auto_skip_4_SUITE,tc2}},
@@ -232,8 +282,9 @@ test_events(auto_skip) ->
 				       {'EXIT',{group,g1,failed}}}}}},
       {?eh,test_stats,{2,0,{0,7}}},
       {?eh,tc_auto_skip,
-       {auto_skip_5_SUITE,end_per_group,{failed,{auto_skip_5_SUITE,init_per_group,
-						 {'EXIT',{group,g1,failed}}}}}}],
+       {auto_skip_5_SUITE,{end_per_group,g1},
+	{failed,{auto_skip_5_SUITE,init_per_group,
+		 {'EXIT',{group,g1,failed}}}}}}],
 
      {?eh,tc_start,{auto_skip_5_SUITE,end_per_suite}},
      {?eh,tc_done,{auto_skip_5_SUITE,end_per_suite,ok}},
@@ -260,8 +311,9 @@ test_events(auto_skip) ->
 				       {'EXIT',{group,g1,failed}}}}}},
       {?eh,test_stats,{2,0,{0,11}}},
       {?eh,tc_auto_skip,
-       {auto_skip_6_SUITE,end_per_group,{failed,{auto_skip_6_SUITE,init_per_group,
-						 {'EXIT',{group,g1,failed}}}}}}],
+       {auto_skip_6_SUITE,{end_per_group,g1},
+	{failed,{auto_skip_6_SUITE,init_per_group,
+		 {'EXIT',{group,g1,failed}}}}}}],
 
      [{?eh,tc_start,{auto_skip_6_SUITE,{init_per_group,g3,[]}}},
       {?eh,tc_done,{auto_skip_6_SUITE,{init_per_group,g3,[]},ok}},
@@ -280,8 +332,9 @@ test_events(auto_skip) ->
 					{'EXIT',{group,g4,failed}}}}}},
        {?eh,test_stats,{3,0,{0,13}}},
        {?eh,tc_auto_skip,
-	{auto_skip_6_SUITE,end_per_group,{failed,{auto_skip_6_SUITE,init_per_group,
-						  {'EXIT',{group,g4,failed}}}}}}],
+	{auto_skip_6_SUITE,{end_per_group,g4},
+	 {failed,{auto_skip_6_SUITE,init_per_group,
+		  {'EXIT',{group,g4,failed}}}}}}],
       {?eh,tc_start,{auto_skip_6_SUITE,tc2}},
       {?eh,tc_done,{auto_skip_6_SUITE,tc2,ok}},
       {?eh,test_stats,{4,0,{0,13}}},
@@ -320,7 +373,7 @@ test_events(auto_skip) ->
      {?eh,test_stats,{5,0,{0,17}}},
      {?eh,tc_start,{auto_skip_9_SUITE,tc2}},
      {?eh,tc_done,{auto_skip_9_SUITE,tc2,
-		   {skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}},
+		   {auto_skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}},
      {?eh,test_stats,{5,0,{0,18}}},
 
      [{?eh,tc_start,{auto_skip_9_SUITE,{init_per_group,g1,[]}}},
@@ -343,7 +396,7 @@ test_events(auto_skip) ->
       {?eh,tc_done,{auto_skip_9_SUITE,{init_per_group,g3,[]},ok}},
       {?eh,tc_start,{auto_skip_9_SUITE,tc5}},
       {?eh,tc_done,{auto_skip_9_SUITE,tc5,
-		    {skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}},
+		    {auto_skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,bad_return}}}}},
       {?eh,test_stats,{7,0,{0,19}}},
       {?eh,tc_start,{auto_skip_9_SUITE,{end_per_group,g3,[]}}},
       {?eh,tc_done,{auto_skip_9_SUITE,{end_per_group,g3,[]},ok}}],
@@ -363,7 +416,7 @@ test_events(auto_skip) ->
 	 {?eh,tc_start,{auto_skip_9_SUITE,tc8}},
 	 {?eh,tc_done,
 	  {auto_skip_9_SUITE,tc8,
-	   {skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,
+	   {auto_skipped,{failed,{auto_skip_9_SUITE,init_per_testcase,
 			     {{badmatch,undefined},'_'}}}}}},
 	 {?eh,tc_start,
 	  {auto_skip_9_SUITE,{end_per_group,g5,[parallel]}}},
@@ -383,26 +436,26 @@ test_events(auto_skip) ->
 
      {?eh,tc_start,{auto_skip_10_SUITE,init_per_suite}},
      {?eh,tc_done,{auto_skip_10_SUITE,init_per_suite,
-		   {skipped,
-		    {require_failed_in_suite0,
-		     {not_available,undefined_config_variable}}}}},
-     {?eh,tc_auto_skip,
-      {auto_skip_10_SUITE,tc1,
-       {require_failed_in_suite0,{not_available,undefined_config_variable}}}},
+		   {auto_skipped,{require_failed_in_suite0,
+				  {not_available,undefined_config_variable}}}}},
+     {?eh,tc_auto_skip,{auto_skip_10_SUITE,tc1,
+			{require_failed_in_suite0,
+			 {not_available,undefined_config_variable}}}},
      {?eh,test_stats,{9,0,{0,21}}},
-     {?eh,tc_auto_skip,
-      {auto_skip_10_SUITE,tc2,
-       {require_failed_in_suite0,{not_available,undefined_config_variable}}}},
+     {?eh,tc_auto_skip,{auto_skip_10_SUITE,tc2,
+			{require_failed_in_suite0,
+			 {not_available,undefined_config_variable}}}},
      {?eh,test_stats,{9,0,{0,22}}},
-     {?eh,tc_auto_skip,
-      {auto_skip_10_SUITE,end_per_suite,
-       {require_failed_in_suite0,{not_available,undefined_config_variable}}}},
+     {?eh,tc_auto_skip,{auto_skip_10_SUITE,end_per_suite,
+			{require_failed_in_suite0,
+			 {not_available,undefined_config_variable}}}},
 
      {?eh,tc_start,{auto_skip_11_SUITE,init_per_suite}},
      {?eh,tc_done,{auto_skip_11_SUITE,init_per_suite,ok}},
      {?eh,tc_start,{auto_skip_11_SUITE,tc1}},
      {?eh,tc_done,{auto_skip_11_SUITE,tc1,
-		   {skipped,{require_failed,{not_available,undefined_config_variable}}}}},
+		   {auto_skipped,{require_failed,
+				  {not_available,undefined_config_variable}}}}},
      {?eh,test_stats,{9,0,{0,23}}},
      {?eh,tc_start,{auto_skip_11_SUITE,tc2}},
      {?eh,tc_done,{auto_skip_11_SUITE,tc2,ok}},
@@ -421,7 +474,8 @@ test_events(auto_skip) ->
 	 {?eh,tc_start,{auto_skip_11_SUITE,tc3}},
 	 {?eh,tc_done,
 	  {auto_skip_11_SUITE,tc3,
-	   {skipped,{require_failed,{not_available,undefined_config_variable}}}}},
+	   {auto_skipped,{require_failed,
+			  {not_available,undefined_config_variable}}}}},
 	 {?eh,test_stats,{10,0,{0,24}}},
 	 {?eh,tc_start,
 	  {auto_skip_11_SUITE,{end_per_group,g2,[parallel]}}},
@@ -434,6 +488,75 @@ test_events(auto_skip) ->
 
      {?eh,tc_start,{auto_skip_11_SUITE,end_per_suite}},
      {?eh,tc_done,{auto_skip_11_SUITE,end_per_suite,ok}},
+     
+     {?eh,tc_start,{auto_skip_12_SUITE,init_per_suite}},
+     {?eh,tc_done,{auto_skip_12_SUITE,init_per_suite,ok}},
+
+     [{?eh,tc_start,{ct_framework,{init_per_group,g1,
+				   [{suite,auto_skip_12_SUITE}]}}},
+      {?eh,tc_done,{ct_framework,{init_per_group,g1,
+				  [{suite,auto_skip_12_SUITE}]},
+		    {auto_skipped,
+		     {require_failed,{not_available,unknown_variable_g1}}}}},
+      {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc1,
+			 {require_failed,{not_available,unknown_variable_g1}}}},
+      {?eh,test_stats,{10,0,{0,25}}},
+      {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc2,
+			 {require_failed,{not_available,unknown_variable_g1}}}},
+      {?eh,test_stats,{10,0,{0,26}}},
+      {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3,
+			 {require_failed,{not_available,unknown_variable_g1}}}},
+      {?eh,test_stats,{10,0,{0,27}}},
+      {?eh,tc_auto_skip,{ct_framework,{end_per_group,g1},
+			 {require_failed,{not_available,unknown_variable_g1}}}}],
+     
+     [{?eh,tc_start,{ct_framework,{init_per_group,g1,
+				   [{suite,auto_skip_12_SUITE}]}}},
+      {?eh,tc_done,{ct_framework,{init_per_group,g1,
+				  [{suite,auto_skip_12_SUITE}]},
+		    {auto_skipped,
+		     {require_failed,{not_available,unknown_variable_g1}}}}},
+      {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc1,
+			 {require_failed,{not_available,unknown_variable_g1}}}},
+      {?eh,test_stats,{10,0,{0,28}}},
+      {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc2,
+			 {require_failed,{not_available,unknown_variable_g1}}}},
+      {?eh,test_stats,{10,0,{0,29}}},
+      {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3,
+			 {require_failed,{not_available,unknown_variable_g1}}}},
+      {?eh,test_stats,{10,0,{0,30}}},
+      {?eh,tc_auto_skip,{ct_framework,{end_per_group,g1},
+			 {require_failed,{not_available,unknown_variable_g1}}}}],
+
+     [{?eh,tc_start,{ct_framework,{init_per_group,g3,
+				   [{suite,auto_skip_12_SUITE}]}}},
+      {?eh,tc_done,{ct_framework,{init_per_group,g3,
+				  [{suite,auto_skip_12_SUITE}]},ok}},
+      {?eh,tc_start,{auto_skip_12_SUITE,tc1}},
+      {?eh,tc_done,{auto_skip_12_SUITE,tc1,ok}},
+      {?eh,test_stats,{11,0,{0,30}}},
+      {?eh,tc_start,{auto_skip_12_SUITE,tc2}},
+      {?eh,tc_done,{auto_skip_12_SUITE,tc2,ok}},
+      {?eh,test_stats,{12,0,{0,30}}},
+      [{?eh,tc_start,{ct_framework,{init_per_group,g4,
+				    [{suite,auto_skip_12_SUITE}]}}},
+       {?eh,tc_done,{ct_framework,{init_per_group,g4,
+				   [{suite,auto_skip_12_SUITE}]},
+		     {auto_skipped,
+		      {require_failed,{not_available,unknown_variable_g4}}}}},
+       {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3,
+			  {require_failed,{not_available,unknown_variable_g4}}}},
+       {?eh,test_stats,{12,0,{0,31}}},
+       {?eh,tc_auto_skip,{ct_framework,{end_per_group,g4},
+			  {require_failed,{not_available,unknown_variable_g4}}}}],
+
+      {?eh,tc_start,{ct_framework,{end_per_group,g3,
+				   [{suite,auto_skip_12_SUITE}]}}},
+      {?eh,tc_done,{ct_framework,{end_per_group,g3,
+				  [{suite,auto_skip_12_SUITE}]},ok}}],
+
+     {?eh,tc_start,{auto_skip_12_SUITE,end_per_suite}},
+     {?eh,tc_done,{auto_skip_12_SUITE,end_per_suite,ok}},
 
      {?eh,test_done,{'DEF','STOP_TIME'}},
      {?eh,stop_logging,[]}
@@ -442,46 +565,46 @@ test_events(auto_skip) ->
 test_events(user_skip) ->
     [{?eh,start_logging,{'DEF','RUNDIR'}},
      {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
-     {?eh,start_info,{5,5,27}},
+     {?eh,start_info,{6,6,35}},
 
      {?eh,tc_start,{user_skip_1_SUITE,init_per_suite}},
      {?eh,tc_done,
       {user_skip_1_SUITE,init_per_suite,{skipped,"Whole suite skipped"}}},
-     {?eh,tc_auto_skip,
+     {?eh,tc_user_skip,
       {user_skip_1_SUITE,tc1,"Whole suite skipped"}},
-     {?eh,test_stats,{0,0,{0,1}}},
-     {?eh,tc_auto_skip,
+     {?eh,test_stats,{0,0,{1,0}}},
+     {?eh,tc_user_skip,
       {user_skip_1_SUITE,tc2,"Whole suite skipped"}},
-     {?eh,test_stats,{0,0,{0,2}}},
-     {?eh,tc_auto_skip,
+     {?eh,test_stats,{0,0,{2,0}}},
+     {?eh,tc_user_skip,
       {user_skip_1_SUITE,tc3,"Whole suite skipped"}},
-     {?eh,test_stats,{0,0,{0,3}}},
-     {?eh,tc_auto_skip,
+     {?eh,test_stats,{0,0,{3,0}}},
+     {?eh,tc_user_skip,
       {user_skip_1_SUITE,tc4,"Whole suite skipped"}},
-     {?eh,test_stats,{0,0,{0,4}}},
-     {?eh,tc_auto_skip,
+     {?eh,test_stats,{0,0,{4,0}}},
+     {?eh,tc_user_skip,
       {user_skip_1_SUITE,end_per_suite,"Whole suite skipped"}},
 
      {?eh,tc_start,{user_skip_2_SUITE,init_per_suite}},
      {?eh,tc_done,{user_skip_2_SUITE,init_per_suite,ok}},
      {?eh,tc_start,{user_skip_2_SUITE,tc1}},
      {?eh,tc_done,{user_skip_2_SUITE,tc1,{skipped,{tc1,skipped}}}},
-     {?eh,test_stats,{0,0,{1,4}}},
+     {?eh,test_stats,{0,0,{5,0}}},
 
      [{?eh,tc_start,{user_skip_2_SUITE,{init_per_group,g1,[]}}},
       {?eh,tc_done,{user_skip_2_SUITE,{init_per_group,g1,[]},ok}},
       {?eh,tc_start,{user_skip_2_SUITE,tc2}},
       {?eh,tc_done,{user_skip_2_SUITE,tc2,ok}},
-      {?eh,test_stats,{1,0,{1,4}}},
+      {?eh,test_stats,{1,0,{5,0}}},
       {?eh,tc_start,{user_skip_2_SUITE,tc3}},
       {?eh,tc_done,{user_skip_2_SUITE,tc3,{skipped,{tc3,skipped}}}},
-      {?eh,test_stats,{1,0,{2,4}}},
+      {?eh,test_stats,{1,0,{6,0}}},
       {?eh,tc_start,{user_skip_2_SUITE,{end_per_group,g1,[]}}},
       {?eh,tc_done,{user_skip_2_SUITE,{end_per_group,g1,[]},ok}}],
 
      {?eh,tc_start,{user_skip_2_SUITE,tc4}},
      {?eh,tc_done,{user_skip_2_SUITE,tc4,ok}},
-     {?eh,test_stats,{2,0,{2,4}}},
+     {?eh,test_stats,{2,0,{6,0}}},
      {?eh,tc_start,{user_skip_2_SUITE,end_per_suite}},
      {?eh,tc_done,{user_skip_2_SUITE,end_per_suite,ok}},
 
@@ -489,16 +612,16 @@ test_events(user_skip) ->
      {?eh,tc_done,{user_skip_3_SUITE,init_per_suite,ok}},
      {?eh,tc_start,{user_skip_3_SUITE,tc1}},
      {?eh,tc_done,{user_skip_3_SUITE,tc1,{skipped,"Test case skipped"}}},
-     {?eh,test_stats,{2,0,{3,4}}},
+     {?eh,test_stats,{2,0,{7,0}}},
 
      [{?eh,tc_start,{user_skip_3_SUITE,{init_per_group,g1,[]}}},
       {?eh,tc_done,{user_skip_3_SUITE,{init_per_group,g1,[]},ok}},
       {?eh,tc_start,{user_skip_3_SUITE,tc2}},
       {?eh,tc_done,{user_skip_3_SUITE,tc2,ok}},
-      {?eh,test_stats,{3,0,{3,4}}},
+      {?eh,test_stats,{3,0,{7,0}}},
       {?eh,tc_start,{user_skip_3_SUITE,tc3}},
       {?eh,tc_done,{user_skip_3_SUITE,tc3,{skipped,"Test case skipped"}}},
-      {?eh,test_stats,{3,0,{4,4}}},
+      {?eh,test_stats,{3,0,{8,0}}},
       {?eh,tc_start,{user_skip_3_SUITE,{end_per_group,g1,[]}}},
       {?eh,tc_done,{user_skip_3_SUITE,{end_per_group,g1,[]},ok}}],
 
@@ -506,7 +629,7 @@ test_events(user_skip) ->
      {?eh,tc_done,{user_skip_3_SUITE,tc4,
 		   {skipped,{proc_info,{{current_function,{user_skip_3_SUITE,tc4,1}},
 					{initial_call,{erlang,apply,2}}}}}}},
-     {?eh,test_stats,{3,0,{5,4}}},
+     {?eh,test_stats,{3,0,{9,0}}},
      {?eh,tc_start,{user_skip_3_SUITE,end_per_suite}},
      {?eh,tc_done,{user_skip_3_SUITE,end_per_suite,ok}},
 
@@ -515,63 +638,219 @@ test_events(user_skip) ->
 
      [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g1,[]}}},
       {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g1,[]},{skipped,"Group skipped"}}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,tc1,"Group skipped"}},   
-      {?eh,test_stats,{3,0,{5,5}}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,tc2,"Group skipped"}},
-      {?eh,test_stats,{3,0,{5,6}}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
+      {?eh,tc_user_skip,{user_skip_4_SUITE,tc1,"Group skipped"}},   
+      {?eh,test_stats,{3,0,{10,0}}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,tc2,"Group skipped"}},
+      {?eh,test_stats,{3,0,{11,0}}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g1},"Group skipped"}}],
 
      [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g2,[]}}},
       {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g2,[]},ok}},
       {?eh,tc_start,{user_skip_4_SUITE,tc3}},
       {?eh,tc_done,{user_skip_4_SUITE,tc3,ok}},
-      {?eh,test_stats,{4,0,{5,6}}},
+      {?eh,test_stats,{4,0,{11,0}}},
       {?eh,tc_start,{user_skip_4_SUITE,tc4}},
       {?eh,tc_done,{user_skip_4_SUITE,tc4,ok}},
-      {?eh,test_stats,{5,0,{5,6}}},
+      {?eh,test_stats,{5,0,{11,0}}},
       {?eh,tc_start,{user_skip_4_SUITE,{end_per_group,g2,[]}}},
       {?eh,tc_done,{user_skip_4_SUITE,{end_per_group,g2,[]},ok}}],
 
      [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g3,[]}}},
       {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g3,[]},{skipped,"Group skipped"}}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,tc5,"Group skipped"}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,tc6,"Group skipped"}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,tc7,"Group skipped"}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,tc8,"Group skipped"}},
-      {?eh,test_stats,{5,0,{5,10}}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
+      {?eh,tc_user_skip,{user_skip_4_SUITE,tc5,"Group skipped"}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,tc6,"Group skipped"}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,tc7,"Group skipped"}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,tc8,"Group skipped"}},
+      {?eh,test_stats,{5,0,{15,0}}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g3},"Group skipped"}}],
 
      [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g5,[]}}},
       {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g5,[]},ok}},
       {?eh,tc_start,{user_skip_4_SUITE,tc9}},
       {?eh,tc_done,{user_skip_4_SUITE,tc9,ok}},
-      {?eh,test_stats,{6,0,{5,10}}},
+      {?eh,test_stats,{6,0,{15,0}}},
       [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g6,[]}}},
        {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g6,[]},{skipped,"Group skipped"}}},
-       {?eh,tc_auto_skip,{user_skip_4_SUITE,tc10,"Group skipped"}},
-       {?eh,test_stats,{6,0,{5,11}}},
-       {?eh,tc_auto_skip,{user_skip_4_SUITE,tc11,"Group skipped"}},
-       {?eh,test_stats,{6,0,{5,12}}},
-       {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
+       {?eh,tc_user_skip,{user_skip_4_SUITE,tc10,"Group skipped"}},
+       {?eh,test_stats,{6,0,{16,0}}},
+       {?eh,tc_user_skip,{user_skip_4_SUITE,tc11,"Group skipped"}},
+       {?eh,test_stats,{6,0,{17,0}}},
+       {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g6},"Group skipped"}}],
        {?eh,tc_start,{user_skip_4_SUITE,{end_per_group,g5,[]}}},
        {?eh,tc_done,{user_skip_4_SUITE,{end_per_group,g5,[]},ok}}],
 
      {?eh,tc_start,{user_skip_4_SUITE,end_per_suite}},
      {?eh,tc_done,{user_skip_4_SUITE,end_per_suite,ok}},
 
-         {ct_test_support_eh,tc_start,{user_skip_5_SUITE,init_per_suite}},
+     {?eh,tc_start,{user_skip_5_SUITE,init_per_suite}},
      {?eh,tc_done,{user_skip_5_SUITE,init_per_suite,
 		   {skipped,{bad,'Whole suite skipped'}}}},
-     {?eh,tc_auto_skip,{user_skip_5_SUITE,tc1,{bad,'Whole suite skipped'}}},
-     {?eh,test_stats,{6,0,{5,13}}},
-     {?eh,tc_auto_skip,{user_skip_5_SUITE,tc2,{bad,'Whole suite skipped'}}},
-     {?eh,test_stats,{6,0,{5,14}}},
-     {?eh,tc_auto_skip,{user_skip_5_SUITE,tc3,{bad,'Whole suite skipped'}}},
-     {?eh,test_stats,{6,0,{5,15}}},
-     {?eh,tc_auto_skip,{user_skip_5_SUITE,tc4,{bad,'Whole suite skipped'}}},
-     {?eh,test_stats,{6,0,{5,16}}},
-     {?eh,tc_auto_skip,{user_skip_5_SUITE,end_per_suite,{bad,'Whole suite skipped'}}},
+     {?eh,tc_user_skip,{user_skip_5_SUITE,tc1,{bad,'Whole suite skipped'}}},
+     {?eh,test_stats,{6,0,{18,0}}},
+     {?eh,tc_user_skip,{user_skip_5_SUITE,tc2,{bad,'Whole suite skipped'}}},
+     {?eh,test_stats,{6,0,{19,0}}},
+     {?eh,tc_user_skip,{user_skip_5_SUITE,tc3,{bad,'Whole suite skipped'}}},
+     {?eh,test_stats,{6,0,{20,0}}},
+     {?eh,tc_user_skip,{user_skip_5_SUITE,tc4,{bad,'Whole suite skipped'}}},
+     {?eh,test_stats,{6,0,{21,0}}},
+     {?eh,tc_user_skip,{user_skip_5_SUITE,end_per_suite,{bad,'Whole suite skipped'}}},
+     
+     {parallel,
+      [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,ptop1,[parallel]}}},
+       {?eh,tc_done,{user_skip_6_SUITE,
+		     {init_per_group,ptop1,[parallel]},
+		     {skipped,"Top group skipped"}}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,tc1,"Top group skipped"}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"Top group skipped"}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"Top group skipped"}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,tc2,"Top group skipped"}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,{end_per_group,ptop1},
+			  "Top group skipped"}}]},
+      
+     {parallel,
+      [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,ptop2,[parallel]}}},
+       {?eh,tc_done,{user_skip_6_SUITE,{init_per_group,ptop2,[parallel]},ok}},
+       {?eh,tc_start,{user_skip_6_SUITE,tc1}},
+       {?eh,tc_done,{user_skip_6_SUITE,tc1,ok}},
+       
+       {parallel,
+	[{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,psub2,[parallel]}}},
+	 {?eh,tc_done,{user_skip_6_SUITE,
+		       {init_per_group,psub2,[parallel]},
+		       {skipped,"Sub group skipped"}}},
+	 {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"Sub group skipped"}},
+	 {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"Sub group skipped"}},
+	 {?eh,tc_user_skip,{user_skip_6_SUITE,{end_per_group,psub2},
+			    "Sub group skipped"}}]},
+       
+       {?eh,tc_start,{user_skip_6_SUITE,tc2}},
+       {?eh,tc_done,{user_skip_6_SUITE,tc2,ok}},
+       {?eh,test_stats,{8,0,{27,0}}},
+       {?eh,tc_start,{user_skip_6_SUITE,{end_per_group,ptop2,[parallel]}}},
+       {?eh,tc_done,{user_skip_6_SUITE,{end_per_group,ptop2,[parallel]},ok}}]},
+       
+       {?eh,test_done,{'DEF','STOP_TIME'}},
+       {?eh,stop_logging,[]}
+    ];
+
+test_events(testspec_skip) ->
+    [
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,start_info,{1,1,4}},
+     {?eh,tc_start,{ct_framework,init_per_suite}},
+     {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+     {parallel,
+      [{?eh,tc_start,
+	{user_skip_7_SUITE,{init_per_group,ptop1,[parallel]}}},
+       {?eh,tc_done,
+	{user_skip_7_SUITE,{init_per_group,ptop1,[parallel]},ok}},
+       {?eh,tc_user_skip,{user_skip_7_SUITE,tc1,"SKIPPED"}},
+       {?eh,test_stats,{0,0,{1,0}}},
+       {parallel,
+	[{?eh,tc_start,
+          {user_skip_7_SUITE,{init_per_group,psub1,[parallel]}}},
+	 {?eh,tc_done,
+          {user_skip_7_SUITE,{init_per_group,psub1,[parallel]},ok}},
+	 {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
+	 {?eh,tc_start,{user_skip_7_SUITE,tc4}},
+	 {?eh,tc_done,{user_skip_7_SUITE,tc4,ok}},
+	 {?eh,test_stats,{1,0,{2,0}}},
+	 {?eh,tc_start,
+	  {user_skip_7_SUITE,{end_per_group,psub1,[parallel]}}},
+	 {?eh,tc_done,
+          {user_skip_7_SUITE,{end_per_group,psub1,[parallel]},ok}}]},
+       {?eh,tc_start,{user_skip_7_SUITE,tc2}},
+       {?eh,tc_done,{user_skip_7_SUITE,tc2,ok}},
+       {?eh,test_stats,{2,0,{2,0}}},
+       {?eh,tc_start,
+	{user_skip_7_SUITE,{end_per_group,ptop1,[parallel]}}},
+       {?eh,tc_done,
+	{user_skip_7_SUITE,{end_per_group,ptop1,[parallel]},ok}}]},
+     {?eh,tc_start,{ct_framework,end_per_suite}},
+     {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]},
+     
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,start_info,{1,1,4}},
+     {?eh,tc_start,{ct_framework,init_per_suite}},
+     {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+     {?eh,tc_user_skip,{user_skip_7_SUITE,{init_per_group,ptop1},"SKIPPED"}},
+     {?eh,tc_user_skip,{user_skip_7_SUITE,tc1,"SKIPPED"}},
+     {?eh,test_stats,{0,0,{1,0}}},
+     {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
+     {?eh,test_stats,{0,0,{2,0}}},
+     {?eh,tc_user_skip,{user_skip_7_SUITE,tc4,"SKIPPED"}},
+     {?eh,test_stats,{0,0,{3,0}}},
+     {?eh,tc_user_skip,{user_skip_7_SUITE,tc2,"SKIPPED"}},
+     {?eh,test_stats,{0,0,{4,0}}},
+     {?eh,tc_user_skip,{user_skip_7_SUITE,{end_per_group,ptop1},"SKIPPED"}},
+     {?eh,tc_start,{ct_framework,end_per_suite}},
+     {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]},
      
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,start_info,{1,1,4}},
+     {?eh,tc_start,{ct_framework,init_per_suite}},
+     {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+     {parallel,
+      [{?eh,tc_start,
+	{user_skip_7_SUITE,{init_per_group,ptop1,[parallel]}}},
+       {?eh,tc_done,
+	{user_skip_7_SUITE,{init_per_group,ptop1,[parallel]},ok}},
+       {?eh,tc_user_skip,
+	{user_skip_7_SUITE,{init_per_group,psub1},"SKIPPED"}},
+       {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
+       {?eh,tc_user_skip,{user_skip_7_SUITE,tc4,"SKIPPED"}},
+       {?eh,test_stats,{0,0,{2,0}}},
+       {?eh,tc_user_skip,{user_skip_7_SUITE,{end_per_group,psub1},"SKIPPED"}},
+       {?eh,tc_start,{user_skip_7_SUITE,tc1}},
+       {?eh,tc_done,{user_skip_7_SUITE,tc1,ok}},
+       {?eh,tc_start,{user_skip_7_SUITE,tc2}},
+       {?eh,tc_done,{user_skip_7_SUITE,tc2,ok}},
+       {?eh,test_stats,{2,0,{2,0}}},
+       {?eh,tc_start,{user_skip_7_SUITE,{end_per_group,ptop1,[parallel]}}},
+       {?eh,tc_done,{user_skip_7_SUITE,{end_per_group,ptop1,[parallel]},ok}}]},
+     {?eh,tc_start,{ct_framework,end_per_suite}},
+     {?eh,tc_done,{ct_framework,end_per_suite,ok}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]},
+
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,start_info,{1,1,0}},
+     {?eh,tc_user_skip,{user_skip_7_SUITE,all,"SKIPPED"}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]},
+
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,start_info,{1,1,4}},
+     {?eh,tc_start,{ct_framework,init_per_suite}},
+     {?eh,tc_done,{ct_framework,init_per_suite,ok}},
+     {parallel,
+      [{?eh,tc_start,{user_skip_6_SUITE,{init_per_group,ptop1,[parallel]}}},
+       {?eh,tc_done,{user_skip_6_SUITE,
+		     {init_per_group,ptop1,[parallel]},
+		     {skipped,"Top group skipped"}}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,tc1,"Top group skipped"}},
+       {?eh,test_stats,{0,0,{1,0}}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"SKIPPED"}},
+       {?eh,test_stats,{0,0,{2,0}}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"SKIPPED"}},
+       {?eh,test_stats,{0,0,{3,0}}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,tc2,"Top group skipped"}},
+       {?eh,test_stats,{0,0,{4,0}}},
+       {?eh,tc_user_skip,
+	{user_skip_6_SUITE,{end_per_group,ptop1},"Top group skipped"}}]},
+     {?eh,tc_start,{ct_framework,end_per_suite}},
+     {?eh,tc_done,{ct_framework,end_per_suite,ok}},
      {?eh,test_done,{'DEF','STOP_TIME'}},
      {?eh,stop_logging,[]}
     ].
+
+