Merge branch 'peppe/common_test/group_events'

* peppe/common_test/group_events:
  Update test suites and fix various remaining problems
  Update event protocol and CT Hooks API
  Introduce group name for skipped cases in events, hooks and overview log

OTP-11732

1 files changed, 59 insertions, 59 deletions

diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl
index b0a6c839a2..6fb803b928 100644
--- a/lib/common_test/test/ct_skip_SUITE.erl
+++ b/lib/common_test/test/ct_skip_SUITE.erl
@@ -153,10 +153,10 @@ testspec_skip(Config) when is_list(Config) ->
 		 {skip_groups, TestDir, user_skip_6_SUITE, psub1, "SKIPPED"}],
 
     {Opts,ERPid} = setup_testspec([{ts1,TestSpec1},
-				   {ts2,TestSpec2},
-				   {ts3,TestSpec3},
-				   {ts4,TestSpec4},
-				   {ts5,TestSpec5}], Config),
+    				   {ts2,TestSpec2},
+    				   {ts3,TestSpec3},
+    				   {ts4,TestSpec4},
+    				   {ts5,TestSpec5}], Config),
 
     ok = ct_test_support:run(Opts, Config),
 
@@ -234,8 +234,8 @@ test_events(auto_skip) ->
      {?eh,tc_done,
       {auto_skip_2_SUITE,init_per_suite,{failed,{error,init_per_suite_failed}}}},
      {?eh,tc_auto_skip,      
-      {auto_skip_2_SUITE,tc1,{failed,{auto_skip_2_SUITE,init_per_suite,
-				      {'EXIT',init_per_suite_failed}}}}},
+      {auto_skip_2_SUITE,{tc1,g1},{failed,{auto_skip_2_SUITE,init_per_suite,
+					   {'EXIT',init_per_suite_failed}}}}},
      {?eh,test_stats,{0,0,{0,3}}},
      {?eh,tc_auto_skip,
       {auto_skip_2_SUITE,end_per_suite,{failed,{auto_skip_2_SUITE,init_per_suite,
@@ -274,12 +274,12 @@ test_events(auto_skip) ->
       {?eh,tc_done,
        {auto_skip_5_SUITE,{init_per_group,g1,[]},{failed,{error,{group,g1,failed}}}}},
       {?eh,tc_auto_skip,
-       {auto_skip_5_SUITE,tc1,{failed,{auto_skip_5_SUITE,init_per_group,
-				       {'EXIT',{group,g1,failed}}}}}},
+       {auto_skip_5_SUITE,{tc1,g1},{failed,{auto_skip_5_SUITE,init_per_group,
+					    {'EXIT',{group,g1,failed}}}}}},
       {?eh,test_stats,{2,0,{0,6}}},    
       {?eh,tc_auto_skip,
-       {auto_skip_5_SUITE,tc2,{failed,{auto_skip_5_SUITE,init_per_group,
-				       {'EXIT',{group,g1,failed}}}}}},
+       {auto_skip_5_SUITE,{tc2,g1},{failed,{auto_skip_5_SUITE,init_per_group,
+					    {'EXIT',{group,g1,failed}}}}}},
       {?eh,test_stats,{2,0,{0,7}}},
       {?eh,tc_auto_skip,
        {auto_skip_5_SUITE,{end_per_group,g1},
@@ -295,20 +295,20 @@ test_events(auto_skip) ->
       {?eh,tc_done,
        {auto_skip_6_SUITE,{init_per_group,g1,[]},{failed,{error,{group,g1,failed}}}}},
       {?eh,tc_auto_skip,
-       {auto_skip_6_SUITE,tc1,{failed,{auto_skip_6_SUITE,init_per_group,
-				       {'EXIT',{group,g1,failed}}}}}},
+       {auto_skip_6_SUITE,{tc1,g1},{failed,{auto_skip_6_SUITE,init_per_group,
+					    {'EXIT',{group,g1,failed}}}}}},
       {?eh,test_stats,{2,0,{0,8}}},
       {?eh,tc_auto_skip,
-       {auto_skip_6_SUITE,tc3,{failed,{auto_skip_6_SUITE,init_per_group,
-				       {'EXIT',{group,g1,failed}}}}}},
+       {auto_skip_6_SUITE,{tc3,g2},{failed,{auto_skip_6_SUITE,init_per_group,
+					    {'EXIT',{group,g1,failed}}}}}},
       {?eh,test_stats,{2,0,{0,9}}},
       {?eh,tc_auto_skip,
-       {auto_skip_6_SUITE,tc4,{failed,{auto_skip_6_SUITE,init_per_group,
-				       {'EXIT',{group,g1,failed}}}}}},
+       {auto_skip_6_SUITE,{tc4,g2},{failed,{auto_skip_6_SUITE,init_per_group,
+					    {'EXIT',{group,g1,failed}}}}}},
       {?eh,test_stats,{2,0,{0,10}}},
       {?eh,tc_auto_skip,
-       {auto_skip_6_SUITE,tc2,{failed,{auto_skip_6_SUITE,init_per_group,
-				       {'EXIT',{group,g1,failed}}}}}},
+       {auto_skip_6_SUITE,{tc2,g1},{failed,{auto_skip_6_SUITE,init_per_group,
+					    {'EXIT',{group,g1,failed}}}}}},
       {?eh,test_stats,{2,0,{0,11}}},
       {?eh,tc_auto_skip,
        {auto_skip_6_SUITE,{end_per_group,g1},
@@ -324,12 +324,12 @@ test_events(auto_skip) ->
        {?eh,tc_done,{auto_skip_6_SUITE,{init_per_group,g4,[]},
 		     {failed,{error,{group,g4,failed}}}}},
        {?eh,tc_auto_skip,
-	{auto_skip_6_SUITE,tc3,{failed,{auto_skip_6_SUITE,init_per_group,
-					{'EXIT',{group,g4,failed}}}}}},
+	{auto_skip_6_SUITE,{tc3,g4},{failed,{auto_skip_6_SUITE,init_per_group,
+					     {'EXIT',{group,g4,failed}}}}}},
        {?eh,test_stats,{3,0,{0,12}}},
        {?eh,tc_auto_skip,
-	{auto_skip_6_SUITE,tc4,{failed,{auto_skip_6_SUITE,init_per_group,
-					{'EXIT',{group,g4,failed}}}}}},
+	{auto_skip_6_SUITE,{tc4,g4},{failed,{auto_skip_6_SUITE,init_per_group,
+					     {'EXIT',{group,g4,failed}}}}}},
        {?eh,test_stats,{3,0,{0,13}}},
        {?eh,tc_auto_skip,
 	{auto_skip_6_SUITE,{end_per_group,g4},
@@ -498,13 +498,13 @@ test_events(auto_skip) ->
 				  [{suite,auto_skip_12_SUITE}]},
 		    {auto_skipped,
 		     {require_failed,{not_available,unknown_variable_g1}}}}},
-      {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc1,
+      {?eh,tc_auto_skip,{auto_skip_12_SUITE,{tc1,g1},
 			 {require_failed,{not_available,unknown_variable_g1}}}},
       {?eh,test_stats,{10,0,{0,25}}},
-      {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc2,
+      {?eh,tc_auto_skip,{auto_skip_12_SUITE,{tc2,g1},
 			 {require_failed,{not_available,unknown_variable_g1}}}},
       {?eh,test_stats,{10,0,{0,26}}},
-      {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3,
+      {?eh,tc_auto_skip,{auto_skip_12_SUITE,{tc3,g2},
 			 {require_failed,{not_available,unknown_variable_g1}}}},
       {?eh,test_stats,{10,0,{0,27}}},
       {?eh,tc_auto_skip,{ct_framework,{end_per_group,g1},
@@ -516,13 +516,13 @@ test_events(auto_skip) ->
 				  [{suite,auto_skip_12_SUITE}]},
 		    {auto_skipped,
 		     {require_failed,{not_available,unknown_variable_g1}}}}},
-      {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc1,
+      {?eh,tc_auto_skip,{auto_skip_12_SUITE,{tc1,g1},
 			 {require_failed,{not_available,unknown_variable_g1}}}},
       {?eh,test_stats,{10,0,{0,28}}},
-      {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc2,
+      {?eh,tc_auto_skip,{auto_skip_12_SUITE,{tc2,g1},
 			 {require_failed,{not_available,unknown_variable_g1}}}},
       {?eh,test_stats,{10,0,{0,29}}},
-      {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3,
+      {?eh,tc_auto_skip,{auto_skip_12_SUITE,{tc3,g2},
 			 {require_failed,{not_available,unknown_variable_g1}}}},
       {?eh,test_stats,{10,0,{0,30}}},
       {?eh,tc_auto_skip,{ct_framework,{end_per_group,g1},
@@ -544,7 +544,7 @@ test_events(auto_skip) ->
 				   [{suite,auto_skip_12_SUITE}]},
 		     {auto_skipped,
 		      {require_failed,{not_available,unknown_variable_g4}}}}},
-       {?eh,tc_auto_skip,{auto_skip_12_SUITE,tc3,
+       {?eh,tc_auto_skip,{auto_skip_12_SUITE,{tc3,g4},
 			  {require_failed,{not_available,unknown_variable_g4}}}},
        {?eh,test_stats,{12,0,{0,31}}},
        {?eh,tc_auto_skip,{ct_framework,{end_per_group,g4},
@@ -574,10 +574,10 @@ test_events(user_skip) ->
       {user_skip_1_SUITE,tc1,"Whole suite skipped"}},
      {?eh,test_stats,{0,0,{1,0}}},
      {?eh,tc_user_skip,
-      {user_skip_1_SUITE,tc2,"Whole suite skipped"}},
+      {user_skip_1_SUITE,{tc2,g1},"Whole suite skipped"}},
      {?eh,test_stats,{0,0,{2,0}}},
      {?eh,tc_user_skip,
-      {user_skip_1_SUITE,tc3,"Whole suite skipped"}},
+      {user_skip_1_SUITE,{tc3,g1},"Whole suite skipped"}},
      {?eh,test_stats,{0,0,{3,0}}},
      {?eh,tc_user_skip,
       {user_skip_1_SUITE,tc4,"Whole suite skipped"}},
@@ -638,9 +638,9 @@ test_events(user_skip) ->
 
      [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g1,[]}}},
       {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g1,[]},{skipped,"Group skipped"}}},
-      {?eh,tc_user_skip,{user_skip_4_SUITE,tc1,"Group skipped"}},   
+      {?eh,tc_user_skip,{user_skip_4_SUITE,{tc1,g1},"Group skipped"}},   
       {?eh,test_stats,{3,0,{10,0}}},
-      {?eh,tc_user_skip,{user_skip_4_SUITE,tc2,"Group skipped"}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,{tc2,g1},"Group skipped"}},
       {?eh,test_stats,{3,0,{11,0}}},
       {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g1},"Group skipped"}}],
 
@@ -657,10 +657,10 @@ test_events(user_skip) ->
 
      [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g3,[]}}},
       {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g3,[]},{skipped,"Group skipped"}}},
-      {?eh,tc_user_skip,{user_skip_4_SUITE,tc5,"Group skipped"}},
-      {?eh,tc_user_skip,{user_skip_4_SUITE,tc6,"Group skipped"}},
-      {?eh,tc_user_skip,{user_skip_4_SUITE,tc7,"Group skipped"}},
-      {?eh,tc_user_skip,{user_skip_4_SUITE,tc8,"Group skipped"}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,{tc5,g3},"Group skipped"}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,{tc6,g4},"Group skipped"}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,{tc7,g4},"Group skipped"}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,{tc8,g3},"Group skipped"}},
       {?eh,test_stats,{5,0,{15,0}}},
       {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g3},"Group skipped"}}],
 
@@ -671,9 +671,9 @@ test_events(user_skip) ->
       {?eh,test_stats,{6,0,{15,0}}},
       [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g6,[]}}},
        {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g6,[]},{skipped,"Group skipped"}}},
-       {?eh,tc_user_skip,{user_skip_4_SUITE,tc10,"Group skipped"}},
+       {?eh,tc_user_skip,{user_skip_4_SUITE,{tc10,g6},"Group skipped"}},
        {?eh,test_stats,{6,0,{16,0}}},
-       {?eh,tc_user_skip,{user_skip_4_SUITE,tc11,"Group skipped"}},
+       {?eh,tc_user_skip,{user_skip_4_SUITE,{tc11,g6},"Group skipped"}},
        {?eh,test_stats,{6,0,{17,0}}},
        {?eh,tc_user_skip,{user_skip_4_SUITE,{end_per_group,g6},"Group skipped"}}],
        {?eh,tc_start,{user_skip_4_SUITE,{end_per_group,g5,[]}}},
@@ -687,9 +687,9 @@ test_events(user_skip) ->
 		   {skipped,{bad,'Whole suite skipped'}}}},
      {?eh,tc_user_skip,{user_skip_5_SUITE,tc1,{bad,'Whole suite skipped'}}},
      {?eh,test_stats,{6,0,{18,0}}},
-     {?eh,tc_user_skip,{user_skip_5_SUITE,tc2,{bad,'Whole suite skipped'}}},
+     {?eh,tc_user_skip,{user_skip_5_SUITE,{tc2,g1},{bad,'Whole suite skipped'}}},
      {?eh,test_stats,{6,0,{19,0}}},
-     {?eh,tc_user_skip,{user_skip_5_SUITE,tc3,{bad,'Whole suite skipped'}}},
+     {?eh,tc_user_skip,{user_skip_5_SUITE,{tc3,g1},{bad,'Whole suite skipped'}}},
      {?eh,test_stats,{6,0,{20,0}}},
      {?eh,tc_user_skip,{user_skip_5_SUITE,tc4,{bad,'Whole suite skipped'}}},
      {?eh,test_stats,{6,0,{21,0}}},
@@ -700,10 +700,10 @@ test_events(user_skip) ->
        {?eh,tc_done,{user_skip_6_SUITE,
 		     {init_per_group,ptop1,[parallel]},
 		     {skipped,"Top group skipped"}}},
-       {?eh,tc_user_skip,{user_skip_6_SUITE,tc1,"Top group skipped"}},
-       {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"Top group skipped"}},
-       {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"Top group skipped"}},
-       {?eh,tc_user_skip,{user_skip_6_SUITE,tc2,"Top group skipped"}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,{tc1,ptop1},"Top group skipped"}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,{tc3,psub1},"Top group skipped"}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,{tc4,psub1},"Top group skipped"}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,{tc2,ptop1},"Top group skipped"}},
        {?eh,tc_user_skip,{user_skip_6_SUITE,{end_per_group,ptop1},
 			  "Top group skipped"}}]},
       
@@ -718,8 +718,8 @@ test_events(user_skip) ->
 	 {?eh,tc_done,{user_skip_6_SUITE,
 		       {init_per_group,psub2,[parallel]},
 		       {skipped,"Sub group skipped"}}},
-	 {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"Sub group skipped"}},
-	 {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"Sub group skipped"}},
+	 {?eh,tc_user_skip,{user_skip_6_SUITE,{tc3,psub2},"Sub group skipped"}},
+	 {?eh,tc_user_skip,{user_skip_6_SUITE,{tc4,psub2},"Sub group skipped"}},
 	 {?eh,tc_user_skip,{user_skip_6_SUITE,{end_per_group,psub2},
 			    "Sub group skipped"}}]},
        
@@ -745,14 +745,14 @@ test_events(testspec_skip) ->
 	{user_skip_7_SUITE,{init_per_group,ptop1,[parallel]}}},
        {?eh,tc_done,
 	{user_skip_7_SUITE,{init_per_group,ptop1,[parallel]},ok}},
-       {?eh,tc_user_skip,{user_skip_7_SUITE,tc1,"SKIPPED"}},
+       {?eh,tc_user_skip,{user_skip_7_SUITE,{tc1,ptop1},"SKIPPED"}},
        {?eh,test_stats,{0,0,{1,0}}},
        {parallel,
 	[{?eh,tc_start,
           {user_skip_7_SUITE,{init_per_group,psub1,[parallel]}}},
 	 {?eh,tc_done,
           {user_skip_7_SUITE,{init_per_group,psub1,[parallel]},ok}},
-	 {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
+	 {?eh,tc_user_skip,{user_skip_7_SUITE,{tc3,psub1},"SKIPPED"}},
 	 {?eh,tc_start,{user_skip_7_SUITE,tc4}},
 	 {?eh,tc_done,{user_skip_7_SUITE,tc4,ok}},
 	 {?eh,test_stats,{1,0,{2,0}}},
@@ -778,13 +778,13 @@ test_events(testspec_skip) ->
      {?eh,tc_start,{ct_framework,init_per_suite}},
      {?eh,tc_done,{ct_framework,init_per_suite,ok}},
      {?eh,tc_user_skip,{user_skip_7_SUITE,{init_per_group,ptop1},"SKIPPED"}},
-     {?eh,tc_user_skip,{user_skip_7_SUITE,tc1,"SKIPPED"}},
+     {?eh,tc_user_skip,{user_skip_7_SUITE,{tc1,ptop1},"SKIPPED"}},
      {?eh,test_stats,{0,0,{1,0}}},
-     {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
+     {?eh,tc_user_skip,{user_skip_7_SUITE,{tc3,psub1},"SKIPPED"}},
      {?eh,test_stats,{0,0,{2,0}}},
-     {?eh,tc_user_skip,{user_skip_7_SUITE,tc4,"SKIPPED"}},
+     {?eh,tc_user_skip,{user_skip_7_SUITE,{tc4,psub1},"SKIPPED"}},
      {?eh,test_stats,{0,0,{3,0}}},
-     {?eh,tc_user_skip,{user_skip_7_SUITE,tc2,"SKIPPED"}},
+     {?eh,tc_user_skip,{user_skip_7_SUITE,{tc2,ptop1},"SKIPPED"}},
      {?eh,test_stats,{0,0,{4,0}}},
      {?eh,tc_user_skip,{user_skip_7_SUITE,{end_per_group,ptop1},"SKIPPED"}},
      {?eh,tc_start,{ct_framework,end_per_suite}},
@@ -804,8 +804,8 @@ test_events(testspec_skip) ->
 	{user_skip_7_SUITE,{init_per_group,ptop1,[parallel]},ok}},
        {?eh,tc_user_skip,
 	{user_skip_7_SUITE,{init_per_group,psub1},"SKIPPED"}},
-       {?eh,tc_user_skip,{user_skip_7_SUITE,tc3,"SKIPPED"}},
-       {?eh,tc_user_skip,{user_skip_7_SUITE,tc4,"SKIPPED"}},
+       {?eh,tc_user_skip,{user_skip_7_SUITE,{tc3,psub1},"SKIPPED"}},
+       {?eh,tc_user_skip,{user_skip_7_SUITE,{tc4,psub1},"SKIPPED"}},
        {?eh,test_stats,{0,0,{2,0}}},
        {?eh,tc_user_skip,{user_skip_7_SUITE,{end_per_group,psub1},"SKIPPED"}},
        {?eh,tc_start,{user_skip_7_SUITE,tc1}},
@@ -837,13 +837,13 @@ test_events(testspec_skip) ->
        {?eh,tc_done,{user_skip_6_SUITE,
 		     {init_per_group,ptop1,[parallel]},
 		     {skipped,"Top group skipped"}}},
-       {?eh,tc_user_skip,{user_skip_6_SUITE,tc1,"Top group skipped"}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,{tc1,ptop1},"Top group skipped"}},
        {?eh,test_stats,{0,0,{1,0}}},
-       {?eh,tc_user_skip,{user_skip_6_SUITE,tc3,"SKIPPED"}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,{tc3,psub1},"SKIPPED"}},
        {?eh,test_stats,{0,0,{2,0}}},
-       {?eh,tc_user_skip,{user_skip_6_SUITE,tc4,"SKIPPED"}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,{tc4,psub1},"SKIPPED"}},
        {?eh,test_stats,{0,0,{3,0}}},
-       {?eh,tc_user_skip,{user_skip_6_SUITE,tc2,"Top group skipped"}},
+       {?eh,tc_user_skip,{user_skip_6_SUITE,{tc2,ptop1},"Top group skipped"}},
        {?eh,test_stats,{0,0,{4,0}}},
        {?eh,tc_user_skip,
 	{user_skip_6_SUITE,{end_per_group,ptop1},"Top group skipped"}}]},