Fix problem with handling Config and FW reports correctly

2 files changed, 56 insertions, 50 deletions

diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 276f902b05..76516e6a72 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -1256,14 +1256,20 @@ report(What,Data) ->
 		{_,{SkipOrFail,_Reason}} ->
 		    add_to_stats(SkipOrFail)
 	    end;
-	tc_user_skip ->	    
-	    %% test case specified as skipped in testspec
+	tc_user_skip ->
+	    %% test case specified as skipped in testspec, or init
+	    %% config func for suite/group has returned {skip,Reason}
 	    %% Data = {Suite,Case,Comment}
 	    ct_event:sync_notify(#event{name=tc_user_skip,
 					node=node(),
 					data=Data}),
-	    ct_hooks:on_tc_skip(What, Data),
-	    add_to_stats(user_skipped);
+	    case Data of
+		{_,Func,_} when Func /= end_per_suite, Func /= end_per_group ->
+		    ct_hooks:on_tc_skip(What, Data),
+		    add_to_stats(user_skipped);
+		_ ->
+		    ok
+	    end;
 	tc_auto_skip ->
 	    %% test case skipped because of error in init_per_suite
 	    %% Data = {Suite,Case,Comment}
diff --git a/lib/common_test/test/ct_skip_SUITE.erl b/lib/common_test/test/ct_skip_SUITE.erl
index b8be55f43a..35ac483309 100644
--- a/lib/common_test/test/ct_skip_SUITE.erl
+++ b/lib/common_test/test/ct_skip_SUITE.erl
@@ -447,41 +447,41 @@ test_events(user_skip) ->
      {?eh,tc_start,{user_skip_1_SUITE,init_per_suite}},
      {?eh,tc_done,
       {user_skip_1_SUITE,init_per_suite,{skipped,"Whole suite skipped"}}},
-     {?eh,tc_auto_skip,
+     {?eh,tc_user_skip,
       {user_skip_1_SUITE,tc1,"Whole suite skipped"}},
-     {?eh,test_stats,{0,0,{0,1}}},
-     {?eh,tc_auto_skip,
+     {?eh,test_stats,{0,0,{1,0}}},
+     {?eh,tc_user_skip,
       {user_skip_1_SUITE,tc2,"Whole suite skipped"}},
-     {?eh,test_stats,{0,0,{0,2}}},
-     {?eh,tc_auto_skip,
+     {?eh,test_stats,{0,0,{2,0}}},
+     {?eh,tc_user_skip,
       {user_skip_1_SUITE,tc3,"Whole suite skipped"}},
-     {?eh,test_stats,{0,0,{0,3}}},
-     {?eh,tc_auto_skip,
+     {?eh,test_stats,{0,0,{3,0}}},
+     {?eh,tc_user_skip,
       {user_skip_1_SUITE,tc4,"Whole suite skipped"}},
-     {?eh,test_stats,{0,0,{0,4}}},
-     {?eh,tc_auto_skip,
+     {?eh,test_stats,{0,0,{4,0}}},
+     {?eh,tc_user_skip,
       {user_skip_1_SUITE,end_per_suite,"Whole suite skipped"}},
 
      {?eh,tc_start,{user_skip_2_SUITE,init_per_suite}},
      {?eh,tc_done,{user_skip_2_SUITE,init_per_suite,ok}},
      {?eh,tc_start,{user_skip_2_SUITE,tc1}},
      {?eh,tc_done,{user_skip_2_SUITE,tc1,{skipped,{tc1,skipped}}}},
-     {?eh,test_stats,{0,0,{1,4}}},
+     {?eh,test_stats,{0,0,{5,0}}},
 
      [{?eh,tc_start,{user_skip_2_SUITE,{init_per_group,g1,[]}}},
       {?eh,tc_done,{user_skip_2_SUITE,{init_per_group,g1,[]},ok}},
       {?eh,tc_start,{user_skip_2_SUITE,tc2}},
       {?eh,tc_done,{user_skip_2_SUITE,tc2,ok}},
-      {?eh,test_stats,{1,0,{1,4}}},
+      {?eh,test_stats,{1,0,{5,0}}},
       {?eh,tc_start,{user_skip_2_SUITE,tc3}},
       {?eh,tc_done,{user_skip_2_SUITE,tc3,{skipped,{tc3,skipped}}}},
-      {?eh,test_stats,{1,0,{2,4}}},
+      {?eh,test_stats,{1,0,{6,0}}},
       {?eh,tc_start,{user_skip_2_SUITE,{end_per_group,g1,[]}}},
       {?eh,tc_done,{user_skip_2_SUITE,{end_per_group,g1,[]},ok}}],
 
      {?eh,tc_start,{user_skip_2_SUITE,tc4}},
      {?eh,tc_done,{user_skip_2_SUITE,tc4,ok}},
-     {?eh,test_stats,{2,0,{2,4}}},
+     {?eh,test_stats,{2,0,{6,0}}},
      {?eh,tc_start,{user_skip_2_SUITE,end_per_suite}},
      {?eh,tc_done,{user_skip_2_SUITE,end_per_suite,ok}},
 
@@ -489,16 +489,16 @@ test_events(user_skip) ->
      {?eh,tc_done,{user_skip_3_SUITE,init_per_suite,ok}},
      {?eh,tc_start,{user_skip_3_SUITE,tc1}},
      {?eh,tc_done,{user_skip_3_SUITE,tc1,{skipped,"Test case skipped"}}},
-     {?eh,test_stats,{2,0,{3,4}}},
+     {?eh,test_stats,{2,0,{7,0}}},
 
      [{?eh,tc_start,{user_skip_3_SUITE,{init_per_group,g1,[]}}},
       {?eh,tc_done,{user_skip_3_SUITE,{init_per_group,g1,[]},ok}},
       {?eh,tc_start,{user_skip_3_SUITE,tc2}},
       {?eh,tc_done,{user_skip_3_SUITE,tc2,ok}},
-      {?eh,test_stats,{3,0,{3,4}}},
+      {?eh,test_stats,{3,0,{7,0}}},
       {?eh,tc_start,{user_skip_3_SUITE,tc3}},
       {?eh,tc_done,{user_skip_3_SUITE,tc3,{skipped,"Test case skipped"}}},
-      {?eh,test_stats,{3,0,{4,4}}},
+      {?eh,test_stats,{3,0,{8,0}}},
       {?eh,tc_start,{user_skip_3_SUITE,{end_per_group,g1,[]}}},
       {?eh,tc_done,{user_skip_3_SUITE,{end_per_group,g1,[]},ok}}],
 
@@ -506,7 +506,7 @@ test_events(user_skip) ->
      {?eh,tc_done,{user_skip_3_SUITE,tc4,
 		   {skipped,{proc_info,{{current_function,{user_skip_3_SUITE,tc4,1}},
 					{initial_call,{erlang,apply,2}}}}}}},
-     {?eh,test_stats,{3,0,{5,4}}},
+     {?eh,test_stats,{3,0,{9,0}}},
      {?eh,tc_start,{user_skip_3_SUITE,end_per_suite}},
      {?eh,tc_done,{user_skip_3_SUITE,end_per_suite,ok}},
 
@@ -515,62 +515,62 @@ test_events(user_skip) ->
 
      [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g1,[]}}},
       {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g1,[]},{skipped,"Group skipped"}}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,tc1,"Group skipped"}},   
-      {?eh,test_stats,{3,0,{5,5}}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,tc2,"Group skipped"}},
-      {?eh,test_stats,{3,0,{5,6}}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
+      {?eh,tc_user_skip,{user_skip_4_SUITE,tc1,"Group skipped"}},   
+      {?eh,test_stats,{3,0,{10,0}}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,tc2,"Group skipped"}},
+      {?eh,test_stats,{3,0,{11,0}}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
 
      [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g2,[]}}},
       {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g2,[]},ok}},
       {?eh,tc_start,{user_skip_4_SUITE,tc3}},
       {?eh,tc_done,{user_skip_4_SUITE,tc3,ok}},
-      {?eh,test_stats,{4,0,{5,6}}},
+      {?eh,test_stats,{4,0,{11,0}}},
       {?eh,tc_start,{user_skip_4_SUITE,tc4}},
       {?eh,tc_done,{user_skip_4_SUITE,tc4,ok}},
-      {?eh,test_stats,{5,0,{5,6}}},
+      {?eh,test_stats,{5,0,{11,0}}},
       {?eh,tc_start,{user_skip_4_SUITE,{end_per_group,g2,[]}}},
       {?eh,tc_done,{user_skip_4_SUITE,{end_per_group,g2,[]},ok}}],
 
      [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g3,[]}}},
       {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g3,[]},{skipped,"Group skipped"}}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,tc5,"Group skipped"}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,tc6,"Group skipped"}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,tc7,"Group skipped"}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,tc8,"Group skipped"}},
-      {?eh,test_stats,{5,0,{5,10}}},
-      {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
+      {?eh,tc_user_skip,{user_skip_4_SUITE,tc5,"Group skipped"}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,tc6,"Group skipped"}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,tc7,"Group skipped"}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,tc8,"Group skipped"}},
+      {?eh,test_stats,{5,0,{15,0}}},
+      {?eh,tc_user_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
 
      [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g5,[]}}},
       {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g5,[]},ok}},
       {?eh,tc_start,{user_skip_4_SUITE,tc9}},
       {?eh,tc_done,{user_skip_4_SUITE,tc9,ok}},
-      {?eh,test_stats,{6,0,{5,10}}},
+      {?eh,test_stats,{6,0,{15,0}}},
       [{?eh,tc_start,{user_skip_4_SUITE,{init_per_group,g6,[]}}},
        {?eh,tc_done,{user_skip_4_SUITE,{init_per_group,g6,[]},{skipped,"Group skipped"}}},
-       {?eh,tc_auto_skip,{user_skip_4_SUITE,tc10,"Group skipped"}},
-       {?eh,test_stats,{6,0,{5,11}}},
-       {?eh,tc_auto_skip,{user_skip_4_SUITE,tc11,"Group skipped"}},
-       {?eh,test_stats,{6,0,{5,12}}},
-       {?eh,tc_auto_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
+       {?eh,tc_user_skip,{user_skip_4_SUITE,tc10,"Group skipped"}},
+       {?eh,test_stats,{6,0,{16,0}}},
+       {?eh,tc_user_skip,{user_skip_4_SUITE,tc11,"Group skipped"}},
+       {?eh,test_stats,{6,0,{17,0}}},
+       {?eh,tc_user_skip,{user_skip_4_SUITE,end_per_group,"Group skipped"}}],
        {?eh,tc_start,{user_skip_4_SUITE,{end_per_group,g5,[]}}},
        {?eh,tc_done,{user_skip_4_SUITE,{end_per_group,g5,[]},ok}}],
 
      {?eh,tc_start,{user_skip_4_SUITE,end_per_suite}},
      {?eh,tc_done,{user_skip_4_SUITE,end_per_suite,ok}},
 
-         {ct_test_support_eh,tc_start,{user_skip_5_SUITE,init_per_suite}},
+     {?eh,tc_start,{user_skip_5_SUITE,init_per_suite}},
      {?eh,tc_done,{user_skip_5_SUITE,init_per_suite,
 		   {skipped,{bad,'Whole suite skipped'}}}},
-     {?eh,tc_auto_skip,{user_skip_5_SUITE,tc1,{bad,'Whole suite skipped'}}},
-     {?eh,test_stats,{6,0,{5,13}}},
-     {?eh,tc_auto_skip,{user_skip_5_SUITE,tc2,{bad,'Whole suite skipped'}}},
-     {?eh,test_stats,{6,0,{5,14}}},
-     {?eh,tc_auto_skip,{user_skip_5_SUITE,tc3,{bad,'Whole suite skipped'}}},
-     {?eh,test_stats,{6,0,{5,15}}},
-     {?eh,tc_auto_skip,{user_skip_5_SUITE,tc4,{bad,'Whole suite skipped'}}},
-     {?eh,test_stats,{6,0,{5,16}}},
-     {?eh,tc_auto_skip,{user_skip_5_SUITE,end_per_suite,{bad,'Whole suite skipped'}}},
+     {?eh,tc_user_skip,{user_skip_5_SUITE,tc1,{bad,'Whole suite skipped'}}},
+     {?eh,test_stats,{6,0,{18,0}}},
+     {?eh,tc_user_skip,{user_skip_5_SUITE,tc2,{bad,'Whole suite skipped'}}},
+     {?eh,test_stats,{6,0,{19,0}}},
+     {?eh,tc_user_skip,{user_skip_5_SUITE,tc3,{bad,'Whole suite skipped'}}},
+     {?eh,test_stats,{6,0,{20,0}}},
+     {?eh,tc_user_skip,{user_skip_5_SUITE,tc4,{bad,'Whole suite skipped'}}},
+     {?eh,test_stats,{6,0,{21,0}}},
+     {?eh,tc_user_skip,{user_skip_5_SUITE,end_per_suite,{bad,'Whole suite skipped'}}},
      
      {?eh,test_done,{'DEF','STOP_TIME'}},
      {?eh,stop_logging,[]}