Merge remote-tracking branch 'origin/peppe/common_test/improve_suite_error_reports'

* origin/peppe/common_test/improve_suite_error_reports:
  Add missing events and hook function calls
  Improve error reports in log when suite compilation fails

OTP-10816

4 files changed, 65 insertions, 35 deletions

diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index a699ec3438..91368d3137 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -113,6 +113,7 @@ init_tc1(?MODULE,_,error_in_suite,[Config0]) when is_list(Config0) ->
     ct_event:notify(#event{name=tc_start,
 			   node=node(),
 			   data={?MODULE,error_in_suite}}),
+    ct_suite_init(?MODULE, error_in_suite, [], Config0),
     case ?val(error, Config0) of
 	undefined ->
 	    {fail,"unknown_error_in_suite"};
@@ -635,7 +636,20 @@ try_set_default(Name,Key,Info,Where) ->
 end_tc(Mod, Fun, Args) ->
     %% Have to keep end_tc/3 for backwards compatibility issues
     end_tc(Mod, Fun, Args, '$end_tc_dummy').
-end_tc(?MODULE,error_in_suite,_, _) ->		% bad start!
+end_tc(?MODULE,error_in_suite,{Result,[Args]},Return) ->
+    %% this clause gets called if CT has encountered a suite that
+    %% can't be executed
+    FinalNotify =
+	case ct_hooks:end_tc(?MODULE, error_in_suite, Args, Result, Return) of
+	    '$ct_no_change' ->
+		Result;
+	    HookResult ->
+		HookResult
+	end,
+    Event = #event{name=tc_done,
+		   node=node(),
+		   data={?MODULE,error_in_suite,tag(FinalNotify)}},
+    ct_event:sync_notify(Event),
     ok;
 end_tc(Mod,Func,{TCPid,Result,[Args]}, Return) when is_pid(TCPid) ->
     end_tc(Mod,Func,TCPid,Result,Args,Return);
@@ -1310,6 +1324,8 @@ report(What,Data) ->
 	    end,
 	    ct_logs:unregister_groupleader(ReportingPid),
 	    case {Func,Result} of
+		{error_in_suite,_} when Suite == ?MODULE ->
+		    ok;
 		{init_per_suite,_} ->
 		    ok;
 		{end_per_suite,_} ->
diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl
index 4d5a75d354..7c8c720e13 100644
--- a/lib/common_test/src/ct_logs.erl
+++ b/lib/common_test/src/ct_logs.erl
@@ -2054,6 +2054,13 @@ runentry(Dir, Totals={Node,Label,Logs,
 					     ?testname_width-3)),
 		lists:flatten(io_lib:format("~ts...",[Trunc]))
 	end,
+    TotMissingStr =
+	if NotBuilt > 0 ->
+		["<font color=\"red\">",
+		 integer_to_list(NotBuilt),"</font>"];
+	   true ->
+		integer_to_list(NotBuilt)
+	end,
     Total = TotSucc+TotFail+AllSkip,
     A = xhtml(["<td align=center><font size=\"-1\">",Node,
 	       "</font></td>\n",
@@ -2073,7 +2080,7 @@ runentry(Dir, Totals={Node,Label,Logs,
 	 "<td align=right>",TotFailStr,"</td>\n",
 	 "<td align=right>",integer_to_list(AllSkip),
 	 " (",UserSkipStr,"/",AutoSkipStr,")</td>\n",
-	 "<td align=right>",integer_to_list(NotBuilt),"</td>\n"],
+	 "<td align=right>",TotMissingStr,"</td>\n"],
     TotalsStr = A++B++C,
     
     XHTML = [xhtml("<tr>\n", ["<tr class=\"",odd_or_even(),"\">\n"]),
diff --git a/lib/common_test/test/ct_auto_compile_SUITE.erl b/lib/common_test/test/ct_auto_compile_SUITE.erl
index cc546ed30d..3e4da31ab4 100644
--- a/lib/common_test/test/ct_auto_compile_SUITE.erl
+++ b/lib/common_test/test/ct_auto_compile_SUITE.erl
@@ -108,6 +108,8 @@ ac_spec(Config) when is_list(Config) ->
     PrivDir = ?config(priv_dir, Config),
     file:copy(filename:join(DataDir, "bad_SUITE.erl"),
 	      filename:join(PrivDir, "bad_SUITE.erl")),
+    Suite = filename:join(DataDir, "dummy_SUITE"),
+    compile:file(Suite, [{outdir,PrivDir}]),
     TestSpec = [{label,ac_spec},
 		{auto_compile,false},
 		{suites,PrivDir,all}],
@@ -160,28 +162,34 @@ events_to_check(Test, N) ->
 
 test_events(ac_flag) ->
     [
-     {ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
-     {ct_test_support_eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
-     {ct_test_support_eh,start_info,{1,1,3}},
-     {ct_test_support_eh,tc_start,{dummy_SUITE,init_per_suite}},
-     {ct_test_support_eh,tc_done,{dummy_SUITE,init_per_suite,ok}},
-     {ct_test_support_eh,test_stats,{1,1,{1,0}}},
-     {ct_test_support_eh,tc_start,{dummy_SUITE,end_per_suite}},
-     {ct_test_support_eh,tc_done,{dummy_SUITE,end_per_suite,ok}},
-     {ct_test_support_eh,test_done,{'DEF','STOP_TIME'}},
-     {ct_test_support_eh,stop_logging,[]}
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,start_info,{1,1,3}},
+     {?eh,tc_start,{ct_framework,error_in_suite}},
+     {?eh,tc_done,{ct_framework,error_in_suite,
+       {failed,{error,'bad_SUITE can not be compiled or loaded'}}}},
+     {?eh,tc_start,{dummy_SUITE,init_per_suite}},
+     {?eh,tc_done,{dummy_SUITE,init_per_suite,ok}},
+     {?eh,test_stats,{1,1,{1,0}}},
+     {?eh,tc_start,{dummy_SUITE,end_per_suite}},
+     {?eh,tc_done,{dummy_SUITE,end_per_suite,ok}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]}
     ];
 
 test_events(ac_spec) ->
     [
-     {ct_test_support_eh,start_logging,{'DEF','RUNDIR'}},
-     {ct_test_support_eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
-     {ct_test_support_eh,start_info,{1,1,3}},
-     {ct_test_support_eh,tc_start,{dummy_SUITE,init_per_suite}},
-     {ct_test_support_eh,tc_done,{dummy_SUITE,init_per_suite,ok}},
-     {ct_test_support_eh,test_stats,{1,1,{1,0}}},
-     {ct_test_support_eh,tc_start,{dummy_SUITE,end_per_suite}},
-     {ct_test_support_eh,tc_done,{dummy_SUITE,end_per_suite,ok}},
-     {ct_test_support_eh,test_done,{'DEF','STOP_TIME'}},
-     {ct_test_support_eh,stop_logging,[]}
+     {?eh,start_logging,{'DEF','RUNDIR'}},
+     {?eh,test_start,{'DEF',{'START_TIME','LOGDIR'}}},
+     {?eh,start_info,{1,1,3}},
+     {?eh,tc_start,{ct_framework,error_in_suite}},
+     {?eh,tc_done,{ct_framework,error_in_suite,
+       {failed,{error,'bad_SUITE can not be compiled or loaded'}}}},
+     {?eh,tc_start,{dummy_SUITE,init_per_suite}},
+     {?eh,tc_done,{dummy_SUITE,init_per_suite,ok}},
+     {?eh,test_stats,{1,1,{1,0}}},
+     {?eh,tc_start,{dummy_SUITE,end_per_suite}},
+     {?eh,tc_done,{dummy_SUITE,end_per_suite,ok}},
+     {?eh,test_done,{'DEF','STOP_TIME'}},
+     {?eh,stop_logging,[]}
     ].
diff --git a/lib/common_test/test/ct_test_server_if_1_SUITE.erl b/lib/common_test/test/ct_test_server_if_1_SUITE.erl
index b6ef3062d4..214cb60c0d 100644
--- a/lib/common_test/test/ct_test_server_if_1_SUITE.erl
+++ b/lib/common_test/test/ct_test_server_if_1_SUITE.erl
@@ -236,14 +236,13 @@ test_events(ts_if_1) ->
       {ts_if_2_SUITE,end_per_suite,
        {failed,{error,{suite0_failed,{exited,suite0_goes_boom}}}}}},
 
-
      {?eh,tc_start,{ct_framework,error_in_suite}},
-     {?eh,test_stats,{2,6,{4,7}}},
-
+     {?eh,tc_done,{ct_framework,error_in_suite,
+		   {failed,{error,'ts_if_3_SUITE:all/0 is missing'}}}},
 
      {?eh,tc_start,{ct_framework,error_in_suite}},
-     {?eh,test_stats,{2,7,{4,7}}},
-
+     {?eh,tc_done,{ct_framework,error_in_suite,
+		   {failed,{error,'Bad return value from ts_if_4_SUITE:all/0'}}}},
 
      {?eh,tc_start,{ts_if_5_SUITE,init_per_suite}},
      {?eh,tc_done,{ts_if_5_SUITE,init_per_suite,
@@ -252,7 +251,7 @@ test_events(ts_if_1) ->
      {?eh,tc_auto_skip,
       {ts_if_5_SUITE,my_test_case,
        {require_failed_in_suite0,{not_available,undef_variable}}}},
-     {?eh,test_stats,{2,7,{4,8}}},
+     {?eh,test_stats,{2,5,{4,8}}},
      {?eh,tc_auto_skip,
       {ts_if_5_SUITE,end_per_suite,
        {require_failed_in_suite0,{not_available,undef_variable}}}},
@@ -264,7 +263,7 @@ test_events(ts_if_1) ->
      {?eh,tc_auto_skip,
       {ts_if_6_SUITE,tc1,
        {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}},
-     {?eh,test_stats,{2,7,{4,9}}},
+     {?eh,test_stats,{2,5,{4,9}}},
      {?eh,tc_auto_skip,
       {ct_framework,end_per_suite,
        {failed,{error,{suite0_failed,{exited,suite0_byebye}}}}}},
@@ -274,13 +273,13 @@ test_events(ts_if_1) ->
      {?eh,tc_done,{ct_framework,init_per_suite,ok}},
      {?eh,tc_done,
       {ts_if_7_SUITE,tc1,{auto_skipped,{testcase0_failed,bad_return_value}}}},
-     {?eh,test_stats,{2,7,{4,10}}},
+     {?eh,test_stats,{2,5,{4,10}}},
      {?eh,tc_done,{ts_if_7_SUITE,
 		   {init_per_group,g1,[]},
 		   {auto_skipped,{group0_failed,bad_return_value}}}},
      {?eh,tc_auto_skip,
       {ts_if_7_SUITE,{tc2,g1},{group0_failed,bad_return_value}}},
-     {?eh,test_stats,{2,7,{4,11}}},
+     {?eh,test_stats,{2,5,{4,11}}},
      {?eh,tc_auto_skip,
       {ts_if_7_SUITE,{end_per_group,g1},{group0_failed,bad_return_value}}},
      
@@ -288,7 +287,7 @@ test_events(ts_if_1) ->
       {?eh,tc_done,{ts_if_7_SUITE,{init_per_group,g2,[]},ok}},
       {?eh,tc_done,{ts_if_7_SUITE,tc2,
 		    {auto_skipped,{testcase0_failed,bad_return_value}}}},
-      {?eh,test_stats,{2,7,{4,12}}},
+      {?eh,test_stats,{2,5,{4,12}}},
       {?eh,tc_start,{ts_if_7_SUITE,{end_per_group,g2,[]}}},
       {?eh,tc_done,{ts_if_7_SUITE,{end_per_group,g2,[]},ok}}],
      
@@ -300,17 +299,17 @@ test_events(ts_if_1) ->
      {?eh,tc_done,{ct_framework,init_per_suite,ok}},
      {?eh,tc_start,{ts_if_8_SUITE,tc1}},
      {?eh,tc_done,{ts_if_8_SUITE,tc1,{failed,{error,failed_on_purpose}}}},
-     {?eh,test_stats,{2,8,{4,12}}},
+     {?eh,test_stats,{2,6,{4,12}}},
      {?eh,tc_start,{ct_framework,end_per_suite}},
      {?eh,tc_done,{ct_framework,end_per_suite,ok}},
      
      
      {?eh,tc_user_skip,{skipped_by_spec_1_SUITE,all,"should be skipped"}},
-     {?eh,test_stats,{2,8,{5,12}}},
+     {?eh,test_stats,{2,6,{5,12}}},
      {?eh,tc_start,{skipped_by_spec_2_SUITE,init_per_suite}},
      {?eh,tc_done,{skipped_by_spec_2_SUITE,init_per_suite,ok}},
      {?eh,tc_user_skip,{skipped_by_spec_2_SUITE,tc1,"should be skipped"}},
-     {?eh,test_stats,{2,8,{6,12}}},
+     {?eh,test_stats,{2,6,{6,12}}},
      {?eh,tc_start,{skipped_by_spec_2_SUITE,end_per_suite}},
      {?eh,tc_done,{skipped_by_spec_2_SUITE,end_per_suite,ok}},