Merge branch 'peppe/common_test/cth_improvements' into maint

* peppe/common_test/cth_improvements:
  Add missing internal hook functions
  Fix remaining issues
  Update ct_hooks test suites
  Introduce new CT hook functions

OTP-13242

1 files changed, 156 insertions, 60 deletions

diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl
index 671674c617..f62eb88ccf 100644
--- a/lib/test_server/src/test_server.erl
+++ b/lib/test_server/src/test_server.erl
@@ -714,10 +714,10 @@ call_end_conf(Mod,Func,TCPid,TCExitReason,Loc,Conf,TVal) ->
 			       Starter ! {self(),{call_end_conf,Data,ok}}
 		       end);
 	true ->
-	    do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal)
+	    do_call_end_conf(Starter,Mod,Func,Data,TCExitReason,Conf,TVal)
     end.
 
-do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) ->
+do_call_end_conf(Starter,Mod,Func,Data,TCExitReason,Conf,TVal) ->
     EndConfProc =
 	fun() ->
 		process_flag(trap_exit,true), % to catch timetraps
@@ -725,16 +725,29 @@ do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) ->
 		EndConfApply =
 		    fun() ->
 			    timetrap(TVal),
-			    try apply(Mod,end_per_testcase,[Func,Conf]) of
+			    %% We can't handle fails or skips here
+			    %% (neither input nor output). The error can
+			    %% be read from Conf though (tc_status).
+			    EndConf =
+				case do_init_tc_call(Mod,{end_per_testcase,Func},
+						     [Conf],
+						     {TCExitReason,[Conf]}) of
+				    {_,[EPTCInit]} when is_list(EPTCInit) ->
+					EPTCInit;
+				    _ ->
+					Conf
+				end,
+			    try apply(Mod,end_per_testcase,[Func,EndConf]) of
 				_ -> ok
 			    catch
 				_:Why ->
 				    timer:sleep(1),
-				    group_leader() ! {printout,12,
-						      "WARNING! "
-						      "~w:end_per_testcase(~w, ~p)"
-						      " crashed!\n\tReason: ~p\n",
-						      [Mod,Func,Conf,Why]}
+				    GLMsg = {printout,12,
+					     "WARNING! "
+					     "~w:end_per_testcase(~w, ~p)"
+					     " crashed!\n\tReason: ~p\n",
+					     [Mod,Func,Conf,Why]},
+				    group_leader() ! GLMsg					
 			    end,
 			    Supervisor ! {self(),end_conf}
 		    end,
@@ -755,28 +768,38 @@ do_call_end_conf(Starter,Mod,Func,Data,Conf,TVal) ->
 	end,
     spawn_link(EndConfProc).
 
-spawn_fw_call(Mod,{init_per_testcase,Func},CurrConf,Pid,
-	      {timetrap_timeout,TVal}=Why,
-	      Loc,SendTo) ->
+spawn_fw_call(Mod,IPTC={init_per_testcase,Func},CurrConf,Pid,
+	      Why,Loc,SendTo) ->
     FwCall =
 	fun() ->
 		Skip = {skip,{failed,{Mod,init_per_testcase,Why}}},
 		%% if init_per_testcase fails, the test case
 		%% should be skipped
-		try do_end_tc_call(Mod,Func, {Pid,Skip,[CurrConf]}, Why) of
+		try begin do_end_tc_call(Mod,IPTC, {Pid,Skip,[CurrConf]}, Why),
+			  do_init_tc_call(Mod,{end_per_testcase,Func},
+					  [CurrConf],{ok,[CurrConf]}),
+			  do_end_tc_call(Mod,{end_per_testcase,Func}, 
+					 {Pid,Skip,[CurrConf]}, Why) end of
 		    _ -> ok
 		catch
 		    _:FwEndTCErr ->
 			exit({fw_notify_done,end_tc,FwEndTCErr})
 		end,
+		Time = case Why of
+			   {timetrap_timeout,TVal} -> TVal/1000;
+			   _                       -> died
+		       end,
+		group_leader() ! {printout,12,
+				  "ERROR! ~w:init_per_testcase(~w, ~p)"
+				  " failed!\n\tReason: ~tp\n",
+				 [Mod,Func,CurrConf,Why]},
 		%% finished, report back
-		SendTo ! {self(),fw_notify_done,
-			  {TVal/1000,Skip,Loc,[],undefined}}
+		SendTo ! {self(),fw_notify_done,{Time,Skip,Loc,[],undefined}}
 	end,
     spawn_link(FwCall);
 
-spawn_fw_call(Mod,{end_per_testcase,Func},EndConf,Pid,
-	      {timetrap_timeout,TVal}=Why,_Loc,SendTo) ->
+spawn_fw_call(Mod,EPTC={end_per_testcase,Func},EndConf,Pid,
+	      Why,_Loc,SendTo) ->
     FwCall =
 	fun() ->
 		{RetVal,Report} =
@@ -790,24 +813,38 @@ spawn_fw_call(Mod,{end_per_testcase,Func},EndConf,Pid,
 			    E = {failed,{Mod,end_per_testcase,Why}},
 			    {Result,E}
 		    end,
-		group_leader() ! {printout,12,
-				  "WARNING! ~w:end_per_testcase(~w, ~p)"
-				  " failed!\n\tReason: timetrap timeout"
-				  " after ~w ms!\n", [Mod,Func,EndConf,TVal]},
-		FailLoc = proplists:get_value(tc_fail_loc, EndConf),
-		try do_end_tc_call(Mod,Func,
-					  {Pid,Report,[EndConf]}, Why) of
+		{Time,Warn} =
+		    case Why of
+			{timetrap_timeout,TVal} ->
+			    group_leader() !
+				{printout,12,
+				 "WARNING! ~w:end_per_testcase(~w, ~p)"
+				 " failed!\n\tReason: timetrap timeout"
+				 " after ~w ms!\n", [Mod,Func,EndConf,TVal]},
+			    W = "<font color=\"red\">"
+				"WARNING: end_per_testcase timed out!</font>",
+			    {TVal/1000,W};
+			_ ->
+			    group_leader() !
+				{printout,12,
+				 "WARNING! ~w:end_per_testcase(~w, ~p)"
+				 " failed!\n\tReason: ~tp\n",
+				 [Mod,Func,EndConf,Why]},
+			    W = "<font color=\"red\">"
+				"WARNING: end_per_testcase failed!</font>",
+			    {died,W}
+		    end,
+		try do_end_tc_call(Mod,EPTC,{Pid,Report,[EndConf]}, Why) of
 		    _ -> ok
 		catch
 		    _:FwEndTCErr ->
 			exit({fw_notify_done,end_tc,FwEndTCErr})
 		end,
-		Warn = "<font color=\"red\">"
-		       "WARNING: end_per_testcase timed out!</font>",
+		FailLoc = proplists:get_value(tc_fail_loc, EndConf),
 		%% finished, report back (if end_per_testcase fails, a warning
 		%% should be printed as part of the comment)
 		SendTo ! {self(),fw_notify_done,
-			  {TVal/1000,RetVal,FailLoc,[],Warn}}
+			  {Time,RetVal,FailLoc,[],Warn}}
 	end,
     spawn_link(FwCall);
 
@@ -830,10 +867,12 @@ spawn_fw_call(FwMod,FwFunc,_,_Pid,{framework_error,FwError},_,SendTo) ->
     spawn_link(FwCall);
 
 spawn_fw_call(Mod,Func,CurrConf,Pid,Error,Loc,SendTo) ->
-    Func1 = case Func of
-		{_InitOrEndPerTC,F} -> F;
-		F -> F
-	    end,
+    {Func1,EndTCFunc} = case Func of
+			    CF when CF == init_per_suite; CF == end_per_suite;
+				    CF == init_per_group; CF == end_per_group ->
+				{CF,CF};
+			    TC -> {TC,{end_per_testcase,TC}}
+			end,
     FwCall =
 	fun() ->
 		try fw_error_notify(Mod,Func1,[],
@@ -845,8 +884,7 @@ spawn_fw_call(Mod,Func,CurrConf,Pid,Error,Loc,SendTo) ->
 			      FwErrorNotifyErr})
 		end,
 		Conf = [{tc_status,{failed,Error}}|CurrConf],
-		try do_end_tc_call(Mod,Func1,
-				   {Pid,Error,[Conf]},Error) of
+		try do_end_tc_call(Mod,EndTCFunc,{Pid,Error,[Conf]},Error) of
 		    _ -> ok
 		catch
 		    _:FwEndTCErr ->
@@ -923,32 +961,38 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit,
     Where = [{Mod,Func}],
     put(test_server_loc, Where),
 
-    FWInitResult = test_server_sup:framework_call(init_tc,[Mod,Func,Args0],
-						  {ok,Args0}),
+    FWInitFunc = case RunInit of
+		     run_init -> {init_per_testcase,Func};
+		     _        -> Func
+		 end,
+
+    FWInitResult0 = do_init_tc_call(Mod,FWInitFunc,Args0,{ok,Args0}),
+
     set_tc_state(running),
     {{Time,Value},Loc,Opts} =
-	case FWInitResult of
+	case FWInitResult0 of
 	    {ok,Args} ->
 		run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback);
 	    Error = {error,_Reason} ->
-		NewResult = do_end_tc_call(Mod,Func, {Error,Args0},
+		NewResult = do_end_tc_call(Mod,FWInitFunc, {Error,Args0},
 					   {auto_skip,{failed,Error}}),
 		{{0,NewResult},Where,[]};
 	    {fail,Reason} ->
-		Conf = [{tc_status,{failed,Reason}} | hd(Args0)],
+		Conf = [{tc_status,{failed,Reason}} | hd(Args0)],		
 		fw_error_notify(Mod, Func, Conf, Reason),
-		NewResult = do_end_tc_call(Mod,Func, {{error,Reason},[Conf]},
+		NewResult = do_end_tc_call(Mod,FWInitFunc,
+					   {{error,Reason},[Conf]},
 					   {fail,Reason}),
 		{{0,NewResult},Where,[]};
 	    Skip = {SkipType,_Reason} when SkipType == skip;
 					   SkipType == skipped ->
-		NewResult = do_end_tc_call(Mod,Func,
+		NewResult = do_end_tc_call(Mod,FWInitFunc,
 					   {Skip,Args0}, Skip),
 		{{0,NewResult},Where,[]};
 	    AutoSkip = {auto_skip,_Reason} ->
 		%% special case where a conf case "pretends" to be skipped
 		NewResult =
-		    do_end_tc_call(Mod,Func, {AutoSkip,Args0}, AutoSkip),
+		    do_end_tc_call(Mod,FWInitFunc, {AutoSkip,Args0}, AutoSkip),
 		{{0,NewResult},Where,[]}
 	end,
     exit({Ref,Time,Value,Loc,Opts}).
@@ -963,31 +1007,41 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
 					      SkipType == skipped ->
 		    Line = get_loc(),
 		    Conf = [{tc_status,{skipped,Reason}}|hd(Args)],
-		    NewRes = do_end_tc_call(Mod,Func,
+		    NewRes = do_end_tc_call(Mod,{init_per_testcase,Func},
 					    {Skip,[Conf]}, Skip),
 		    {{0,NewRes},Line,[]};
 		{skip_and_save,Reason,SaveCfg} ->
 		    Line = get_loc(),
 		    Conf = [{tc_status,{skipped,Reason}},
 			    {save_config,SaveCfg}|hd(Args)],
-		    NewRes = do_end_tc_call(Mod,Func, {{skip,Reason},[Conf]},
+		    NewRes = do_end_tc_call(Mod,{init_per_testcase,Func},
+					    {{skip,Reason},[Conf]},
 					    {skip,Reason}),
 		    {{0,NewRes},Line,[]};
 		FailTC = {fail,Reason} ->       % user fails the testcase
 		    EndConf = [{tc_status,{failed,Reason}} | hd(Args)],
 		    fw_error_notify(Mod, Func, EndConf, Reason),
-		    NewRes = do_end_tc_call(Mod,Func,
+		    NewRes = do_end_tc_call(Mod,{init_per_testcase,Func},
 					    {{error,Reason},[EndConf]},
 					    FailTC),
 		    {{0,NewRes},[{Mod,Func}],[]};
 		{ok,NewConf} ->
-		    %% call user callback function if defined
-		    NewConf1 =
-			user_callback(TCCallback, Mod, Func, init, NewConf),
-		    %% save current state in controller loop
-		    set_tc_state(tc, NewConf1),
-		    %% execute the test case
-		    {{T,Return},Loc} = {ts_tc(Mod,Func,[NewConf1]), get_loc()},
+		    IPTCEndRes = do_end_tc_call(Mod,{init_per_testcase,Func},
+						{ok,[NewConf]}, NewConf),
+		    {{T,Return},Loc,NewConf1} =
+			if not is_list(IPTCEndRes) ->
+				%% received skip or fail, not config
+				{{0,IPTCEndRes},undefined,NewConf};
+			   true ->
+				%% call user callback function if defined
+				NewConfUC =
+				    user_callback(TCCallback, Mod, Func,
+						  init, IPTCEndRes),
+				%% save current state in controller loop
+				set_tc_state(tc, NewConfUC),
+				%% execute the test case
+				{ts_tc(Mod,Func,[NewConfUC]),get_loc(),NewConfUC}
+			end,
 		    {EndConf,TSReturn,FWReturn} =
 			case Return of
 			    {E,TCError} when E=='EXIT' ; E==failed ->
@@ -1013,36 +1067,47 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
 		    %% call user callback function if defined
 		    EndConf1 =
 			user_callback(TCCallback, Mod, Func, 'end', EndConf),
+
+		    %% We can't handle fails or skips here
+		    EndConf2 =
+			case do_init_tc_call(Mod,{end_per_testcase,Func},
+					     [EndConf1],{ok,[EndConf1]}) of
+			    {ok,[EPTCInitRes]} when is_list(EPTCInitRes) ->
+				EPTCInitRes;
+			    _ ->
+				EndConf1
+			end,
+
 		    %% update current state in controller loop
-		    {FWReturn1,TSReturn1,EndConf2} =
-			case end_per_testcase(Mod, Func, EndConf1) of
+		    {FWReturn1,TSReturn1,EndConf3} =
+			case end_per_testcase(Mod, Func, EndConf2) of
 			    SaveCfg1={save_config,_} ->
 				{FWReturn,TSReturn,
 				 [SaveCfg1|lists:keydelete(save_config,1,
-							   EndConf1)]};
+							   EndConf2)]};
 			    {fail,ReasonToFail} ->
 				%% user has failed the testcase
-				fw_error_notify(Mod, Func, EndConf1,
+				fw_error_notify(Mod, Func, EndConf2,
 						ReasonToFail),
 				{{error,ReasonToFail},
 				 {failed,ReasonToFail},
-				 EndConf1};
+				 EndConf2};
 			    {failed,{_,end_per_testcase,_}} = Failure when
 				  FWReturn == ok ->
 				%% unexpected termination in end_per_testcase
 				%% report this as the result to the framework
-				{Failure,TSReturn,EndConf1};
+				{Failure,TSReturn,EndConf2};
 			    _ ->
 				%% test case result should be reported to
 				%% framework no matter the status of
 				%% end_per_testcase
-				{FWReturn,TSReturn,EndConf1}
+				{FWReturn,TSReturn,EndConf2}
 			end,
 		    %% clear current state in controller loop
-		    case do_end_tc_call(Mod,Func,
-					{FWReturn1,[EndConf2]}, TSReturn1) of
+		    case do_end_tc_call(Mod,{end_per_testcase,Func},
+					{FWReturn1,[EndConf3]}, TSReturn1) of
 			{failed,Reason} = NewReturn ->
-			    fw_error_notify(Mod,Func,EndConf2, Reason),
+			    fw_error_notify(Mod,Func,EndConf3, Reason),
 			    {{T,NewReturn},[{Mod,Func}],[]};
 			NewReturn ->
 			    {{T,NewReturn},Loc,[]}
@@ -1068,7 +1133,38 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
 	    {{T,Return2},Loc,Opts}
     end.
 
+do_init_tc_call(Mod, Func, Res, Return) ->
+    test_server_sup:framework_call(init_tc,[Mod,Func,Res],Return).
+
+do_end_tc_call(Mod, IPTC={init_per_testcase,Func}, Res, Return) ->
+     case Return of
+	 {NOk,_} when NOk == auto_skip; NOk == fail;
+		      NOk == skip ; NOk == skipped ->
+	     {_,Args} = Res,
+	     IPTCEndRes =
+		 case do_end_tc_call1(Mod, IPTC, Res, Return) of
+		     IPTCEndConfig when is_list(IPTCEndConfig) ->
+			 IPTCEndConfig;
+		     _ ->
+			 Args
+		 end,
+	     EPTCInitRes =
+		 case do_init_tc_call(Mod,{end_per_testcase,Func},
+				      IPTCEndRes,Return) of
+		     {ok,EPTCInitConfig} when is_list(EPTCInitConfig) ->
+			 {Return,EPTCInitConfig};
+		     _ ->
+			 Return
+		 end,
+	     do_end_tc_call1(Mod, {end_per_testcase,Func},
+			     EPTCInitRes, Return);
+	 _Ok ->
+	     do_end_tc_call1(Mod, IPTC, Res, Return)
+     end;
 do_end_tc_call(Mod, Func, Res, Return) ->
+    do_end_tc_call1(Mod, Func, Res, Return).
+
+do_end_tc_call1(Mod, Func, Res, Return) ->
     FwMod = os:getenv("TEST_SERVER_FRAMEWORK"),
     Ref = make_ref(),
     if FwMod == "ct_framework" ; FwMod == "undefined"; FwMod == false ->