Merge branch 'lukas/common_test/suite_callback/OTP-8851' into dev

* lukas/common_test/suite_callback/OTP-8851: (54 commits)
  Update example cth spec to reflect the implementation
  Cleanup code to fix dialyzer warning
  Rename Suite Callback to Common Test hook in documentation
  Rename Suite Callback to Common Test Hook in code and testcases
  Fix bug where the state of an SCB was altered when no on_tc_* existed
  Update SCBs to use a specific id/1 function for SCb overriding instead of returning it from init/1.
  Add documentation for SCBs
  Update suite callback test timeout so that beam debug test runs do not timeout
  Update test support so that if common test fails to be stopped in a suite, the ct node is restarted and the test case fails
  Add tests for SCB's with same id and fixed bug relating to it
  Add test suites for failing in init/1 function
  Added tests for starting SCB's with arguments and fixed a bug with how SCB's are parsed from the command line
  Add so that failures in SCB:init/1 causes the entire scb scope to fail
  Fix bug which caused scb to be added too much when initiated in the suite info function
  Started work on documenting suite callbacks, this is a partial commit
  Add locking mechanism for scb state when using parallel groups
  Add state update tests for suite callbacks
  Update ct_framework calls to allow manipulation of test results in end_tc without breaking backwards compatability (I hope)
  Add test case for init_per_suite recover scenario
  Add testcase for config updates
  ...

2 files changed, 77 insertions, 57 deletions

diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl
index ee121e5bb6..13ff02cc6c 100644
--- a/lib/test_server/src/test_server.erl
+++ b/lib/test_server/src/test_server.erl
@@ -935,8 +935,7 @@ spawn_fw_call(Mod,{init_per_testcase,Func},Pid,{timetrap_timeout,TVal}=Why,
 	    Skip = {skip,{failed,{Mod,init_per_testcase,Why}}},
 	    %% if init_per_testcase fails, the test case
 	    %% should be skipped
-	    case catch test_server_sup:framework_call(
-			 end_tc,[?pl2a(Mod),Func,{Pid,Skip,[[]]}]) of
+	    case catch do_end_tc_call(Mod,Func,{Pid,Skip,[[]]},Why) of
 		{'EXIT',FwEndTCErr} ->
 		    exit({fw_notify_done,end_tc,FwEndTCErr});
 		_ ->
@@ -955,11 +954,9 @@ spawn_fw_call(Mod,{end_per_testcase,Func},Pid,{timetrap_timeout,TVal}=Why,
 	    Conf = [{tc_status,ok}],
 	    %% if end_per_testcase fails, the test case should be
 	    %% reported successful with a warning printed as comment
-	    case catch test_server_sup:framework_call(end_tc,
-						      [?pl2a(Mod),Func,
-						       {Pid,
-							{failed,{Mod,end_per_testcase,Why}},
-							[Conf]}]) of
+	    case catch do_end_tc_call(Mod,Func,{Pid,
+						{failed,{Mod,end_per_testcase,Why}},
+						[Conf]}, Why) of
 		{'EXIT',FwEndTCErr} ->
 		    exit({fw_notify_done,end_tc,FwEndTCErr});
 		_ ->
@@ -1001,9 +998,7 @@ spawn_fw_call(Mod,Func,Pid,Error,Loc,SendTo,Comment) ->
 			ok
 		end,
 		Conf = [{tc_status,{failed,timetrap_timeout}}],
-		case catch test_server_sup:framework_call(end_tc,
-							  [?pl2a(Mod),Func,
-							   {Pid,Error,[Conf]}]) of
+		case catch do_end_tc_call(Mod,Func,{Pid,Error,[Conf]},Error) of
 		    {'EXIT',FwEndTCErr} ->
 			exit({fw_notify_done,end_tc,FwEndTCErr});
 		    _ ->
@@ -1069,27 +1064,27 @@ run_test_case_eval(Mod, Func, Args0, Name, Ref, RunInit,
 
     {{Time,Value},Loc,Opts} =
 	case test_server_sup:framework_call(init_tc,[?pl2a(Mod),Func,Args0],
-					    {ok,Args0}) of
+					    {ok, Args0}) of
 	    {ok,Args} ->
 		run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback);
 	    Error = {error,_Reason} ->
-		test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func,{Error,Args0}]),
-		{{0,{skip,{failed,Error}}},{Mod,Func},[]};
+		NewResult = do_end_tc_call(Mod,Func,{Error,Args0},
+					   {skip,{failed,Error}}),
+		{{0,NewResult},{Mod,Func},[]};
 	    {fail,Reason} ->
 		[Conf] = Args0,
 		Conf1 = [{tc_status,{failed,Reason}} | Conf],
 		fw_error_notify(Mod, Func, Conf, Reason),
-		test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func,
-						       {{error,Reason},[Conf1]}]),
-		{{0,{failed,Reason}},{Mod,Func},[]};
+		NewResult = do_end_tc_call(Mod,Func, {{error,Reason},[Conf1]},
+					   {fail, Reason}),
+		{{0,NewResult},{Mod,Func},[]};
 	    Skip = {skip,_Reason} ->
-		test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func,{Skip,Args0}]),
-		{{0,Skip},{Mod,Func},[]};
+		NewResult = do_end_tc_call(Mod,Func,{Skip,Args0},Skip),
+		{{0,NewResult},{Mod,Func},[]};
 	    {auto_skip,Reason} ->
-		test_server_sup:framework_call(end_tc,[?pl2a(Mod),
-						       Func,
-						       {{skip,Reason},Args0}]),
-		{{0,{skip,{fw_auto_skip,Reason}}},{Mod,Func},[]}
+		NewResult = do_end_tc_call(Mod, Func, {{skip,Reason},Args0},
+					   {skip, {fw_auto_skip,Reason}}),
+		{{0,NewResult},{Mod,Func},[]}
 	end,
     exit({Ref,Time,Value,Loc,Opts}).
 
@@ -1103,14 +1098,14 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
 		Skip = {skip,Reason} ->
 		    Line = get_loc(),
 		    Conf = [{tc_status,{skipped,Reason}}],
-		    test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func,{Skip,[Conf]}]),
-		    {{0,{skip,Reason}},Line,[]};
+		    NewRes = do_end_tc_call(Mod,Func,{Skip,[Conf]}, Skip),
+		    {{0,NewRes},Line,[]};
 		{skip_and_save,Reason,SaveCfg} ->
 		    Line = get_loc(),
 		    Conf = [{tc_status,{skipped,Reason}},{save_config,SaveCfg}],
-		    test_server_sup:framework_call(end_tc,[?pl2a(Mod),Func,
-							   {{skip,Reason},[Conf]}]),
-		    {{0,{skip,Reason}},Line,[]};
+		    NewRes = do_end_tc_call(Mod, Func, {{skip, Reason}, [Conf]},
+					    {skip, Reason}),
+		    {{0,NewRes},Line,[]};
 		{ok,NewConf} ->
 		    put(test_server_init_or_end_conf,undefined),
 		    %% call user callback function if defined
@@ -1155,13 +1150,12 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
 				{FWReturn,TSReturn,EndConf1}
 			end,
 		    put(test_server_init_or_end_conf,undefined),
-		    case test_server_sup:framework_call(end_tc, [?pl2a(Mod), Func,
-								 {FWReturn1,[EndConf2]}]) of
-			{fail,Reason} ->
-			    fw_error_notify(Mod, Func, EndConf2, Reason),
-			    {{T,{failed,Reason}},{Mod,Func},[]};
-			_ ->
-			    {{T,TSReturn1},Loc,[]}
+		    case do_end_tc_call(Mod, Func, {FWReturn1,[EndConf2]}, TSReturn1) of
+			{failed,Reason} = NewReturn ->
+			    fw_error_notify(Mod,Func,EndConf2, Reason),
+			    {{T,NewReturn},{Mod,Func},[]};
+			NewReturn ->
+			    {{T,NewReturn},Loc,[]}
 		    end
 	    end;
 	skip_init ->
@@ -1179,10 +1173,36 @@ run_test_case_eval1(Mod, Func, Args, Name, RunInit, TCCallback) ->
 	    {{T,Return},Loc} = {ts_tc(Mod, Func, Args2),get_loc()},
 	    %% call user callback function if defined
 	    Return1 = user_callback(TCCallback, Mod, Func, 'end', Return),
-	    {Return2,Opts} = process_return_val([Return1], Mod,Func,Args1, Loc, Return1),
+	    {Return2,Opts} = process_return_val([Return1], Mod, Func,
+						Args1, Loc, Return1),
 	    {{T,Return2},Loc,Opts}
     end.
 
+do_end_tc_call(M,F,Res,Return) ->
+    Ref = make_ref(),
+    case test_server_sup:framework_call(
+	   end_tc, [?pl2a(M),F,Res], Ref) of
+	{fail,FWReason} ->
+	    {failed,FWReason};
+	Ref ->
+	    case test_server_sup:framework_call(
+		   end_tc, [?pl2a(M),F,Res, Return], ok) of
+		{fail,FWReason} ->
+		    {failed,FWReason};
+		ok ->
+		    case Return of
+			{fail,Reason} ->
+			    {failed,Reason};
+			Return ->
+			    Return
+		    end;
+		NewReturn ->
+		    NewReturn
+	    end;
+	_ ->
+	    Return
+    end.
+
 %% the return value is a list and we have to check if it contains
 %% the result of an end conf case or if it's a Config list
 process_return_val([Return], M,F,A, Loc, Final) when is_list(Return) ->
@@ -1197,13 +1217,13 @@ process_return_val([Return], M,F,A, Loc, Final) when is_list(Return) ->
 		   end, Return) of
 	true ->		     % must be return value from end conf case
 	    process_return_val1(Return, M,F,A, Loc, Final, []);
-	false ->	     % must be Config value from init conf case
-	    case test_server_sup:framework_call(end_tc, [?pl2a(M),F,{ok,A}]) of
-		{fail,FWReason} ->
+	false -> % must be Config value from init conf case
+	    case do_end_tc_call(M,F,{ok,A}, Return) of
+		{failed, FWReason} = Failed ->
 		    fw_error_notify(M,F,A, FWReason),
-		    {{failed,FWReason},[]};
-		_ ->
-		    {Return,[]}
+		    {Failed, []};
+		NewReturn ->
+		    {NewReturn, []}
 	    end
     end;
 %% the return value is not a list, so it's the return value from an
@@ -1211,16 +1231,16 @@ process_return_val([Return], M,F,A, Loc, Final) when is_list(Return) ->
 process_return_val(Return, M,F,A, Loc, Final) ->
     process_return_val1(Return, M,F,A, Loc, Final, []).
 
-process_return_val1([Failed={E,TCError}|_], M,F,A=[Args], Loc, _, SaveOpts) when E=='EXIT';
-										 E==failed ->
+process_return_val1([Failed={E,TCError}|_], M,F,A=[Args], Loc, _, SaveOpts)
+  when E=='EXIT';
+       E==failed ->
     fw_error_notify(M,F,A, TCError, mod_loc(Loc)),
-    case test_server_sup:framework_call(end_tc,
-					[?pl2a(M),F,{{error,TCError},
-						     [[{tc_status,{failed,TCError}}|Args]]}]) of
-	{fail,FWReason} ->
+    case do_end_tc_call(M,F,{{error,TCError},
+			     [[{tc_status,{failed,TCError}}|Args]]}, Failed) of
+	{failed,FWReason} ->
 	    {{failed,FWReason},SaveOpts};
-	_ ->
-	    {Failed,SaveOpts}
+	NewReturn ->
+	    {NewReturn,SaveOpts}
     end;
 process_return_val1([SaveCfg={save_config,_}|Opts], M,F,[Args], Loc, Final, SaveOpts) ->
     process_return_val1(Opts, M,F,[[SaveCfg|Args]], Loc, Final, SaveOpts);
@@ -1234,11 +1254,11 @@ process_return_val1([RetVal={Tag,_}|Opts], M,F,A, Loc, _, SaveOpts) when Tag==sk
 process_return_val1([_|Opts], M,F,A, Loc, Final, SaveOpts) ->
     process_return_val1(Opts, M,F,A, Loc, Final, SaveOpts);
 process_return_val1([], M,F,A, _Loc, Final, SaveOpts) ->
-    case test_server_sup:framework_call(end_tc, [?pl2a(M),F,{Final,A}]) of
-	{fail,FWReason} ->
+    case do_end_tc_call(M,F,{Final,A}, Final) of
+	{failed,FWReason} ->
 	    {{failed,FWReason},SaveOpts};
-	_ ->
-	    {Final,lists:reverse(SaveOpts)}
+	NewReturn ->
+	    {NewReturn,lists:reverse(SaveOpts)}
     end.
 
 user_callback(undefined, _, _, _, Args) ->
@@ -1263,7 +1283,7 @@ init_per_testcase(Mod, Func, Args) ->
 	false -> code:load_file(Mod);
 	_ -> ok
     end,
-    %% init_per_testcase defined, returns new configuration
+%% init_per_testcase defined, returns new configuration
     case erlang:function_exported(Mod,init_per_testcase,2) of
 	true ->
 	    case catch my_apply(Mod, init_per_testcase, [Func|Args]) of
@@ -1306,8 +1326,8 @@ init_per_testcase(Mod, Func, Args) ->
 		    {skip,{failed,{Mod,init_per_testcase,Other}}}
 	    end;
 	false ->
-	    %% Optional init_per_testcase not defined
-	    %% keep quiet.
+%% Optional init_per_testcase not defined
+%% keep quiet.
 	    [Config] = Args,
 	    {ok, Config}
     end.
diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl
index 1dc5646184..977a2b1785 100644
--- a/lib/test_server/src/test_server_ctrl.erl
+++ b/lib/test_server/src/test_server_ctrl.erl
@@ -3552,7 +3552,7 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit, Where,
 	    {_,{'EXIT',Reason}} ->
 		progress(failed, Num, Mod, Func, Loc, Reason,
 			 Time, Comment, Style);
-	    {_, {failed, Reason}} ->
+	    {_, {Fail, Reason}} when Fail =:= fail; Fail =:= failed ->
 		progress(failed, Num, Mod, Func, Loc, Reason,
 			 Time, Comment, Style);
 	    {_, {Skip, Reason}} when Skip==skip; Skip==skipped ->