Merge remote branch 'origin/peppe/common_test/exit_status' into maint

* origin/peppe/common_test/exit_status:
  Implement support for returning valid exit status from ct_run

OTP-9865
OTP-10087

3 files changed, 42 insertions, 14 deletions

diff --git a/lib/common_test/test/ct_misc_1_SUITE.erl b/lib/common_test/test/ct_misc_1_SUITE.erl
index cb17af9ab5..d2318de445 100644
--- a/lib/common_test/test/ct_misc_1_SUITE.erl
+++ b/lib/common_test/test/ct_misc_1_SUITE.erl
@@ -106,7 +106,7 @@ beam_me_up(Config) when is_list(Config) ->
 
     {Opts,ERPid} = setup([{suite,Suites},{auto_compile,false}], Config),
 
-    ok = ct_test_support:run(ct, run_test, [Opts], Config),
+    {_Ok,_Fail,_Skip} = ct_test_support:run(ct, run_test, [Opts], Config),
     Events = ct_test_support:get_events(ERPid, Config),
 
     ct_test_support:log_events(beam_me_up,
diff --git a/lib/common_test/test/ct_test_support.erl b/lib/common_test/test/ct_test_support.erl
index 02246b5763..80cca4a1cc 100644
--- a/lib/common_test/test/ct_test_support.erl
+++ b/lib/common_test/test/ct_test_support.erl
@@ -35,6 +35,8 @@
 	 verify_events/3, reformat/2, log_events/4,
 	 join_abs_dirs/2]).
 
+-export([ct_test_halt/1]).
+
 -include_lib("kernel/include/file.hrl").
 
 %%%-----------------------------------------------------------------
@@ -229,8 +231,9 @@ run(Opts, Config) when is_list(Opts) ->
     %% use ct interface
     test_server:format(Level, "~n[RUN #1] Calling ct:run_test(~p) on ~p~n",
 		       [Opts, CTNode]),
-    Result1 = rpc:call(CTNode, ct, run_test, [Opts]),
-
+    CtRunTestResult = rpc:call(CTNode, ct, run_test, [Opts]),
+    test_server:format(Level, "~n[RUN #1] Got return value ~p~n",
+		       [CtRunTestResult]),
     case rpc:call(CTNode, erlang, whereis, [ct_util_server]) of
 	undefined ->
 	    ok;
@@ -242,17 +245,36 @@ run(Opts, Config) when is_list(Opts) ->
 	    undefined = rpc:call(CTNode, erlang, whereis, [ct_util_server])
     end,
     %% use run_test interface (simulated)
-    test_server:format(Level, "Saving start opts on ~p: ~p~n", [CTNode,Opts]),
-    rpc:call(CTNode, application, set_env, [common_test, run_test_start_opts, Opts]),
-    test_server:format(Level, "[RUN #2] Calling ct_run:script_start() on ~p~n", [CTNode]),
-    Result2 = rpc:call(CTNode, ct_run, script_start, []),
-    case {Result1,Result2} of
-	{ok,ok} ->
+    Opts1 = [{halt_with,{?MODULE,ct_test_halt}} | Opts],
+    test_server:format(Level, "Saving start opts on ~p: ~p~n",
+		       [CTNode, Opts1]),
+    rpc:call(CTNode, application, set_env,
+	     [common_test, run_test_start_opts, Opts1]),
+    test_server:format(Level, "[RUN #2] Calling ct_run:script_start() on ~p~n",
+		       [CTNode]),
+    ExitStatus = rpc:call(CTNode, ct_run, script_start, []),
+    test_server:format(Level, "[RUN #2] Got exit status value ~p~n",
+		       [ExitStatus]),
+    case {CtRunTestResult,ExitStatus} of
+	{{_Ok,Failed,{_UserSkipped,_AutoSkipped}},1} when Failed > 0 ->
+	    ok;
+	{{_Ok,0,{_UserSkipped,AutoSkipped}},ExitStatus} when AutoSkipped > 0 ->
+	    case proplists:get_value(exit_status, Opts1) of
+		ignore_config when ExitStatus == 1 ->
+		    {error,{wrong_exit_status,ExitStatus}};
+		_ ->
+		    ok
+	    end;
+	{{error,_}=Error,ExitStatus} ->
+	    if ExitStatus /= 2 ->
+		    {error,{wrong_exit_status,ExitStatus}};
+	       ExitStatus == 2 ->
+		    Error
+	    end;
+	{{_Ok,0,{_UserSkipped,_AutoSkipped}},0} ->
 	    ok;
-	{E,_} when E =/= ok ->
-	    E;
-	{_,E} when E =/= ok ->
-	    E
+	Unexpected ->
+	    {error,{unexpected_return_value,Unexpected}}
     end.
 
 run(M, F, A, Config) ->
@@ -272,6 +294,10 @@ run({M,F,A}, InitCalls, Config) ->
 		       [M, F, A, CTNode]),
     rpc:call(CTNode, M, F, A).
 
+%% this is the last function that ct_run:script_start() calls, so the
+%% return value here is what rpc:call/4 above returns
+ct_test_halt(ExitStatus) ->
+    ExitStatus.	    
 
 %%%-----------------------------------------------------------------
 %%% wait_for_ct_stop/1
diff --git a/lib/common_test/test/ct_testspec_1_SUITE.erl b/lib/common_test/test/ct_testspec_1_SUITE.erl
index b6dcf63fdf..693e8c6567 100644
--- a/lib/common_test/test/ct_testspec_1_SUITE.erl
+++ b/lib/common_test/test/ct_testspec_1_SUITE.erl
@@ -621,7 +621,9 @@ setup_and_execute(TCName, TestSpec, Config) ->
     ok = ct_test_support:run(Opts, Config),
     TestSpec1 = [{logdir,proplists:get_value(logdir,Opts)},
 		 {label,proplists:get_value(label,TestTerms)} | TestSpec],
-    ok = ct_test_support:run(ct, run_testspec, [TestSpec1], Config),
+    {_Ok,_Failed,{_USkipped,_ASkipped}} = 
+	ct_test_support:run(ct, run_testspec, [TestSpec1], Config),
+
     Events = ct_test_support:get_events(ERPid, Config),
 
     ct_test_support:log_events(TCName,