From 827732949ba9cd844347bca1a46b2e2055810e69 Mon Sep 17 00:00:00 2001
From: Siri Hansen <siri@erlang.org>
Date: Thu, 16 Feb 2017 12:16:21 +0100
Subject: [ct] Correctly handle process died in init and end_per_testcase

Documentation says that a test case is skipped if init_per_testcase
fails. This is correct if init_per_testcase fails in a normal way, but
if it gets an exit signal causing the process to die, then the case
was earlier marked as failed with reason {skip,...}. This is now
corrected, so the case is auto skipped.

If end_per_testcase crashes, it does not change the result of a test
case, it only adds a warning in the comment field. But if it gets an
exit signal causing the process to die while running end_per_testcase,
then the case was earlier marked as failed in addition to this
warning. This is now corrected, so the result of the test case is not
changed by an exit signal during end_per_testcase.
---
 lib/common_test/src/test_server_ctrl.erl | 53 ++++++++++++++++++++------------
 1 file changed, 33 insertions(+), 20 deletions(-)

(limited to 'lib/common_test/src')

diff --git a/lib/common_test/src/test_server_ctrl.erl b/lib/common_test/src/test_server_ctrl.erl
index d3a0f74f7a..39c523f8b3 100644
--- a/lib/common_test/src/test_server_ctrl.erl
+++ b/lib/common_test/src/test_server_ctrl.erl
@@ -2939,22 +2939,21 @@ run_test_cases_loop([{Mod,Func,Args}|Cases], Config, TimetrapData, Mode, Status)
 	    exit(framework_error);
 	%% sequential execution of test case finished
 	{Time,RetVal,_} ->
+            RetTag =
+                if is_tuple(RetVal) -> element(1,RetVal);
+                   true -> undefined
+                end,
 	    {Failed,Status1} =
-		case Time of
-		    died ->
-			{true,update_status(failed, Mod, Func, Status)};
-		    _ when is_tuple(RetVal) ->
-			case element(1, RetVal) of
-			    R when R=='EXIT'; R==failed ->
-				{true,update_status(failed, Mod, Func, Status)};
-			    R when R==skip; R==skipped ->
-				{false,update_status(skipped, Mod, Func, Status)};
-			    _ ->
-				{false,update_status(ok, Mod, Func, Status)}
-			end;
-		    _ ->
-			{false,update_status(ok, Mod, Func, Status)}
-		end,
+                case RetTag of
+                    Skip when Skip==skip; Skip==skipped ->
+                        {false,update_status(skipped, Mod, Func, Status)};
+                    Fail when Fail=='EXIT'; Fail==failed ->
+                        {true,update_status(failed, Mod, Func, Status)};
+                    _ when Time==died, RetVal=/=ok ->
+                        {true,update_status(failed, Mod, Func, Status)};
+                    _ ->
+                        {false,update_status(ok, Mod, Func, Status)}
+                end,
 	    case check_prop(sequence, Mode) of
 		false ->
 		    stop_minor_log_file(),
@@ -3809,7 +3808,15 @@ run_test_case1(Ref, Num, Mod, Func, Args, RunInit,
 	    {died,{timetrap_timeout,TimetrapTimeout}} ->
 		progress(failed, Num, Mod, Func, GrName, Loc,
 			 timetrap_timeout, TimetrapTimeout, Comment, Style);
-	    {died,Reason} ->
+	    {died,{Skip,Reason}} when Skip==skip; Skip==skipped ->
+                %% died in init_per_testcase
+		progress(skip, Num, Mod, Func, GrName, Loc, Reason,
+			 Time, Comment, Style);
+	    {died,Reason} when Reason=/=ok ->
+                %% (If Reason==ok it means that process died in
+                %% end_per_testcase after successfully completing the
+                %% test case itself - then we shall not fail, but a
+                %% warning will be issued in the comment field.)
 		progress(failed, Num, Mod, Func, GrName, Loc, Reason,
 			 Time, Comment, Style);
 	    {_,{'EXIT',{Skip,Reason}}} when Skip==skip; Skip==skipped;
@@ -3958,6 +3965,9 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
 	  [get_info_str(Mod,Func, CaseNum, get(test_server_cases))]),
     test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},
 						     {ReportTag,Reason1}}]),
+    TimeStr = io_lib:format(if is_float(Time) -> "~.3fs";
+			       true -> "~w"
+			    end, [Time]),
     ReasonStr = escape_chars(reason_to_string(Reason1)),
     ReasonStr1 = lists:flatten([string:strip(S,left) ||
 				S <- string:tokens(ReasonStr,[$\n])]),
@@ -3972,10 +3982,10 @@ progress(skip, CaseNum, Mod, Func, GrName, Loc, Reason, Time,
 		   _ -> xhtml("<br>(","<br />(") ++ to_string(Comment) ++ ")"
 	       end,
     print(html,
-	  "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>"
+	  "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>"
 	  "<td><font color=\"~ts\">SKIPPED</font></td>"
 	  "<td>~ts~ts</td></tr>\n",
-	  [Time,Color,ReasonStr2,Comment1]),
+	  [TimeStr,Color,ReasonStr2,Comment1]),
     FormatLoc = test_server_sup:format_loc(Loc),
     print(minor, "=== Location: ~ts", [FormatLoc]),
     print(minor, "=== Reason: ~ts", [ReasonStr1]),
@@ -4113,6 +4123,9 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time,
 	 Comment0, {St0,St1}) ->
     print(minor, "successfully completed test case", []),
     test_server_sup:framework_call(report, [tc_done,{Mod,{Func,GrName},ok}]),
+    TimeStr = io_lib:format(if is_float(Time) -> "~.3fs";
+			       true -> "~w"
+			    end, [Time]),
     Comment =
 	case RetVal of
 	    {comment,RetComment} ->
@@ -4131,10 +4144,10 @@ progress(ok, _CaseNum, Mod, Func, GrName, _Loc, RetVal, Time,
 	end,
     print(major, "=elapsed       ~p", [Time]),
     print(html,
-	  "<td>" ++ St0 ++ "~.3fs" ++ St1 ++ "</td>"
+	  "<td>" ++ St0 ++ "~ts" ++ St1 ++ "</td>"
 	  "<td><font color=\"green\">Ok</font></td>"
 	  "~ts</tr>\n",
-	  [Time,Comment]),
+	  [TimeStr,Comment]),
     print(minor,
 	  escape_chars(io_lib:format("=== Returned value: ~tp", [RetVal])),
 	  []),
-- 
cgit v1.2.3