[common_test] Don't abort test run if connection process crashes

Earlier ct_util_server would terminate and thus abort the complete
test run if a connection process (ct_gen_conn) crashed. This is now
changed so that ct_util will only print an error report (in the test
case log) and continue the rest of the test.

2 files changed, 30 insertions, 5 deletions

diff --git a/lib/common_test/src/ct_util.erl b/lib/common_test/src/ct_util.erl
index 9d6ee3c8b9..66ecb142ca 100644
--- a/lib/common_test/src/ct_util.erl
+++ b/lib/common_test/src/ct_util.erl
@@ -369,11 +369,23 @@ loop(Mode,TestData,StartDir) ->
 	{'EXIT',_Pid,normal} ->
 	    loop(Mode,TestData,StartDir);
 	{'EXIT',Pid,Reason} ->
-	    %% Let process crash in case of error, this shouldn't happen!
-	    io:format("\n\nct_util_server got EXIT from ~p: ~p\n\n",
-		      [Pid,Reason]),
-	    file:set_cwd(StartDir),
-	    exit(Reason)
+	    case ets:lookup(?conn_table,Pid) of
+		[#conn{address=A,callback=CB}] ->
+		    %% A connection crashed - remove the connection but don't die
+		    ct_logs:tc_log_async(ct_error_notify,
+					 "Connection process died: "
+					 "Pid: ~p, Address: ~p, Callback: ~p\n"
+					 "Reason: ~p\n\n",
+					 [Pid,A,CB,Reason]),
+		    catch CB:close(Pid),
+		    loop(Mode,TestData,StartDir);
+		_ ->
+		    %% Let process crash in case of error, this shouldn't happen!
+		    io:format("\n\nct_util_server got EXIT from ~p: ~p\n\n",
+			      [Pid,Reason]),
+		    file:set_cwd(StartDir),
+		    exit(Reason)
+	    end
     end.
 
 
diff --git a/lib/common_test/test/ct_netconfc_SUITE.erl b/lib/common_test/test/ct_netconfc_SUITE.erl
index 8880d8b618..a7df38108f 100644
--- a/lib/common_test/test/ct_netconfc_SUITE.erl
+++ b/lib/common_test/test/ct_netconfc_SUITE.erl
@@ -106,6 +106,7 @@ all() ->
 	     receive_chunked_data,
 	     timeout_receive_chunked_data,
 	     close_while_waiting_for_chunked_data,
+	     connection_crash,
 	     get_event_streams,
 	     create_subscription,
 	     receive_event]
@@ -766,6 +767,18 @@ close_while_waiting_for_chunked_data(Config) ->
     {error,closed} = ct_netconfc:get(Client,{server,[{xmlns,"myns"}],[]},2000),
     ok.
 
+connection_crash(Config) ->
+    DataDir = ?config(data_dir,Config),
+    {ok,Client} = open_success(DataDir),
+
+    %% Test that if the test survives killing the connection
+    %% process. Earlier this caused ct_util_server to terminate, and
+    %% this aborting the complete test run.
+    spawn(fun() -> timer:sleep(500),exit(Client,kill) end),
+    ?NS:expect(get),
+    {error,{closed,killed}}=ct_netconfc:get(Client,{server,[{xmlns,"myns"}],[]}),
+    ok.
+
 get_event_streams(Config) ->
     DataDir = ?config(data_dir,Config),
     {ok,Client} = open_success(DataDir),