Merge branch 'ingela/maint/ssl/close-alert-ERL-738/OTP-15337' into maint-21

* ingela/maint/ssl/close-alert-ERL-738/OTP-15337:
  ssl: ERL-738 - Correct alert handling with new TLS sender process

7 files changed, 152 insertions, 115 deletions

diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl
index 5ea1924d40..9f876add6c 100644
--- a/lib/ssl/src/ssl_connection.erl
+++ b/lib/ssl/src/ssl_connection.erl
@@ -334,17 +334,12 @@ prf(ConnectionPid, Secret, Label, Seed, WantedLength) ->
 %%====================================================================
 %% Alert and close handling
 %%====================================================================
-handle_own_alert(Alert, Version, StateName, 
+handle_own_alert(Alert, _, StateName,
 		 #state{role = Role,
-                        transport_cb = Transport,
-                        socket = Socket,
                         protocol_cb = Connection,
-                        connection_states = ConnectionStates,
                         ssl_options = SslOpts} = State) ->
     try %% Try to tell the other side
-	{BinMsg, _} =
-            Connection:encode_alert(Alert, Version, ConnectionStates),
-	Connection:send(Transport, Socket, BinMsg)
+        send_alert(Alert, StateName, State)
     catch _:_ ->  %% Can crash if we are in a uninitialized state
 	    ignore
     end,
@@ -1160,24 +1155,20 @@ handle_call({close, {Pid, Timeout}}, From, StateName, State0, Connection) when i
     %% we must recive the close alert from the peer before releasing the 
     %% transport socket.
     {next_state, downgrade, State#state{terminated = true}, [{timeout, Timeout, downgrade}]};
-handle_call({close, _} = Close, From, StateName, State, Connection) ->
+handle_call({close, _} = Close, From, StateName, State, _Connection) ->
     %% Run terminate before returning so that the reuseaddr
     %% inet-option works properly
-    Result = Connection:terminate(Close, StateName, State#state{terminated = true}),
+    Result = terminate(Close, StateName, State),
     stop_and_reply(
       {shutdown, normal},
-      {reply, From, Result}, State);
-handle_call({shutdown, How0}, From, _,
+      {reply, From, Result}, State#state{terminated = true});
+handle_call({shutdown, How0}, From, StateName,
 	    #state{transport_cb = Transport,
-		   negotiated_version = Version,
-		   connection_states = ConnectionStates,
-		   socket = Socket} = State, Connection) ->
+		   socket = Socket} = State, _) ->
     case How0 of
 	How when How == write; How == both ->	    
-	    Alert = ?ALERT_REC(?WARNING, ?CLOSE_NOTIFY),
-	    {BinMsg, _} =
-		Connection:encode_alert(Alert, Version, ConnectionStates),
-	    Connection:send(Transport, Socket, BinMsg);
+            send_alert(?ALERT_REC(?WARNING, ?CLOSE_NOTIFY),
+                       StateName, State);
 	_ ->
 	    ok
     end,
@@ -1343,14 +1334,20 @@ terminate({shutdown, own_alert}, _StateName, #state{
 	_ ->
 	    Connection:close({timeout, ?DEFAULT_TIMEOUT}, Socket, Transport, undefined, undefined)
     end;
+terminate(downgrade = Reason, connection, #state{protocol_cb = Connection,
+                                        transport_cb = Transport, socket = Socket
+                                       } = State) ->
+    handle_trusted_certs_db(State),
+    Connection:close(Reason, Socket, Transport, undefined, undefined);
 terminate(Reason, connection, #state{protocol_cb = Connection,
-				     connection_states = ConnectionStates, 
-				     ssl_options = #ssl_options{padding_check = Check},
-				     transport_cb = Transport, socket = Socket
-				    } = State) ->
+                                                connection_states = ConnectionStates,
+                                                ssl_options = #ssl_options{padding_check = Check},
+                                                transport_cb = Transport, socket = Socket
+                                               } = State) ->
     handle_trusted_certs_db(State),
     Alert = terminate_alert(Reason),
-    ok = Connection:send_alert_in_connection(Alert, State),
+    %% Send the termination ALERT if possible
+    catch (ok = Connection:send_alert_in_connection(Alert, State)),
     Connection:close(Reason, Socket, Transport, ConnectionStates, Check);
 terminate(Reason, _StateName, #state{transport_cb = Transport, protocol_cb = Connection,
 				     socket = Socket 
@@ -1387,6 +1384,11 @@ format_status(terminate, [_, StateName, State]) ->
 %%--------------------------------------------------------------------
 %%% Internal functions
 %%--------------------------------------------------------------------
+send_alert(Alert, connection, #state{protocol_cb = Connection} = State) ->
+     Connection:send_alert_in_connection(Alert, State);
+send_alert(Alert, _, #state{protocol_cb = Connection} = State) ->
+    Connection:send_alert(Alert, State).
+
 connection_info(#state{sni_hostname = SNIHostname, 
 		       session = #session{session_id = SessionId,
                                           cipher_suite = CipherSuite, ecc = ECCCurve},
diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl
index 2fde17a0fd..adb4f6d9ea 100644
--- a/lib/ssl/src/tls_connection.erl
+++ b/lib/ssl/src/tls_connection.erl
@@ -56,7 +56,9 @@
          empty_connection_state/2]).
 
 %% Alert and close handling
--export([send_alert/2, send_alert_in_connection/2, encode_alert/3, close/5, protocol_name/0]).
+-export([send_alert/2, send_alert_in_connection/2,
+         send_sync_alert/2,
+         encode_alert/3, close/5, protocol_name/0]).
 
 %% Data handling
 -export([encode_data/3, passive_receive/2, next_record_if_active/1, 
@@ -346,16 +348,34 @@ encode_alert(#alert{} = Alert, Version, ConnectionStates) ->
 
 send_alert(Alert, #state{negotiated_version = Version,
                          socket = Socket,
-                         protocol_cb = Connection, 
                          transport_cb = Transport,
                          connection_states = ConnectionStates0} = StateData0) ->
     {BinMsg, ConnectionStates} =
-	Connection:encode_alert(Alert, Version, ConnectionStates0),
-    Connection:send(Transport, Socket, BinMsg),
+        encode_alert(Alert, Version, ConnectionStates0),
+    send(Transport, Socket, BinMsg),
     StateData0#state{connection_states = ConnectionStates}.
 
-send_alert_in_connection(Alert, #state{protocol_specific = #{sender := Sender}}) ->
+%% If an ALERT sent in the connection state, should cause the TLS
+%% connection to end, we need to synchronize with the tls_sender
+%% process so that the ALERT if possible (that is the tls_sender process is
+%% not blocked) is sent before the connection process terminates and
+%% thereby closes the transport socket.
+send_alert_in_connection(#alert{level = ?FATAL} = Alert, State) ->
+    send_sync_alert(Alert, State);
+send_alert_in_connection(#alert{description = ?CLOSE_NOTIFY} = Alert, State) ->
+    send_sync_alert(Alert, State);
+send_alert_in_connection(Alert,
+                         #state{protocol_specific = #{sender := Sender}}) ->
     tls_sender:send_alert(Sender, Alert).
+send_sync_alert(Alert, #state{protocol_specific = #{sender := Sender}}= State) ->
+    tls_sender:send_and_ack_alert(Sender, Alert),
+    receive
+        {Sender, ack_alert} ->
+            ok
+    after ?DEFAULT_TIMEOUT ->
+            %% Sender is blocked terminate anyway
+            throw({stop, {shutdown, own_alert}, State})
+    end.
 
 %% User closes or recursive call!
 close({close, Timeout}, Socket, Transport = gen_tcp, _,_) ->
@@ -505,7 +525,9 @@ hello(internal, #client_hello{client_version = ClientVersion} = Hello,
     case tls_handshake:hello(Hello, SslOpts, {Port, Session0, Cache, CacheCb,
 					      ConnectionStates0, Cert, KeyExAlg}, Renegotiation) of
         #alert{} = Alert ->
-            ssl_connection:handle_own_alert(Alert, ClientVersion, hello, State);
+            ssl_connection:handle_own_alert(Alert, ClientVersion, hello,
+                                            State#state{negotiated_version
+                                                        = ClientVersion});
         {Version, {Type, Session},
 	 ConnectionStates, Protocol0, ServerHelloExt, HashSign} ->
 	    Protocol = case Protocol0 of
@@ -528,7 +550,8 @@ hello(internal, #server_hello{} = Hello,
 	     ssl_options = SslOptions} = State) ->
     case tls_handshake:hello(Hello, SslOptions, ConnectionStates0, Renegotiation) of
 	#alert{} = Alert ->
-	    ssl_connection:handle_own_alert(Alert, ReqVersion, hello, State);
+	    ssl_connection:handle_own_alert(Alert, ReqVersion, hello,
+                                            State#state{negotiated_version = ReqVersion});
 	{Version, NewId, ConnectionStates, ProtoExt, Protocol} ->
 	    ssl_connection:handle_session(Hello, 
 					  Version, NewId, ConnectionStates, ProtoExt, Protocol, State)
@@ -636,8 +659,8 @@ callback_mode() ->
     state_functions.
 
 terminate(Reason, StateName, State) ->
-    ensure_sender_terminate(Reason, State),
-    catch ssl_connection:terminate(Reason, StateName, State).
+    catch ssl_connection:terminate(Reason, StateName, State),
+    ensure_sender_terminate(Reason, State).
 
 format_status(Type, Data) ->
     ssl_connection:format_status(Type, Data).
@@ -788,8 +811,8 @@ handle_info({CloseTag, Socket}, StateName,
             %% and then receive the final message.
             next_event(StateName, no_record, State)
     end;
-handle_info({'EXIT', Pid, Reason}, _,
-            #state{protocol_specific = Pid} = State) ->
+handle_info({'EXIT', Sender, Reason}, _,
+            #state{protocol_specific = #{sender := Sender}} = State) ->
     {stop, {shutdown, sender_died, Reason}, State};
 handle_info(Msg, StateName, State) ->
     ssl_connection:StateName(info, Msg, State, ?MODULE).
diff --git a/lib/ssl/src/tls_sender.erl b/lib/ssl/src/tls_sender.erl
index db67d7ddff..ec03000b33 100644
--- a/lib/ssl/src/tls_sender.erl
+++ b/lib/ssl/src/tls_sender.erl
@@ -28,7 +28,8 @@
 -include("ssl_api.hrl").
 
 %% API
--export([start/0, start/1, initialize/2, send_data/2, send_alert/2, renegotiate/1,
+-export([start/0, start/1, initialize/2, send_data/2, send_alert/2,
+         send_and_ack_alert/2, renegotiate/1,
          update_connection_state/3, dist_tls_socket/1, dist_handshake_complete/3]).
 
 %% gen_statem callbacks
@@ -89,13 +90,21 @@ send_data(Pid, AppData) ->
 
 %%--------------------------------------------------------------------
 -spec send_alert(pid(), #alert{}) -> _. 
-%% Description: TLS connection process wants to end an Alert
+%% Description: TLS connection process wants to send an Alert
 %% in the connection state.
 %%--------------------------------------------------------------------
 send_alert(Pid, Alert) ->
     gen_statem:cast(Pid, Alert).
 
 %%--------------------------------------------------------------------
+-spec send_and_ack_alert(pid(), #alert{}) -> ok.
+%% Description: TLS connection process wants to send an Alert
+%% in the connection state and recive an ack.
+%%--------------------------------------------------------------------
+send_and_ack_alert(Pid, Alert) ->
+    gen_statem:cast(Pid, {ack_alert, Alert}).
+
+%%--------------------------------------------------------------------
 -spec renegotiate(pid()) -> {ok, WriteState::map()} | {error, closed}.
 %% Description: So TLS connection process can synchronize the 
 %% encryption state to be used when handshaking.
@@ -207,6 +216,10 @@ connection({call, From}, {dist_handshake_complete, _Node, DHandle}, #data{connec
     process_flag(priority, normal),
     Events = dist_data_events(DHandle, []),
     {next_state, ?FUNCTION_NAME, StateData#data{dist_handle = DHandle}, [{reply, From, ok} | Events]};
+connection(cast, {ack_alert, #alert{} = Alert}, #data{connection_pid = Pid} =StateData0) ->
+    StateData = send_tls_alert(Alert, StateData0),
+    Pid ! {self(), ack_alert},
+    {next_state, ?FUNCTION_NAME, StateData};
 connection(cast, #alert{} = Alert, StateData0) ->
     StateData = send_tls_alert(Alert, StateData0),
     {next_state, ?FUNCTION_NAME, StateData};
diff --git a/lib/ssl/test/ssl_alpn_handshake_SUITE.erl b/lib/ssl/test/ssl_alpn_handshake_SUITE.erl
index 27062d4801..04c4b257d9 100644
--- a/lib/ssl/test/ssl_alpn_handshake_SUITE.erl
+++ b/lib/ssl/test/ssl_alpn_handshake_SUITE.erl
@@ -155,7 +155,7 @@ empty_client(Config) when is_list(Config) ->
     run_failing_handshake(Config,
         [{alpn_advertised_protocols, []}],
         [{alpn_preferred_protocols, [<<"spdy/2">>, <<"spdy/3">>, <<"http/2">>]}],
-        {connect_failed,{tls_alert,"no application protocol"}}).
+        {error,{tls_alert,"no application protocol"}}).
 
 %--------------------------------------------------------------------------------
 
@@ -163,7 +163,7 @@ empty_server(Config) when is_list(Config) ->
     run_failing_handshake(Config,
         [{alpn_advertised_protocols, [<<"http/1.0">>, <<"http/1.1">>]}],
         [{alpn_preferred_protocols, []}],
-        {connect_failed,{tls_alert,"no application protocol"}}).
+        {error,{tls_alert,"no application protocol"}}).
 
 %--------------------------------------------------------------------------------
 
@@ -171,7 +171,7 @@ empty_client_empty_server(Config) when is_list(Config) ->
     run_failing_handshake(Config,
         [{alpn_advertised_protocols, []}],
         [{alpn_preferred_protocols, []}],
-        {connect_failed,{tls_alert,"no application protocol"}}).
+        {error,{tls_alert,"no application protocol"}}).
 
 %--------------------------------------------------------------------------------
 
@@ -179,7 +179,7 @@ no_matching_protocol(Config) when is_list(Config) ->
     run_failing_handshake(Config,
         [{alpn_advertised_protocols, [<<"http/1.0">>, <<"http/1.1">>]}],
         [{alpn_preferred_protocols, [<<"spdy/2">>, <<"spdy/3">>, <<"http/2">>]}],
-        {connect_failed,{tls_alert,"no application protocol"}}).
+        {error,{tls_alert,"no application protocol"}}).
 
 %--------------------------------------------------------------------------------
 
@@ -342,18 +342,19 @@ run_failing_handshake(Config, ClientExtraOpts, ServerExtraOpts, ExpectedResult)
     ServerOpts = ServerExtraOpts ++ ssl_test_lib:ssl_options(server_rsa_opts, Config),
 
     {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config),
-    Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0},
+    Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0},
                     {from, self()},
                     {mfa, {?MODULE, placeholder, []}},
                     {options, ServerOpts}]),
 
     Port = ssl_test_lib:inet_port(Server),
-    ExpectedResult
-        = ssl_test_lib:start_client([{node, ClientNode}, {port, Port},
-               {host, Hostname},
-               {from, self()},
-               {mfa, {?MODULE, placeholder, []}},
-               {options, ClientOpts}]).
+    Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port},
+                                           {host, Hostname},
+                                           {from, self()},
+                                           {mfa, {?MODULE, placeholder, []}},
+                                           {options, ClientOpts}]),
+    ssl_test_lib:check_result(Server, ExpectedResult,
+                              Client, ExpectedResult).
 
 run_handshake(Config, ClientExtraOpts, ServerExtraOpts, ExpectedProtocol) ->
     Data = "hello world",
diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl
index cae491b882..4585ea7306 100644
--- a/lib/ssl/test/ssl_basic_SUITE.erl
+++ b/lib/ssl/test/ssl_basic_SUITE.erl
@@ -1183,16 +1183,16 @@ fallback(Config) when is_list(Config) ->
     
     Port = ssl_test_lib:inet_port(Server),
     
-    Client =
-	ssl_test_lib:start_client_error([{node, ClientNode}, 
-					 {port, Port}, {host, Hostname},
-					 {from, self()},  {options, 
-							   [{fallback, true}, 
-							    {versions, ['tlsv1']} 
-							    | ClientOpts]}]),
+    Client = 
+        ssl_test_lib:start_client_error([{node, ClientNode},
+                                         {port, Port}, {host, Hostname},
+                                         {from, self()},  {options,
+                                                           [{fallback, true},
+                                                            {versions, ['tlsv1']}
+                                                            | ClientOpts]}]),
     
-    ssl_test_lib:check_result(Server, {error,{tls_alert,"inappropriate fallback"}}, 
-			      Client, {error,{tls_alert,"inappropriate fallback"}}).
+    ssl_test_lib:check_result(Server, {error,{tls_alert,"inappropriate fallback"}},
+                              Client, {error,{tls_alert,"inappropriate fallback"}}).
 
 %%--------------------------------------------------------------------
 cipher_format() ->
@@ -2645,14 +2645,14 @@ default_reject_anonymous(Config) when is_list(Config) ->
 					      {options, ServerOpts}]),
     Port = ssl_test_lib:inet_port(Server),
     Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port},
-					{host, Hostname},
-			   {from, self()},
-			   {options,
-			    [{ciphers,[CipherSuite]} |
-			     ClientOpts]}]),
+                                              {host, Hostname},
+                                              {from, self()},
+                                              {options,
+                                               [{ciphers,[CipherSuite]} |
+                                                ClientOpts]}]),
 
     ssl_test_lib:check_result(Server, {error, {tls_alert, "insufficient security"}},
-			      Client, {error, {tls_alert, "insufficient security"}}).
+                              Client, {error, {tls_alert, "insufficient security"}}).
 
 %%--------------------------------------------------------------------
 ciphers_ecdsa_signed_certs() ->
@@ -3605,14 +3605,14 @@ no_common_signature_algs(Config) when is_list(Config) ->
 							 | ServerOpts]}]),
     Port = ssl_test_lib:inet_port(Server),
     Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port},
-					      {host, Hostname},
-					      {from, self()}, 
-					      {options, [{signature_algs, [{sha384, rsa}]}
-							 | ClientOpts]}]),
+                                              {host, Hostname},
+                                              {from, self()},
+                                              {options, [{signature_algs, [{sha384, rsa}]}
+                                                         | ClientOpts]}]),
     
     ssl_test_lib:check_result(Server, {error, {tls_alert, "insufficient security"}},
-			      Client, {error, {tls_alert, "insufficient security"}}).
-    						
+                              Client, {error, {tls_alert, "insufficient security"}}).
+
 %%--------------------------------------------------------------------
 
 tls_dont_crash_on_handshake_garbage() ->
diff --git a/lib/ssl/test/ssl_certificate_verify_SUITE.erl b/lib/ssl/test/ssl_certificate_verify_SUITE.erl
index b387feb97a..588ca153a9 100644
--- a/lib/ssl/test/ssl_certificate_verify_SUITE.erl
+++ b/lib/ssl/test/ssl_certificate_verify_SUITE.erl
@@ -620,8 +620,8 @@ cert_expired(Config) when is_list(Config) ->
 					      {from, self()},
 					      {options, [{verify, verify_peer}, {active, Active}  | ClientOpts]}]),    
     
-    tcp_delivery_workaround(Server, {error, {tls_alert, "certificate expired"}},
-			    Client, {error, {tls_alert, "certificate expired"}}).
+    ssl_test_lib:check_result(Server, {error, {tls_alert, "certificate expired"}},
+                              Client, {error, {tls_alert, "certificate expired"}}).
 
 two_digits_str(N) when N < 10 ->
     lists:flatten(io_lib:format("0~p", [N]));
@@ -729,8 +729,8 @@ critical_extension_verify_server(Config) when is_list(Config) ->
     %% This certificate has a critical extension that we don't
     %% understand.  Therefore, verification should fail.      
 
-    tcp_delivery_workaround(Server, {error, {tls_alert, "unsupported certificate"}},
-			    Client, {error, {tls_alert, "unsupported certificate"}}),
+    ssl_test_lib:check_result(Server, {error, {tls_alert, "unsupported certificate"}},
+                              Client, {error, {tls_alert, "unsupported certificate"}}),
     
     ssl_test_lib:close(Server).
 %%--------------------------------------------------------------------
@@ -909,8 +909,8 @@ invalid_signature_server(Config) when is_list(Config) ->
 					      {from, self()},
 					      {options, [{verify, verify_peer} | ClientOpts]}]),
 
-    tcp_delivery_workaround(Server, {error, {tls_alert, "unknown ca"}},
-			    Client, {error, {tls_alert, "unknown ca"}}).
+    ssl_test_lib:check_result(Server, {error, {tls_alert, "unknown ca"}},
+                              Client, {error, {tls_alert, "unknown ca"}}).
 
 %%--------------------------------------------------------------------
 
@@ -946,8 +946,8 @@ invalid_signature_client(Config) when is_list(Config) ->
 					      {from, self()},
 					      {options, NewClientOpts}]),
 
-    tcp_delivery_workaround(Server, {error, {tls_alert, "unknown ca"}},
-			    Client, {error, {tls_alert, "unknown ca"}}).
+    ssl_test_lib:check_result(Server, {error, {tls_alert, "unknown ca"}},
+                              Client, {error, {tls_alert, "unknown ca"}}).
 
 
 %%--------------------------------------------------------------------
@@ -1236,41 +1236,3 @@ incomplete_chain(Config) when is_list(Config) ->
 %% Internal functions ------------------------------------------------
 %%--------------------------------------------------------------------
 
-tcp_delivery_workaround(Server, ServerMsg, Client, ClientMsg) ->
-    receive
-	{Server, ServerMsg} ->
-	    client_msg(Client, ClientMsg);
-	{Client, ClientMsg} ->
-	    server_msg(Server, ServerMsg);
-	{Client, {error,closed}} ->
-	    server_msg(Server, ServerMsg);
-	{Server, {error,closed}} ->
-	    client_msg(Client, ClientMsg)
-    end.
-
-client_msg(Client, ClientMsg) ->
-    receive
-	{Client, ClientMsg} ->
-	    ok;
-	{Client, {error,closed}} ->
-	    ct:log("client got close"),
-	    ok;
-	{Client, {error, Reason}} ->
-	    ct:log("client got econnaborted: ~p", [Reason]),
-	    ok;
-	Unexpected ->
-	    ct:fail(Unexpected)
-    end.
-server_msg(Server, ServerMsg) ->
-    receive
-	{Server, ServerMsg} ->
-	    ok;
-	{Server, {error,closed}} ->
-	    ct:log("server got close"),
-	    ok;
-	{Server, {error, Reason}} ->
-	    ct:log("server got econnaborted: ~p", [Reason]),
-	    ok;
-	Unexpected ->
-	    ct:fail(Unexpected)
-    end.
diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl
index f3235f5614..39a5bcaad6 100644
--- a/lib/ssl/test/ssl_test_lib.erl
+++ b/lib/ssl/test/ssl_test_lib.erl
@@ -1003,7 +1003,6 @@ ecc_test_error(COpts, SOpts, CECCOpts, SECCOpts, Config) ->
     Error = {error, {tls_alert, "insufficient security"}},
     check_result(Server, Error, Client, Error).
 
-
 start_client(openssl, Port, ClientOpts, Config) ->
     Cert = proplists:get_value(certfile, ClientOpts),
     Key = proplists:get_value(keyfile, ClientOpts),
@@ -2061,3 +2060,40 @@ hardcode_dsa_key(3) ->
        y =  48598545580251057979126570873881530215432219542526130654707948736559463436274835406081281466091739849794036308281564299754438126857606949027748889019480936572605967021944405048011118039171039273602705998112739400664375208228641666852589396502386172780433510070337359132965412405544709871654840859752776060358,
        x = 1457508827177594730669011716588605181448418352823}.
 
+tcp_delivery_workaround(Server, ServerMsg, Client, ClientMsg) ->
+    receive
+	{Server, ServerMsg} ->
+	    client_msg(Client, ClientMsg);
+	{Client, ClientMsg} ->
+	    server_msg(Server, ServerMsg);
+	{Client, {error,closed}} ->
+	    server_msg(Server, ServerMsg);
+	{Server, {error,closed}} ->
+	    client_msg(Client, ClientMsg)
+    end.
+client_msg(Client, ClientMsg) ->
+    receive
+	{Client, ClientMsg} ->
+	    ok;
+	{Client, {error,closed}} ->
+	    ct:log("client got close"),
+	    ok;
+	{Client, {error, Reason}} ->
+	    ct:log("client got econnaborted: ~p", [Reason]),
+	    ok;
+	Unexpected ->
+	    ct:fail(Unexpected)
+    end.
+server_msg(Server, ServerMsg) ->
+    receive
+	{Server, ServerMsg} ->
+	    ok;
+	{Server, {error,closed}} ->
+	    ct:log("server got close"),
+	    ok;
+	{Server, {error, Reason}} ->
+	    ct:log("server got econnaborted: ~p", [Reason]),
+	    ok;
+	Unexpected ->
+	    ct:fail(Unexpected)
+    end.