ssl: ERL-738 - Correct alert handling with new TLS sender process

With the new TLS sender process, solving ERL-622, TLS ALERTs sent in
the connection state must be encrypted and sent by the TLS sender
process. This to make sure that the correct encryption state is used
to encode the ALERTS. Care must also be taken to ensure a graceful
close down behavior both for normal shutdown and downgrading from TLS
to TCP.

The original TR ERL-738 is verified by cowboy tests, and close down
behavior by our tests. However we alas have not been able to yet
create a minimal test case for the originating problem.

Also it seems it has become less likely that we run in to the TCP
delivery problem, that is the guarantee is only on transport level,
not application level. Keep work around function in ssl_test_lib but
we can have better test as long as we do not get to much wobbling
tests.

7 files changed, 152 insertions, 115 deletions

diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl
index 5ea1924d40..9f876add6c 100644
--- a/lib/ssl/src/ssl_connection.erl
+++ b/lib/ssl/src/ssl_connection.erl
@@ -334,17 +334,12 @@ prf(ConnectionPid, Secret, Label, Seed, WantedLength) ->
 %%====================================================================
 %% Alert and close handling
 %%====================================================================
-handle_own_alert(Alert, Version, StateName, 
+handle_own_alert(Alert, _, StateName,
 		 #state{role = Role,
-                        transport_cb = Transport,
-                        socket = Socket,
                         protocol_cb = Connection,
-                        connection_states = ConnectionStates,
                         ssl_options = SslOpts} = State) ->
     try %% Try to tell the other side
-	{BinMsg, _} =
-            Connection:encode_alert(Alert, Version, ConnectionStates),
-	Connection:send(Transport, Socket, BinMsg)
+        send_alert(Alert, StateName, State)
     catch _:_ ->  %% Can crash if we are in a uninitialized state
 	    ignore
     end,
@@ -1160,24 +1155,20 @@ handle_call({close, {Pid, Timeout}}, From, StateName, State0, Connection) when i
     %% we must recive the close alert from the peer before releasing the 
     %% transport socket.
     {next_state, downgrade, State#state{terminated = true}, [{timeout, Timeout, downgrade}]};
-handle_call({close, _} = Close, From, StateName, State, Connection) ->
+handle_call({close, _} = Close, From, StateName, State, _Connection) ->
     %% Run terminate before returning so that the reuseaddr
     %% inet-option works properly
-    Result = Connection:terminate(Close, StateName, State#state{terminated = true}),
+    Result = terminate(Close, StateName, State),
     stop_and_reply(
       {shutdown, normal},
-      {reply, From, Result}, State);
-handle_call({shutdown, How0}, From, _,
+      {reply, From, Result}, State#state{terminated = true});
+handle_call({shutdown, How0}, From, StateName,
 	    #state{transport_cb = Transport,
-		   negotiated_version = Version,
-		   connection_states = ConnectionStates,
-		   socket = Socket} = State, Connection) ->
+		   socket = Socket} = State, _) ->
     case How0 of
 	How when How == write; How == both ->	    
-	    Alert = ?ALERT_REC(?WARNING, ?CLOSE_NOTIFY),
-	    {BinMsg, _} =
-		Connection:encode_alert(Alert, Version, ConnectionStates),
-	    Connection:send(Transport, Socket, BinMsg);
+            send_alert(?ALERT_REC(?WARNING, ?CLOSE_NOTIFY),
+                       StateName, State);
 	_ ->
 	    ok
     end,
@@ -1343,14 +1334,20 @@ terminate({shutdown, own_alert}, _StateName, #state{
 	_ ->
 	    Connection:close({timeout, ?DEFAULT_TIMEOUT}, Socket, Transport, undefined, undefined)
     end;
+terminate(downgrade = Reason, connection, #state{protocol_cb = Connection,
+                                        transport_cb = Transport, socket = Socket
+                                       } = State) ->
+    handle_trusted_certs_db(State),
+    Connection:close(Reason, Socket, Transport, undefined, undefined);
 terminate(Reason, connection, #state{protocol_cb = Connection,
-				     connection_states = ConnectionStates, 
-				     ssl_options = #ssl_options{padding_check = Check},
-				     transport_cb = Transport, socket = Socket
-				    } = State) ->
+                                                connection_states = ConnectionStates,
+                                                ssl_options = #ssl_options{padding_check = Check},
+                                                transport_cb = Transport, socket = Socket
+                                               } = State) ->
     handle_trusted_certs_db(State),
     Alert = terminate_alert(Reason),
-    ok = Connection:send_alert_in_connection(Alert, State),
+    %% Send the termination ALERT if possible
+    catch (ok = Connection:send_alert_in_connection(Alert, State)),
     Connection:close(Reason, Socket, Transport, ConnectionStates, Check);
 terminate(Reason, _StateName, #state{transport_cb = Transport, protocol_cb = Connection,
 				     socket = Socket 
@@ -1387,6 +1384,11 @@ format_status(terminate, [_, StateName, State]) ->
 %%--------------------------------------------------------------------
 %%% Internal functions
 %%--------------------------------------------------------------------
+send_alert(Alert, connection, #state{protocol_cb = Connection} = State) ->
+     Connection:send_alert_in_connection(Alert, State);
+send_alert(Alert, _, #state{protocol_cb = Connection} = State) ->
+    Connection:send_alert(Alert, State).
+
 connection_info(#state{sni_hostname = SNIHostname, 
 		       session = #session{session_id = SessionId,
                                           cipher_suite = CipherSuite, ecc = ECCCurve},
diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl
index 2fde17a0fd..adb4f6d9ea 100644
--- a/lib/ssl/src/tls_connection.erl
+++ b/lib/ssl/src/tls_connection.erl
@@ -56,7 +56,9 @@
          empty_connection_state/2]).
 
 %% Alert and close handling
--export([send_alert/2, send_alert_in_connection/2, encode_alert/3, close/5, protocol_name/0]).
+-export([send_alert/2, send_alert_in_connection/2,
+         send_sync_alert/2,
+         encode_alert/3, close/5, protocol_name/0]).
 
 %% Data handling
 -export([encode_data/3, passive_receive/2, next_record_if_active/1, 
@@ -346,16 +348,34 @@ encode_alert(#alert{} = Alert, Version, ConnectionStates) ->
 
 send_alert(Alert, #state{negotiated_version = Version,
                          socket = Socket,
-                         protocol_cb = Connection, 
                          transport_cb = Transport,
                          connection_states = ConnectionStates0} = StateData0) ->
     {BinMsg, ConnectionStates} =
-	Connection:encode_alert(Alert, Version, ConnectionStates0),
-    Connection:send(Transport, Socket, BinMsg),
+        encode_alert(Alert, Version, ConnectionStates0),
+    send(Transport, Socket, BinMsg),
     StateData0#state{connection_states = ConnectionStates}.
 
-send_alert_in_connection(Alert, #state{protocol_specific = #{sender := Sender}}) ->
+%% If an ALERT sent in the connection state, should cause the TLS
+%% connection to end, we need to synchronize with the tls_sender
+%% process so that the ALERT if possible (that is the tls_sender process is
+%% not blocked) is sent before the connection process terminates and
+%% thereby closes the transport socket.
+send_alert_in_connection(#alert{level = ?FATAL} = Alert, State) ->
+    send_sync_alert(Alert, State);
+send_alert_in_connection(#alert{description = ?CLOSE_NOTIFY} = Alert, State) ->
+    send_sync_alert(Alert, State);
+send_alert_in_connection(Alert,
+                         #state{protocol_specific = #{sender := Sender}}) ->
     tls_sender:send_alert(Sender, Alert).
+send_sync_alert(Alert, #state{protocol_specific = #{sender := Sender}}= State) ->
+    tls_sender:send_and_ack_alert(Sender, Alert),
+    receive
+        {Sender, ack_alert} ->
+            ok
+    after ?DEFAULT_TIMEOUT ->
+            %% Sender is blocked terminate anyway
+            throw({stop, {shutdown, own_alert}, State})
+    end.
 
 %% User closes or recursive call!
 close({close, Timeout}, Socket, Transport = gen_tcp, _,_) ->
@@ -505,7 +525,9 @@ hello(internal, #client_hello{client_version = ClientVersion} = Hello,
     case tls_handshake:hello(Hello, SslOpts, {Port, Session0, Cache, CacheCb,
 					      ConnectionStates0, Cert, KeyExAlg}, Renegotiation) of
         #alert{} = Alert ->
-            ssl_connection:handle_own_alert(Alert, ClientVersion, hello, State);
+            ssl_connection:handle_own_alert(Alert, ClientVersion, hello,
+                                            State#state{negotiated_version
+                                                        = ClientVersion});
         {Version, {Type, Session},
 	 ConnectionStates, Protocol0, ServerHelloExt, HashSign} ->
 	    Protocol = case Protocol0 of
@@ -528,7 +550,8 @@ hello(internal, #server_hello{} = Hello,
 	     ssl_options = SslOptions} = State) ->
     case tls_handshake:hello(Hello, SslOptions, ConnectionStates0, Renegotiation) of
 	#alert{} = Alert ->
-	    ssl_connection:handle_own_alert(Alert, ReqVersion, hello, State);
+	    ssl_connection:handle_own_alert(Alert, ReqVersion, hello,
+                                            State#state{negotiated_version = ReqVersion});
 	{Version, NewId, ConnectionStates, ProtoExt, Protocol} ->
 	    ssl_connection:handle_session(Hello, 
 					  Version, NewId, ConnectionStates, ProtoExt, Protocol, State)
@@ -636,8 +659,8 @@ callback_mode() ->
     state_functions.
 
 terminate(Reason, StateName, State) ->
-    ensure_sender_terminate(Reason, State),
-    catch ssl_connection:terminate(Reason, StateName, State).
+    catch ssl_connection:terminate(Reason, StateName, State),
+    ensure_sender_terminate(Reason, State).
 
 format_status(Type, Data) ->
     ssl_connection:format_status(Type, Data).
@@ -788,8 +811,8 @@ handle_info({CloseTag, Socket}, StateName,
             %% and then receive the final message.
             next_event(StateName, no_record, State)
     end;
-handle_info({'EXIT', Pid, Reason}, _,
-            #state{protocol_specific = Pid} = State) ->
+handle_info({'EXIT', Sender, Reason}, _,
+            #state{protocol_specific = #{sender := Sender}} = State) ->
     {stop, {shutdown, sender_died, Reason}, State};
 handle_info(Msg, StateName, State) ->
     ssl_connection:StateName(info, Msg, State, ?MODULE).
diff --git a/lib/ssl/src/tls_sender.erl b/lib/ssl/src/tls_sender.erl
index db67d7ddff..ec03000b33 100644
--- a/lib/ssl/src/tls_sender.erl
+++ b/lib/ssl/src/tls_sender.erl
@@ -28,7 +28,8 @@
 -include("ssl_api.hrl").
 
 %% API
--export([start/0, start/1, initialize/2, send_data/2, send_alert/2, renegotiate/1,
+-export([start/0, start/1, initialize/2, send_data/2, send_alert/2,
+         send_and_ack_alert/2, renegotiate/1,
          update_connection_state/3, dist_tls_socket/1, dist_handshake_complete/3]).
 
 %% gen_statem callbacks
@@ -89,13 +90,21 @@ send_data(Pid, AppData) ->
 
 %%--------------------------------------------------------------------
 -spec send_alert(pid(), #alert{}) -> _. 
-%% Description: TLS connection process wants to end an Alert
+%% Description: TLS connection process wants to send an Alert
 %% in the connection state.
 %%--------------------------------------------------------------------
 send_alert(Pid, Alert) ->
     gen_statem:cast(Pid, Alert).
 
 %%--------------------------------------------------------------------
+-spec send_and_ack_alert(pid(), #alert{}) -> ok.
+%% Description: TLS connection process wants to send an Alert
+%% in the connection state and recive an ack.
+%%--------------------------------------------------------------------
+send_and_ack_alert(Pid, Alert) ->
+    gen_statem:cast(Pid, {ack_alert, Alert}).
+
+%%--------------------------------------------------------------------
 -spec renegotiate(pid()) -> {ok, WriteState::map()} | {error, closed}.
 %% Description: So TLS connection process can synchronize the 
 %% encryption state to be used when handshaking.
@@ -207,6 +216,10 @@ connection({call, From}, {dist_handshake_complete, _Node, DHandle}, #data{connec
     process_flag(priority, normal),
     Events = dist_data_events(DHandle, []),
     {next_state, ?FUNCTION_NAME, StateData#data{dist_handle = DHandle}, [{reply, From, ok} | Events]};
+connection(cast, {ack_alert, #alert{} = Alert}, #data{connection_pid = Pid} =StateData0) ->
+    StateData = send_tls_alert(Alert, StateData0),
+    Pid ! {self(), ack_alert},
+    {next_state, ?FUNCTION_NAME, StateData};
 connection(cast, #alert{} = Alert, StateData0) ->
     StateData = send_tls_alert(Alert, StateData0),
     {next_state, ?FUNCTION_NAME, StateData};
diff --git a/lib/ssl/test/ssl_alpn_handshake_SUITE.erl b/lib/ssl/test/ssl_alpn_handshake_SUITE.erl
index 27062d4801..04c4b257d9 100644
--- a/lib/ssl/test/ssl_alpn_handshake_SUITE.erl
+++ b/lib/ssl/test/ssl_alpn_handshake_SUITE.erl
@@ -155,7 +155,7 @@ empty_client(Config) when is_list(Config) ->
     run_failing_handshake(Config,
         [{alpn_advertised_protocols, []}],
         [{alpn_preferred_protocols, [<<"spdy/2">>, <<"spdy/3">>, <<"http/2">>]}],
-        {connect_failed,{tls_alert,"no application protocol"}}).
+        {error,{tls_alert,"no application protocol"}}).
 
 %--------------------------------------------------------------------------------
 
@@ -163,7 +163,7 @@ empty_server(Config) when is_list(Config) ->
     run_failing_handshake(Config,
         [{alpn_advertised_protocols, [<<"http/1.0">>, <<"http/1.1">>]}],
         [{alpn_preferred_protocols, []}],
-        {connect_failed,{tls_alert,"no application protocol"}}).
+        {error,{tls_alert,"no application protocol"}}).
 
 %--------------------------------------------------------------------------------
 
@@ -171,7 +171,7 @@ empty_client_empty_server(Config) when is_list(Config) ->
     run_failing_handshake(Config,
         [{alpn_advertised_protocols, []}],
         [{alpn_preferred_protocols, []}],
-        {connect_failed,{tls_alert,"no application protocol"}}).
+        {error,{tls_alert,"no application protocol"}}).
 
 %--------------------------------------------------------------------------------
 
@@ -179,7 +179,7 @@ no_matching_protocol(Config) when is_list(Config) ->
     run_failing_handshake(Config,
         [{alpn_advertised_protocols, [<<"http/1.0">>, <<"http/1.1">>]}],
         [{alpn_preferred_protocols, [<<"spdy/2">>, <<"spdy/3">>, <<"http/2">>]}],
-        {connect_failed,{tls_alert,"no application protocol"}}).
+        {error,{tls_alert,"no application protocol"}}).
 
 %--------------------------------------------------------------------------------
 
@@ -342,18 +342,19 @@ run_failing_handshake(Config, ClientExtraOpts, ServerExtraOpts, ExpectedResult)
     ServerOpts = ServerExtraOpts ++ ssl_test_lib:ssl_options(server_rsa_opts, Config),
 
     {ClientNode, ServerNode, Hostname} = ssl_test_lib:run_where(Config),
-    Server = ssl_test_lib:start_server([{node, ServerNode}, {port, 0},
+    Server = ssl_test_lib:start_server_error([{node, ServerNode}, {port, 0},
                     {from, self()},
                     {mfa, {?MODULE, placeholder, []}},
                     {options, ServerOpts}]),
 
     Port = ssl_test_lib:inet_port(Server),
-    ExpectedResult
-        = ssl_test_lib:start_client([{node, ClientNode}, {port, Port},
-               {host, Hostname},
-               {from, self()},
-               {mfa, {?MODULE, placeholder, []}},
-               {options, ClientOpts}]).
+    Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port},
+                                           {host, Hostname},
+                                           {from, self()},
+                                           {mfa, {?MODULE, placeholder, []}},
+                                           {options, ClientOpts}]),
+    ssl_test_lib:check_result(Server, ExpectedResult,
+                              Client, ExpectedResult).
 
 run_handshake(Config, ClientExtraOpts, ServerExtraOpts, ExpectedProtocol) ->
     Data = "hello world",
diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl
index cae491b882..4585ea7306 100644
--- a/lib/ssl/test/ssl_basic_SUITE.erl
+++ b/lib/ssl/test/ssl_basic_SUITE.erl
@@ -1183,16 +1183,16 @@ fallback(Config) when is_list(Config) ->
     
     Port = ssl_test_lib:inet_port(Server),
     
-    Client =
-	ssl_test_lib:start_client_error([{node, ClientNode}, 
-					 {port, Port}, {host, Hostname},
-					 {from, self()},  {options, 
-							   [{fallback, true}, 
-							    {versions, ['tlsv1']} 
-							    | ClientOpts]}]),
+    Client = 
+        ssl_test_lib:start_client_error([{node, ClientNode},
+                                         {port, Port}, {host, Hostname},
+                                         {from, self()},  {options,
+                                                           [{fallback, true},
+                                                            {versions, ['tlsv1']}
+                                                            | ClientOpts]}]),
     
-    ssl_test_lib:check_result(Server, {error,{tls_alert,"inappropriate fallback"}}, 
-			      Client, {error,{tls_alert,"inappropriate fallback"}}).
+    ssl_test_lib:check_result(Server, {error,{tls_alert,"inappropriate fallback"}},
+                              Client, {error,{tls_alert,"inappropriate fallback"}}).
 
 %%--------------------------------------------------------------------
 cipher_format() ->
@@ -2645,14 +2645,14 @@ default_reject_anonymous(Config) when is_list(Config) ->
 					      {options, ServerOpts}]),
     Port = ssl_test_lib:inet_port(Server),
     Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port},
-					{host, Hostname},
-			   {from, self()},
-			   {options,
-			    [{ciphers,[CipherSuite]} |
-			     ClientOpts]}]),
+                                              {host, Hostname},
+                                              {from, self()},
+                                              {options,
+                                               [{ciphers,[CipherSuite]} |
+                                                ClientOpts]}]),
 
     ssl_test_lib:check_result(Server, {error, {tls_alert, "insufficient security"}},
-			      Client, {error, {tls_alert, "insufficient security"}}).
+                              Client, {error, {tls_alert, "insufficient security"}}).
 
 %%--------------------------------------------------------------------
 ciphers_ecdsa_signed_certs() ->
@@ -3605,14 +3605,14 @@ no_common_signature_algs(Config) when is_list(Config) ->
 							 | ServerOpts]}]),
     Port = ssl_test_lib:inet_port(Server),
     Client = ssl_test_lib:start_client_error([{node, ClientNode}, {port, Port},
-					      {host, Hostname},
-					      {from, self()}, 
-					      {options, [{signature_algs, [{sha384, rsa}]}
-							 | ClientOpts]}]),
+                                              {host, Hostname},
+                                              {from, self()},
+                                              {options, [{signature_algs, [{sha384, rsa}]}
+                                                         | ClientOpts]}]),
     
     ssl_test_lib:check_result(Server, {error, {tls_alert, "insufficient security"}},
-			      Client, {error, {tls_alert, "insufficient security"}}).
-    						
+                              Client, {error, {tls_alert, "insufficient security"}}).
+
 %%--------------------------------------------------------------------
 
 tls_dont_crash_on_handshake_garbage() ->
diff --git a/lib/ssl/test/ssl_certificate_verify_SUITE.erl b/lib/ssl/test/ssl_certificate_verify_SUITE.erl
index b387feb97a..588ca153a9 100644
--- a/lib/ssl/test/ssl_certificate_verify_SUITE.erl
+++ b/lib/ssl/test/ssl_certificate_verify_SUITE.erl
@@ -620,8 +620,8 @@ cert_expired(Config) when is_list(Config) ->
 					      {from, self()},
 					      {options, [{verify, verify_peer}, {active, Active}  | ClientOpts]}]),    
     
-    tcp_delivery_workaround(Server, {error, {tls_alert, "certificate expired"}},
-			    Client, {error, {tls_alert, "certificate expired"}}).
+    ssl_test_lib:check_result(Server, {error, {tls_alert, "certificate expired"}},
+                              Client, {error, {tls_alert, "certificate expired"}}).
 
 two_digits_str(N) when N < 10 ->
     lists:flatten(io_lib:format("0~p", [N]));
@@ -729,8 +729,8 @@ critical_extension_verify_server(Config) when is_list(Config) ->
     %% This certificate has a critical extension that we don't
     %% understand.  Therefore, verification should fail.      
 
-    tcp_delivery_workaround(Server, {error, {tls_alert, "unsupported certificate"}},
-			    Client, {error, {tls_alert, "unsupported certificate"}}),
+    ssl_test_lib:check_result(Server, {error, {tls_alert, "unsupported certificate"}},
+                              Client, {error, {tls_alert, "unsupported certificate"}}),
     
     ssl_test_lib:close(Server).
 %%--------------------------------------------------------------------
@@ -909,8 +909,8 @@ invalid_signature_server(Config) when is_list(Config) ->
 					      {from, self()},
 					      {options, [{verify, verify_peer} | ClientOpts]}]),
 
-    tcp_delivery_workaround(Server, {error, {tls_alert, "unknown ca"}},
-			    Client, {error, {tls_alert, "unknown ca"}}).
+    ssl_test_lib:check_result(Server, {error, {tls_alert, "unknown ca"}},
+                              Client, {error, {tls_alert, "unknown ca"}}).
 
 %%--------------------------------------------------------------------
 
@@ -946,8 +946,8 @@ invalid_signature_client(Config) when is_list(Config) ->
 					      {from, self()},
 					      {options, NewClientOpts}]),
 
-    tcp_delivery_workaround(Server, {error, {tls_alert, "unknown ca"}},
-			    Client, {error, {tls_alert, "unknown ca"}}).
+    ssl_test_lib:check_result(Server, {error, {tls_alert, "unknown ca"}},
+                              Client, {error, {tls_alert, "unknown ca"}}).
 
 
 %%--------------------------------------------------------------------
@@ -1236,41 +1236,3 @@ incomplete_chain(Config) when is_list(Config) ->
 %% Internal functions ------------------------------------------------
 %%--------------------------------------------------------------------
 
-tcp_delivery_workaround(Server, ServerMsg, Client, ClientMsg) ->
-    receive
-	{Server, ServerMsg} ->
-	    client_msg(Client, ClientMsg);
-	{Client, ClientMsg} ->
-	    server_msg(Server, ServerMsg);
-	{Client, {error,closed}} ->
-	    server_msg(Server, ServerMsg);
-	{Server, {error,closed}} ->
-	    client_msg(Client, ClientMsg)
-    end.
-
-client_msg(Client, ClientMsg) ->
-    receive
-	{Client, ClientMsg} ->
-	    ok;
-	{Client, {error,closed}} ->
-	    ct:log("client got close"),
-	    ok;
-	{Client, {error, Reason}} ->
-	    ct:log("client got econnaborted: ~p", [Reason]),
-	    ok;
-	Unexpected ->
-	    ct:fail(Unexpected)
-    end.
-server_msg(Server, ServerMsg) ->
-    receive
-	{Server, ServerMsg} ->
-	    ok;
-	{Server, {error,closed}} ->
-	    ct:log("server got close"),
-	    ok;
-	{Server, {error, Reason}} ->
-	    ct:log("server got econnaborted: ~p", [Reason]),
-	    ok;
-	Unexpected ->
-	    ct:fail(Unexpected)
-    end.
diff --git a/lib/ssl/test/ssl_test_lib.erl b/lib/ssl/test/ssl_test_lib.erl
index f3235f5614..39a5bcaad6 100644
--- a/lib/ssl/test/ssl_test_lib.erl
+++ b/lib/ssl/test/ssl_test_lib.erl
@@ -1003,7 +1003,6 @@ ecc_test_error(COpts, SOpts, CECCOpts, SECCOpts, Config) ->
     Error = {error, {tls_alert, "insufficient security"}},
     check_result(Server, Error, Client, Error).
 
-
 start_client(openssl, Port, ClientOpts, Config) ->
     Cert = proplists:get_value(certfile, ClientOpts),
     Key = proplists:get_value(keyfile, ClientOpts),
@@ -2061,3 +2060,40 @@ hardcode_dsa_key(3) ->
        y =  48598545580251057979126570873881530215432219542526130654707948736559463436274835406081281466091739849794036308281564299754438126857606949027748889019480936572605967021944405048011118039171039273602705998112739400664375208228641666852589396502386172780433510070337359132965412405544709871654840859752776060358,
        x = 1457508827177594730669011716588605181448418352823}.
 
+tcp_delivery_workaround(Server, ServerMsg, Client, ClientMsg) ->
+    receive
+	{Server, ServerMsg} ->
+	    client_msg(Client, ClientMsg);
+	{Client, ClientMsg} ->
+	    server_msg(Server, ServerMsg);
+	{Client, {error,closed}} ->
+	    server_msg(Server, ServerMsg);
+	{Server, {error,closed}} ->
+	    client_msg(Client, ClientMsg)
+    end.
+client_msg(Client, ClientMsg) ->
+    receive
+	{Client, ClientMsg} ->
+	    ok;
+	{Client, {error,closed}} ->
+	    ct:log("client got close"),
+	    ok;
+	{Client, {error, Reason}} ->
+	    ct:log("client got econnaborted: ~p", [Reason]),
+	    ok;
+	Unexpected ->
+	    ct:fail(Unexpected)
+    end.
+server_msg(Server, ServerMsg) ->
+    receive
+	{Server, ServerMsg} ->
+	    ok;
+	{Server, {error,closed}} ->
+	    ct:log("server got close"),
+	    ok;
+	{Server, {error, Reason}} ->
+	    ct:log("server got econnaborted: ~p", [Reason]),
+	    ok;
+	Unexpected ->
+	    ct:fail(Unexpected)
+    end.