ssl: ERL-738 - Correct alert handling with new TLS sender process

With the new TLS sender process, solving ERL-622, TLS ALERTs sent in
the connection state must be encrypted and sent by the TLS sender
process. This to make sure that the correct encryption state is used
to encode the ALERTS. Care must also be taken to ensure a graceful
close down behavior both for normal shutdown and downgrading from TLS
to TCP.

The original TR ERL-738 is verified by cowboy tests, and close down
behavior by our tests. However we alas have not been able to yet
create a minimal test case for the originating problem.

Also it seems it has become less likely that we run in to the TCP
delivery problem, that is the guarantee is only on transport level,
not application level. Keep work around function in ssl_test_lib but
we can have better test as long as we do not get to much wobbling
tests.

3 files changed, 74 insertions, 36 deletions

diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl
index 5ea1924d40..9f876add6c 100644
--- a/lib/ssl/src/ssl_connection.erl
+++ b/lib/ssl/src/ssl_connection.erl
@@ -334,17 +334,12 @@ prf(ConnectionPid, Secret, Label, Seed, WantedLength) ->
 %%====================================================================
 %% Alert and close handling
 %%====================================================================
-handle_own_alert(Alert, Version, StateName, 
+handle_own_alert(Alert, _, StateName,
 		 #state{role = Role,
-                        transport_cb = Transport,
-                        socket = Socket,
                         protocol_cb = Connection,
-                        connection_states = ConnectionStates,
                         ssl_options = SslOpts} = State) ->
     try %% Try to tell the other side
-	{BinMsg, _} =
-            Connection:encode_alert(Alert, Version, ConnectionStates),
-	Connection:send(Transport, Socket, BinMsg)
+        send_alert(Alert, StateName, State)
     catch _:_ ->  %% Can crash if we are in a uninitialized state
 	    ignore
     end,
@@ -1160,24 +1155,20 @@ handle_call({close, {Pid, Timeout}}, From, StateName, State0, Connection) when i
     %% we must recive the close alert from the peer before releasing the 
     %% transport socket.
     {next_state, downgrade, State#state{terminated = true}, [{timeout, Timeout, downgrade}]};
-handle_call({close, _} = Close, From, StateName, State, Connection) ->
+handle_call({close, _} = Close, From, StateName, State, _Connection) ->
     %% Run terminate before returning so that the reuseaddr
     %% inet-option works properly
-    Result = Connection:terminate(Close, StateName, State#state{terminated = true}),
+    Result = terminate(Close, StateName, State),
     stop_and_reply(
       {shutdown, normal},
-      {reply, From, Result}, State);
-handle_call({shutdown, How0}, From, _,
+      {reply, From, Result}, State#state{terminated = true});
+handle_call({shutdown, How0}, From, StateName,
 	    #state{transport_cb = Transport,
-		   negotiated_version = Version,
-		   connection_states = ConnectionStates,
-		   socket = Socket} = State, Connection) ->
+		   socket = Socket} = State, _) ->
     case How0 of
 	How when How == write; How == both ->	    
-	    Alert = ?ALERT_REC(?WARNING, ?CLOSE_NOTIFY),
-	    {BinMsg, _} =
-		Connection:encode_alert(Alert, Version, ConnectionStates),
-	    Connection:send(Transport, Socket, BinMsg);
+            send_alert(?ALERT_REC(?WARNING, ?CLOSE_NOTIFY),
+                       StateName, State);
 	_ ->
 	    ok
     end,
@@ -1343,14 +1334,20 @@ terminate({shutdown, own_alert}, _StateName, #state{
 	_ ->
 	    Connection:close({timeout, ?DEFAULT_TIMEOUT}, Socket, Transport, undefined, undefined)
     end;
+terminate(downgrade = Reason, connection, #state{protocol_cb = Connection,
+                                        transport_cb = Transport, socket = Socket
+                                       } = State) ->
+    handle_trusted_certs_db(State),
+    Connection:close(Reason, Socket, Transport, undefined, undefined);
 terminate(Reason, connection, #state{protocol_cb = Connection,
-				     connection_states = ConnectionStates, 
-				     ssl_options = #ssl_options{padding_check = Check},
-				     transport_cb = Transport, socket = Socket
-				    } = State) ->
+                                                connection_states = ConnectionStates,
+                                                ssl_options = #ssl_options{padding_check = Check},
+                                                transport_cb = Transport, socket = Socket
+                                               } = State) ->
     handle_trusted_certs_db(State),
     Alert = terminate_alert(Reason),
-    ok = Connection:send_alert_in_connection(Alert, State),
+    %% Send the termination ALERT if possible
+    catch (ok = Connection:send_alert_in_connection(Alert, State)),
     Connection:close(Reason, Socket, Transport, ConnectionStates, Check);
 terminate(Reason, _StateName, #state{transport_cb = Transport, protocol_cb = Connection,
 				     socket = Socket 
@@ -1387,6 +1384,11 @@ format_status(terminate, [_, StateName, State]) ->
 %%--------------------------------------------------------------------
 %%% Internal functions
 %%--------------------------------------------------------------------
+send_alert(Alert, connection, #state{protocol_cb = Connection} = State) ->
+     Connection:send_alert_in_connection(Alert, State);
+send_alert(Alert, _, #state{protocol_cb = Connection} = State) ->
+    Connection:send_alert(Alert, State).
+
 connection_info(#state{sni_hostname = SNIHostname, 
 		       session = #session{session_id = SessionId,
                                           cipher_suite = CipherSuite, ecc = ECCCurve},
diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl
index 2fde17a0fd..adb4f6d9ea 100644
--- a/lib/ssl/src/tls_connection.erl
+++ b/lib/ssl/src/tls_connection.erl
@@ -56,7 +56,9 @@
          empty_connection_state/2]).
 
 %% Alert and close handling
--export([send_alert/2, send_alert_in_connection/2, encode_alert/3, close/5, protocol_name/0]).
+-export([send_alert/2, send_alert_in_connection/2,
+         send_sync_alert/2,
+         encode_alert/3, close/5, protocol_name/0]).
 
 %% Data handling
 -export([encode_data/3, passive_receive/2, next_record_if_active/1, 
@@ -346,16 +348,34 @@ encode_alert(#alert{} = Alert, Version, ConnectionStates) ->
 
 send_alert(Alert, #state{negotiated_version = Version,
                          socket = Socket,
-                         protocol_cb = Connection, 
                          transport_cb = Transport,
                          connection_states = ConnectionStates0} = StateData0) ->
     {BinMsg, ConnectionStates} =
-	Connection:encode_alert(Alert, Version, ConnectionStates0),
-    Connection:send(Transport, Socket, BinMsg),
+        encode_alert(Alert, Version, ConnectionStates0),
+    send(Transport, Socket, BinMsg),
     StateData0#state{connection_states = ConnectionStates}.
 
-send_alert_in_connection(Alert, #state{protocol_specific = #{sender := Sender}}) ->
+%% If an ALERT sent in the connection state, should cause the TLS
+%% connection to end, we need to synchronize with the tls_sender
+%% process so that the ALERT if possible (that is the tls_sender process is
+%% not blocked) is sent before the connection process terminates and
+%% thereby closes the transport socket.
+send_alert_in_connection(#alert{level = ?FATAL} = Alert, State) ->
+    send_sync_alert(Alert, State);
+send_alert_in_connection(#alert{description = ?CLOSE_NOTIFY} = Alert, State) ->
+    send_sync_alert(Alert, State);
+send_alert_in_connection(Alert,
+                         #state{protocol_specific = #{sender := Sender}}) ->
     tls_sender:send_alert(Sender, Alert).
+send_sync_alert(Alert, #state{protocol_specific = #{sender := Sender}}= State) ->
+    tls_sender:send_and_ack_alert(Sender, Alert),
+    receive
+        {Sender, ack_alert} ->
+            ok
+    after ?DEFAULT_TIMEOUT ->
+            %% Sender is blocked terminate anyway
+            throw({stop, {shutdown, own_alert}, State})
+    end.
 
 %% User closes or recursive call!
 close({close, Timeout}, Socket, Transport = gen_tcp, _,_) ->
@@ -505,7 +525,9 @@ hello(internal, #client_hello{client_version = ClientVersion} = Hello,
     case tls_handshake:hello(Hello, SslOpts, {Port, Session0, Cache, CacheCb,
 					      ConnectionStates0, Cert, KeyExAlg}, Renegotiation) of
         #alert{} = Alert ->
-            ssl_connection:handle_own_alert(Alert, ClientVersion, hello, State);
+            ssl_connection:handle_own_alert(Alert, ClientVersion, hello,
+                                            State#state{negotiated_version
+                                                        = ClientVersion});
         {Version, {Type, Session},
 	 ConnectionStates, Protocol0, ServerHelloExt, HashSign} ->
 	    Protocol = case Protocol0 of
@@ -528,7 +550,8 @@ hello(internal, #server_hello{} = Hello,
 	     ssl_options = SslOptions} = State) ->
     case tls_handshake:hello(Hello, SslOptions, ConnectionStates0, Renegotiation) of
 	#alert{} = Alert ->
-	    ssl_connection:handle_own_alert(Alert, ReqVersion, hello, State);
+	    ssl_connection:handle_own_alert(Alert, ReqVersion, hello,
+                                            State#state{negotiated_version = ReqVersion});
 	{Version, NewId, ConnectionStates, ProtoExt, Protocol} ->
 	    ssl_connection:handle_session(Hello, 
 					  Version, NewId, ConnectionStates, ProtoExt, Protocol, State)
@@ -636,8 +659,8 @@ callback_mode() ->
     state_functions.
 
 terminate(Reason, StateName, State) ->
-    ensure_sender_terminate(Reason, State),
-    catch ssl_connection:terminate(Reason, StateName, State).
+    catch ssl_connection:terminate(Reason, StateName, State),
+    ensure_sender_terminate(Reason, State).
 
 format_status(Type, Data) ->
     ssl_connection:format_status(Type, Data).
@@ -788,8 +811,8 @@ handle_info({CloseTag, Socket}, StateName,
             %% and then receive the final message.
             next_event(StateName, no_record, State)
     end;
-handle_info({'EXIT', Pid, Reason}, _,
-            #state{protocol_specific = Pid} = State) ->
+handle_info({'EXIT', Sender, Reason}, _,
+            #state{protocol_specific = #{sender := Sender}} = State) ->
     {stop, {shutdown, sender_died, Reason}, State};
 handle_info(Msg, StateName, State) ->
     ssl_connection:StateName(info, Msg, State, ?MODULE).
diff --git a/lib/ssl/src/tls_sender.erl b/lib/ssl/src/tls_sender.erl
index db67d7ddff..ec03000b33 100644
--- a/lib/ssl/src/tls_sender.erl
+++ b/lib/ssl/src/tls_sender.erl
@@ -28,7 +28,8 @@
 -include("ssl_api.hrl").
 
 %% API
--export([start/0, start/1, initialize/2, send_data/2, send_alert/2, renegotiate/1,
+-export([start/0, start/1, initialize/2, send_data/2, send_alert/2,
+         send_and_ack_alert/2, renegotiate/1,
          update_connection_state/3, dist_tls_socket/1, dist_handshake_complete/3]).
 
 %% gen_statem callbacks
@@ -89,13 +90,21 @@ send_data(Pid, AppData) ->
 
 %%--------------------------------------------------------------------
 -spec send_alert(pid(), #alert{}) -> _. 
-%% Description: TLS connection process wants to end an Alert
+%% Description: TLS connection process wants to send an Alert
 %% in the connection state.
 %%--------------------------------------------------------------------
 send_alert(Pid, Alert) ->
     gen_statem:cast(Pid, Alert).
 
 %%--------------------------------------------------------------------
+-spec send_and_ack_alert(pid(), #alert{}) -> ok.
+%% Description: TLS connection process wants to send an Alert
+%% in the connection state and recive an ack.
+%%--------------------------------------------------------------------
+send_and_ack_alert(Pid, Alert) ->
+    gen_statem:cast(Pid, {ack_alert, Alert}).
+
+%%--------------------------------------------------------------------
 -spec renegotiate(pid()) -> {ok, WriteState::map()} | {error, closed}.
 %% Description: So TLS connection process can synchronize the 
 %% encryption state to be used when handshaking.
@@ -207,6 +216,10 @@ connection({call, From}, {dist_handshake_complete, _Node, DHandle}, #data{connec
     process_flag(priority, normal),
     Events = dist_data_events(DHandle, []),
     {next_state, ?FUNCTION_NAME, StateData#data{dist_handle = DHandle}, [{reply, From, ok} | Events]};
+connection(cast, {ack_alert, #alert{} = Alert}, #data{connection_pid = Pid} =StateData0) ->
+    StateData = send_tls_alert(Alert, StateData0),
+    Pid ! {self(), ack_alert},
+    {next_state, ?FUNCTION_NAME, StateData};
 connection(cast, #alert{} = Alert, StateData0) ->
     StateData = send_tls_alert(Alert, StateData0),
     {next_state, ?FUNCTION_NAME, StateData};