ssl: introduce the notion of flights for dtls and tls

The flight concept was introduced by DTLS (RFC 4347) to optimize
the packing of DTLS records into UDP packets. This change
implments the flight concept in the the generic SSL connection
logic and add the queue logic to the TLS and DTLS stack.

The DTLS required resend handling is not implemented yet.

While the flight handling is only required for DTSL, it turns
out that the same mechanism can be usefull to TCP based TLS as
well.
With the current scheme each TLS record will be mapped into a
separate TCP frame. This causes more TCP frames to be generate
that necessary. On fast network this will have no impact, but
reducing the number of frames and thereby the number of
round trips can result in significant speedups on slow and
unreliable networks.

Conflicts:
	lib/ssl/src/tls_connection.erl

4 files changed, 89 insertions, 63 deletions

diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl
index 60a61bc901..b7436ff15a 100644
--- a/lib/ssl/src/dtls_connection.erl
+++ b/lib/ssl/src/dtls_connection.erl
@@ -42,9 +42,8 @@
 -export([next_record/1, next_event/3]).
 
 %% Handshake handling
--export([%%renegotiate/2, 
-	 send_handshake/2, send_change_cipher/2]).
-
+-export([%%renegotiate/2,
+        send_handshake/2, queue_handshake/2, queue_change_cipher/2]).
 
 %% Alert and close handling
 -export([%%send_alert/2, handle_own_alert/4, handle_close_alert/3,
@@ -100,14 +99,37 @@ start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = true},_} = Opts,
 	    Error
     end.
 
-send_handshake(Handshake, #state{negotiated_version = Version,
-				 tls_handshake_history = Hist0,
-				 connection_states = ConnectionStates0} = State0) ->
-    {BinHandshake, ConnectionStates, Hist} =
+send_handshake(Handshake, State) ->
+    send_handshake_flight(queue_handshake(Handshake, State)).
+
+queue_flight_buffer(Msg, #state{negotiated_version = Version,
+				connection_states = #connection_states{
+				  current_write =
+				      #connection_state{epoch = Epoch}},
+				flight_buffer = Flight} = State) ->
+    State#state{flight_buffer = Flight ++ [{Version, Epoch, Msg}]}.
+
+queue_handshake(Handshake, #state{negotiated_version = Version,
+				  tls_handshake_history = Hist0,
+				  connection_states = ConnectionStates0} = State0) ->
+    {Frag, ConnectionStates, Hist} =
 	encode_handshake(Handshake, Version, ConnectionStates0, Hist0),
-    send_flight(BinHandshake, State0#state{connection_states = ConnectionStates,
-					   tls_handshake_history = Hist
-					  }).
+    queue_flight_buffer(Frag, State0#state{connection_states = ConnectionStates,
+					   tls_handshake_history = Hist}).
+
+send_handshake_flight(#state{socket = Socket,
+			     transport_cb = Transport,
+			     flight_buffer = Flight,
+			     connection_states = ConnectionStates0} = State0) ->
+
+    {Encoded, ConnectionStates} =
+	encode_handshake_flight(Flight, ConnectionStates0),
+
+    Transport:send(Socket, Encoded),
+    State0#state{flight_buffer = [], connection_states = ConnectionStates}.
+
+queue_change_cipher(Msg, State) ->
+    queue_flight_buffer(Msg, State).
 
 send_alert(Alert, #state{negotiated_version = Version,
 			 socket = Socket,
@@ -118,15 +140,6 @@ send_alert(Alert, #state{negotiated_version = Version,
     Transport:send(Socket, BinMsg),
     State0#state{connection_states = ConnectionStates}.
 
-send_change_cipher(Msg, #state{connection_states = ConnectionStates0,
-			       socket = Socket,
-			       negotiated_version = Version,
-			       transport_cb = Transport} = State0) ->
-    {BinChangeCipher, ConnectionStates} =
-	encode_change_cipher(Msg, Version, ConnectionStates0),
-    Transport:send(Socket, BinChangeCipher),
-    State0#state{connection_states = ConnectionStates}.
-
 %%====================================================================
 %% tls_connection_sup API
 %%====================================================================
diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl
index 90e0810241..53282998d0 100644
--- a/lib/ssl/src/ssl_connection.erl
+++ b/lib/ssl/src/ssl_connection.erl
@@ -1049,9 +1049,9 @@ format_status(terminate, [_, StateName, State]) ->
 					       srp_params = ?SECRET_PRINTOUT,
 					       srp_keys =  ?SECRET_PRINTOUT,
 					       premaster_secret =  ?SECRET_PRINTOUT,
-					       ssl_options = NewOptions}
+					       ssl_options = NewOptions,
+					       flight_buffer =  ?SECRET_PRINTOUT}
 		       }}]}].
-
 %%--------------------------------------------------------------------
 %%% Internal functions
 %%--------------------------------------------------------------------
@@ -1120,7 +1120,7 @@ resumed_server_hello(#state{session = Session,
 server_hello(ServerHello, State0, Connection) ->
     CipherSuite = ServerHello#server_hello.cipher_suite,
     {KeyAlgorithm, _, _, _} = ssl_cipher:suite_definition(CipherSuite),
-    State = Connection:send_handshake(ServerHello, State0),
+    State = Connection:queue_handshake(ServerHello, State0),
     State#state{key_algorithm = KeyAlgorithm}.
 
 server_hello_done(State, Connection) ->
@@ -1166,7 +1166,7 @@ certify_client(#state{client_certificate_requested = true, role = client,
 		      session = #session{own_certificate = OwnCert}}
 	       = State, Connection) ->
     Certificate = ssl_handshake:certificate(OwnCert, CertDbHandle, CertDbRef, client),
-    Connection:send_handshake(Certificate, State);
+    Connection:queue_handshake(Certificate, State);
 
 certify_client(#state{client_certificate_requested = false} = State, _) ->
     State.
@@ -1182,7 +1182,7 @@ verify_client_cert(#state{client_certificate_requested = true, role = client,
     case ssl_handshake:client_certificate_verify(OwnCert, MasterSecret,
 						 Version, HashSign, PrivateKey, Handshake0) of
         #certificate_verify{} = Verified ->
-           Connection:send_handshake(Verified, State);
+           Connection:queue_handshake(Verified, State);
 	ignore ->
 	    State;
 	#alert{} = Alert ->
@@ -1276,7 +1276,7 @@ certify_server(#state{cert_db = CertDbHandle,
 		      session = #session{own_certificate = OwnCert}} = State, Connection) ->
     case ssl_handshake:certificate(OwnCert, CertDbHandle, CertDbRef, server) of
 	Cert = #certificate{} ->
-	    Connection:send_handshake(Cert, State);
+	    Connection:queue_handshake(Cert, State);
 	Alert = #alert{} ->
 	    throw(Alert)
     end.
@@ -1303,7 +1303,7 @@ key_exchange(#state{role = server, key_algorithm = Algo,
 					       HashSignAlgo, ClientRandom,
 					       ServerRandom,
 					       PrivateKey}),
-    State = Connection:send_handshake(Msg, State0),
+    State = Connection:queue_handshake(Msg, State0),
     State#state{diffie_hellman_keys = DHKeys};
 
 key_exchange(#state{role = server, private_key = Key, key_algorithm = Algo} = State, _)
@@ -1328,7 +1328,7 @@ key_exchange(#state{role = server, key_algorithm = Algo,
 							HashSignAlgo, ClientRandom,
 							ServerRandom,
 							PrivateKey}),
-    State = Connection:send_handshake(Msg, State0),
+    State = Connection:queue_handshake(Msg, State0),
     State#state{diffie_hellman_keys = ECDHKeys};
 
 key_exchange(#state{role = server, key_algorithm = psk,
@@ -1350,7 +1350,7 @@ key_exchange(#state{role = server, key_algorithm = psk,
 						       HashSignAlgo, ClientRandom,
 						       ServerRandom,
 						       PrivateKey}),
-    Connection:send_handshake(Msg, State0);
+    Connection:queue_handshake(Msg, State0);
 
 key_exchange(#state{role = server, key_algorithm = dhe_psk,
 		    ssl_options = #ssl_options{psk_identity = PskIdentityHint},
@@ -1371,7 +1371,7 @@ key_exchange(#state{role = server, key_algorithm = dhe_psk,
 							HashSignAlgo, ClientRandom,
 							ServerRandom,
 							PrivateKey}),
-    State = Connection:send_handshake(Msg, State0),
+    State = Connection:queue_handshake(Msg, State0),
     State#state{diffie_hellman_keys = DHKeys};
 
 key_exchange(#state{role = server, key_algorithm = rsa_psk,
@@ -1393,7 +1393,7 @@ key_exchange(#state{role = server, key_algorithm = rsa_psk,
 					       HashSignAlgo, ClientRandom,
 					       ServerRandom,
 					       PrivateKey}),
-    Connection:send_handshake(Msg, State0);
+    Connection:queue_handshake(Msg, State0);
 
 key_exchange(#state{role = server, key_algorithm = Algo,
 		    ssl_options = #ssl_options{user_lookup_fun = LookupFun},
@@ -1422,7 +1422,7 @@ key_exchange(#state{role = server, key_algorithm = Algo,
 					       HashSignAlgo, ClientRandom,
 					       ServerRandom,
 					       PrivateKey}),
-    State = Connection:send_handshake(Msg, State0),
+    State = Connection:queue_handshake(Msg, State0),
     State#state{srp_params = SrpParams,
 		srp_keys = Keys};
 
@@ -1432,7 +1432,7 @@ key_exchange(#state{role = client,
 		    negotiated_version = Version,
 		    premaster_secret = PremasterSecret} = State0, Connection) ->
     Msg = rsa_key_exchange(Version, PremasterSecret, PublicKeyInfo),
-    Connection:send_handshake(Msg, State0);
+    Connection:queue_handshake(Msg, State0);
 
 key_exchange(#state{role = client,
 		    key_algorithm = Algorithm,
@@ -1443,7 +1443,7 @@ key_exchange(#state{role = client,
        Algorithm == dhe_rsa;
        Algorithm == dh_anon ->
     Msg =  ssl_handshake:key_exchange(client, Version, {dh, DhPubKey}),
-    Connection:send_handshake(Msg, State0);
+    Connection:queue_handshake(Msg, State0);
 
 key_exchange(#state{role = client,
 		    key_algorithm = Algorithm,
@@ -1453,7 +1453,7 @@ key_exchange(#state{role = client,
        Algorithm == ecdh_ecdsa; Algorithm == ecdh_rsa;
        Algorithm == ecdh_anon ->
     Msg = ssl_handshake:key_exchange(client, Version, {ecdh, Keys}),
-    Connection:send_handshake(Msg, State0);
+    Connection:queue_handshake(Msg, State0);
 
 key_exchange(#state{role = client,
 		    ssl_options = SslOpts,
@@ -1461,7 +1461,7 @@ key_exchange(#state{role = client,
 		    negotiated_version = Version} = State0, Connection) ->
     Msg =  ssl_handshake:key_exchange(client, Version, 
 				      {psk, SslOpts#ssl_options.psk_identity}),
-    Connection:send_handshake(Msg, State0);
+    Connection:queue_handshake(Msg, State0);
 
 key_exchange(#state{role = client,
 		    ssl_options = SslOpts,
@@ -1471,7 +1471,7 @@ key_exchange(#state{role = client,
     Msg =  ssl_handshake:key_exchange(client, Version,
 				      {dhe_psk, 
 				       SslOpts#ssl_options.psk_identity, DhPubKey}),
-    Connection:send_handshake(Msg, State0);
+    Connection:queue_handshake(Msg, State0);
 key_exchange(#state{role = client,
 		    ssl_options = SslOpts,
 		    key_algorithm = rsa_psk,
@@ -1481,7 +1481,7 @@ key_exchange(#state{role = client,
 	     = State0, Connection) ->
     Msg = rsa_psk_key_exchange(Version, SslOpts#ssl_options.psk_identity,
 			       PremasterSecret, PublicKeyInfo),
-    Connection:send_handshake(Msg, State0);
+    Connection:queue_handshake(Msg, State0);
 
 key_exchange(#state{role = client,
 		    key_algorithm = Algorithm,
@@ -1492,7 +1492,7 @@ key_exchange(#state{role = client,
        Algorithm == srp_rsa;
        Algorithm == srp_anon ->
     Msg =  ssl_handshake:key_exchange(client, Version, {srp, ClientPubKey}),
-    Connection:send_handshake(Msg, State0).
+    Connection:queue_handshake(Msg, State0).
 
 rsa_key_exchange(Version, PremasterSecret, PublicKeyInfo = {Algorithm, _, _})
   when Algorithm == ?rsaEncryption;
@@ -1539,7 +1539,7 @@ request_client_cert(#state{ssl_options = #ssl_options{verify = verify_peer,
     HashSigns = ssl_handshake:available_signature_algs(SupportedHashSigns, Version, [Version]),
     Msg = ssl_handshake:certificate_request(CipherSuite, CertDbHandle, CertDbRef, 
 					    HashSigns, Version),
-    State = Connection:send_handshake(Msg, State0),
+    State = Connection:queue_handshake(Msg, State0),
     State#state{client_certificate_requested = true};
 
 request_client_cert(#state{ssl_options = #ssl_options{verify = verify_none}} =
@@ -1583,10 +1583,10 @@ next_protocol(#state{expecting_next_protocol_negotiation = false} = State, _) ->
     State;
 next_protocol(#state{negotiated_protocol = NextProtocol} = State0, Connection) ->
     NextProtocolMessage = ssl_handshake:next_protocol(NextProtocol),
-    Connection:send_handshake(NextProtocolMessage, State0).
+    Connection:queue_handshake(NextProtocolMessage, State0).
 
 cipher_protocol(State, Connection) ->
-    Connection:send_change_cipher(#change_cipher_spec{}, State).
+    Connection:queue_change_cipher(#change_cipher_spec{}, State).
 
 finished(#state{role = Role, negotiated_version = Version,
 		session = Session,
diff --git a/lib/ssl/src/ssl_connection.hrl b/lib/ssl/src/ssl_connection.hrl
index 7682cb86ea..4b54943ddf 100644
--- a/lib/ssl/src/ssl_connection.hrl
+++ b/lib/ssl/src/ssl_connection.hrl
@@ -84,7 +84,12 @@
 	  client_ecc,          % {Curves, PointFmt}
 	  tracker              :: pid() | 'undefined', %% Tracker process for listen socket
 	  sni_hostname = undefined,
-	  downgrade
+	  downgrade,
+	  flight_buffer = []   :: list()  %% Buffer of TLS/DTLS records, used during the TLS handshake
+				          %% to when possible pack more than on TLS record into the 
+                                          %% underlaying packet format. Introduced by DTLS - RFC 4347.
+				          %% The mecahnism is also usefull in TLS although we do not
+				          %% need to worry about packet loss in TLS.
 	 }).
 
 -define(DEFAULT_DIFFIE_HELLMAN_PARAMS,
diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl
index eaf2dd002d..9880befa94 100644
--- a/lib/ssl/src/tls_connection.erl
+++ b/lib/ssl/src/tls_connection.erl
@@ -49,8 +49,9 @@
 -export([next_record/1, next_event/3]).
 
 %% Handshake handling
--export([renegotiate/2, send_handshake/2, send_change_cipher/2,
-	reinit_handshake_data/1, handle_sni_extension/2]).
+-export([renegotiate/2, send_handshake/2, 
+	 queue_handshake/2, queue_change_cipher/2,
+	 reinit_handshake_data/1,  handle_sni_extension/2]).
 
 %% Alert and close handling
 -export([send_alert/2, handle_own_alert/4, handle_close_alert/3,
@@ -102,17 +103,32 @@ start_fsm(Role, Host, Port, Socket, {#ssl_options{erl_dist = true},_, Tracker} =
 	    Error
     end.
 
-send_handshake(Handshake, #state{negotiated_version = Version,
-				 socket = Socket,
-				 transport_cb = Transport,
-				 tls_handshake_history = Hist0,
-				 connection_states = ConnectionStates0} = State0) ->
+send_handshake(Handshake, State) ->
+    send_handshake_flight(queue_handshake(Handshake, State)).
+
+queue_handshake(Handshake, #state{negotiated_version = Version,
+				  tls_handshake_history = Hist0,
+				  flight_buffer = Flight0,
+				  connection_states = ConnectionStates0} = State0) ->
     {BinHandshake, ConnectionStates, Hist} =
 	encode_handshake(Handshake, Version, ConnectionStates0, Hist0),
-    Transport:send(Socket, BinHandshake),
     State0#state{connection_states = ConnectionStates,
-		tls_handshake_history = Hist
-	       }.
+		 tls_handshake_history = Hist,
+		 flight_buffer = Flight0 ++ [BinHandshake]}.
+
+send_handshake_flight(#state{socket = Socket,
+			     transport_cb = Transport,
+			     flight_buffer = Flight} = State0) ->
+    Transport:send(Socket, Flight),
+    State0#state{flight_buffer = []}.
+
+queue_change_cipher(Msg, #state{negotiated_version = Version,
+				  flight_buffer = Flight0,
+				  connection_states = ConnectionStates0} = State0) ->
+    {BinChangeCipher, ConnectionStates} =
+	encode_change_cipher(Msg, Version, ConnectionStates0),
+    State0#state{connection_states = ConnectionStates,
+		 flight_buffer = Flight0 ++ [BinChangeCipher]}.
 
 send_alert(Alert, #state{negotiated_version = Version,
 			 socket = Socket,
@@ -123,15 +139,6 @@ send_alert(Alert, #state{negotiated_version = Version,
     Transport:send(Socket, BinMsg),
     State0#state{connection_states = ConnectionStates}.
 
-send_change_cipher(Msg, #state{connection_states = ConnectionStates0,
-			       socket = Socket,
-			       negotiated_version = Version,
-			       transport_cb = Transport} = State0) ->
-    {BinChangeCipher, ConnectionStates} =
-	encode_change_cipher(Msg, Version, ConnectionStates0),
-    Transport:send(Socket, BinChangeCipher),
-    State0#state{connection_states = ConnectionStates}.
-
 reinit_handshake_data(State) ->
     %% premaster_secret, public_key_info and tls_handshake_info 
     %% are only needed during the handshake phase. 
@@ -141,7 +148,7 @@ reinit_handshake_data(State) ->
        public_key_info = undefined,
        tls_handshake_history = ssl_handshake:init_handshake_history()
      }.
-			  
+
 %%====================================================================
 %% tls_connection_sup API
 %%====================================================================
@@ -504,7 +511,8 @@ initial_state(Role, Host, Port, Socket, {SSLOptions, SocketOptions, Tracker}, Us
 	   allow_renegotiate = SSLOptions#ssl_options.client_renegotiation,
 	   start_or_recv_from = undefined,
 	   protocol_cb = ?MODULE,
-	   tracker = Tracker
+	   tracker = Tracker,
+	   flight_buffer = []
 	  }.