Merge branch 'raimo/ssl-dist-bench/OTP-14657'

* origin/raimo/ssl-dist-bench/OTP-14657:
  Write SSL distribution benchmarks
  Polish SSL distribution
  Handle whitebox test message
  Correct distribution doc
  Use SNI when connecting
  Use -ssl_dist_optfile options
  Read in -ssl_dist_optfile to ETS
  Facilitate test certs with common root
  Stop checking DNS name for SNI

6 files changed, 224 insertions, 98 deletions

diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl
index 073cb4009b..e5760e7951 100644
--- a/lib/ssl/src/dtls_connection.erl
+++ b/lib/ssl/src/dtls_connection.erl
@@ -467,7 +467,8 @@ init(Type, Event, State) ->
 error(enter, _, State) ->
     {keep_state, State};     
 error({call, From}, {start, _Timeout}, {Error, State}) ->
-    {stop_and_reply, normal, {reply, From, {error, Error}}, State};
+    ssl_connection:stop_and_reply(
+      normal, {reply, From, {error, Error}}, State);
 error({call, _} = Call, Msg, State) ->
     gen_handshake(?FUNCTION_NAME, Call, Msg, State);
 error(_, _, _) ->
@@ -821,7 +822,7 @@ handle_info({Protocol, _, _, _, Data}, StateName,
 	    next_event(StateName, Record, State);
 	#alert{} = Alert ->
 	    ssl_connection:handle_normal_shutdown(Alert, StateName, State0), 
-	    {stop, {shutdown, own_alert}}
+	    ssl_connection:stop({shutdown, own_alert}, State0)
     end;
 handle_info({CloseTag, Socket}, StateName,
 	    #state{socket = Socket, 
@@ -846,7 +847,7 @@ handle_info({CloseTag, Socket}, StateName,
                     ok
             end,
             ssl_connection:handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State),
-            {stop, {shutdown, transport_closed}};
+            ssl_connection:stop({shutdown, transport_closed}, State);
         true ->
             %% Fixes non-delivery of final DTLS record in {active, once}.
             %% Basically allows the application the opportunity to set {active, once} again
@@ -872,7 +873,7 @@ handle_state_timeout(flight_retransmission_timeout, StateName,
 
 handle_alerts([], Result) ->
     Result;
-handle_alerts(_, {stop,_} = Stop) ->
+handle_alerts(_, {stop, _, _} = Stop) ->
     Stop;
 handle_alerts([Alert | Alerts], {next_state, StateName, State}) ->
      handle_alerts(Alerts, ssl_connection:handle_alert(Alert, StateName, State));
diff --git a/lib/ssl/src/inet_tls_dist.erl b/lib/ssl/src/inet_tls_dist.erl
index 96782dcfc0..8e605bec65 100644
--- a/lib/ssl/src/inet_tls_dist.erl
+++ b/lib/ssl/src/inet_tls_dist.erl
@@ -324,12 +324,13 @@ do_accept(Driver, Kernel, AcceptPid, DistCtrl, MyNode, Allowed, SetupTime) ->
                           timer = Timer,
                           this_flags = 0,
                           allowed = Allowed},
+                    link(DistCtrl),
 		    dist_util:handshake_other_started(trace(HSData));
 		{false,IP} ->
 		    error_logger:error_msg(
                       "** Connection attempt from "
                       "disallowed IP ~w ** ~n", [IP]),
-		    ?shutdown(trace(no_node))
+		    ?shutdown2(no_node, trace({disallowed, IP}))
 	    end
     end.
 
@@ -357,7 +358,11 @@ do_setup(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
 	    ErlEpmd = net_kernel:epmd_module(),
 	    case ErlEpmd:port_please(Name, Ip) of
 		{port, TcpPort, Version} ->
-                    Opts = trace(connect_options(get_ssl_options(client))),
+                    Opts =
+                        trace(
+                          connect_options(
+                            [{server_name_indication, atom_to_list(Node)}
+                             |get_ssl_options(client)])),
 		    dist_util:reset_timer(Timer),
                     case ssl:connect(
                            Address, TcpPort,
@@ -378,21 +383,26 @@ do_setup(Driver, Kernel, Node, Type, MyNode, LongOrShortNames, SetupTime) ->
                                   this_flags = 0,
                                   other_version = Version,
                                   request_type = Type},
+                            link(DistCtrl),
 			    dist_util:handshake_we_started(trace(HSData));
 			Other ->
 			    %% Other Node may have closed since 
 			    %% port_please !
 			    ?shutdown2(
                                Node,
-                               trace({shutdown, {connect_failed, Other}}))
+                               trace(
+                                 {ssl_connect_failed, Ip, TcpPort, Other}))
 		    end;
 		Other ->
 		    ?shutdown2(
                        Node,
-                       trace({shutdown, {port_please_failed, Other}}))
+                       trace(
+                         {port_please_failed, ErlEpmd, Name, Ip, Other}))
 	    end;
 	Other ->
-	    ?shutdown2(Node, trace({shutdown, {getaddr_failed, Other}}))
+	    ?shutdown2(
+               Node,
+               trace({getaddr_failed, Driver, Address, Other}))
     end.
 
 close(Socket) ->
@@ -411,8 +421,9 @@ check_ip(Driver, SslSocket) ->
 	    case get_ifs(SslSocket) of
 		{ok, IFs, IP} ->
 		    check_ip(Driver, IFs, IP);
-		_ ->
-		    ?shutdown(no_node)
+		Other ->
+		    ?shutdown2(
+                       no_node, trace({check_ip_failed, SslSocket, Other}))
 	    end;
 	_ ->
 	    true
@@ -441,23 +452,22 @@ get_ifs(#sslsocket{fd = {gen_tcp, Socket, _}}) ->
 
 %% If Node is illegal terminate the connection setup!!
 splitnode(Driver, Node, LongOrShortNames) ->
-    case split_node(atom_to_list(Node), $@, []) of
-	[Name|Tail] when Tail =/= [] ->
-	    Host = lists:append(Tail),
+    case string:split(atom_to_list(Node), "@") of
+	[Name, Host] when Host =/= [] ->
 	    check_node(Driver, Name, Node, Host, LongOrShortNames);
 	[_] ->
 	    error_logger:error_msg(
               "** Nodename ~p illegal, no '@' character **~n",
               [Node]),
-	    ?shutdown(Node);
+	    ?shutdown2(Node, trace({illegal_node_n@me, Node}));
 	_ ->
 	    error_logger:error_msg(
               "** Nodename ~p illegal **~n", [Node]),
-	    ?shutdown(Node)
+	    ?shutdown2(Node, trace({illegal_node_name, Node}))
     end.
 
 check_node(Driver, Name, Node, Host, LongOrShortNames) ->
-    case split_node(Host, $., []) of
+    case string:split(Host, ".") of
 	[_] when LongOrShortNames == longnames ->
 	    case Driver:parse_address(Host) of
 		{ok, _} ->
@@ -468,35 +478,28 @@ check_node(Driver, Name, Node, Host, LongOrShortNames) ->
                       "fully qualified hostnames **~n"
                       "** Hostname ~s is illegal **~n",
                       [Host]),
-		    ?shutdown(Node)
+		    ?shutdown2(Node, trace({not_longnames, Host}))
 	    end;
-	[_, _ | _] when LongOrShortNames == shortnames ->
+	[_, _] when LongOrShortNames == shortnames ->
 	    error_logger:error_msg(
               "** System NOT running to use "
               "fully qualified hostnames **~n"
               "** Hostname ~s is illegal **~n",
               [Host]),
-	    ?shutdown(Node);
+	    ?shutdown2(Node, trace({not_shortnames, Host}));
 	_ ->
 	    [Name, Host]
     end.
 
 split_node(Node) when is_atom(Node) ->
-    case split_node(atom_to_list(Node), $@, []) of
-        [_, Host] ->
+    case string:split(atom_to_list(Node), "@") of
+        [Name, Host] when Name =/= [], Host =/= [] ->
             Host;
         _ ->
             false
     end;
 split_node(_) ->
     false.
-%%
-split_node([Chr|T], Chr, Ack) -> 
-    [lists:reverse(Ack)|split_node(T, Chr, [])];
-split_node([H|T], Chr, Ack) -> 
-    split_node(T, Chr, [H|Ack]);
-split_node([], _, Ack) -> 
-    [lists:reverse(Ack)].
 
 %% -------------------------------------------------------------------------
 
@@ -524,6 +527,17 @@ nodelay() ->
 
 
 get_ssl_options(Type) ->
+    try ets:lookup(ssl_dist_opts, Type) of
+        [{Type, Opts}] ->
+            [{erl_dist, true} | Opts];
+        _ ->
+            get_ssl_dist_arguments(Type)
+    catch
+        error:badarg ->
+            get_ssl_dist_arguments(Type)
+    end.
+
+get_ssl_dist_arguments(Type) ->
     case init:get_argument(ssl_dist_opt) of
 	{ok, Args} ->
 	    [{erl_dist, true} | ssl_options(Type, lists:append(Args))];
diff --git a/lib/ssl/src/ssl.erl b/lib/ssl/src/ssl.erl
index 4007e44a83..4bff9fdf39 100644
--- a/lib/ssl/src/ssl.erl
+++ b/lib/ssl/src/ssl.erl
@@ -992,17 +992,21 @@ validate_option(next_protocols_advertised, Value) when is_list(Value) ->
     Value;
 validate_option(next_protocols_advertised, undefined) ->
     undefined;
-validate_option(server_name_indication = Opt, Value) when is_list(Value) ->
+validate_option(server_name_indication, Value) when is_list(Value) ->
     %% RFC 6066, Section 3: Currently, the only server names supported are
     %% DNS hostnames
-     case inet_parse:domain(Value) of
-        false -> 
-           throw({error, {options, {{Opt, Value}}}});
-        true -> 
-            Value
-     end;
-validate_option(server_name_indication, undefined = Value) ->
+    %% case inet_parse:domain(Value) of
+    %%     false ->
+    %%         throw({error, {options, {{Opt, Value}}}});
+    %%     true ->
+    %%         Value
+    %% end;
+    %%
+    %% But the definition seems very diffuse, so let all strings through
+    %% and leave it up to public_key to decide...
     Value;
+validate_option(server_name_indication, undefined) ->
+    undefined;
 validate_option(server_name_indication, disable) ->
     disable;
 
diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl
index 5d6428a4e0..b9b463eeb6 100644
--- a/lib/ssl/src/ssl_connection.erl
+++ b/lib/ssl/src/ssl_connection.erl
@@ -49,7 +49,7 @@
 
 %% Alert and close handling
 -export([handle_own_alert/4, handle_alert/3, 
-	 handle_normal_shutdown/3 
+	 handle_normal_shutdown/3, stop/2, stop_and_reply/3
 	]).
 
 %% Data handling
@@ -316,7 +316,7 @@ handle_own_alert(Alert, Version, StateName,
     catch _:_ ->
 	    ok
     end,
-    {stop, {shutdown, own_alert}}.
+    stop({shutdown, own_alert}, State).
 
 handle_normal_shutdown(Alert, _, #state{socket = Socket,
 					transport_cb = Transport,
@@ -340,24 +340,24 @@ handle_alert(#alert{level = ?FATAL} = Alert, StateName,
 		    protocol_cb = Connection,
 		    ssl_options = SslOpts, start_or_recv_from = From, host = Host,
 		    port = Port, session = Session, user_application = {_Mon, Pid},
-		    role = Role, socket_options = Opts, tracker = Tracker}) ->
+		    role = Role, socket_options = Opts, tracker = Tracker} = State) ->
     invalidate_session(Role, Host, Port, Session),
     log_alert(SslOpts#ssl_options.log_alert, Role, Connection:protocol_name(), 
               StateName, Alert#alert{role = opposite_role(Role)}),
     alert_user(Transport, Tracker, Socket, StateName, Opts, Pid, From, Alert, Role, Connection),
-    {stop, normal};
+    stop(normal, State);
 
 handle_alert(#alert{level = ?WARNING, description = ?CLOSE_NOTIFY} = Alert, 
 	     StateName, State) -> 
     handle_normal_shutdown(Alert, StateName, State),
-    {stop, {shutdown, peer_close}};
+    stop({shutdown, peer_close}, State);
 
 handle_alert(#alert{level = ?WARNING, description = ?NO_RENEGOTIATION} = Alert, StateName, 
 	     #state{role = Role, ssl_options = SslOpts, protocol_cb = Connection, renegotiation = {true, internal}} = State) ->
     log_alert(SslOpts#ssl_options.log_alert, Role, 
               Connection:protocol_name(), StateName, Alert#alert{role = opposite_role(Role)}),
     handle_normal_shutdown(Alert, StateName, State),
-    {stop, {shutdown, peer_close}};
+    stop({shutdown, peer_close}, State);
 
 handle_alert(#alert{level = ?WARNING, description = ?NO_RENEGOTIATION} = Alert, StateName, 
 	     #state{role = Role,
@@ -404,7 +404,7 @@ write_application_data(Data0, {FromPid, _} = From,
                 ok when FromPid =:= self() ->
                     hibernate_after(connection, NewState, []);
                 Error when FromPid =:= self() ->
-                    {stop, {shutdown, Error}, NewState};
+                    stop({shutdown, Error}, NewState);
                 ok ->
                     hibernate_after(connection, NewState, [{reply, From, ok}]);
                 Result ->
@@ -446,8 +446,8 @@ read_application_data(Data, #state{user_application = {_Mon, Pid},
                             Connection:next_record_if_active(State);
                         _ -> %% We have more data
                             read_application_data(<<>>, State)
-                    catch _:Reason ->
-                            death_row(State, Reason)
+                    catch error:_ ->
+                            death_row(State, disconnect)
                     end;
                 _ ->
                     SocketOpt =
@@ -479,7 +479,7 @@ read_application_data(Data, #state{user_application = {_Mon, Pid},
 	    Connection:next_record_if_active(State0#state{user_data_buffer = Buffer});
 	{error,_Reason} -> %% Invalid packet in packet mode
 	    deliver_packet_error(Transport, Socket, SOpts, Buffer1, Pid, RecvFrom, Tracker, Connection),
-            stop_normal(State0)
+            stop(normal, State0)
     end.
 %%====================================================================
 %% Help functions for tls|dtls_connection.erl
@@ -581,7 +581,7 @@ init({call, From}, {start, {Opts, EmOpts}, Timeout},
 	init({call, From}, {start, Timeout}, 
 	     State#state{ssl_options = SslOpts, socket_options = new_emulated(EmOpts, SockOpts)}, Connection)
     catch throw:Error ->
-	    {stop_and_reply, normal, {reply, From, {error, Error}}}
+	    stop_and_reply(normal, {reply, From, {error, Error}}, State0)
     end;
 init({call, From}, Msg, State, Connection) ->
     handle_call(Msg, From, ?FUNCTION_NAME, State, Connection);
@@ -966,7 +966,7 @@ connection({call, {FromPid, _} = From}, {application_data, Data},
      catch throw:Error ->
              case self() of
                  FromPid ->
-                     {stop, {shutdown, Error}};
+                     stop({shutdown, Error}, State);
                  _ ->
                      hibernate_after(
                        ?FUNCTION_NAME, State, [{reply, From, Error}])
@@ -1017,8 +1017,8 @@ connection(
                       ProtocolSpecific#{d_handle => DHandle}},
             {Record, NewerState} = Connection:next_record_if_active(NewState),
             Connection:next_event(connection, Record, NewerState, [{reply, From, ok}])
-    catch _:Reason ->
-            death_row(State, Reason)
+    catch error:_ ->
+            death_row(State, disconnect)
     end;
 connection({call, From}, Msg, State, Connection) ->
     handle_call(Msg, From, ?FUNCTION_NAME, State, Connection);
@@ -1030,10 +1030,24 @@ connection(
   _) ->
     eat_msgs(Msg),
     try send_dist_data(?FUNCTION_NAME, State, DHandle, [])
-    catch _:Reason ->
-            death_row(State, Reason)
+    catch error:_ ->
+            death_row(State, disconnect)
     end;
 connection(
+  info, {send, From, Ref, Data},
+  #state{
+     ssl_options = #ssl_options{erl_dist = true},
+     protocol_specific = #{d_handle := _}},
+  _) ->
+    %% This is for testing only!
+    %%
+    %% Needed by some OTP distribution
+    %% test suites...
+    From ! {Ref, ok},
+    {keep_state_and_data,
+     [{next_event, {call, {self(), undefined}},
+       {application_data, iolist_to_binary(Data)}}]};
+connection(
   info, tick = Msg,
   #state{
      ssl_options = #ssl_options{erl_dist = true},
@@ -1058,20 +1072,22 @@ connection(Type, Msg, State, Connection) ->
 %% or the socket may die too
 death_row(
   info, {'DOWN', MonitorRef, _, _, Reason},
-  #state{user_application={MonitorRef,_Pid} = State},
+  #state{user_application={MonitorRef,_Pid}},
   _) ->
-    {stop, {shutdown, Reason}, State};
+    {stop, {shutdown, Reason}};
 death_row(
-  info, {'EXIT', Socket, Reason}, #state{socket = Socket} = State, _) ->
-    {stop, {shutdown, Reason}, State};
+  info, {'EXIT', Socket, Reason}, #state{socket = Socket}, _) ->
+    {stop, {shutdown, Reason}};
 death_row(state_timeout, Reason, _State, _Connection) ->
     {stop, {shutdown,Reason}};
-death_row(_Type, _Msg, State, _Connection) ->
-    {keep_state, State, [postpone]}.
+death_row(_Type, _Msg, _State, _Connection) ->
+    %% Waste all other events
+    keep_state_and_data.
 
 %% State entry function
 death_row(State, Reason) ->
-    {next_state, death_row, State, [{state_timeout, 5000, Reason}]}.
+    {next_state, death_row, State,
+     [{state_timeout, 5000, Reason}]}.
 
 %%--------------------------------------------------------------------
 -spec downgrade(gen_statem:event_type(), term(), 
@@ -1084,10 +1100,10 @@ downgrade(internal, #alert{description = ?CLOSE_NOTIFY},
     tls_socket:setopts(Transport, Socket, [{active, false}, {packet, 0}, {mode, binary}]),
     Transport:controlling_process(Socket, Pid),
     gen_statem:reply(From, {ok, Socket}),
-    stop_normal(State);
+    stop(normal, State);
 downgrade(timeout, downgrade, #state{downgrade = {_, From}} = State, _) ->
     gen_statem:reply(From, {error, timeout}),
-    stop_normal(State);
+    stop(normal, State);
 downgrade(Type, Event, State, Connection) ->
     handle_common_event(Type, Event, ?FUNCTION_NAME, State, Connection).
 
@@ -1102,7 +1118,7 @@ handle_common_event(internal, {handshake, {#hello_request{} = Handshake, _}}, co
 handle_common_event(internal, {handshake, {#hello_request{}, _}}, StateName, #state{role = client}, _) 
   when StateName =/= connection ->
     {keep_state_and_data};
-handle_common_event(internal, {handshake, {Handshake, Raw}}, StateName, 
+handle_common_event(internal, {handshake, {Handshake, Raw}}, StateName,
 		    #state{tls_handshake_history = Hs0,
 			   ssl_options = #ssl_options{v2_hello_compatible = V2HComp}} = State0, 
 		    Connection) ->
@@ -1121,8 +1137,8 @@ handle_common_event(timeout, hibernate, _, _, _) ->
     {keep_state_and_data, [hibernate]};
 handle_common_event(internal, {application_data, Data}, StateName, State0, Connection) ->
     case read_application_data(Data, State0) of
-	{stop, Reason, State} ->
-   	    {stop, Reason, State};
+	{stop, _, _} = Stop->
+            Stop;
 	{Record, State} ->
    	    Connection:next_event(StateName, Record, State)
     end;
@@ -1151,8 +1167,9 @@ handle_call({close, _} = Close, From, StateName, State, Connection) ->
     %% Run terminate before returning so that the reuseaddr
     %% inet-option works properly
     Result = Connection:terminate(Close, StateName, State#state{terminated = true}),
-    {stop_and_reply, {shutdown, normal},  
-     {reply, From, Result}, State};
+    stop_and_reply(
+      {shutdown, normal},
+      {reply, From, Result}, State);
 handle_call({shutdown, How0}, From, _,
 	    #state{transport_cb = Transport,
 		   negotiated_version = Version,
@@ -1173,7 +1190,7 @@ handle_call({shutdown, How0}, From, _,
 	    {keep_state_and_data, [{reply, From, ok}]};
 	Error ->
 	    gen_statem:reply(From, {error, Error}),
-            stop_normal(State)
+            stop(normal, State)
     end;
 handle_call({recv, _N, _Timeout}, From, _,  
 		  #state{socket_options = 
@@ -1253,33 +1270,50 @@ handle_info({ErrorTag, Socket, econnaborted}, StateName,
 		   tracker = Tracker} = State)  when StateName =/= connection ->
     alert_user(Transport, Tracker,Socket, 
 	       StartFrom, ?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), Role, Connection),
-    stop_normal(State);
+    stop(normal, State);
 
 handle_info({ErrorTag, Socket, Reason}, StateName, #state{socket = Socket,
 							  error_tag = ErrorTag} = State)  ->
     Report = io_lib:format("SSL: Socket error: ~p ~n", [Reason]),
     error_logger:error_report(Report),
     handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State),
-    stop_normal(State);
+    stop(normal, State);
 
 handle_info(
+  {'DOWN', MonitorRef, _, _, Reason}, _,
+  #state{
+     user_application = {MonitorRef, _Pid},
+     ssl_options = #ssl_options{erl_dist = true}}) ->
+    {stop, {shutdown, Reason}};
+handle_info(
   {'DOWN', MonitorRef, _, _, _}, _,
-  #state{user_application={MonitorRef,_Pid}} = State) ->
-    stop_normal(State);
+  #state{user_application = {MonitorRef, _Pid}}) ->
+    {stop, normal};
+handle_info(
+  {'EXIT', Pid, _Reason}, StateName,
+  #state{user_application = {_MonitorRef, Pid}} = State) ->
+    %% It seems the user application has linked to us
+    %% - ignore that and let the monitor handle this
+    {next_state, StateName, State};
+
 %%% So that terminate will be run when supervisor issues shutdown
 handle_info({'EXIT', _Sup, shutdown}, _StateName, State) ->
-    {stop, shutdown, State};
+    stop(shutdown, State);
 handle_info({'EXIT', Socket, normal}, _StateName, #state{socket = Socket} = State) ->
     %% Handle as transport close"
-    {stop, {shutdown, transport_closed}, State};
+    stop({shutdown, transport_closed}, State);
 handle_info({'EXIT', Socket, Reason}, _StateName, #state{socket = Socket} = State) ->
-    {stop, {shutdown, Reason}, State};
+    stop({shutdown, Reason}, State);
+
 handle_info(allow_renegotiate, StateName, State) ->
     {next_state, StateName, State#state{allow_renegotiate = true}};
+
 handle_info({cancel_start_or_recv, StartFrom}, StateName,
 	    #state{renegotiation = {false, first}} = State) when StateName =/= connection ->
-    {stop_and_reply, {shutdown, user_timeout},  
-     {reply, StartFrom, {error, timeout}}, State#state{timer = undefined}};
+    stop_and_reply(
+      {shutdown, user_timeout},
+      {reply, StartFrom, {error, timeout}},
+      State#state{timer = undefined});
 handle_info({cancel_start_or_recv, RecvFrom}, StateName, 
 	    #state{start_or_recv_from = RecvFrom} = State) when RecvFrom =/= undefined ->
     {next_state, StateName, State#state{start_or_recv_from = undefined,
@@ -2423,8 +2457,8 @@ handle_active_option(_, connection = StateName0, To, Reply, #state{protocol_cb =
 	    hibernate_after(StateName, State, [{reply, To, Reply}]);
 	{next_state, StateName, State, Actions} -> 
 	    hibernate_after(StateName, State, [{reply, To, Reply} | Actions]);
-	{stop, Reason, State} ->
-	    {stop, Reason, State}
+	{stop, _, _} = Stop ->
+	    Stop
     end;
 handle_active_option(_, StateName, To, Reply, #state{user_data_buffer = <<>>} = State) ->
     %% Active once already set 
@@ -2433,8 +2467,8 @@ handle_active_option(_, StateName, To, Reply, #state{user_data_buffer = <<>>} =
 %% user_data_buffer =/= <<>>
 handle_active_option(_, StateName0, To, Reply, #state{protocol_cb = Connection} = State0) -> 
     case read_application_data(<<>>, State0) of
-	{stop, Reason, State} ->
-	    {stop, Reason, State};
+	{stop, _, _} = Stop ->
+	    Stop;
 	{Record, State1} ->
 	    %% Note: Renogotiation may cause StateName0 =/= StateName
 	    case Connection:next_event(StateName0, Record, State1) of
@@ -2592,7 +2626,8 @@ send_or_reply(_, Pid, _From, Data) ->
     send_user(Pid, Data).
 
 send_user(Pid, Msg) ->
-    Pid ! Msg.
+    Pid ! Msg,
+    ok.
 
 alert_user(Transport, Tracker, Socket, connection, Opts, Pid, From, Alert, Role, Connection) ->
     alert_user(Transport, Tracker, Socket, Opts#socket_options.active, Pid, From, Alert, Role, Connection);
@@ -2705,14 +2740,22 @@ eat_msgs(Msg) ->
     after 0 -> ok
     end.
 
-%% When running with erl_dist the stop reason 'normal'
-%% would be too silent and prevent cleanup
-stop_normal(State) ->
-    Reason =
-        case State of
-            #state{ssl_options = #ssl_options{erl_dist = true}} ->
-                {shutdown, normal};
-            _ ->
-                normal
-        end,
-    {stop, Reason, State}.
+%% When acting as distribution controller map the exit reason
+%% to follow the documented nodedown_reason for net_kernel
+stop(Reason, State) ->
+    {stop, erl_dist_stop_reason(Reason, State), State}.
+
+stop_and_reply(Reason, Replies, State) ->
+    {stop_and_reply, erl_dist_stop_reason(Reason, State), Replies, State}.
+
+erl_dist_stop_reason(
+  Reason, #state{ssl_options = #ssl_options{erl_dist = true}}) ->
+    case Reason of
+        normal ->
+            %% We can not exit with normal since that will not bring
+            %% down the rest of the distribution processes
+            {shutdown, normal};
+        _ -> Reason
+    end;
+erl_dist_stop_reason(Reason, _State) ->
+    Reason.
diff --git a/lib/ssl/src/ssl_dist_sup.erl b/lib/ssl/src/ssl_dist_sup.erl
index c241a9bced..bea67935d8 100644
--- a/lib/ssl/src/ssl_dist_sup.erl
+++ b/lib/ssl/src/ssl_dist_sup.erl
@@ -30,6 +30,9 @@
 %% Supervisor callback
 -export([init/1]).
 
+%% Debug
+-export([consult/1]).
+
 %%%=========================================================================
 %%%  API
 %%%=========================================================================
@@ -37,7 +40,18 @@
 -spec start_link() -> {ok, pid()} | ignore | {error, term()}.
 			
 start_link() ->
-    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+    case init:get_argument(ssl_dist_optfile) of
+        {ok, [File]} ->
+            DistOpts = consult(File),
+            TabOpts = [set, protected, named_table],
+            Tab = ets:new(ssl_dist_opts, TabOpts),
+            true = ets:insert(Tab, DistOpts),
+            supervisor:start_link({local, ?MODULE}, ?MODULE, []);
+        {ok, BadArg} ->
+            error({bad_ssl_dist_optfile, BadArg});
+        error ->
+            supervisor:start_link({local, ?MODULE}, ?MODULE, [])
+    end.
 
 %%%=========================================================================
 %%%  Supervisor callback
@@ -68,3 +82,52 @@ ssl_connection_sup() ->
     Modules = [ssl_connection_sup],
     Type = supervisor,
     {Name, StartFunc, Restart, Shutdown, Type, Modules}.
+
+consult(File) ->
+    case erl_prim_loader:get_file(File) of
+        {ok, Binary, _FullName} ->
+            Encoding =
+                case epp:read_encoding_from_binary(Binary) of
+                    none -> latin1;
+                    Enc -> Enc
+                end,
+            case unicode:characters_to_list(Binary, Encoding) of
+                {error, _String, Rest} ->
+                    error(
+                      {bad_ssl_dist_optfile, {encoding_error, Rest}});
+                {incomplete, _String, Rest} ->
+                    error(
+                      {bad_ssl_dist_optfile, {encoding_incomplete, Rest}});
+                String when is_list(String) ->
+                    consult_string(String)
+            end;
+        error ->
+            error({bad_ssl_dist_optfile, File})
+    end.
+
+consult_string(String) ->
+    case erl_scan:string(String) of
+        {error, Info, Location} ->
+            error({bad_ssl_dist_optfile, {scan_error, Info, Location}});
+        {ok, Tokens, _EndLocation} ->
+            consult_tokens(Tokens)
+    end.
+
+consult_tokens(Tokens) ->
+    case erl_parse:parse_exprs(Tokens) of
+        {error, Info} ->
+            error({bad_ssl_dist_optfile, {parse_error, Info}});
+        {ok, [Expr]} ->
+            consult_expr(Expr);
+        {ok, Other} ->
+            error({bad_ssl_dist_optfile, {parse_error, Other}})
+    end.
+
+consult_expr(Expr) ->
+    {value, Value, Bs} = erl_eval:expr(Expr, erl_eval:new_bindings()),
+    case erl_eval:bindings(Bs) of
+        [] ->
+            Value;
+        Other ->
+            error({bad_ssl_dist_optfile, {bindings, Other}})
+    end.
diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl
index f1db75d5e9..406a095d2e 100644
--- a/lib/ssl/src/tls_connection.erl
+++ b/lib/ssl/src/tls_connection.erl
@@ -437,7 +437,8 @@ init(Type, Event, State) ->
 %%--------------------------------------------------------------------
 
 error({call, From}, {start, _Timeout}, {Error, State}) ->
-    {stop_and_reply, normal, {reply, From, {error, Error}}, State};
+    ssl_connection:stop_and_reply(
+      normal, {reply, From, {error, Error}}, State);
 error({call, _} = Call, Msg, State) ->
     gen_handshake(?FUNCTION_NAME, Call, Msg, State);
 error(_, _, _) ->
@@ -659,7 +660,7 @@ handle_info({Protocol, _, Data}, StateName,
 	    next_event(StateName, Record, State);
 	#alert{} = Alert ->
 	    ssl_connection:handle_normal_shutdown(Alert, StateName, State0), 
-	    {stop, {shutdown, own_alert}}
+	    ssl_connection:stop({shutdown, own_alert}, State0)
     end;
 handle_info({CloseTag, Socket}, StateName,
             #state{socket = Socket, close_tag = CloseTag,
@@ -686,7 +687,7 @@ handle_info({CloseTag, Socket}, StateName,
             end,
 
             ssl_connection:handle_normal_shutdown(?ALERT_REC(?FATAL, ?CLOSE_NOTIFY), StateName, State),
-            {stop, {shutdown, transport_closed}};
+            ssl_connection:stop({shutdown, transport_closed}, State);
         true ->
             %% Fixes non-delivery of final TLS record in {active, once}.
             %% Basically allows the application the opportunity to set {active, once} again
@@ -698,7 +699,7 @@ handle_info(Msg, StateName, State) ->
 
 handle_alerts([], Result) ->
     Result;
-handle_alerts(_, {stop,_} = Stop) ->
+handle_alerts(_, {stop, _, _} = Stop) ->
     Stop;
 handle_alerts([Alert | Alerts], {next_state, StateName, State}) ->
      handle_alerts(Alerts, ssl_connection:handle_alert(Alert, StateName, State));