ssl: Improve error handling

When doing ssl:controlling_process on a ssl socket that has not
performed the TLS/DTLS handshake that call will succeed even though
the documentation stated otherwise. However if some other ssl option
was incorrect the call would hang. Now {error, closed} will be
returned in the latter case, which is logical independent on if it
should succeed or not in the former case. The former case will continue
to succeed, as it is not dependent of the TLS/DTLS connection being
established, and the documentation is altered slightly to not
explicitly disallow it. If the TLS/DTLS connection later fails and
the socket mode is active, the new controlling process will be
notified as expected.

3 files changed, 12 insertions, 14 deletions

diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl
index 53b46542e7..bf3ff3a9a7 100644
--- a/lib/ssl/src/dtls_connection.erl
+++ b/lib/ssl/src/dtls_connection.erl
@@ -91,13 +91,14 @@ start_link(Role, Host, Port, Socket, Options, User, CbInfo) ->
 
 init([Role, Host, Port, Socket, Options,  User, CbInfo]) ->
     process_flag(trap_exit, true),
-    State0 =  initial_state(Role, Host, Port, Socket, Options, User, CbInfo),
+    State0 = #state{protocol_specific = Map} = initial_state(Role, Host, Port, Socket, Options, User, CbInfo),
     try
 	State = ssl_connection:ssl_config(State0#state.ssl_options, Role, State0),
 	gen_statem:enter_loop(?MODULE, [], init, State)
     catch
 	throw:Error ->
-	    gen_statem:enter_loop(?MODULE, [], error, {Error,State0})
+            EState = State0#state{protocol_specific = Map#{error => Error}},
+	    gen_statem:enter_loop(?MODULE, [], error, EState)
     end.
 %%====================================================================
 %% State transition handling
@@ -470,7 +471,8 @@ init(Type, Event, State) ->
 %%--------------------------------------------------------------------
 error(enter, _, State) ->
     {keep_state, State};     
-error({call, From}, {start, _Timeout}, {Error, State}) ->
+error({call, From}, {start, _Timeout}, 
+      #state{protocol_specific = #{error := Error}} = State) ->
     ssl_connection:stop_and_reply(
       normal, {reply, From, {error, Error}}, State);
 error({call, _} = Call, Msg, State) ->
diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl
index c5f75894cd..e2adf35440 100644
--- a/lib/ssl/src/ssl_connection.erl
+++ b/lib/ssl/src/ssl_connection.erl
@@ -634,8 +634,8 @@ init(_Type, _Event, _State, _Connection) ->
             tls_connection | dtls_connection) ->
 		   gen_statem:state_function_result().
 %%--------------------------------------------------------------------
-error({call, From}, Msg, State, Connection) ->
-    handle_call(Msg, From, ?FUNCTION_NAME, State, Connection).
+error({call, From}, _Msg, State, _Connection) ->
+    {next_state, ?FUNCTION_NAME, State, [{reply, From, {error, closed}}]}.
 
 %%--------------------------------------------------------------------
 -spec hello(gen_statem:event_type(),
diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl
index a3002830d1..4d1122f804 100644
--- a/lib/ssl/src/tls_connection.erl
+++ b/lib/ssl/src/tls_connection.erl
@@ -111,12 +111,13 @@ start_link(Role, Host, Port, Socket, Options, User, CbInfo) ->
 
 init([Role, Host, Port, Socket, Options,  User, CbInfo]) ->
     process_flag(trap_exit, true),
-    State0 = initial_state(Role, Host, Port, Socket, Options, User, CbInfo),
+    State0 = #state{protocol_specific = Map} = initial_state(Role, Host, Port, Socket, Options, User, CbInfo),
     try 
 	State = ssl_connection:ssl_config(State0#state.ssl_options, Role, State0),
 	gen_statem:enter_loop(?MODULE, [], init, State)
     catch throw:Error ->
-	gen_statem:enter_loop(?MODULE, [], error, {Error, State0}) 
+            EState = State0#state{protocol_specific = Map#{error => Error}},
+            gen_statem:enter_loop(?MODULE, [], error, EState) 
     end.
 %%====================================================================
 %% State transition handling
@@ -432,17 +433,12 @@ init(Type, Event, State) ->
 	   {start, timeout()} | term(), #state{}) ->
 		   gen_statem:state_function_result().
 %%--------------------------------------------------------------------
-
-error({call, From}, {start, _Timeout}, {Error, State}) ->
-    ssl_connection:stop_and_reply(
-      normal, {reply, From, {error, Error}}, State);
 error({call, From}, {start, _Timeout}, 
       #state{protocol_specific = #{error := Error}} = State) ->
     ssl_connection:stop_and_reply(
       normal, {reply, From, {error, Error}}, State);
-error({call, _} = Call, Msg, {Error, #state{protocol_specific = Map} = State}) ->
-    gen_handshake(?FUNCTION_NAME, Call, Msg, 
-                  State#state{protocol_specific = Map#{error => Error}});
+error({call, _} = Call, Msg, State) ->
+    gen_handshake(?FUNCTION_NAME, Call, Msg, State);
 error(_, _, _) ->
      {keep_state_and_data, [postpone]}.