ssl: Improve error handling

When doing ssl:controlling_process on a ssl socket that has not
performed the TLS/DTLS handshake that call will succeed even though
the documentation stated otherwise. However if some other ssl option
was incorrect the call would hang. Now {error, closed} will be
returned in the latter case, which is logical independent on if it
should succeed or not in the former case. The former case will continue
to succeed, as it is not dependent of the TLS/DTLS connection being
established, and the documentation is altered slightly to not
explicitly disallow it. If the TLS/DTLS connection later fails and
the socket mode is active, the new controlling process will be
notified as expected.

5 files changed, 48 insertions, 18 deletions

diff --git a/lib/ssl/doc/src/ssl.xml b/lib/ssl/doc/src/ssl.xml
index 6e124c3513..437510b54d 100644
--- a/lib/ssl/doc/src/ssl.xml
+++ b/lib/ssl/doc/src/ssl.xml
@@ -1516,9 +1516,9 @@ fun(srp, Username :: string(), UserState :: term()) ->
 	to complete handshaking, that is,
 	establishing the SSL/TLS/DTLS connection.</p>
         <warning>
-          <p>The socket returned can only be used with
-	  <seealso marker="#handshake-2"> handshake/[2,3]</seealso>.
-	  No traffic can be sent or received before that call.</p>
+          <p>Most API functions require that the TLS/DTLS 
+          connection is established to work as expected.
+	  </p>
         </warning>
         <p>The accepted socket inherits the options set for
 	<c>ListenSocket</c> in
diff --git a/lib/ssl/src/dtls_connection.erl b/lib/ssl/src/dtls_connection.erl
index 53b46542e7..bf3ff3a9a7 100644
--- a/lib/ssl/src/dtls_connection.erl
+++ b/lib/ssl/src/dtls_connection.erl
@@ -91,13 +91,14 @@ start_link(Role, Host, Port, Socket, Options, User, CbInfo) ->
 
 init([Role, Host, Port, Socket, Options,  User, CbInfo]) ->
     process_flag(trap_exit, true),
-    State0 =  initial_state(Role, Host, Port, Socket, Options, User, CbInfo),
+    State0 = #state{protocol_specific = Map} = initial_state(Role, Host, Port, Socket, Options, User, CbInfo),
     try
 	State = ssl_connection:ssl_config(State0#state.ssl_options, Role, State0),
 	gen_statem:enter_loop(?MODULE, [], init, State)
     catch
 	throw:Error ->
-	    gen_statem:enter_loop(?MODULE, [], error, {Error,State0})
+            EState = State0#state{protocol_specific = Map#{error => Error}},
+	    gen_statem:enter_loop(?MODULE, [], error, EState)
     end.
 %%====================================================================
 %% State transition handling
@@ -470,7 +471,8 @@ init(Type, Event, State) ->
 %%--------------------------------------------------------------------
 error(enter, _, State) ->
     {keep_state, State};     
-error({call, From}, {start, _Timeout}, {Error, State}) ->
+error({call, From}, {start, _Timeout}, 
+      #state{protocol_specific = #{error := Error}} = State) ->
     ssl_connection:stop_and_reply(
       normal, {reply, From, {error, Error}}, State);
 error({call, _} = Call, Msg, State) ->
diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl
index c5f75894cd..e2adf35440 100644
--- a/lib/ssl/src/ssl_connection.erl
+++ b/lib/ssl/src/ssl_connection.erl
@@ -634,8 +634,8 @@ init(_Type, _Event, _State, _Connection) ->
             tls_connection | dtls_connection) ->
 		   gen_statem:state_function_result().
 %%--------------------------------------------------------------------
-error({call, From}, Msg, State, Connection) ->
-    handle_call(Msg, From, ?FUNCTION_NAME, State, Connection).
+error({call, From}, _Msg, State, _Connection) ->
+    {next_state, ?FUNCTION_NAME, State, [{reply, From, {error, closed}}]}.
 
 %%--------------------------------------------------------------------
 -spec hello(gen_statem:event_type(),
diff --git a/lib/ssl/src/tls_connection.erl b/lib/ssl/src/tls_connection.erl
index a3002830d1..4d1122f804 100644
--- a/lib/ssl/src/tls_connection.erl
+++ b/lib/ssl/src/tls_connection.erl
@@ -111,12 +111,13 @@ start_link(Role, Host, Port, Socket, Options, User, CbInfo) ->
 
 init([Role, Host, Port, Socket, Options,  User, CbInfo]) ->
     process_flag(trap_exit, true),
-    State0 = initial_state(Role, Host, Port, Socket, Options, User, CbInfo),
+    State0 = #state{protocol_specific = Map} = initial_state(Role, Host, Port, Socket, Options, User, CbInfo),
     try 
 	State = ssl_connection:ssl_config(State0#state.ssl_options, Role, State0),
 	gen_statem:enter_loop(?MODULE, [], init, State)
     catch throw:Error ->
-	gen_statem:enter_loop(?MODULE, [], error, {Error, State0}) 
+            EState = State0#state{protocol_specific = Map#{error => Error}},
+            gen_statem:enter_loop(?MODULE, [], error, EState) 
     end.
 %%====================================================================
 %% State transition handling
@@ -432,17 +433,12 @@ init(Type, Event, State) ->
 	   {start, timeout()} | term(), #state{}) ->
 		   gen_statem:state_function_result().
 %%--------------------------------------------------------------------
-
-error({call, From}, {start, _Timeout}, {Error, State}) ->
-    ssl_connection:stop_and_reply(
-      normal, {reply, From, {error, Error}}, State);
 error({call, From}, {start, _Timeout}, 
       #state{protocol_specific = #{error := Error}} = State) ->
     ssl_connection:stop_and_reply(
       normal, {reply, From, {error, Error}}, State);
-error({call, _} = Call, Msg, {Error, #state{protocol_specific = Map} = State}) ->
-    gen_handshake(?FUNCTION_NAME, Call, Msg, 
-                  State#state{protocol_specific = Map#{error => Error}});
+error({call, _} = Call, Msg, State) ->
+    gen_handshake(?FUNCTION_NAME, Call, Msg, State);
 error(_, _, _) ->
      {keep_state_and_data, [postpone]}.
  
diff --git a/lib/ssl/test/ssl_basic_SUITE.erl b/lib/ssl/test/ssl_basic_SUITE.erl
index e525ed138e..c63a6dbacc 100644
--- a/lib/ssl/test/ssl_basic_SUITE.erl
+++ b/lib/ssl/test/ssl_basic_SUITE.erl
@@ -242,7 +242,8 @@ error_handling_tests()->
     [close_transport_accept,
      recv_active,
      recv_active_once,
-     recv_error_handling
+     recv_error_handling,
+     call_in_error_state
     ].
 
 error_handling_tests_tls()->
@@ -4000,6 +4001,37 @@ recv_error_handling(Config) when is_list(Config) ->
     ssl:close(SslSocket),
     ssl_test_lib:check_result(Server, ok).
 
+
+
+%%--------------------------------------------------------------------
+call_in_error_state() ->
+    [{doc,"Special case of call error handling"}].
+call_in_error_state(Config) when is_list(Config) ->
+    ServerOpts0 = ssl_test_lib:ssl_options(server_opts, Config),
+    ClientOpts = ssl_test_lib:ssl_options(client_opts, Config),
+    ServerOpts = [{cacertfile, "foo.pem"} | proplists:delete(cacertfile, ServerOpts0)],
+    Pid = spawn_link(?MODULE, run_error_server, [[self() | ServerOpts]]),
+    receive
+        {Pid, Port} ->
+            spawn_link(?MODULE, run_client_error, [[Port, ClientOpts]])
+    end,
+    receive
+        {error, closed} ->
+            ok;
+        Other ->
+            ct:fail(Other)
+    end.
+
+run_client_error([Port, Opts]) ->
+    ssl:connect("localhost", Port, Opts).
+    
+run_error_server([ Pid | Opts]) ->
+    {ok, Listen} = ssl:listen(0, Opts),
+    {ok,{_, Port}} = ssl:sockname(Listen),
+    Pid ! {self(), Port},
+    {ok, Socket} = ssl:transport_accept(Listen),
+    Pid ! ssl:controlling_process(Socket, self()).
+
 %%--------------------------------------------------------------------
 
 rizzo() ->