Cache invalidation and consistent user closing

Added cache invalidation control of ssl certificates so that
sessions will not be reused if file content is changed.

There was a glitch in ssl:close that made it possible to
to get eaddrinuse even though reuseadder-option was used.
Also improved tests for better user-close handling.

3 files changed, 62 insertions, 37 deletions

diff --git a/lib/ssl/src/ssl_certificate_db.erl b/lib/ssl/src/ssl_certificate_db.erl
index 019f73fc80..f34459de37 100644
--- a/lib/ssl/src/ssl_certificate_db.erl
+++ b/lib/ssl/src/ssl_certificate_db.erl
@@ -27,7 +27,7 @@
 
 -export([create/0, remove/1, add_trusted_certs/3, 
 	 remove_trusted_certs/2, lookup_trusted_cert/3, issuer_candidate/1,
-	 lookup_cached_certs/1, cache_pem_file/4, uncache_pem_file/2, ref_count/3]).
+	 lookup_cached_certs/1, cache_pem_file/4, uncache_pem_file/2, lookup/2]).
 
 -type time()      :: {non_neg_integer(), non_neg_integer(), non_neg_integer()}.
 
@@ -122,10 +122,13 @@ cache_pem_file(Pid, File, Time, [CertsDb, _FileToRefDb, PidToFileDb]) ->
 %% but with different content.
 %% --------------------------------------------------------------------
 uncache_pem_file(File, [_CertsDb, _FileToRefDb, PidToFileDb]) ->
-    Pids = select(PidToFileDb, [{{'$1', File},[],['$$']}]),
+    [Pids] = select(PidToFileDb, [{{'$1', File},[],['$$']}]),
     lists:foreach(fun(Pid) ->
 			  exit(Pid, shutdown)
 		  end, Pids).
+
+
+
 %%--------------------------------------------------------------------
 -spec remove_trusted_certs(pid(), certdb_ref()) -> term().
 				  
@@ -191,6 +194,22 @@ issuer_candidate(PrevCandidateKey) ->
     end.
 
 %%--------------------------------------------------------------------
+-spec lookup(term(), term()) -> term() | undefined.
+%%
+%% Description: Looks up an element in a certificat <Db>.
+%%--------------------------------------------------------------------
+lookup(Key, Db) ->
+    case ets:lookup(Db, Key) of
+	[] ->
+	    undefined;
+	Contents  ->
+	    Pick = fun({_, Data}) -> Data;
+		      ({_,_,Data}) -> Data
+		   end,
+	    [Pick(Data) || Data <- Contents]
+    end.
+
+%%--------------------------------------------------------------------
 %%% Internal functions
 %%--------------------------------------------------------------------
 certificate_db_name() ->
@@ -208,17 +227,6 @@ ref_count(Key, Db,N) ->
 delete(Key, Db) ->
     _ = ets:delete(Db, Key).
 
-lookup(Key, Db) ->
-    case ets:lookup(Db, Key) of
-	[] ->
-	    undefined;
-	Contents  ->
-	    Pick = fun({_, Data}) -> Data;
-		      ({_,_,Data}) -> Data
-		   end,
-	    [Pick(Data) || Data <- Contents]
-    end.
-
 select(Db, MatchSpec)->
     ets:select(Db, MatchSpec).
 
diff --git a/lib/ssl/src/ssl_connection.erl b/lib/ssl/src/ssl_connection.erl
index 478f465705..675e5e44bd 100644
--- a/lib/ssl/src/ssl_connection.erl
+++ b/lib/ssl/src/ssl_connection.erl
@@ -90,7 +90,8 @@
 	  log_alert,           % boolean() 
 	  renegotiation,        % {boolean(), From | internal | peer}
 	  recv_during_renegotiation,  %boolean() 
-	  send_queue           % queue()
+	  send_queue,           % queue()
+	  terminated = false   %
 	 }).
 
 -define(DEFAULT_DIFFIE_HELLMAN_PARAMS, 
@@ -781,8 +782,12 @@ handle_sync_event(start, _, connection, State) ->
 handle_sync_event(start, From, StateName, State) ->
     {next_state, StateName, State#state{from = From}};
 
-handle_sync_event(close, _, _StateName, State) ->
-    {stop, normal, ok, State};
+handle_sync_event(close, _, StateName, State) ->
+    %% Run terminate before returning
+    %% so that the reuseaddr inet-option will work
+    %% as intended.
+    (catch terminate(user_close, StateName, State)),
+    {stop, normal, ok, State#state{terminated = true}};
 
 handle_sync_event({shutdown, How0}, _, StateName,
 		  #state{transport_cb = Transport,
@@ -970,6 +975,11 @@ handle_info(Msg, StateName, State) ->
 %% necessary cleaning up. When it returns, the gen_fsm terminates with
 %% Reason. The return value is ignored.
 %%--------------------------------------------------------------------
+terminate(_, _, #state{terminated = true}) ->
+    %% Happens when user closes the connection using ssl:close/1
+    %% we want to guarantee that Transport:close has been called
+    %% when ssl:close/1 returns.
+    ok;
 terminate(Reason, connection, #state{negotiated_version = Version,
 				      connection_states = ConnectionStates,
 				      transport_cb = Transport,
@@ -979,14 +989,14 @@ terminate(Reason, connection, #state{negotiated_version = Version,
     notify_renegotiater(Renegotiate),
     BinAlert = terminate_alert(Reason, Version, ConnectionStates),
     Transport:send(Socket, BinAlert),
-    workaround_transport_delivery_problems(Socket, Transport),
+    workaround_transport_delivery_problems(Socket, Transport, Reason),
     Transport:close(Socket);
-terminate(_Reason, _StateName, #state{transport_cb = Transport, 
+terminate(Reason, _StateName, #state{transport_cb = Transport,
 				      socket = Socket, send_queue = SendQueue,
 				      renegotiation = Renegotiate}) ->
     notify_senders(SendQueue),
     notify_renegotiater(Renegotiate),
-    workaround_transport_delivery_problems(Socket, Transport),
+    workaround_transport_delivery_problems(Socket, Transport, Reason),
     Transport:close(Socket).
 
 %%--------------------------------------------------------------------
@@ -2189,7 +2199,8 @@ notify_renegotiater({true, From}) when not is_atom(From)  ->
 notify_renegotiater(_) ->
     ok.
 
-terminate_alert(Reason, Version, ConnectionStates) when Reason == normal; Reason == shutdown ->
+terminate_alert(Reason, Version, ConnectionStates) when Reason == normal; Reason == shutdown;
+							Reason == user_close ->
     {BinAlert, _} = encode_alert(?ALERT_REC(?WARNING, ?CLOSE_NOTIFY),
 				 Version, ConnectionStates),
     BinAlert;
@@ -2198,10 +2209,13 @@ terminate_alert(_, Version, ConnectionStates) ->
 				 Version, ConnectionStates),
     BinAlert.
 
-workaround_transport_delivery_problems(Socket, Transport) ->
+workaround_transport_delivery_problems(_,_, user_close) ->
+    ok;
+workaround_transport_delivery_problems(Socket, Transport, _) ->
     %% Standard trick to try to make sure all
     %% data sent to to tcp port is really sent
-    %% before tcp port is closed.
+    %% before tcp port is closed so that the peer will
+    %% get a correct error message.
     inet:setopts(Socket, [{active, false}]),
     Transport:shutdown(Socket, write),
     Transport:recv(Socket, 0).
diff --git a/lib/ssl/src/ssl_manager.erl b/lib/ssl/src/ssl_manager.erl
index dc613eec11..f845b1ecc0 100644
--- a/lib/ssl/src/ssl_manager.erl
+++ b/lib/ssl/src/ssl_manager.erl
@@ -35,7 +35,7 @@
 	 invalidate_session/3]).
 
 % Spawn export
--export([init_session_validator/1, recache_pem/4]).
+-export([init_session_validator/1]).
 
 %% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
@@ -229,8 +229,8 @@ handle_call({{cache_pem, File, LastWrite}, Pid}, _,
     end;
 handle_call({{recache_pem, File, LastWrite}, Pid}, From,
 	    #state{certificate_db = Db} = State) ->
-    ssl_certificate_db:uncache_pem_file(File, Pid, Db),
-    spawn_link(?MODULE, recache_pem, [File, Db, LastWrite, From]),
+    ssl_certificate_db:uncache_pem_file(File, Db),
+    cast({recache_pem, File, LastWrite, Pid, From}),
     {noreply, State}.
 
 %%--------------------------------------------------------------------
@@ -269,7 +269,21 @@ handle_cast({invalidate_session, Port, #session{session_id = ID}},
 	    #state{session_cache = Cache,
 		   session_cache_cb = CacheCb} = State) ->
     CacheCb:delete(Cache, {Port, ID}),
-    {noreply, State}.
+    {noreply, State};
+
+handle_cast({recache_pem, File, LastWrite, Pid, From},
+	    #state{certificate_db = [_, FileToRefDb, _]} = State0) ->
+    case ssl_certificate_db:lookup(File, FileToRefDb) of
+	undefined ->
+	    {reply, Msg, State} = handle_call({{cache_pem, File, LastWrite}, Pid}, From, State0),
+	    gen_server:reply(From, Msg),
+	    {noreply, State};
+	_ -> %% Send message to self letting cleanup messages be handled
+	     %% first so that no reference to the old version of file
+             %% exists when we cache the new one.
+	    cast({recache_pem, File, LastWrite, Pid, From}),
+	    {noreply, State0}
+    end.
 
 %%--------------------------------------------------------------------
 -spec handle_info(msg(), #state{}) -> {noreply, #state{}}.
@@ -387,14 +401,3 @@ cache_pem_file(File, LastWrite) ->
 	[] ->
 	    call({cache_pem, File, LastWrite})
     end.
-
-
-recache_pem(File, Db, LastWrite, From) ->
-    case ssl_certificate_db:ref_count(File, Db, 0) of
-	0 ->
-	    Result = call({cache_pem, File, LastWrite}),
-	    gen_server:reply(From, Result);
-	_ ->
-	    timer:sleep(1000),
-	    recache_pem(File, Db, LastWrite, From)
-    end.