Fix faulty connection handling

OTP-10126

3 files changed, 444 insertions, 0 deletions

diff --git a/lib/common_test/test/ct_gen_conn_SUITE_data/conn.conf b/lib/common_test/test/ct_gen_conn_SUITE_data/conn.conf
new file mode 100644
index 0000000000..09f3c11e10
--- /dev/null
+++ b/lib/common_test/test/ct_gen_conn_SUITE_data/conn.conf
@@ -0,0 +1,8 @@
+{multi_conn_pid, [{addr,"localhost"},
+		  {port,8383},
+		  {multiple_conn_pids,true}]}.
+
+{single_conn_pid, [{addr,"localhost"},
+		   {port,8383},
+		   {multiple_conn_pids,false},
+		   {conn_mgr_name,conn_mgr}]}.
diff --git a/lib/common_test/test/ct_gen_conn_SUITE_data/conn_SUITE.erl b/lib/common_test/test/ct_gen_conn_SUITE_data/conn_SUITE.erl
new file mode 100644
index 0000000000..c0707160cf
--- /dev/null
+++ b/lib/common_test/test/ct_gen_conn_SUITE_data/conn_SUITE.erl
@@ -0,0 +1,240 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2008-2010. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File    : 
+%%% Description : 
+%%%-------------------------------------------------------------------
+-module(conn_SUITE).
+
+%% Note: This directive should only be used in test suites.
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%--------------------------------------------------------------------
+%% COMMON TEST CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+
+suite() ->
+    [{timetrap,{seconds,5}}].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(_Config) ->
+    ok.
+
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
+all() -> 
+    [handles_to_multi_conn_pids, handles_to_single_conn_pids,
+     names_to_multi_conn_pids, names_to_single_conn_pids].
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+handles_to_multi_conn_pids() ->
+    [{require,multi_conn_pid}].
+
+handles_to_multi_conn_pids(Config) ->
+    application:set_env(ct_test, reconnect, true),
+
+    Handle1 = proto:open(multi_conn_pid),
+    ConnPid1 = ct_gen_conn:get_conn_pid(Handle1),
+    {true,true} = {is_process_alive(Handle1),is_process_alive(ConnPid1)},
+    Handle2 = proto:open(multi_conn_pid),
+    ConnPid2 = ct_gen_conn:get_conn_pid(Handle2),
+    {true,true} = {is_process_alive(Handle2),is_process_alive(ConnPid2)},
+    Handle3 = proto:open(multi_conn_pid),
+    ConnPid3 = ct_gen_conn:get_conn_pid(Handle3),
+    {true,true} = {is_process_alive(Handle3),is_process_alive(ConnPid3)},
+    
+    ok = proto:close(Handle1),
+    timer:sleep(100),
+    {false,false} = {is_process_alive(Handle1),is_process_alive(ConnPid1)},
+    {true,true} = {is_process_alive(Handle2),is_process_alive(ConnPid2)},
+
+    ok = proto:kill_conn_proc(Handle2),
+    timer:sleep(100),
+    {true,false} = {is_process_alive(Handle2),is_process_alive(ConnPid2)},
+    ConnPid2x = ct_gen_conn:get_conn_pid(Handle2),
+    true = is_process_alive(ConnPid2x),
+
+    ok = proto:close(Handle2),
+    timer:sleep(100),
+    {false,false} = {is_process_alive(Handle2),is_process_alive(ConnPid2x)},
+
+    application:set_env(ct_test, reconnect, false),
+    ok = proto:kill_conn_proc(Handle3),
+    timer:sleep(100),
+    {false,false} = {is_process_alive(Handle3),is_process_alive(ConnPid3)},
+
+    ok.
+
+handles_to_single_conn_pids() ->
+    [{require,single_conn_pid}].
+
+handles_to_single_conn_pids(Config) ->
+    application:set_env(ct_test, reconnect, true),
+
+    Handle1 = proto:open(single_conn_pid),
+    ConnPid = ct_gen_conn:get_conn_pid(Handle1),
+    {true,true} = {is_process_alive(Handle1),is_process_alive(ConnPid)},
+    Handle2 = proto:open(single_conn_pid),
+    ConnPid = ct_gen_conn:get_conn_pid(Handle2),
+    {true,true} = {is_process_alive(Handle2),is_process_alive(ConnPid)},
+    Handle3 = proto:open(single_conn_pid),
+    ConnPid = ct_gen_conn:get_conn_pid(Handle3),
+    {true,true} = {is_process_alive(Handle3),is_process_alive(ConnPid)},
+
+    Conns = [{undefined,Handle3,_,_},
+	     {undefined,Handle2,_,_},
+	     {undefined,Handle1,_,_}] = ct_util:get_connections(ConnPid),
+    ct:pal("CONNS = ~n~p", [Conns]),
+
+    ok = proto:close(Handle1),
+    timer:sleep(100),
+    {false,true} = {is_process_alive(Handle1),is_process_alive(ConnPid)},
+
+    ok = proto:kill_conn_proc(Handle2),
+    timer:sleep(100),
+    NewConnPid = ct_gen_conn:get_conn_pid(Handle2),
+    NewConnPid = ct_gen_conn:get_conn_pid(Handle3),
+    true = is_process_alive(Handle2),
+    true = is_process_alive(Handle3),
+
+    ok = proto:close(Handle2),
+    timer:sleep(100),
+    {false,true} = {is_process_alive(Handle2),is_process_alive(NewConnPid)},
+
+    application:set_env(ct_test, reconnect, false),
+    ok = proto:kill_conn_proc(Handle3),
+    timer:sleep(100),
+    {false,false} = {is_process_alive(Handle3),is_process_alive(NewConnPid)},    
+
+    ok.
+
+names_to_multi_conn_pids() ->
+    [{require,conn1,multi_conn_pid},
+     {require,conn2,multi_conn_pid},
+     {require,conn3,multi_conn_pid}].
+
+names_to_multi_conn_pids(Config) ->
+    application:set_env(ct_test, reconnect, true),
+
+    Handle1 = proto:open(conn1),
+    ConnPid1 = ct_gen_conn:get_conn_pid(Handle1),
+    {true,true} = {is_process_alive(Handle1),is_process_alive(ConnPid1)},
+    Handle2 = proto:open(conn2),
+    ConnPid2 = ct_gen_conn:get_conn_pid(Handle2),
+    {true,true} = {is_process_alive(Handle2),is_process_alive(ConnPid2)},
+    Handle3 = proto:open(conn3),
+    ConnPid3 = ct_gen_conn:get_conn_pid(Handle3),
+    {true,true} = {is_process_alive(Handle3),is_process_alive(ConnPid3)},
+
+    Handle1 = proto:open(conn1),
+
+    ok = proto:close(conn1),
+    timer:sleep(100),
+    {false,false} = {is_process_alive(Handle1),is_process_alive(ConnPid1)},
+
+    ok = proto:kill_conn_proc(Handle2),
+    timer:sleep(100),
+    Handle2 = proto:open(conn2),  % should've been reconnected already
+    {true,false} = {is_process_alive(Handle2),is_process_alive(ConnPid2)},
+    ConnPid2x = ct_gen_conn:get_conn_pid(Handle2),
+    true = is_process_alive(ConnPid2x),
+
+    ok = proto:close(conn2),
+    timer:sleep(100),
+    {false,false} = {is_process_alive(Handle2),is_process_alive(ConnPid2x)},
+    Handle2y = proto:open(conn2),
+    ConnPid2y = ct_gen_conn:get_conn_pid(Handle2y),
+    {true,true} = {is_process_alive(Handle2y),is_process_alive(ConnPid2y)},
+    ok = proto:close(conn2),
+    timer:sleep(100),
+    {false,false} = {is_process_alive(Handle2y),is_process_alive(ConnPid2y)},
+
+    application:set_env(ct_test, reconnect, false),
+    ok = proto:kill_conn_proc(Handle3),
+    timer:sleep(100),
+    {false,false} = {is_process_alive(Handle3),is_process_alive(ConnPid3)},
+
+    ok.
+
+names_to_single_conn_pids() ->
+    [{require,conn1,single_conn_pid},
+     {require,conn2,single_conn_pid},
+     {require,conn3,single_conn_pid}].
+
+names_to_single_conn_pids(Config) ->
+    application:set_env(ct_test, reconnect, true),
+
+    Handle1 = proto:open(conn1),
+    ConnPid = ct_gen_conn:get_conn_pid(Handle1),
+    {true,true} = {is_process_alive(Handle1),is_process_alive(ConnPid)},
+    Handle2 = proto:open(conn2),
+    ConnPid = ct_gen_conn:get_conn_pid(Handle2),
+    {true,true} = {is_process_alive(Handle2),is_process_alive(ConnPid)},
+    Handle3 = proto:open(conn3),
+    ConnPid = ct_gen_conn:get_conn_pid(Handle3),
+    {true,true} = {is_process_alive(Handle3),is_process_alive(ConnPid)},
+
+    Handle1 = proto:open(conn1),
+
+    Conns = [{conn1,Handle1,_,_},
+	     {conn2,Handle2,_,_},
+	     {conn3,Handle3,_,_}] = lists:sort(ct_util:get_connections(ConnPid)),
+    ct:pal("CONNS on ~p = ~n~p", [ConnPid,Conns]),
+
+    ok = proto:close(conn1),
+    timer:sleep(100),
+    {false,true} = {is_process_alive(Handle1),is_process_alive(ConnPid)},
+
+    ok = proto:kill_conn_proc(Handle2),
+    timer:sleep(100),
+    {true,false} = {is_process_alive(Handle2),is_process_alive(ConnPid)},
+    Handle2 = proto:open(conn2),  % should've been reconnected already
+    NewConnPid = ct_gen_conn:get_conn_pid(Handle2),
+    true = is_process_alive(NewConnPid),
+    
+    Conns1 = [{conn2,Handle2,_,_},
+	      {conn3,Handle3,_,_}] = 
+	lists:sort(ct_util:get_connections(NewConnPid)),
+    ct:pal("CONNS on ~p = ~n~p", [NewConnPid,Conns1]),
+
+    ok = proto:close(conn2),
+    timer:sleep(100),
+    {false,true} = {is_process_alive(Handle2),is_process_alive(NewConnPid)},
+
+    application:set_env(ct_test, reconnect, false),
+    ok = proto:kill_conn_proc(Handle3),
+    timer:sleep(100),
+    {false,false} = {is_process_alive(Handle3),is_process_alive(NewConnPid)},
+
+    ok.
+
+
diff --git a/lib/common_test/test/ct_gen_conn_SUITE_data/proto.erl b/lib/common_test/test/ct_gen_conn_SUITE_data/proto.erl
new file mode 100644
index 0000000000..8fcd35e0a4
--- /dev/null
+++ b/lib/common_test/test/ct_gen_conn_SUITE_data/proto.erl
@@ -0,0 +1,196 @@
+%%% @author Peter Andersson <[email protected]>
+%%% @copyright (C) 2013, Peter Andersson
+%%% @doc
+%%%
+%%% @end
+%%% Created : 24 May 2013 by Peter Andersson <[email protected]>
+
+-module(proto).
+
+-compile(export_all).
+
+-record(conn_state, {id, pid, ref, data}).
+
+%% TEST1: N connections (same key) -> N conn pids
+%% TEST2: N connections (same key) -> 1 conn pid
+%% TEST3: N aliases (same key) -> N conn pids
+%% TEST4: N aliases (same key) -> 1 conn pid
+
+open(KeyOrAlias) ->
+    case ct:get_config(KeyOrAlias) of
+	undefined ->
+	    {error,{not_available,KeyOrAlias}};
+	ConnData ->
+	    io:format("Opening connection with ~p~n", [ConnData]),
+
+	    %% if KeyOrAlias == Key, each call returns unique handle
+	    %% if KeyOrAlias == Alias, successive calls return same handle
+	    {ok,Handle} = ct_gen_conn:start(ConnData,
+					    [],
+					    ?MODULE,
+					    [{name,KeyOrAlias}]),	    
+	    io:format("Handle for ~p = ~p~n", [KeyOrAlias,Handle]),
+	    Handle
+    end.
+
+close(AliasOrHandle) ->
+    Handle = get_handle(AliasOrHandle),
+    io:format("Closing connection for ~p (~p)~n", [AliasOrHandle,Handle]),
+    case ct_gen_conn:stop(Handle) of
+	E = {error,_} ->
+	    E;
+	Result ->
+	    Result
+    end.
+
+kill_conn_proc(AliasOrHandle) ->
+    ConnPid = ct_gen_conn:get_conn_pid(get_handle(AliasOrHandle)),
+    io:format("Killing connection process ~p~n", [ConnPid]),
+    ConnPid ! fail,
+    ok.
+
+send(_) ->
+    ok.
+
+%%%-----------------------------------------------------------------
+%%% 
+
+init(KeyOrAlias, ConnData, []) ->
+    Addr = proplists:get_value(addr, ConnData),
+    Port = proplists:get_value(port, ConnData),
+    Ref = make_ref(),
+    Starter = self(),
+    MultConnPids = proplists:get_value(multiple_conn_pids, ConnData),
+    ConnPid =
+	case MultConnPids of
+	    true ->
+		spawn(fun() -> active_conn(Starter, KeyOrAlias, Ref, 
+					   ConnData) end);
+	    _ ->
+		ConnMgr = proplists:get_value(conn_mgr_name, ConnData),
+		case whereis(ConnMgr) of
+		    undefined ->
+			MgrPid =
+			    spawn(fun() -> active_conn(Starter, KeyOrAlias,
+						       Ref, ConnData) end),
+			receive MgrPid -> 
+				MgrPid
+			end;			
+		    MgrPid when is_pid(MgrPid) ->
+			MgrPid ! {connect,Ref},
+			MgrPid
+		end
+	end,
+    io:format("Connection ~p opened on ~p:~p -> ~p (~p)~n",
+	      [KeyOrAlias,Addr,Port,ConnPid,Ref]),
+    {ok,ConnPid,#conn_state{id=KeyOrAlias, pid=ConnPid, ref=Ref, data=ConnData}}.
+
+
+terminate(ConnPid, #conn_state{id=Id, pid=ConnPid, ref = Ref, data=Data}) ->
+    case proplists:get_value(multiple_conn_pids, Data) of
+	true ->
+	    ConnPid ! close;
+	_ ->
+	    ConnPid ! {close,Ref}
+    end,
+    io:format("Connection ~p on ~p (~p) closing!~n", [Id,ConnPid,Ref]),
+    ok.
+
+
+reconnect(ConnData, State = #conn_state{id=Id, ref=DeadRef}) ->    
+    io:format("Reconnect for ~p initiated...~n", [DeadRef]),
+    case application:get_env(ct_test, reconnect) of
+	{ok,true} ->
+	    ConnMgr = proplists:get_value(conn_mgr_name, ConnData),
+	    NewRef = make_ref(),
+	    Starter = self(),
+	    ConnPid =
+		case proplists:get_value(multiple_conn_pids, ConnData) of
+		    true ->
+			spawn(fun() ->
+				      active_conn(Starter, Id, NewRef,
+						  ConnData)
+			      end);
+		    _ ->
+			case whereis(ConnMgr) of
+			    undefined ->
+				MgrPid =
+				    spawn(fun() ->
+						  active_conn(Starter, Id,
+							      NewRef, ConnData)
+					  end),
+				receive MgrPid -> 
+					MgrPid
+				end;			
+			    MgrPid ->
+				MgrPid ! {reconnect,DeadRef,NewRef},
+				MgrPid
+			end
+		end,
+	    io:format("Connection ~p reopened on ~p (~p)~n",
+		      [Id,ConnPid,NewRef]),
+	    {ok,ConnPid,State#conn_state{pid=ConnPid, ref=NewRef}};
+	_ ->
+	    {error,no_reconnection_allowed}
+    end.
+
+%%%-----------------------------------------------------------------
+%%% 
+
+active_conn(Starter, Id, Ref, ConnData) ->
+    ConnMgr = proplists:get_value(conn_mgr_name, ConnData),
+    case proplists:get_value(multiple_conn_pids, ConnData) of
+	true ->
+	    ok;
+	_ ->
+	    register(ConnMgr,self()),
+	    io:format("Connection manager ~p on ~p started for "
+		      "~p and ~p~n",
+		      [ConnMgr,self(),Id,Ref])
+    end,
+    Starter ! self(),
+    active_conn_loop(ConnData, [Ref]).
+
+active_conn_loop(ConnData, Conns) ->
+    receive
+	{connect,Ref} ->
+	    io:format("Connecting ~p on ~p~n",
+		      [Ref,self()]),	    
+	    active_conn_loop(ConnData, [Ref | Conns]);
+	{reconnect,DeadRef,NewRef} ->
+	    Conns1 = [NewRef | lists:delete(DeadRef, Conns)],
+	    io:format("Reconnecting on ~p: ~p -> ~p~n",
+		      [self(),DeadRef,NewRef]),
+	    active_conn_loop(ConnData, Conns1);
+	close ->
+	    io:format("Conn process ~p shutting down~n", [self()]),
+	    ok;
+	{close,Ref} ->
+	    io:format("Closing connection ~p on ~p~n", [Ref,self()]),
+	    case proplists:delete(Ref, Conns) of
+		[] ->
+		    io:format("Last connection on ~p closed, "
+			      "now stopping~n", [self()]),
+		    ok;
+		Conns1 ->
+		    active_conn_loop(ConnData, Conns1)
+	    end;
+	fail ->
+	    io:format("Connection process not feeling good...~n", []),
+	    exit(kaboom);
+	{respond,To} ->
+	    To ! {self(),hello},
+	    active_conn_loop(ConnData, Conns)
+    end.
+    
+%%%-----------------------------------------------------------------
+%%% 
+
+get_handle(AliasOrHandle) when is_pid(AliasOrHandle) ->
+    AliasOrHandle;
+
+get_handle(AliasOrHandle) ->
+    {ok,{H,_}} = ct_util:get_connection(AliasOrHandle,
+					?MODULE),
+    H.
+