Merge branch 'anders/diameter/testsuites/OTP-9620'

* anders/diameter/testsuites/OTP-9620:
  Add failover suite
  Use util to simplify connection establishment in suites
  Move certificate generation into own testcase
  Add beam target to makefile
  Add util functions for managing connections
  Use tcp/sctp port resolution from testsuites

8 files changed, 460 insertions, 215 deletions

diff --git a/lib/diameter/test/Makefile b/lib/diameter/test/Makefile
index dba1f126dc..04e686c969 100644
--- a/lib/diameter/test/Makefile
+++ b/lib/diameter/test/Makefile
@@ -77,7 +77,7 @@ ERL_COMPILE_FLAGS += $(DIAMETER_ERL_COMPILE_FLAGS) \
 
 all: $(SUITES)
 
-tests debug opt: $(TARGET_FILES)
+beam tests debug opt: $(TARGET_FILES)
 
 clean:
 	rm -f $(TARGET_FILES) 
diff --git a/lib/diameter/test/diameter_failover_SUITE.erl b/lib/diameter/test/diameter_failover_SUITE.erl
new file mode 100644
index 0000000000..c25e9682f0
--- /dev/null
+++ b/lib/diameter/test/diameter_failover_SUITE.erl
@@ -0,0 +1,262 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%
+%% Tests of traffic between six Diameter nodes in three realms,
+%% connected as follows.
+%%
+%%                  ----- SERVER1.REALM2
+%%                /
+%%               /  ----- SERVER2.REALM2
+%%              | /
+%%   CLIENT.REALM1 ------ SERVER3.REALM2
+%%              | \
+%%              |  \
+%%               \   ---- SERVER1.REALM3
+%%                \
+%%                  ----- SERVER2.REALM3
+%%
+
+-module(diameter_failover_SUITE).
+
+-export([suite/0,
+         all/0]).
+
+%% testcases
+-export([start/1,
+         start_services/1,
+         connect/1,
+         send_ok/1,
+         send_nok/1,
+         stop_services/1,
+         stop/1]).
+
+%% diameter callbacks
+-export([peer_up/3,
+         peer_down/3,
+         pick_peer/4,
+         prepare_request/3,
+         prepare_retransmit/3,
+         handle_answer/4,
+         handle_error/4,
+         handle_request/3]).
+
+-ifdef(DIAMETER_CT).
+-include("diameter_gen_base_rfc3588.hrl").
+-else.
+-include_lib("diameter/include/diameter_gen_base_rfc3588.hrl").
+-endif.
+
+-include_lib("diameter/include/diameter.hrl").
+-include("diameter_ct.hrl").
+
+%% ===========================================================================
+
+-define(util, diameter_util).
+
+-define(ADDR, {127,0,0,1}).
+
+-define(CLIENT,  "CLIENT.REALM1").
+-define(SERVER1, "SERVER1.REALM2").
+-define(SERVER2, "SERVER2.REALM2").
+-define(SERVER3, "SERVER3.REALM2").
+-define(SERVER4, "SERVER1.REALM3").
+-define(SERVER5, "SERVER2.REALM3").
+
+-define(SERVICES, [?CLIENT, ?SERVER1, ?SERVER2, ?SERVER3, ?SERVER4, ?SERVER5]).
+
+-define(DICT_COMMON,  ?DIAMETER_DICT_COMMON).
+
+-define(APP_ALIAS, the_app).
+-define(APP_ID, ?DICT_COMMON:id()).
+
+%% Config for diameter:start_service/2.
+-define(SERVICE(Host, Dict),
+        [{'Origin-Host', Host},
+         {'Origin-Realm', realm(Host)},
+         {'Host-IP-Address', [?ADDR]},
+         {'Vendor-Id', 12345},
+         {'Product-Name', "OTP/diameter"},
+         {'Acct-Application-Id', [Dict:id()]},
+         {application, [{alias, ?APP_ALIAS},
+                        {dictionary, Dict},
+                        {module, ?MODULE},
+                        {answer_errors, callback}]}]).
+
+-define(SUCCESS, 2001).
+
+-define(LOGOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_DIAMETER_LOGOUT').
+
+%% ===========================================================================
+
+suite() ->
+    [{timetrap, {seconds, 10}}].
+
+all() ->
+    [start,
+     start_services,
+     connect,
+     send_ok,
+     send_nok,
+     stop_services,
+     stop].
+
+%% ===========================================================================
+%% start/stop testcases
+
+start(_Config) ->
+    ok = diameter:start().
+
+start_services(_Config) ->
+    S = [server(N, ?DICT_COMMON) || N <- tl(?SERVICES)],
+
+    ok = diameter:start_service(?CLIENT, ?SERVICE(?CLIENT, ?DICT_COMMON)),
+
+    {save_config, [{?CLIENT, S}]}.
+
+connect(Config) ->
+    {_, Conns} = proplists:get_value(saved_config, Config),
+
+    lists:foreach(fun({CN,Ss}) -> connect(CN, Ss) end, Conns).
+
+stop_services(_Config) ->
+    [] = [{H,T} || H <- ?SERVICES,
+                   T <- [diameter:stop_service(H)],
+                   T /= ok].
+
+stop(_Config) ->
+    ok = diameter:stop().
+
+%% ----------------------------------------
+
+server(Name, Dict) ->
+    ok = diameter:start_service(Name, ?SERVICE(Name, Dict)),
+    {Name, ?util:listen(Name, tcp)}.
+
+connect(Name, Refs) ->
+    [{{Name, ?util:connect(Name, tcp, LRef)}, T} || {_, LRef} = T <- Refs].
+
+%% ===========================================================================
+%% traffic testcases
+
+%% Send an STR and expect success after SERVER3 answers after a couple
+%% of failovers.
+send_ok(_Config) ->
+    Req = ['STR', {'Destination-Realm', realm(?SERVER1)},
+                  {'Termination-Cause', ?LOGOUT},
+                  {'Auth-Application-Id', ?APP_ID}],
+    #diameter_base_STA{'Result-Code' = ?SUCCESS,
+                       'Origin-Host' = ?SERVER3}
+        = call(Req, [{filter, realm}]).
+
+%% Send an STR and expect failure when both servers fail.
+send_nok(_Config) ->
+    Req = ['STR', {'Destination-Realm', realm(?SERVER4)},
+                  {'Termination-Cause', ?LOGOUT},
+                  {'Auth-Application-Id', ?APP_ID}],
+    {error, failover} = call(Req, [{filter, realm}]).
+
+%% ===========================================================================
+
+realm(Host) ->
+    tl(lists:dropwhile(fun(C) -> C /= $. end, Host)).
+
+call(Req, Opts) ->
+    diameter:call(?CLIENT, ?APP_ALIAS, Req, Opts).
+    
+set([H|T], Vs) ->
+    [H | Vs ++ T].
+
+%% ===========================================================================
+%% diameter callbacks
+
+%% peer_up/3
+
+peer_up(_SvcName, _Peer, State) ->
+    State.
+
+%% peer_down/3
+
+peer_down(_SvcName, _Peer, State) ->
+    State.
+
+%% pick_peer/4
+
+%% Choose a server other than SERVER3 or SERVER5 if possible.
+pick_peer(Peers, _, ?CLIENT, _State) ->
+    case lists:partition(fun({_, #diameter_caps{origin_host = {_, OH}}}) ->
+                                 OH /= ?SERVER3 andalso OH /= ?SERVER5
+                         end,
+                         Peers)
+    of
+        {[], [Peer]} ->
+            {ok, Peer};
+        {[Peer | _], _} ->
+            {ok, Peer}
+    end.
+
+%% prepare_request/3
+
+prepare_request(Pkt, ?CLIENT, {_Ref, Caps}) ->
+    {send, prepare(Pkt, Caps)}.
+
+prepare(#diameter_packet{msg = Req}, Caps) ->
+    #diameter_caps{origin_host  = {OH, _},
+                   origin_realm = {OR, _}}
+        = Caps,
+    set(Req, [{'Session-Id', diameter:session_id(OH)},
+              {'Origin-Host',  OH},
+              {'Origin-Realm', OR}]).
+
+%% prepare_retransmit/3
+
+prepare_retransmit(Pkt, ?CLIENT, _Peer) ->
+    {send, Pkt}.
+
+%% handle_answer/4
+
+handle_answer(Pkt, _Req, ?CLIENT, _Peer) ->
+    #diameter_packet{msg = Rec, errors = []} = Pkt,
+    Rec.
+
+%% handle_error/4
+
+handle_error(Reason, _Req, ?CLIENT, _Peer) ->
+    {error, Reason}.
+
+%% handle_request/3
+
+%% Only SERVER3 actually answers.
+handle_request(Pkt, ?SERVER3, {_, Caps}) ->
+    #diameter_packet{msg = #diameter_base_STR{'Session-Id' = SId,
+                                              'Origin-Host' = ?CLIENT}}
+        = Pkt,
+    #diameter_caps{origin_host  = {OH, _},
+                   origin_realm = {OR, _}}
+        = Caps,
+
+    {reply, #diameter_base_STA{'Result-Code' = ?SUCCESS,
+                               'Session-Id' = SId,
+                               'Origin-Host' = OH,
+                               'Origin-Realm' = OR}};
+
+%% Others kill the transport to force failover.
+handle_request(_, _, {TPid, _}) ->
+    exit(TPid, kill),
+    discard.
diff --git a/lib/diameter/test/diameter_relay_SUITE.erl b/lib/diameter/test/diameter_relay_SUITE.erl
index 60babd0b9a..03f1115496 100644
--- a/lib/diameter/test/diameter_relay_SUITE.erl
+++ b/lib/diameter/test/diameter_relay_SUITE.erl
@@ -110,19 +110,6 @@
                         {module, ?MODULE},
                         {answer_errors, callback}]}]).
 
-%% Config for diameter:add_transport/2. In the listening case, listen
-%% on a free port that we then lookup using the implementation detail
-%% that diameter_tcp registers the port with diameter_reg.
--define(CONNECT(PortNr),
-        {connect, [{transport_module, diameter_tcp},
-                   {transport_config, [{raddr, ?ADDR},
-                                       {rport, PortNr},
-                                       {ip, ?ADDR},
-                                       {port, 0}]}]}).
--define(LISTEN,
-        {listen, [{transport_module, diameter_tcp},
-                  {transport_config, [{ip, ?ADDR}, {port, 0}]}]}).
-
 -define(SUCCESS, 2001).
 -define(LOOP_DETECTED, 3005).
 -define(UNABLE_TO_DELIVER, 3002).
@@ -130,9 +117,6 @@
 -define(LOGOUT, ?'DIAMETER_BASE_TERMINATION-CAUSE_DIAMETER_LOGOUT').
 -define(AUTHORIZE_ONLY, ?'DIAMETER_BASE_RE-AUTH-REQUEST-TYPE_AUTHORIZE_ONLY').
 
--define(A, list_to_atom).
--define(L, atom_to_list).
-
 %% ===========================================================================
 
 suite() ->
@@ -173,47 +157,29 @@ start(_Config) ->
     ok = diameter:start().
 
 start_services(_Config) ->
-    S = [server(N, ?DICT_COMMON) || N <- [?SERVER1,
-                                          ?SERVER2,
-                                          ?SERVER3,
-                                          ?SERVER4]],
-    R = [server(N, ?DICT_RELAY) || N <- [?RELAY1, ?RELAY2]],
+    [S1,S2,S3,S4] = [server(N, ?DICT_COMMON) || N <- [?SERVER1,
+                                                      ?SERVER2,
+                                                      ?SERVER3,
+                                                      ?SERVER4]],
+    [R1,R2] = [server(N, ?DICT_RELAY) || N <- [?RELAY1, ?RELAY2]],
 
     ok = diameter:start_service(?CLIENT, ?SERVICE(?CLIENT, ?DICT_COMMON)),
 
-    {save_config, S ++ R}.
+    {save_config, [{?RELAY1, [S1,S2,R2]},
+                   {?RELAY2, [S3,S4]},
+                   {?CLIENT, [R1,R2]}]}.
 
 connect(Config) ->
-    {_, [S1,S2,S3,S4,R1,R2] = SR} = proplists:get_value(saved_config, Config),
-
-    true = diameter:subscribe(?RELAY1),
-    true = diameter:subscribe(?RELAY2),
-    true = diameter:subscribe(?CLIENT),
+    {_, Conns} = proplists:get_value(saved_config, Config),
 
-    [R1S1,R1S2] = connect(?RELAY1, [S1,S2]),
-    [R2S3,R2S4] = connect(?RELAY2, [S3,S4]),
-    [CR1,CR2]   = connect(?CLIENT, [R1,R2]),
+    ?util:write_priv(Config,
+                     "cfg",
+                     lists:flatmap(fun({CN,Ss}) -> connect(CN, Ss) end,
+                                   Conns)).
 
-    R1R2 = connect(?RELAY1, R2),
-
-    ?util:write_priv(Config, "cfg", SR ++ [R1S1,R1S2,R2S3,R2S4,CR1,CR2,R1R2]).
-
-%% Remove the client transports and expect the corresponding server
-%% transport to go down.
 disconnect(Config) ->
-    [S1,S2,S3,S4,R1,R2,R1S1,R1S2,R2S3,R2S4,CR1,CR2,R1R2]
-        = ?util:read_priv(Config, "cfg"),
-
-    [?CLIENT | Svcs] = ?SERVICES,
-    [] = [{S,T} || S <- Svcs, T <- [diameter:subscribe(S)], T /= true],
-
-    disconnect(?RELAY1, S1, R1S1),
-    disconnect(?RELAY1, S2, R1S2),
-    disconnect(?RELAY2, S3, R2S3),
-    disconnect(?RELAY2, S4, R2S4),
-    disconnect(?CLIENT, R1, CR1),
-    disconnect(?CLIENT, R2, CR2),
-    disconnect(?RELAY1, R2, R1R2).
+    lists:foreach(fun({{CN,CR},{SN,SR}}) -> ?util:disconnect(CN,CR,SN,SR) end,
+                  ?util:read_priv(Config, "cfg")).
 
 stop_services(_Config) ->
     [] = [{H,T} || H <- ?SERVICES,
@@ -223,6 +189,15 @@ stop_services(_Config) ->
 stop(_Config) ->
     ok = diameter:stop().
 
+%% ----------------------------------------
+
+server(Name, Dict) ->
+    ok = diameter:start_service(Name, ?SERVICE(Name, Dict)),
+    {Name, ?util:listen(Name, tcp)}.
+
+connect(Name, Refs) ->
+    [{{Name, ?util:connect(Name, tcp, LRef)}, T} || {_, LRef} = T <- Refs].
+
 %% ===========================================================================
 %% traffic testcases
 
@@ -266,45 +241,6 @@ send_timeout(Tmo) ->
 realm(Host) ->
     tl(lists:dropwhile(fun(C) -> C /= $. end, Host)).
 
-server(Host, Dict) ->
-    ok = diameter:start_service(Host, ?SERVICE(Host, Dict)),
-    {ok, LRef} = diameter:add_transport(Host, ?LISTEN),
-    {LRef, portnr(LRef)}.
-
-portnr(LRef) ->
-    portnr(LRef, 20).
-
-portnr(LRef, N)
-  when 0 < N ->
-    case diameter_reg:match({diameter_tcp, listener, {LRef, '_'}}) of
-        [{T, _Pid}] ->
-            {_, _, {LRef, {_Addr, LSock}}} = T,
-            {ok, PortNr} = inet:port(LSock),
-            PortNr;
-        [] ->
-            receive after 50 -> ok end,
-            portnr(LRef, N-1)
-    end.
-
-connect(Host, {_LRef, PortNr}) ->
-    {ok, Ref} = diameter:add_transport(Host, ?CONNECT(PortNr)),
-    ok = receive
-             #diameter_event{service = Host,
-                             info = {up, Ref, _, _, #diameter_packet{}}} ->
-                 ok
-         after 2000 ->
-                 false
-         end,
-    Ref;
-connect(Host, Ports) ->
-    [connect(Host, P) || P <- Ports].
-
-disconnect(Client, {LRef, _PortNr}, CRef) ->
-    ok = diameter:remove_transport(Client, CRef),
-    ok = receive #diameter_event{info = {down, LRef, _, _}} -> ok
-         after 2000 -> false
-         end.
-
 call(Server) ->
     Realm = realm(Server),
     Req = ['STR', {'Destination-Realm', Realm},
@@ -340,7 +276,7 @@ peer_down(_SvcName, _Peer, State) ->
 pick_peer([Peer | _], _, Svc, _State)
   when Svc == ?RELAY1;
        Svc == ?RELAY2;
-       Svc == ?CLIENT->
+       Svc == ?CLIENT ->
     {ok, Peer}.
 
 %% prepare_request/3
diff --git a/lib/diameter/test/diameter_tls_SUITE.erl b/lib/diameter/test/diameter_tls_SUITE.erl
index 90e32c834f..99f92ca0e0 100644
--- a/lib/diameter/test/diameter_tls_SUITE.erl
+++ b/lib/diameter/test/diameter_tls_SUITE.erl
@@ -44,6 +44,7 @@
 %% testcases
 -export([start_ssl/1,
          start_diameter/1,
+         make_certs/1, make_certs/0,
          start_services/1,
          add_transports/1,
          send1/1,
@@ -134,11 +135,12 @@
 %% ===========================================================================
 
 suite() ->
-    [{timetrap, {seconds, 15}}].
+    [{timetrap, {seconds, 10}}].
 
 all() ->
     [start_ssl,
      start_diameter,
+     make_certs,
      start_services,
      add_transports]
         ++ [{group, N} || {N, _, _} <- groups()]
@@ -184,6 +186,18 @@ start_ssl(_Config) ->
 start_diameter(_Config) ->
     ok = diameter:start().
 
+make_certs() ->
+    [{timetrap, {seconds, 30}}].
+
+make_certs(Config) ->
+    Dir = proplists:get_value(priv_dir, Config),
+
+    [] = ?util:run([[fun make_cert/2, Dir, B] || B <- ["server1",
+                                                       "server2",
+                                                       "server4",
+                                                       "server5",
+                                                       "client"]]).
+
 start_services(Config) ->
     Dir = proplists:get_value(priv_dir, Config),
     Servers = [server(S, sopts(S, Dir)) || S <- ?SERVERS],
@@ -325,8 +339,12 @@ inband_security(Ids) ->
     [{'Inband-Security-Id', Ids}].
 
 ssl_options(Dir, Base) ->
-    {Key, Cert} = make_cert(Dir, Base ++ "_key.pem", Base ++ "_ca.pem"),
-    [{ssl_options, [{certfile, Cert}, {keyfile, Key}]}].
+    Root = filename:join([Dir, Base]),
+    [{ssl_options, [{certfile, Root ++ "_ca.pem"},
+                    {keyfile,  Root ++ "_key.pem"}]}].
+
+make_cert(Dir, Base) ->
+    make_cert(Dir, Base ++ "_key.pem", Base ++ "_ca.pem").
 
 make_cert(Dir, Keyfile, Certfile) ->
     [K,C] = Paths = [filename:join([Dir, F]) || F <- [Keyfile, Certfile]],
@@ -351,7 +369,7 @@ join(Strs) ->
 server(Host, {Caps, Opts}) ->
     ok = diameter:start_service(Host, ?SERVICE(Host, ?DICT_COMMON)),
     {ok, LRef} = diameter:add_transport(Host, ?LISTEN(Caps, Opts)),
-    {LRef, portnr(LRef)}.
+    {LRef, hd([_] = ?util:lport(tcp, LRef, 20))}.
 
 sopts(?SERVER1, Dir) ->
     {inband_security([?TLS]),
@@ -369,32 +387,6 @@ sopts(?SERVER5, Dir) ->
 ssl([{ssl_options = T, Opts}]) ->
     [{T, true} | Opts].
 
-portnr(LRef) ->
-    portnr(LRef, 20).
-
-portnr(LRef, N)
-  when 0 < N ->
-    case diameter_reg:match({diameter_tcp, listener, {LRef, '_'}}) of
-        [{T, _Pid}] ->
-            {_, _, {LRef, {_Addr, LSock}}} = T,
-            {ok, PortNr} = to_portnr(LSock) ,
-            PortNr;
-        [] ->
-            receive after 500 -> ok end,
-            portnr(LRef, N-1)
-    end.
-
-to_portnr(Sock)
-  when is_port(Sock) ->
-    inet:port(Sock);
-to_portnr(Sock) ->
-    case ssl:sockname(Sock) of
-        {ok, {_,N}} ->
-            {ok, N};
-        No ->
-            No
-    end.
-
 %% connect/3
 
 connect(Host, {_LRef, PortNr}, {Caps, Opts}) ->
diff --git a/lib/diameter/test/diameter_traffic_SUITE.erl b/lib/diameter/test/diameter_traffic_SUITE.erl
index f6905473b7..6704f24532 100644
--- a/lib/diameter/test/diameter_traffic_SUITE.erl
+++ b/lib/diameter/test/diameter_traffic_SUITE.erl
@@ -127,19 +127,6 @@
                         {module, ?MODULE},
                         {answer_errors, callback}]}]).
 
-%% Config for diameter:add_transport/2. In the listening case, listen
-%% on a free port that we then lookup using the implementation detail
-%% that diameter_tcp registers the port with diameter_reg.
--define(CONNECT(PortNr),
-        {connect, [{transport_module, diameter_tcp},
-                   {transport_config, [{raddr, ?ADDR},
-                                       {rport, PortNr},
-                                       {ip, ?ADDR},
-                                       {port, 0}]}]}).
--define(LISTEN,
-        {listen, [{transport_module, diameter_tcp},
-                  {transport_config, [{ip, ?ADDR}, {port, 0}]}]}).
-
 -define(SUCCESS,
         ?'DIAMETER_BASE_RESULT-CODE_DIAMETER_SUCCESS').
 -define(COMMAND_UNSUPPORTED,
@@ -253,21 +240,6 @@ tc() ->
      send_multiple_filters_3,
      send_anything].
 
-portnr() ->
-    portnr(20).
-
-portnr(N)
-  when 0 < N ->
-    case diameter_reg:match({diameter_tcp, listener, '_'}) of
-        [{T, _Pid}] ->
-            {_, _, {_LRef, {_Addr, LSock}}} = T,
-            {ok, PortNr} = inet:port(LSock),
-            PortNr;
-        [] ->
-            receive after 50 -> ok end,
-            portnr(N-1)
-    end.
-
 %% ===========================================================================
 %% start/stop testcases
 
@@ -279,26 +251,13 @@ start_services(_Config) ->
     ok = diameter:start_service(?CLIENT, ?SERVICE(?CLIENT)).
 
 add_transports(Config) ->
-    {ok, LRef} = diameter:add_transport(?SERVER, ?LISTEN),
-    true = diameter:subscribe(?CLIENT),
-    {ok, CRef} = diameter:add_transport(?CLIENT, ?CONNECT(portnr())),
-    {up, CRef, _Peer, _Cfg, #diameter_packet{}}
-        = receive #diameter_event{service = ?CLIENT, info = I} -> I
-          after 2000 -> false
-          end,
-    true = diameter:unsubscribe(?CLIENT),
+    LRef = ?util:listen(?SERVER, tcp),
+    CRef = ?util:connect(?CLIENT, tcp, LRef),
     ?util:write_priv(Config, "transport", {LRef, CRef}).
 
-%% Remove the client transport and expect the server transport to
-%% go down.
 remove_transports(Config) ->
     {LRef, CRef} = ?util:read_priv(Config, "transport"),
-    true = diameter:subscribe(?SERVER),
-    ok = diameter:remove_transport(?CLIENT, CRef),
-    {down, LRef, _, _}
-        = receive #diameter_event{service = ?SERVER, info = I} -> I
-          after 2000 -> false
-          end.
+    ?util:disconnect(?CLIENT, CRef, ?SERVER, LRef).
 
 stop_services(_Config) ->
     ok = diameter:stop_service(?CLIENT),
diff --git a/lib/diameter/test/diameter_transport_SUITE.erl b/lib/diameter/test/diameter_transport_SUITE.erl
index d556a903e5..a9520ef5bd 100644
--- a/lib/diameter/test/diameter_transport_SUITE.erl
+++ b/lib/diameter/test/diameter_transport_SUITE.erl
@@ -69,16 +69,6 @@
                                       = #diameter_caps{host_ip_address
                                                        = Addrs}}).
 
-%% The term diameter_tcp/sctp registers after opening a listening
-%% socket. This is an implementation detail that should probably be
-%% replaced by some documented way of getting at the port number of
-%% the listening socket, which is what we're after since we specify
-%% port 0 to get something unused.
--define(TCP_LISTENER(Ref, Addr, LSock),
-        {diameter_tcp, listener, {Ref, {Addr, LSock}}}).
--define(SCTP_LISTENER(Ref, Addr, LSock),
-        {diameter_sctp, listener, {Ref, {[Addr], LSock}}}).
-
 %% The term we register after open a listening port with gen_tcp.
 -define(TEST_LISTENER(Ref, PortNr),
         {?MODULE, listen, Ref, PortNr}).
@@ -227,7 +217,7 @@ init(accept, {Prot, Ref}) ->
 
 init(gen_connect, {Prot, Ref}) ->
     %% Lookup the peer's listening socket.
-    {ok, PortNr} = inet:port(lsock(Prot, Ref)),
+    [PortNr] = ?util:lport(Prot, Ref, 20),
 
     %% Connect, send a message and receive it back.
     {ok, Sock} = gen_connect(Prot, PortNr, Ref),
@@ -264,22 +254,16 @@ init(connect, {Prot, Ref}) ->
     MRef = erlang:monitor(process, TPid),
     ?RECV({'DOWN', MRef, process, _, _}).
 
-lsock(sctp, Ref) ->
-    [{?SCTP_LISTENER(_ , _, LSock), _}]
-        = match(?SCTP_LISTENER(Ref, ?ADDR, '_')),
-    LSock;
-lsock(tcp, Ref) ->
-    [{?TCP_LISTENER(_ , _, LSock), _}]
-        = match(?TCP_LISTENER(Ref, ?ADDR, '_')),
-    LSock.
-
 match(Pat) ->
-    case diameter_reg:match(Pat) of
-        [] ->
+    match(Pat, 20).
+
+match(Pat, T) ->
+    L = diameter_reg:match(Pat),
+    if [] /= L orelse 1 == T ->
+            L;
+       true ->
             ?WAIT(50),
-            match(Pat);
-        L ->
-            L
+            match(Pat, T-1)
     end.
 
 bin(sctp, #diameter_packet{bin = Bin}) ->
@@ -343,7 +327,7 @@ start_accept(Prot, Ref) ->
 
     %% Configure the same port number for transports on the same
     %% reference.
-    PortNr = portnr(Prot, Ref),
+    [PortNr | _] = ?util:lport(Prot, Ref) ++ [0],
     {Mod, Opts} = tmod(Prot),
 
     try
@@ -373,23 +357,6 @@ tmod(sctp) ->
 tmod(tcp) ->
     {diameter_tcp, []}.
 
-portnr(sctp, Ref) ->
-    case diameter_reg:match(?SCTP_LISTENER(Ref, ?ADDR, '_')) of
-        [{?SCTP_LISTENER(_, _, LSock), _}] ->
-            {ok, N} = inet:port(LSock),
-            N;
-        [] ->
-            0
-    end;
-portnr(tcp, Ref) ->
-    case diameter_reg:match(?TCP_LISTENER(Ref, ?ADDR, '_')) of
-        [{?TCP_LISTENER(_, _, LSock), _}] ->
-            {ok, N} = inet:port(LSock),
-            N;
-        [] ->
-            0
-    end.
-
 %% ===========================================================================
 
 %% gen_connect/3
diff --git a/lib/diameter/test/diameter_util.erl b/lib/diameter/test/diameter_util.erl
index f9942c3408..3fe8ea5363 100644
--- a/lib/diameter/test/diameter_util.erl
+++ b/lib/diameter/test/diameter_util.erl
@@ -23,16 +23,28 @@
 %% Utility functions.
 %%
 
+%% generic
 -export([consult/2,
          run/1,
          fold/3,
          foldl/3,
-         scramble/1,
-         write_priv/3,
-         read_priv/2]).
+         scramble/1]).
+
+%% diameter-specific
+-export([lport/2,
+         lport/3,
+         listen/2,
+         connect/3,
+         disconnect/4]).
+
+%% common_test-specific
+-export([write_priv/3,
+         read_priv/2,
+         map_priv/3]).
 
 -define(L, atom_to_list).
 
+%% ---------------------------------------------------------------------------
 %% consult/2
 %%
 %% Extract info from the app/appup file (presumably) of the named
@@ -57,6 +69,7 @@ consult(Path) ->
 %% Name/Path in the return value distinguish the errors and allow for
 %% a useful badmatch.
 
+%% ---------------------------------------------------------------------------
 %% run/1
 %%
 %% Evaluate functions in parallel and return a list of those that
@@ -72,6 +85,7 @@ cons(true, _, _, Acc) ->
 cons(false, F, RC, Acc) ->
     [{F, RC} | Acc].
 
+%% ---------------------------------------------------------------------------
 %% fold/3
 %%
 %% Parallel fold. Results are folded in the order received.
@@ -117,6 +131,7 @@ down(MRef) ->
 down() ->
     receive {'DOWN', MRef, process, _, Reason} -> {MRef, Reason} end.
 
+%% ---------------------------------------------------------------------------
 %% foldl/3
 %%
 %% Parallel fold. Results are folded in order of the function list.
@@ -132,6 +147,7 @@ recvl([{MRef, F} | L], Ref, Fun, Acc) ->
     R = down(MRef),
     recvl(L, Ref, Fun, acc(R, Ref, F, Fun, Acc)).
 
+%% ---------------------------------------------------------------------------
 %% scramble/1
 %%
 %% Sort a list into random order.
@@ -151,7 +167,10 @@ s(Acc, L) ->
     {H, [T|Rest]} = lists:split(random:uniform(length(L)) - 1, L),
     s([T|Acc], H ++ Rest).
 
+%% ---------------------------------------------------------------------------
 %% eval/1
+%%
+%% Evaluate a function in one of a number of forms.
 
 eval({M,[F|A]})
   when is_atom(F) ->
@@ -172,17 +191,126 @@ eval(F)
   when is_function(F,0) ->
     F().
 
+%% ---------------------------------------------------------------------------
 %% write_priv/3
+%%
+%% Write an arbitrary term to a named file.
 
 write_priv(Config, Name, Term) ->
-    Dir = proplists:get_value(priv_dir, Config),
-    Path = filename:join([Dir, Name]),
+    write(path(Config, Name), Term).
+
+write(Path, Term) ->
     ok = file:write_file(Path, term_to_binary(Term)).
 
 %% read_priv/2
+%%
+%% Read a term from a file.
 
 read_priv(Config, Name) ->
-    Dir = proplists:get_value(priv_dir, Config),
-    Path = filename:join([Dir, Name]),
+    read(path(Config, Name)).
+
+read(Path) ->
     {ok, Bin} = file:read_file(Path),
     binary_to_term(Bin).
+    
+%% map_priv/3
+%%
+%% Modify a term in a file and return both old and new values.
+
+map_priv(Config, Name, Fun1) ->
+    map(path(Config, Name), Fun1).
+
+map(Path, Fun1) ->
+    T0 = read(Path),
+    T1 = Fun1(T0),
+    write(Path, T1),
+    {T0, T1}.
+
+path(Config, Name)
+  when is_atom(Name) ->
+    path(Config, ?L(Name));
+path(Config, Name) ->
+    Dir = proplists:get_value(priv_dir, Config),
+    filename:join([Dir, Name]).
+
+%% ---------------------------------------------------------------------------
+%% lport/2-3
+%%
+%% Lookup the port number of a tcp/sctp listening transport.
+
+lport(M, Ref) ->
+    lport(M, Ref, 1).
+
+lport(M, Ref, Tries) ->
+    lp(tmod(M), Ref, Tries).
+
+lp(M, Ref, T) ->
+    L = [N || {listen, N, _} <- M:ports(Ref)],
+    if [] /= L orelse T =< 1 ->
+            L;
+       true ->
+            receive after 50 -> ok end,
+            lp(M, Ref, T-1)
+    end.
+
+%% ---------------------------------------------------------------------------
+%% listen/2
+%%
+%% Add a listening transport on the loopback address and a free port.
+
+listen(SvcName, Prot) ->
+    add_transport(SvcName, {listen, opts(Prot, listen)}).
+
+%% ---------------------------------------------------------------------------
+%% connect/3
+%%
+%% Add a connecting transport on and connect to a listening transport
+%% with the specified reference.
+
+connect(Client, Prot, LRef) ->
+    [PortNr] = lport(Prot, LRef, 20),
+    Ref = add_transport(Client, {connect, opts(Prot, PortNr)}),
+    true = diameter:subscribe(Client),
+    ok = receive
+             {diameter_event, Client, {up, Ref, _, _, _}} -> ok
+         after 2000 ->
+                 {Client, Prot, PortNr, process_info(self(), messages)}
+         end,
+    Ref.
+
+%% ---------------------------------------------------------------------------
+%% disconnect/4
+%%
+%% Remove the client transport and expect the server transport to go
+%% down.
+
+disconnect(Client, Ref, Server, LRef) ->
+    true = diameter:subscribe(Server),
+    ok = diameter:remove_transport(Client, Ref),
+    ok = receive
+             {diameter_event, Server, {down, LRef, _, _}} -> ok
+         after 2000 ->
+                 {Client, Ref, Server, LRef, process_info(self(), messages)}
+         end.
+
+%% ---------------------------------------------------------------------------
+
+-define(ADDR, {127,0,0,1}).
+
+add_transport(SvcName, T) ->
+    {ok, Ref} = diameter:add_transport(SvcName, T),
+    Ref.
+
+tmod(tcp) ->
+    diameter_tcp;
+tmod(sctp) ->
+    diameter_sctp.
+
+opts(Prot, T) ->
+    [{transport_module, tmod(Prot)},
+     {transport_config, [{ip, ?ADDR}, {port, 0} | opts(T)]}].
+
+opts(listen) ->
+    [];
+opts(PortNr) ->
+    [{raddr, ?ADDR}, {rport, PortNr}].
diff --git a/lib/diameter/test/modules.mk b/lib/diameter/test/modules.mk
index 7c691c302b..531aca2799 100644
--- a/lib/diameter/test/modules.mk
+++ b/lib/diameter/test/modules.mk
@@ -35,7 +35,8 @@ MODULES = \
 	diameter_transport_SUITE \
 	diameter_traffic_SUITE \
 	diameter_relay_SUITE \
-	diameter_tls_SUITE
+	diameter_tls_SUITE \
+	diameter_failover_SUITE
 
 INTERNAL_HRL_FILES = \
 	diameter_ct.hrl