Merge branch 'anders/diameter/pool/OTP-12428' into maint

* anders/diameter/pool/OTP-12428:
  Fix SCTP match blunder in suites
  Be backwards compatible with diameter_sctp listener state
  Add gen_tcp testcase that fails sporadically
  Simplify transport suite
  Remove (ancient) dead code
  Don't orphan slave nodes in example suite
  Refresh example code
  Improve language consistency in diameter(1)
  Add pool suite to test transport_opt() pool_size
  Adapt tcp/sctp transport modules for pool_size > 1
  Add transport_opt() pool_size

7 files changed, 258 insertions, 104 deletions

diff --git a/lib/diameter/test/diameter_examples_SUITE.erl b/lib/diameter/test/diameter_examples_SUITE.erl
index aef4bc35ef..dc0b466b59 100644
--- a/lib/diameter/test/diameter_examples_SUITE.erl
+++ b/lib/diameter/test/diameter_examples_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2013-2014. All Rights Reserved.
+%% Copyright Ericsson AB 2013-2015. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -397,4 +397,4 @@ stop(Name)
 
 stop(Config) ->
     Prot = proplists:get_value(group, Config),
-    [] = [RC || N <- ?NODES, RC <- [stop(concat(Prot, N))], RC /= ok].
+    [] = [RC || N <- ?NODES, RC <- [catch stop(concat(Prot, N))], RC /= ok].
diff --git a/lib/diameter/test/diameter_gen_sctp_SUITE.erl b/lib/diameter/test/diameter_gen_sctp_SUITE.erl
index 51ccb1e6ec..25017272b5 100644
--- a/lib/diameter/test/diameter_gen_sctp_SUITE.erl
+++ b/lib/diameter/test/diameter_gen_sctp_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2010-2013. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2015. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -119,10 +119,10 @@ send_not_from_controlling_process(_) ->
 
 send_not_from_controlling_process() ->
     FPid = self(),
-    {L, MRef} = spawn_monitor(fun() -> listen(FPid) end),%% listening process
+    {L, MRef} = spawn_monitor(fun() -> listen(FPid) end),
     receive
         {?MODULE, C, S} ->
-            erlang:demonitor(MRef, [flush]),
+            demonitor(MRef, [flush]),
             [L,C,S];
         {'DOWN', MRef, process, _, _} = T ->
             error(T)
@@ -137,13 +137,7 @@ listen(FPid) ->
     LPid = self(),
     spawn(fun() -> connect1(PortNr, FPid, LPid) end), %% connecting process
     Id = assoc(Sock),
-    ?SCTP(Sock, {[#sctp_sndrcvinfo{assoc_id = Id}], _Bin})
-        = recv(). %% Waits with this as current_function.
-
-%% recv/0
-
-recv() ->
-    receive T -> T end.
+    recv(Sock, Id).
 
 %% connect1/3
 
@@ -154,7 +148,7 @@ connect1(PortNr, FPid, LPid) ->
     FPid ! {?MODULE,
             self(),
             spawn(fun() -> send(Sock, Id) end)}, %% sending process
-    MRef = erlang:monitor(process, LPid),
+    MRef = monitor(process, LPid),
     down(MRef).  %% Waits with this as current_function.
 
 %% down/1
@@ -277,7 +271,8 @@ acc(N, Acc) ->
 
 loop(Sock, MRef, Bin) ->
     receive
-        ?SCTP(Sock, {[#sctp_sndrcvinfo{assoc_id = Id}], B}) ->
+        ?SCTP(Sock, {[#sctp_sndrcvinfo{assoc_id = Id}], B})
+          when is_binary(B) ->
             Sz = size(Bin),
             {Sz, Bin} = {size(B), B},  %% assert
             ok = send(Sock, Id, mark(Bin)),
@@ -291,7 +286,7 @@ loop(Sock, MRef, Bin) ->
 %% connect2/3
 
 connect2(Pid, PortNr, Bin) ->
-    erlang:monitor(process, Pid),
+    monitor(process, Pid),
 
     {ok, Sock} = open(),
     ok = gen_sctp:connect_init(Sock, ?ADDR, PortNr, []),
@@ -311,9 +306,14 @@ connect2(Pid, PortNr, Bin) ->
 
 recv(Sock, Id) ->
     receive
-        ?SCTP(Sock, {[#sctp_sndrcvinfo{assoc_id = Id}], Bin}) ->
+        ?SCTP(Sock, {[#sctp_sndrcvinfo{assoc_id = I}], Bin})
+          when is_binary(Bin) ->
+            Id = I,   %% assert
             Bin;
-        T ->  %% eg. 'DOWN'
+        ?SCTP(S, _) ->
+            Sock = S, %% assert
+            recv(Sock, Id);
+        T ->
             exit(T)
     end.
 
diff --git a/lib/diameter/test/diameter_gen_tcp_SUITE.erl b/lib/diameter/test/diameter_gen_tcp_SUITE.erl
index 7e232edb44..4b542e0156 100644
--- a/lib/diameter/test/diameter_gen_tcp_SUITE.erl
+++ b/lib/diameter/test/diameter_gen_tcp_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2013. All Rights Reserved.
+%% Copyright Ericsson AB 2014-2015. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -18,10 +18,10 @@
 %%
 
 %%
-%% Some gen_sctp-specific tests demonstrating problems that were
+%% Some gen_tcp-specific tests demonstrating problems that were
 %% encountered during diameter development but have nothing
-%% specifically to do with diameter. At least one of them can cause
-%% diameter_traffic_SUITE testcases to fail.
+%% specifically to do with diameter. These can cause testcases in
+%% other suites to fail.
 %%
 
 -module(diameter_gen_tcp_SUITE).
@@ -30,7 +30,8 @@
          all/0]).
 
 %% testcases
--export([send_long/1]).
+-export([send_long/1,
+         connect/1]).
 
 -define(LOOPBACK, {127,0,0,1}).
 -define(GEN_OPTS, [binary, {active, true}, {ip, ?LOOPBACK}]).
@@ -41,7 +42,8 @@ suite() ->
     [{timetrap, {minutes, 2}}].
 
 all() ->
-    [send_long].
+    [connect,     %% Appears to fail only when run first.
+     send_long].
 
 %% ===========================================================================
 
@@ -87,15 +89,6 @@ connect(PortNr, LPid) ->
     LPid ! {self(), fun(B) -> send(Sock, B) end},
     down(LPid).
 
-%% down/1
-
-down(Pid)
-  when is_pid(Pid) ->
-    down(erlang:monitor(process, Pid));
-
-down(MRef) ->
-    receive {'DOWN', MRef, process, _, Reason} -> Reason end.
-
 %% send/2
 %%
 %% Send from a spawned process just to avoid sending from the
@@ -104,3 +97,47 @@ down(MRef) ->
 send(Sock, Bin) ->
     {_, MRef} = spawn_monitor(fun() -> exit(gen_tcp:send(Sock, Bin)) end),
     down(MRef).
+
+%% ===========================================================================
+
+%% connect/1
+%%
+%% Test that simultaneous connections succeed. This fails sporadically
+%% on OS X at the time of writing, when gen_tcp:connect/2 returns
+%% {error, econnreset}.
+
+connect(_) ->
+    {ok, LSock} = gen_tcp:listen(0, ?GEN_OPTS),
+    {ok, {_,PortNr}} = inet:sockname(LSock),
+    Count = lists:seq(1,8),  %% 8 simultaneous connects
+    As = [gen_accept(LSock) || _ <- Count],
+    %% Wait for spawned processes to have called gen_tcp:accept/1
+    %% (presumably).
+    receive after 2000 -> ok end,
+    Cs = [gen_connect(PortNr) || _ <- Count],
+    [] = failures(Cs),
+    [] = failures(As).
+
+failures(Monitors) ->
+    [RC || {_, MRef} <- Monitors, RC <- [down(MRef)], ok /= element(1, RC)].
+
+gen_accept(LSock) ->
+    spawn_monitor(fun() ->
+                          exit(gen_tcp:accept(LSock))
+                  end).
+
+gen_connect(PortNr) ->
+    spawn_monitor(fun() ->
+                          exit(gen_tcp:connect(?LOOPBACK, PortNr, ?GEN_OPTS))
+                  end).
+
+%% ===========================================================================
+
+%% down/1
+
+down(Pid)
+  when is_pid(Pid) ->
+    down(monitor(process, Pid));
+
+down(MRef) ->
+    receive {'DOWN', MRef, process, _, Reason} -> Reason end.
diff --git a/lib/diameter/test/diameter_pool_SUITE.erl b/lib/diameter/test/diameter_pool_SUITE.erl
new file mode 100644
index 0000000000..a59cd66a2e
--- /dev/null
+++ b/lib/diameter/test/diameter_pool_SUITE.erl
@@ -0,0 +1,133 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2015. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%
+%% Test of the pool_size option in connecting nodes with multiple
+%% connections.
+%%
+
+-module(diameter_pool_SUITE).
+
+-export([suite/0,
+         all/0,
+         init_per_testcase/2,
+         end_per_testcase/2,
+         init_per_suite/1,
+         end_per_suite/1]).
+
+%% testcases
+-export([tcp_connect/1,
+         sctp_connect/1,
+         any_connect/1]).
+
+%% ===========================================================================
+
+-define(util, diameter_util).
+
+%% Config for diameter:start_service/2.
+-define(SERVICE(Host),
+        [{'Origin-Host', Host ++ ".ericsson.com"},
+         {'Origin-Realm', "ericsson.com"},
+         {'Host-IP-Address', [{127,0,0,1}]},
+         {'Vendor-Id', 12345},
+         {'Product-Name', "OTP/diameter"},
+         {'Auth-Application-Id', [0]},  %% common
+         {'Acct-Application-Id', [3]},  %% accounting
+         {restrict_connections, false},
+         {application, [{alias, common},
+                        {dictionary, diameter_gen_base_rfc6733},
+                        {module, diameter_callback}]},
+         {application, [{alias, accounting},
+                        {dictionary, diameter_gen_acct_rfc6733},
+                        {module, diameter_callback}]}]).
+
+%% ===========================================================================
+
+suite() ->
+    [{timetrap, {seconds, 30}}].
+
+all() ->
+    [tcp_connect,
+     sctp_connect,
+     any_connect].
+
+init_per_testcase(_Name, Config) ->
+    Config.
+
+end_per_testcase(_Name, _Config) ->
+    diameter:stop().
+
+init_per_suite(Config) ->
+    [{sctp, ?util:have_sctp()} | Config].
+
+end_per_suite(_Config) ->
+    ok.
+
+%% ===========================================================================
+
+tcp_connect(_Config) ->
+    connect(tcp, tcp).
+
+sctp_connect(Config) ->
+    case lists:member({sctp, true}, Config) of
+       true  -> connect(sctp, sctp);
+       false -> {skip, no_sctp}
+    end.
+
+any_connect(_Config) ->
+    connect(any, tcp).
+
+%% connect/2
+
+%% Establish multiple connections between a client and server.
+connect(ClientProt, ServerProt) ->
+    ok = diameter:start(),
+    [] = [{S,T} || S <- ["server", "client"],
+                   T <- [diameter:start_service(S, ?SERVICE(S))],
+                   T /= ok],
+    %% Listen with a single transport with pool_size = 4. Ensure the
+    %% expected number of transport processes are started.
+    LRef = ?util:listen("server", ServerProt, [{pool_size, 4}]),
+    {4,0} = count("server", LRef, accept), %% 4 transports, no connections
+    %% Establish 5 connections.
+    Ref = ?util:connect("client", ClientProt, LRef, [{pool_size, 5}]),
+    {5,5} = count("client", Ref, pool),    %% 5 connections
+    %% Ensure the server has started replacement transports within a
+    %% reasonable time. Sleepsince there's no guarantee the
+    %% replacements have been started before the client has received
+    %% 'up' events. (Although it's likely.)
+    sleep(),
+    {9,5} = count("server", LRef, accept), %% 5 connections + 4 accepting
+    %% Ensure ther are still the expected number of accepting transports
+    %% after stopping the client service.
+    ok = diameter:stop_service("client"),
+    sleep(),
+    {4,0} = count("server", LRef, accept), %% 4 transports, no connections
+    %% Done.
+    ok = diameter:stop_service("server").
+
+count(Name, Ref, Key) ->
+    [{transport, [[{ref, Ref} | T]]},
+     {connections, Cs}]
+        = diameter:service_info(Name, [transport, connections]),
+    {Key, Ps} = lists:keyfind(Key, 1, T),
+    {length(Ps), length(Cs)}.  %% number of processes, connections
+
+sleep() ->
+    receive after 1000 -> ok end.
diff --git a/lib/diameter/test/diameter_transport_SUITE.erl b/lib/diameter/test/diameter_transport_SUITE.erl
index fcffa69c24..84ab6cbdc4 100644
--- a/lib/diameter/test/diameter_transport_SUITE.erl
+++ b/lib/diameter/test/diameter_transport_SUITE.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2010-2013. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2015. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -104,7 +104,7 @@ tc() ->
      reconnect].
 
 init_per_suite(Config) ->
-    [{sctp, have_sctp()} | Config].
+    [{sctp, ?util:have_sctp()} | Config].
 
 end_per_suite(_Config) ->
     ok.
@@ -127,7 +127,10 @@ tcp_accept(_) ->
     accept(tcp).
 
 sctp_accept(Config) ->
-    if_sctp(fun accept/1, Config).
+    case lists:member({sctp, true}, Config) of
+        true  -> accept(sctp);
+        false -> {skip, no_sctp}
+    end.
 
 %% Start multiple accepting transport processes that are connected to
 %% with an equal number of connecting processes using gen_tcp/sctp
@@ -157,7 +160,10 @@ tcp_connect(_) ->
     connect(tcp).
 
 sctp_connect(Config) ->
-    if_sctp(fun connect/1, Config).
+    case lists:member({sctp, true}, Config) of
+        true  -> connect(sctp);
+        false -> {skip, no_sctp}
+    end.
 
 connect(Prot) ->
     T = {Prot, make_ref()},
@@ -251,28 +257,6 @@ abort(SvcName, LRef, Ref)
 %% ===========================================================================
 %% ===========================================================================
 
-%% have_sctp/0
-
-have_sctp() ->
-    case gen_sctp:open() of
-        {ok, Sock} ->
-            gen_sctp:close(Sock),
-            true;
-        {error, E} when E == eprotonosupport;
-                        E == esocktnosupport -> %% fail on any other reason
-            false
-    end.
-
-%% if_sctp/2
-
-if_sctp(F, Config) ->
-    case proplists:get_value(sctp, Config) of
-        true ->
-            F(sctp);
-        false ->
-            {skip, no_sctp}
-    end.
-
 %% init/2
 
 init(accept, {Prot, Ref}) ->
@@ -381,37 +365,14 @@ start_connect(tcp, T, Svc, Opts) ->
     diameter_tcp:start(T, Svc, Opts).
 
 %% start_accept/2
-%%
-%% Start transports sequentially by having each wait for a message
-%% from a job in a queue before commencing. Only one transport with a
-%% pending accept is started at a time since diameter_{tcp,sctp}
-%% currently assume (and diameter currently implements) this.
 
 start_accept(Prot, Ref) ->
-    Pid = sync(accept, Ref),
     {Mod, Opts} = tmod(Prot),
-
-    try
-        {ok, TPid, [?ADDR]} = Mod:start({accept, Ref},
-                                        ?SVC([?ADDR]),
-                                        [{port, 0} | Opts]),
-        ?RECV(?TMSG({TPid, connected})),
-        TPid
-    after
-        Pid ! Ref
-    end.
-
-sync(What, Ref) ->
-    ok = diameter_sync:cast({?MODULE, What, Ref},
-                            [fun lock/2, Ref, self()],
-                            infinity,
-                            infinity),
-    receive {start, Ref, Pid} -> Pid end.
-
-lock(Ref, Pid) ->
-    Pid ! {start, Ref, self()},
-    erlang:monitor(process, Pid),
-    Ref = receive T -> T end.
+    {ok, TPid, [?ADDR]} = Mod:start({accept, Ref},
+                                    ?SVC([?ADDR]),
+                                    [{port, 0} | Opts]),
+    ?RECV(?TMSG({TPid, connected})),
+    TPid.
 
 tmod(sctp) ->
     {diameter_sctp, [{sctp_initmsg, ?SCTP_INIT}]};
@@ -463,7 +424,11 @@ gen_send(tcp, Sock, Bin) ->
 
 gen_recv(sctp, Sock) ->
     {_OS, _IS, Id} = getr(assoc),
-    ?RECV(?SCTP(Sock, {[#sctp_sndrcvinfo{assoc_id = Id}], Bin}), Bin);
+    receive
+        ?SCTP(Sock, {[#sctp_sndrcvinfo{assoc_id = Id}], Bin})
+          when is_binary(Bin) ->
+            Bin
+    end;
 gen_recv(tcp, Sock) ->
     tcp_recv(Sock, <<>>).
 
diff --git a/lib/diameter/test/diameter_util.erl b/lib/diameter/test/diameter_util.erl
index 92c72c84e7..e95cbfceb5 100644
--- a/lib/diameter/test/diameter_util.erl
+++ b/lib/diameter/test/diameter_util.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2010-2013. All Rights Reserved.
+%% Copyright Ericsson AB 2010-2015. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -29,7 +29,8 @@
          run/1,
          fold/3,
          foldl/3,
-         scramble/1]).
+         scramble/1,
+         have_sctp/0]).
 
 %% diameter-specific
 -export([lport/2,
@@ -184,6 +185,19 @@ s(Acc, L) ->
     s([T|Acc], H ++ Rest).
 
 %% ---------------------------------------------------------------------------
+%% have_sctp/0
+
+have_sctp() ->
+    case gen_sctp:open() of
+        {ok, Sock} ->
+            gen_sctp:close(Sock),
+            true;
+        {error, E} when E == eprotonosupport;
+                        E == esocktnosupport -> %% fail on any other reason
+            false
+    end.
+
+%% ---------------------------------------------------------------------------
 %% eval/1
 %%
 %% Evaluate a function in one of a number of forms.
@@ -254,13 +268,12 @@ path(Config, Name) ->
 %%
 %% Lookup the port number of a tcp/sctp listening transport.
 
-lport(M, {Node, Ref}) ->
-    rpc:call(Node, ?MODULE, lport, [M, Ref]);
+lport(Prot, {Node, Ref}) ->
+    rpc:call(Node, ?MODULE, lport, [Prot, Ref]);
 
 lport(Prot, Ref) ->
-    Mod = tmod(Prot),
     [_] = diameter_reg:wait({'_', listener, {Ref, '_'}}),
-    [N || {listen, N, _} <- Mod:ports(Ref)].
+    [N || M <- tmod(Prot), {listen, N, _} <- M:ports(Ref)].
 
 %% ---------------------------------------------------------------------------
 %% listen/2-3
@@ -292,13 +305,17 @@ connect(Client, Prot, LRef, Opts) ->
     Ref = add_transport(Client, {connect, opts(Prot, PortNr) ++ Opts}),
     true = transport(Client, Ref),                 %% assert
 
-    ok = receive
-             {diameter_event, Client, {up, Ref, _, _, _}} -> ok
-         after 10000 ->
-                 {Client, Prot, PortNr, process_info(self(), messages)}
-         end,
+    diameter_lib:for_n(fun(_) -> ok = up(Client, Ref, Prot, PortNr) end,
+                       proplists:get_value(pool_size, Opts, 1)),
     Ref.
 
+up(Client, Ref, Prot, PortNr) ->
+    receive
+        {diameter_event, Client, {up, Ref, _, _, _}} -> ok
+    after 10000 ->
+            {Client, Prot, PortNr, process_info(self(), messages)}
+    end.
+
 transport(SvcName, Ref) ->
     [Ref] == [R || [{ref, R} | _] <- diameter:service_info(SvcName, transport),
                    R == Ref].
@@ -327,13 +344,15 @@ add_transport(SvcName, T) ->
     Ref.
 
 tmod(tcp) ->
-    diameter_tcp;
+    [diameter_tcp];
 tmod(sctp) ->
-    diameter_sctp.
+    [diameter_sctp];
+tmod(any) ->
+    [diameter_sctp, diameter_tcp].
 
 opts(Prot, T) ->
-    [{transport_module, tmod(Prot)},
-     {transport_config, [{ip, ?ADDR}, {port, 0} | opts(T)]}].
+    [{transport_module, M} || M <- tmod(Prot)]
+        ++ [{transport_config, [{ip, ?ADDR}, {port, 0} | opts(T)]}].
 
 opts(listen) ->
     [{accept, M} || M <- [{256,0,0,1}, ["256.0.0.1", ["^.+$"]]]];
diff --git a/lib/diameter/test/modules.mk b/lib/diameter/test/modules.mk
index 4fea62461c..6da96bd676 100644
--- a/lib/diameter/test/modules.mk
+++ b/lib/diameter/test/modules.mk
@@ -1,8 +1,7 @@
-#-*-makefile-*-   ; force emacs to enter makefile-mode
 
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 2010-2013. All Rights Reserved.
+# Copyright Ericsson AB 2010-2015. All Rights Reserved.
 #
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -40,6 +39,7 @@ MODULES = \
 	diameter_gen_sctp_SUITE \
 	diameter_gen_tcp_SUITE \
 	diameter_length_SUITE \
+	diameter_pool_SUITE \
 	diameter_reg_SUITE \
 	diameter_relay_SUITE \
 	diameter_stats_SUITE \