Make testsuites more robust in case of init failure

In particular, move code out of init_per_suite since failure
causes end_per_suite to be skipped. Cleanup is simpler if both
init and cleanup happen as testcases.

4 files changed, 197 insertions, 144 deletions

diff --git a/lib/diameter/test/diameter_relay_SUITE.erl b/lib/diameter/test/diameter_relay_SUITE.erl
index d3d1fe690a..60babd0b9a 100644
--- a/lib/diameter/test/diameter_relay_SUITE.erl
+++ b/lib/diameter/test/diameter_relay_SUITE.erl
@@ -37,20 +37,22 @@
          all/0,
          groups/0,
          init_per_group/2,
-         end_per_group/2,
-         init_per_suite/1,
-         end_per_suite/1]).
+         end_per_group/2]).
 
 %% testcases
--export([send1/1,
+-export([start/1,
+         start_services/1,
+         connect/1,
+         send1/1,
          send2/1,
          send3/1,
          send4/1,
          send_loop/1,
          send_timeout_1/1,
          send_timeout_2/1,
-         remove_transports/1,
-         stop_services/1]).
+         disconnect/1,
+         stop_services/1,
+         stop/1]).
 
 %% diameter callbacks
 -export([peer_up/3,
@@ -73,6 +75,8 @@
 
 %% ===========================================================================
 
+-define(util, diameter_util).
+
 -define(ADDR, {127,0,0,1}).
 
 -define(CLIENT,  "CLIENT.REALM1").
@@ -83,6 +87,10 @@
 -define(SERVER3, "SERVER1.REALM3").
 -define(SERVER4, "SERVER2.REALM3").
 
+-define(SERVICES, [?CLIENT,
+                   ?RELAY1, ?RELAY2,
+                   ?SERVER1, ?SERVER2, ?SERVER3, ?SERVER4]).
+
 -define(DICT_COMMON,  ?DIAMETER_DICT_COMMON).
 -define(DICT_RELAY,   ?DIAMETER_DICT_RELAY).
 
@@ -131,13 +139,15 @@ suite() ->
     [{timetrap, {seconds, 10}}].
 
 all() ->
-    [{group, N} || {N, _, _} <- groups()]
-        ++ [remove_transports, stop_services].
+    [start, start_services, connect]
+        ++ tc()
+        ++ [{group, all},
+            disconnect,
+            stop_services,
+            stop].
 
 groups() ->
-    Ts = tc(),
-    [{all, [], Ts},
-     {p, [parallel], Ts}].
+    [{all, [parallel], tc()}].
 
 init_per_group(_, Config) ->
     Config.
@@ -145,32 +155,7 @@ init_per_group(_, Config) ->
 end_per_group(_, _) ->
     ok.
 
-init_per_suite(Config) ->
-    ok = diameter:start(),
-    [S1,S2,S3,S4] = S = [server(N, ?DICT_COMMON) || N <- [?SERVER1,
-                                                          ?SERVER2,
-                                                          ?SERVER3,
-                                                          ?SERVER4]],
-    [R1,R2] = R = [server(N, ?DICT_RELAY) || N <- [?RELAY1, ?RELAY2]],
-    
-    ok = diameter:start_service(?CLIENT, ?SERVICE(?CLIENT, ?DICT_COMMON)),
-
-    true = diameter:subscribe(?RELAY1),
-    true = diameter:subscribe(?RELAY2),
-    true = diameter:subscribe(?CLIENT),
-
-    [C1,C2] = connect(?RELAY1, [S1,S2]),
-    [C3,C4] = connect(?RELAY2, [S3,S4]),
-    [C5,C6] = connect(?CLIENT, [R1,R2]),
-
-    C7 = connect(?RELAY1, R2),
-
-    [{transports, {S, R, [C1,C2,C3,C4,C5,C6,C7]}} | Config].
-
-end_per_suite(_Config) ->
-    ok = diameter:stop().
-
-%% Testcases to run when services are started and connections
+%% Traffic cases run when services are started and connections
 %% established.
 tc() ->
     [send1,
@@ -181,43 +166,65 @@ tc() ->
      send_timeout_1,
      send_timeout_2].
 
-server(Host, Dict) ->
-    ok = diameter:start_service(Host, ?SERVICE(Host, Dict)),
-    {ok, LRef} = diameter:add_transport(Host, ?LISTEN),
-    {LRef, portnr(LRef)}.
+%% ===========================================================================
+%% start/stop testcases
 
-connect(Host, {_LRef, PortNr}) ->
-    {ok, Ref} = diameter:add_transport(Host, ?CONNECT(PortNr)),
-    ok = receive
-             #diameter_event{service = Host,
-                             info = {up, Ref, _, _, #diameter_packet{}}} ->
-                 ok
-         after 2000 ->
-                 false
-         end,
-    Ref;
-connect(Host, Ports) ->
-    [connect(Host, P) || P <- Ports].
+start(_Config) ->
+    ok = diameter:start().
 
-portnr(LRef) ->
-    portnr(LRef, 20).
+start_services(_Config) ->
+    S = [server(N, ?DICT_COMMON) || N <- [?SERVER1,
+                                          ?SERVER2,
+                                          ?SERVER3,
+                                          ?SERVER4]],
+    R = [server(N, ?DICT_RELAY) || N <- [?RELAY1, ?RELAY2]],
 
-portnr(LRef, N)
-  when 0 < N ->
-    case diameter_reg:match({diameter_tcp, listener, {LRef, '_'}}) of
-        [{T, _Pid}] ->
-            {_, _, {LRef, {_Addr, LSock}}} = T,
-            {ok, PortNr} = inet:port(LSock),
-            PortNr;
-        [] ->
-            receive after 50 -> ok end,
-            portnr(LRef, N-1)
-    end.
+    ok = diameter:start_service(?CLIENT, ?SERVICE(?CLIENT, ?DICT_COMMON)),
 
-realm(Host) ->
-    tl(lists:dropwhile(fun(C) -> C /= $. end, Host)).
+    {save_config, S ++ R}.
+
+connect(Config) ->
+    {_, [S1,S2,S3,S4,R1,R2] = SR} = proplists:get_value(saved_config, Config),
+
+    true = diameter:subscribe(?RELAY1),
+    true = diameter:subscribe(?RELAY2),
+    true = diameter:subscribe(?CLIENT),
+
+    [R1S1,R1S2] = connect(?RELAY1, [S1,S2]),
+    [R2S3,R2S4] = connect(?RELAY2, [S3,S4]),
+    [CR1,CR2]   = connect(?CLIENT, [R1,R2]),
+
+    R1R2 = connect(?RELAY1, R2),
+
+    ?util:write_priv(Config, "cfg", SR ++ [R1S1,R1S2,R2S3,R2S4,CR1,CR2,R1R2]).
+
+%% Remove the client transports and expect the corresponding server
+%% transport to go down.
+disconnect(Config) ->
+    [S1,S2,S3,S4,R1,R2,R1S1,R1S2,R2S3,R2S4,CR1,CR2,R1R2]
+        = ?util:read_priv(Config, "cfg"),
+
+    [?CLIENT | Svcs] = ?SERVICES,
+    [] = [{S,T} || S <- Svcs, T <- [diameter:subscribe(S)], T /= true],
+
+    disconnect(?RELAY1, S1, R1S1),
+    disconnect(?RELAY1, S2, R1S2),
+    disconnect(?RELAY2, S3, R2S3),
+    disconnect(?RELAY2, S4, R2S4),
+    disconnect(?CLIENT, R1, CR1),
+    disconnect(?CLIENT, R2, CR2),
+    disconnect(?RELAY1, R2, R1R2).
+
+stop_services(_Config) ->
+    [] = [{H,T} || H <- ?SERVICES,
+                   T <- [diameter:stop_service(H)],
+                   T /= ok].
+
+stop(_Config) ->
+    ok = diameter:stop().
 
 %% ===========================================================================
+%% traffic testcases
 
 %% Send an STR intended for a specific server and expect success.
 send1(_Config) ->
@@ -254,40 +261,50 @@ send_timeout(Tmo) ->
                   {'Re-Auth-Request-Type', ?AUTHORIZE_ONLY}],
     call(Req, [{filter, realm}, {timeout, Tmo}]).
 
-%% Remove the client transports and expect the corresponding server
-%% transport to go down.
-remove_transports(Config) ->
-    {[S1,S2,S3,S4], [R1,R2], [C1,C2,C3,C4,C5,C6,C7]}
-        = proplists:get_value(transports, Config),
-
-    true = diameter:subscribe(?SERVER1),
-    true = diameter:subscribe(?SERVER2),
-    true = diameter:subscribe(?SERVER3),
-    true = diameter:subscribe(?SERVER4),
-    true = diameter:subscribe(?RELAY1),
-    true = diameter:subscribe(?RELAY2),
+%% ===========================================================================
+
+realm(Host) ->
+    tl(lists:dropwhile(fun(C) -> C /= $. end, Host)).
+
+server(Host, Dict) ->
+    ok = diameter:start_service(Host, ?SERVICE(Host, Dict)),
+    {ok, LRef} = diameter:add_transport(Host, ?LISTEN),
+    {LRef, portnr(LRef)}.
+
+portnr(LRef) ->
+    portnr(LRef, 20).
+
+portnr(LRef, N)
+  when 0 < N ->
+    case diameter_reg:match({diameter_tcp, listener, {LRef, '_'}}) of
+        [{T, _Pid}] ->
+            {_, _, {LRef, {_Addr, LSock}}} = T,
+            {ok, PortNr} = inet:port(LSock),
+            PortNr;
+        [] ->
+            receive after 50 -> ok end,
+            portnr(LRef, N-1)
+    end.
 
-    disconnect(S1, ?RELAY1, C1),
-    disconnect(S2, ?RELAY1, C2),
-    disconnect(S3, ?RELAY2, C3),
-    disconnect(S4, ?RELAY2, C4),
-    disconnect(R1, ?CLIENT, C5),
-    disconnect(R2, ?CLIENT, C6),
-    disconnect(R2, ?RELAY1, C7).
+connect(Host, {_LRef, PortNr}) ->
+    {ok, Ref} = diameter:add_transport(Host, ?CONNECT(PortNr)),
+    ok = receive
+             #diameter_event{service = Host,
+                             info = {up, Ref, _, _, #diameter_packet{}}} ->
+                 ok
+         after 2000 ->
+                 false
+         end,
+    Ref;
+connect(Host, Ports) ->
+    [connect(Host, P) || P <- Ports].
 
-disconnect({LRef, _PortNr}, Client, CRef) ->
+disconnect(Client, {LRef, _PortNr}, CRef) ->
     ok = diameter:remove_transport(Client, CRef),
     ok = receive #diameter_event{info = {down, LRef, _, _}} -> ok
          after 2000 -> false
          end.
 
-stop_services(_Config) ->
-    S = [?CLIENT, ?RELAY1, ?RELAY2, ?SERVER1, ?SERVER2, ?SERVER3, ?SERVER4],
-    Ok = [ok || _ <- S],
-    Ok = [diameter:stop_service(H) || H <- S].
-
-%% ===========================================================================
-
 call(Server) ->
     Realm = realm(Server),
     Req = ['STR', {'Destination-Realm', Realm},
diff --git a/lib/diameter/test/diameter_traffic_SUITE.erl b/lib/diameter/test/diameter_traffic_SUITE.erl
index 8c85323222..f6905473b7 100644
--- a/lib/diameter/test/diameter_traffic_SUITE.erl
+++ b/lib/diameter/test/diameter_traffic_SUITE.erl
@@ -26,15 +26,16 @@
 -export([suite/0,
          all/0,
          groups/0,
-         init_per_suite/1,
-         end_per_suite/1,
          init_per_group/2,
          end_per_group/2,
          init_per_testcase/2,
          end_per_testcase/2]).
 
 %% testcases
--export([result_codes/1,
+-export([start/1,
+         start_services/1,
+         add_transports/1,
+         result_codes/1,
          send_ok/1,
          send_arbitrary/1,
          send_unknown/1,
@@ -73,7 +74,8 @@
          send_multiple_filters_3/1,
          send_anything/1,
          remove_transports/1,
-         stop_services/1]).
+         stop_services/1,
+         stop/1]).
 
 %% diameter callbacks
 -export([peer_up/3,
@@ -96,6 +98,8 @@
 
 %% ===========================================================================
 
+-define(util, diameter_util).
+
 -define(ADDR, {127,0,0,1}).
 
 -define(CLIENT, "CLIENT").
@@ -177,30 +181,18 @@ suite() ->
     [{timetrap, {seconds, 10}}].
 
 all() ->
-    [result_codes | [{group, N} || {N, _, _} <- groups()]]
-        ++ [remove_transports, stop_services].
+    [start, start_services, add_transports, result_codes
+     | [{group, N} || {N, _, _} <- groups()]]
+        ++ [remove_transports, stop_services, stop].
 
 groups() ->
     Ts = tc(),
-    [{E, [], Ts} || E <- ?ENCODINGS]
-        ++ [{?P(E), [parallel], Ts} || E <- ?ENCODINGS].
+    [{grp(E,P), P, Ts} || E <- ?ENCODINGS, P <- [[], [parallel]]].
 
-init_per_suite(Config) ->
-    ok = diameter:start(),
-    ok = diameter:start_service(?SERVER, ?SERVICE(?SERVER)),
-    ok = diameter:start_service(?CLIENT, ?SERVICE(?CLIENT)),
-    {ok, LRef} = diameter:add_transport(?SERVER, ?LISTEN),
-    true = diameter:subscribe(?CLIENT),
-    {ok, CRef} = diameter:add_transport(?CLIENT, ?CONNECT(portnr())),
-    {up, CRef, _Peer, _Config, #diameter_packet{}}
-        = receive #diameter_event{service = ?CLIENT, info = I} -> I
-          after 2000 -> false
-          end,
-    true = diameter:unsubscribe(?CLIENT),
-    [{transports, {LRef, CRef}} | Config].
-
-end_per_suite(_Config) ->
-    ok = diameter:stop().
+grp(E, []) ->
+    E;
+grp(E, [parallel]) ->
+    ?P(E).
 
 init_per_group(Name, Config) ->
     E = case ?L(Name) of
@@ -277,6 +269,45 @@ portnr(N)
     end.
 
 %% ===========================================================================
+%% start/stop testcases
+
+start(_Config) ->
+    ok = diameter:start().
+
+start_services(_Config) ->
+    ok = diameter:start_service(?SERVER, ?SERVICE(?SERVER)),
+    ok = diameter:start_service(?CLIENT, ?SERVICE(?CLIENT)).
+
+add_transports(Config) ->
+    {ok, LRef} = diameter:add_transport(?SERVER, ?LISTEN),
+    true = diameter:subscribe(?CLIENT),
+    {ok, CRef} = diameter:add_transport(?CLIENT, ?CONNECT(portnr())),
+    {up, CRef, _Peer, _Cfg, #diameter_packet{}}
+        = receive #diameter_event{service = ?CLIENT, info = I} -> I
+          after 2000 -> false
+          end,
+    true = diameter:unsubscribe(?CLIENT),
+    ?util:write_priv(Config, "transport", {LRef, CRef}).
+
+%% Remove the client transport and expect the server transport to
+%% go down.
+remove_transports(Config) ->
+    {LRef, CRef} = ?util:read_priv(Config, "transport"),
+    true = diameter:subscribe(?SERVER),
+    ok = diameter:remove_transport(?CLIENT, CRef),
+    {down, LRef, _, _}
+        = receive #diameter_event{service = ?SERVER, info = I} -> I
+          after 2000 -> false
+          end.
+
+stop_services(_Config) ->
+    ok = diameter:stop_service(?CLIENT),
+    ok = diameter:stop_service(?SERVER).
+
+stop(_Config) ->
+    ok = diameter:stop().
+
+%% ===========================================================================
 
 %% Ensure that result codes have the expected values.
 result_codes(_Config) ->
@@ -532,21 +563,6 @@ send_anything(Config) ->
     #diameter_base_STA{'Result-Code' = ?SUCCESS}
         = call(Config, anything).
     
-%% Remove the client transport and expect the server transport to
-%% go down.
-remove_transports(Config) ->
-    {LRef, CRef} = proplists:get_value(transports, Config),
-    true = diameter:subscribe(?SERVER),
-    ok = diameter:remove_transport(?CLIENT, CRef),
-    {down, LRef, _, _}
-        = receive #diameter_event{service = ?SERVER, info = I} -> I
-          after 2000 -> false
-          end.
-
-stop_services(_Config) ->
-    {ok, ok} = {diameter:stop_service(?CLIENT),
-                diameter:stop_service(?SERVER)}.
-
 %% ===========================================================================
 
 call(Config, Req) ->
diff --git a/lib/diameter/test/diameter_transport_SUITE.erl b/lib/diameter/test/diameter_transport_SUITE.erl
index d545859fe8..064e5caa67 100644
--- a/lib/diameter/test/diameter_transport_SUITE.erl
+++ b/lib/diameter/test/diameter_transport_SUITE.erl
@@ -33,10 +33,12 @@
          end_per_suite/1]).
 
 %% testcases
--export([tcp_accept/1,
+-export([start/1,
+         tcp_accept/1,
          tcp_connect/1,
          sctp_accept/1,
-         sctp_connect/1]).
+         sctp_connect/1,
+         stop/1]).
 
 -export([accept/1,
          connect/1,
@@ -101,7 +103,7 @@ suite() ->
     [{timetrap, {minutes, 2}}].
 
 all() ->
-    [{group, all} | tc()].
+    [start | tc()] ++ [{group, all}, stop].
 
 groups() ->
     [{all, [parallel], tc()}].
@@ -119,10 +121,17 @@ end_per_group(_, _) ->
     ok.
 
 init_per_suite(Config) ->
-    ok = diameter:start(),
     [{sctp, have_sctp()} | Config].
 
 end_per_suite(_Config) ->
+    ok.
+
+%% ===========================================================================
+
+start(_Config) ->
+    ok = diameter:start().
+
+stop(_Config) ->
     ok = diameter:stop().
 
 %% ===========================================================================
diff --git a/lib/diameter/test/diameter_util.erl b/lib/diameter/test/diameter_util.erl
index 99f4fa1977..f9942c3408 100644
--- a/lib/diameter/test/diameter_util.erl
+++ b/lib/diameter/test/diameter_util.erl
@@ -28,7 +28,8 @@
          fold/3,
          foldl/3,
          scramble/1,
-         ps/0]).
+         write_priv/3,
+         read_priv/2]).
 
 -define(L, atom_to_list).
 
@@ -150,11 +151,6 @@ s(Acc, L) ->
     {H, [T|Rest]} = lists:split(random:uniform(length(L)) - 1, L),
     s([T|Acc], H ++ Rest).
 
-%% ps/0
-
-ps() ->
-    [{P, process_info(P)} || P <- erlang:processes()].
-
 %% eval/1
 
 eval({M,[F|A]})
@@ -175,3 +171,18 @@ eval(L)
 eval(F)
   when is_function(F,0) ->
     F().
+
+%% write_priv/3
+
+write_priv(Config, Name, Term) ->
+    Dir = proplists:get_value(priv_dir, Config),
+    Path = filename:join([Dir, Name]),
+    ok = file:write_file(Path, term_to_binary(Term)).
+
+%% read_priv/2
+
+read_priv(Config, Name) ->
+    Dir = proplists:get_value(priv_dir, Config),
+    Path = filename:join([Dir, Name]),
+    {ok, Bin} = file:read_file(Path),
+    binary_to_term(Bin).