diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/diameter/test/diameter_watchdog_SUITE.erl | 274 |
1 files changed, 201 insertions, 73 deletions
diff --git a/lib/diameter/test/diameter_watchdog_SUITE.erl b/lib/diameter/test/diameter_watchdog_SUITE.erl index 388177a0bf..82244a1c7f 100644 --- a/lib/diameter/test/diameter_watchdog_SUITE.erl +++ b/lib/diameter/test/diameter_watchdog_SUITE.erl @@ -30,10 +30,14 @@ end_per_suite/1]). %% testcases --export([reopen/1, reopen/4, reopen/6]). +-export([reopen/0, reopen/1, reopen/4, reopen/6, + suspect/1, suspect/4, + okay/1, okay/4]). -export([id/1, %% jitter callback - run1/1]). + run1/1, + abuse/1, + abuse/2]). %% diameter_app callbacks -export([peer_up/3, @@ -75,14 +79,14 @@ %% Watchdog timers used by the testcases. -define(WD_TIMERS, [10000, ?WD(10000)]). -%% Watchdog timer of the misbehaving node. Longer than the peer's so -%% that DWR is never sent at watchdog timeout. --define(PEER_WD, 20000). +%% Watchdog timer of the misbehaving node. +-define(PEER_WD, 10000). %% A timeout that ensures one watchdog. To ensure only one watchdog %% requires (Wd + 2000) + 1000 < 2*(Wd - 2000) ==> 7000 < Wd for the %% case with random jitter. -define(ONE_WD(Wd), jitter(Wd,2000) + 1000). +-define(INFO(T), #diameter_event{info = T}). %% Receive an event message from diameter. -define(EVENT(T), @@ -97,16 +101,19 @@ -define(WD_EVENT(Ref), log_wd(element(4, ?EVENT({watchdog, Ref, _, _, _})))). %% Log to make failures identifiable. --define(LOG(T), ?LOG("~p", [T])). --define(LOG(F,A), ct:pal("~p: " ++ F, [self() | A])). +-define(LOG(T), ?LOG("~p", [T])). +-define(LOG(F,A), ct:pal("~p: " ++ F, [self() | A])). +-define(WARN(F,A), ct:pal(error, "~p: " ++ F, [self() | A])). %% =========================================================================== suite() -> - [{timetrap, {minutes, 5}}]. %% 20 watchdogs @ 15 sec + [{timetrap, {seconds, 90}}]. all() -> - [reopen]. + [reopen, + suspect, + okay]. init_per_suite(Config) -> ok = diameter:start(), @@ -122,6 +129,9 @@ end_per_suite(_Config) -> %% Test the watchdog state machine for the required failover, failback %% and reopen behaviour by examining watchdog events. +reopen() -> + [{timetrap, {minutes, 5}}]. %% 20 watchdogs @ 15 sec + reopen(_) -> [] = run([[reopen, T, W, N, M] || T <- [listen, connect], %% watchdog to test @@ -138,44 +148,19 @@ reopen(Test, Wd, N, M) -> %% reopen/6 reopen(Type, Test, Ref, Wd, N, M) -> - {SvcName, TRef} = start(Type, Ref, opts(Type, Test, Ref, Wd)), + {SvcName, TRef} = start(Type, Ref, cfg(Type, Test, Wd)), reopen(Type, Test, SvcName, TRef, Wd, N, M). -%% start/3 - -start(Type, Ref, Opts) -> - Name = hostname(), - true = diameter:subscribe(Name), - ok = diameter:start_service(Name, [{monitor, self()} | ?SERVICE(Name)]), - {ok, TRef} = diameter:add_transport(Name, {Type, Opts}), - true = diameter_reg:add_new({Type, Ref, Name}), - {Name, TRef}. - -opts(Type, Test, Ref, Wd) -> - {Timer, Counts, Mod} = opts(Type, Test, Wd), - [{transport_module, diameter_tcp}, - {transport_config, Mod ++ [{ip, ?ADDR}, {port, 0}] ++ cfg(Type, Ref)}, - {watchdog_timer, Timer}, - {watchdog_counts, Counts}]. - -cfg(listen, _) -> - []; -cfg(connect, Ref) -> - [{{_, _, SvcName}, _Pid}] = diameter_reg:wait({listen, Ref, '_'}), - [[{ref, LRef} | _]] = diameter:service_info(SvcName, transport), - [LP] = ?util:lport(tcp, LRef, 20), - [{raddr, ?ADDR}, {rport, LP}]. - -opts(Type, Type, Wd) -> +cfg(Type, Type, Wd) -> {Wd, [], []}; -opts(_Type, _Test, _Wd) -> - {?WD(?PEER_WD), [{open, 0}], [{module, ?MODULE}]}. +cfg(_Type, _Test, _Wd) -> + {?WD(?PEER_WD), [{okay, 0}], [{module, ?MODULE}]}. %% reopen/7 %% The watchdog to be tested. reopen(Type, Type, SvcName, Ref, Wd, N, M) -> - ?LOG("node, ~p, ~p", [Type, Ref]), + ?LOG("node ~p", [[Type, SvcName, Ref, Wd, N, M]]), %% Connection should come up immediately as a consequence of %% starting the watchdog process. In the accepting case this @@ -309,24 +294,24 @@ reopen(Type, Type, SvcName, Ref, Wd, N, M) -> %% The misbehaving peer. reopen(Type, _, SvcName, Ref, Wd, N, M) -> - ?LOG("peer, ~p, ~p", [Type, Ref]), + ?LOG("peer ~p", [[Type, SvcName, Ref, Wd, N, M]]), %% First transport process. {initial, okay} = ?WD_EVENT(Ref), ?EVENT({up, Ref, _, _, #diameter_packet{}}), - reg(Type, Ref, SvcName, {SvcName, {Wd,N,M}}), + reg(Ref, SvcName, {SvcName, {Wd,N,M}}), {okay, down} = ?WD_EVENT(Ref), %% Second transport process. - ?EVENT({watchdog, Ref, _, {_, reopen}, _}), - reg(Type, Ref, SvcName, 3), + ?EVENT({watchdog, Ref, _, {_, okay}, _}), + reg(Ref, SvcName, 3), %% answer 3 watchdogs then fall silent ?EVENT({watchdog, Ref, _, {_, down}, _}), %% Third transport process. - ?EVENT({watchdog, Ref, _, {_, reopen}, _}), - reg(Type, Ref, SvcName, 0), + ?EVENT({watchdog, Ref, _, {_, okay}, _}), + reg(Ref, SvcName, 0), %% disable outgoing DWA ?EVENT({watchdog, Ref, _, {_, down}, _}), ok. @@ -340,14 +325,6 @@ log_event(E) -> T == watchdog orelse ?LOG("~p", [T]), E. -%% wd_counts/1 - -wd_counts(SvcName) -> - [Info] = diameter:service_info(SvcName, transport), - {_, Counters} = lists:keyfind(statistics, 1, Info), - [proplists:get_value({{0,280,R}, D}, Counters, 0) || D <- [send,recv], - R <- [1,0]]. - %% recv_reopen/2 recv_reopen(connect, Ref) -> @@ -357,32 +334,32 @@ recv_reopen(connect, Ref) -> recv_reopen(listen, Ref) -> {_, reopen} = ?WD_EVENT(Ref). -%% reg/4 +%% reg/3 %% %% Lookup the pid of the transport process and publish a term for %% send/2 to lookup. -reg(Type, Ref, SvcName, T) -> - TPid = tpid(Type, Ref, diameter:service_info(SvcName, transport)), +reg(TRef, SvcName, T) -> + TPid = tpid(TRef, diameter:service_info(SvcName, transport)), true = diameter_reg:add_new({?MODULE, TPid, T}). -%% tpid/3 - -tpid(connect, Ref, [[{ref, Ref}, - {type, connect}, - {options, _}, - {watchdog, _}, - {peer, _}, - {apps, _}, - {caps, _}, - {port, [{owner, TPid} | _]} - | _]]) -> +%% tpid/2 + +tpid(Ref, [[{ref, Ref}, + {type, connect}, + {options, _}, + {watchdog, _}, + {peer, _}, + {apps, _}, + {caps, _}, + {port, [{owner, TPid} | _]} + | _]]) -> TPid; -tpid(listen, Ref, [[{ref, Ref}, - {type, listen}, - {options, _}, - {accept, As} - | _]]) -> +tpid(Ref, [[{ref, Ref}, + {type, listen}, + {options, _}, + {accept, As} + | _]]) -> [[{watchdog, _}, {peer, _}, {apps, _}, @@ -396,6 +373,156 @@ tpid(listen, Ref, [[{ref, Ref}, TPid. %% =========================================================================== +%% # suspect/1 +%% =========================================================================== + +%% Configure transports to require a set number of watchdogs before +%% moving from OKAY to SUSPECT. + +suspect(_) -> + [] = run([[abuse, [suspect, N]] || N <- [0,1,3]]). + +suspect(Type, Fake, Ref, N) + when is_reference(Ref) -> + {SvcName, TRef} + = start(Type, Ref, {?WD(10000), [{suspect, N}], mod(Fake)}), + {initial, okay} = ?WD_EVENT(TRef), + suspect(TRef, Fake, SvcName, N); + +suspect(TRef, true, SvcName, _) -> + reg(TRef, SvcName, 0), %% disable outgoing DWA + {okay, _} = ?WD_EVENT(TRef); + +suspect(TRef, false, SvcName, 0) -> %% SUSPECT disabled + %% Wait 2+ watchdogs and see that two unanswered watchdogs have + %% been sent. + [2,0,0,0] = receive + ?INFO({watchdog, TRef, _, _, _} = T) -> T + after 28000 -> + wd_counts(SvcName) + end; + +suspect(TRef, false, SvcName, N) -> + {okay, suspect} = ?WD_EVENT(TRef), + [N,0,0,0] = wd_counts(SvcName), + {suspect, down} = ?WD_EVENT(TRef), + [N,0,0,0] = wd_counts(SvcName). + +%% abuse/1 + +abuse(F) -> + [] = run([[abuse, F, T] || T <- [listen, connect]]). + +abuse(F, [_,_,_|_] = Args) -> + ?LOG("~p", [Args]), + apply(?MODULE, F, Args); + +abuse([F|A], Test) -> + Ref = make_ref(), + [] = run([[abuse, F, [T, T == Test, Ref] ++ A] + || T <- [listen, connect]]); + +abuse(F, Test) -> + abuse([F], Test). + +mod(true) -> + [{module, ?MODULE}]; +mod(false) -> + []. + +%% =========================================================================== +%% # okay/1 +%% =========================================================================== + +%% Configure the number of watchdog exchanges before moving from +%% REOPEN to OKAY. + +okay(_) -> + [] = run([[abuse, [okay, N]] || N <- [0,2,3]]). + +okay(Type, Fake, Ref, N) + when is_reference(Ref) -> + {SvcName, TRef} + = start(Type, Ref, {?WD(10000), + [{okay, choose(Fake, 0, N)}], + mod(Fake)}), + {initial, okay} = ?WD_EVENT(TRef), + okay(TRef, + Fake, + SvcName, + choose(Type == listen, initial, down), + N). + +okay(TRef, true, SvcName, Down, _) -> + reg(TRef, SvcName, 0), %% disable outgoing DWA + {okay, down} = ?WD_EVENT(TRef), + {Down, okay} = ?WD_EVENT(TRef), + reg(TRef, SvcName, -1), %% enable outgoing DWA + {okay, down} = ?WD_EVENT(TRef); + +okay(TRef, false, SvcName, Down, N) -> + {okay, suspect} = ?WD_EVENT(TRef), + [1,0,0,0] = wd_counts(SvcName), + {suspect, down} = ?WD_EVENT(TRef), + ok(TRef, SvcName, Down, N). + +ok(TRef, SvcName, Down, 0) -> + %% Connection comes up without watchdog exchange. + {Down, okay} = ?WD_EVENT(TRef), + [1,0,0,0] = wd_counts(SvcName), + %% Wait 2+ watchdog timeout to see that the connection stays up and + %% two watchdogs are exchanged. + ok = receive ?INFO({watchdog, TRef, _, _, _} = T) -> + T + after 28000 -> + ok + end, + [3,0,0,2] = wd_counts(SvcName); + +ok(TRef, SvcName, Down, N) -> + %% Connection required watchdog exchange before reaching OKAY. + {Down, reopen} = ?WD_EVENT(TRef), + {reopen, okay} = ?WD_EVENT(TRef), + %% One DWR was sent in moving to expect, plus N more to reopen the + %% connection. + N1 = N+1, + [N1,0,0,N] = wd_counts(SvcName). + +%% =========================================================================== + +%% wd_counts/1 + +wd_counts(SvcName) -> + [Info] = diameter:service_info(SvcName, transport), + {_, Counters} = lists:keyfind(statistics, 1, Info), + [proplists:get_value({{0,280,R}, D}, Counters, 0) || D <- [send,recv], + R <- [1,0]]. + +%% start/3 + +start(Type, Ref, T) -> + Name = hostname(), + true = diameter:subscribe(Name), + ok = diameter:start_service(Name, [{monitor, self()} | ?SERVICE(Name)]), + {ok, TRef} = diameter:add_transport(Name, {Type, opts(Type, Ref, T)}), + true = diameter_reg:add_new({Type, Ref, Name}), + {Name, TRef}. + +opts(Type, Ref, {Timer, Config, Mod}) -> + [{transport_module, diameter_tcp}, + {transport_config, Mod ++ [{ip, ?ADDR}, {port, 0}] ++ cfg(Type, Ref)}, + {watchdog_timer, Timer}, + {watchdog_config, Config}]. + +cfg(listen, _) -> + []; +cfg(connect, Ref) -> + [{{_, _, SvcName}, _Pid}] = diameter_reg:wait({listen, Ref, '_'}), + [[{ref, LRef} | _]] = diameter:service_info(SvcName, transport), + [LP] = ?util:lport(tcp, LRef, 20), + [{raddr, ?ADDR}, {rport, LP}]. + +%% =========================================================================== listen(PortNr, Opts) -> gen_tcp:listen(PortNr, Opts). @@ -417,6 +544,7 @@ send(Sock, Bin) -> %% First outgoing message from a new transport process is CER/CEA. %% Remaining outgoing messages are either DWR or DWA. send(undefined, Sock, Bin) -> + <<_:32, _:8, 257:24, _/binary>> = Bin, putr(config, init), gen_tcp:send(Sock, Bin); @@ -526,7 +654,7 @@ run1([F|A]) -> catch E:R -> S = erlang:get_stacktrace(), - io:format("~p~n", [{A, E, R, S}]), + ?WARN("~p", [{A, E, R, S}]), S end. |