Add watchdog suite

5 files changed, 621 insertions, 300 deletions

diff --git a/lib/diameter/test/Makefile b/lib/diameter/test/Makefile
index 1004055839..0bd3c0493c 100644
--- a/lib/diameter/test/Makefile
+++ b/lib/diameter/test/Makefile
@@ -17,40 +17,39 @@
 # %CopyrightEnd%
 
 ifneq ($(ERL_TOP),)
-include $(ERL_TOP)/make/target.mk
-include $(ERL_TOP)/make/$(TARGET)/otp.mk
+TOP = $(ERL_TOP)
+DIAMETER_TOP = $(TOP)/lib/diameter
 else
-include $(DIAMETER_TOP)/make/target.mk
-include $(DIAMETER_TOP)/make/$(TARGET)/rules.mk
+TOP = $(DIAMETER_TOP)
 endif
 
+include $(TOP)/make/target.mk
+include $(TOP)/make/$(TARGET)/otp.mk
+
 # ----------------------------------------------------
 # Application version
 # ----------------------------------------------------
+
 include ../vsn.mk
-VSN=$(DIAMETER_VSN)
 
+VSN = $(DIAMETER_VSN)
 
 # ----------------------------------------------------
 # Release directory specification
 # ----------------------------------------------------
+
 RELSYSDIR = $(RELEASE_PATH)/diameter_test
 
 ifeq ($(findstring win32,$(TARGET)),win32)
-
 MAKEFILE_SRC = Makefile.win32.src
-
 else
-
 MAKEFILE_SRC = Makefile.src
-
 endif
 
 ifeq ($(TT_DIR),)
 TT_DIR = /tmp
 endif
 
-
 # ----------------------------------------------------
 # Target Specs
 # ----------------------------------------------------
@@ -59,30 +58,17 @@ include modules.mk
 
 EBIN = .
 
-HRL_FILES = diameter_test_lib.hrl
-
+HRL_FILES = $(INTERNAL_HRL_FILES)
 ERL_FILES = $(MODULES:%=%.erl)
 
 SOURCE = $(HRL_FILES) $(ERL_FILES) 
-
 TARGET_FILES = $(MODULES:%=%.$(EMULATOR))
 
-APP_CASES   = app
-
-TRANSPORT_CASES = tcp
-
-ALL_CASES   = \
-	$(APP_CASES) \
-	compiler conf sync stats reg peer \
-	$(TRANSPORT_CASES)
-
+SUITE_MODULES = $(filter diameter_%_SUITE, $(MODULES))
+SUITES = $(SUITE_MODULES:diameter_%_SUITE=%)
 
 EMAKEFILE  = Emakefile
-ifneq ($(ERL_TOP),)
-MAKE_EMAKE = $(wildcard $(ERL_TOP)/make/make_emakefile)
-else
-MAKE_EMAKE = $(wildcard $(DIAMETER_TOP)/make/make_emakefile)
-endif
+MAKE_EMAKE = $(wildcard $(TOP)/make/make_emakefile)
 
 ifeq ($(MAKE_EMAKE),)
 BUILDTARGET   = $(TARGET_FILES)
@@ -92,7 +78,6 @@ BUILDTARGET   = emakebuild
 RELTEST_FILES = $(EMAKEFILE) $(TEST_SPEC_FILE) $(COVER_SPEC_FILE) $(SOURCE)
 endif
 
-
 # ----------------------------------------------------
 # FLAGS
 # ----------------------------------------------------
@@ -107,59 +92,14 @@ ifeq ($(USE_DIAMETER_HIPE),true)
 ERL_COMPILE_FLAGS += +native -DDIAMETER_hipe_special=true
 endif
 
-ifneq ($(ERL_TOP),)
-ERL_COMPILE_FLAGS += \
-	$(DIAMETER_ERL_COMPILE_FLAGS)       \
-	-pa $(ERL_TOP)/lib/test_server/ebin \
-	-I$(ERL_TOP)/lib/test_server/include
-else
-ERL_COMPILE_FLAGS += \
-	$(DIAMETER_ERL_COMPILE_FLAGS)       \
-	-pa $(TEST_SERVER_DIR)/ebin \
-	-I$(TEST_SERVER_DIR)/include
-endif
-
-ERL_PATH = \
-	-pa ../../$(APPLICATION)/ebin \
-	-pa ../../et/ebin
-
-ifndef SUITE
-SUITE = diameter_SUITE
-endif
-
-ESTOP = -s init stop
-
-ifeq ($(DONT_STOP),true)
-MAYBE_ESTOP =
-else
-MAYBE_ESTOP = $(ESTOP)
-endif
-
-ETVIEW = -s et_viewer
-ifeq ($(USE_ET_VIEWER),true)
-MAYBE_ETVIEW =
-else
-MAYBE_ETVIEW = $(ETVIEW)
-endif
-
-ifeq ($(MERL),)
-MERL = $(ERL)
-endif
-
-ARGS += -noshell
-
-ifeq ($(DISABLE_TC_TIMEOUT),true)
-ARGS += -diameter_test_timeout
-endif
-
-
-DIAMETER_TEST_SERVER = diameter_test_server
-
+ERL_COMPILE_FLAGS += $(DIAMETER_ERL_COMPILE_FLAGS)
 
 # ----------------------------------------------------
 # Targets
 # ----------------------------------------------------
 
+test: $(SUITES)
+
 tests debug opt: $(BUILDTARGET)
 
 targets: $(TARGET_FILES)
@@ -169,8 +109,9 @@ targets: $(TARGET_FILES)
 emakebuild: $(EMAKEFILE)
 
 $(EMAKEFILE): 
-	$(MAKE_EMAKE) $(ERL_COMPILE_FLAGS) -o$(EBIN) '*_SUITE_make' | grep -v Warning > $(EMAKEFILE)
-	$(MAKE_EMAKE) $(ERL_COMPILE_FLAGS) -o$(EBIN) $(MODULES) | grep -v Warning >> $(EMAKEFILE)
+	$(MAKE_EMAKE) $(ERL_COMPILE_FLAGS) -o $(EBIN) '*_SUITE_make' $(MODULES) \
+	| grep -v Warning \
+	> $(EMAKEFILE)
 
 clean:
 	rm -f $(EMAKEFILE)
@@ -180,121 +121,64 @@ clean:
 docs:
 
 info:
-	@echo "MAKE_EMAKE        = $(MAKE_EMAKE)"
-	@echo "EMAKEFILE         = $(EMAKEFILE)"
-	@echo "BUILDTARGET       = $(BUILDTARGET)"
-	@echo ""
+	@echo "MAKE_EMAKE  = $(MAKE_EMAKE)"
+	@echo "EMAKEFILE   = $(EMAKEFILE)"
+	@echo "BUILDTARGET = $(BUILDTARGET)"
+	@echo
 	@echo "ERL_COMPILE_FLAGS = $(ERL_COMPILE_FLAGS)"
 	@echo "ERL  = $(ERL)"
 	@echo "ERLC = $(ERLC)"
-	@echo "MERL = $(MERL)"
-	@echo ""
-	@echo "ARGS = $(ARGS)"
-	@echo ""
+	@echo
 	@echo "HRL_FILES    = $(HRL_FILES)"
 	@echo "ERL_FILES    = $(ERL_FILES)"
 	@echo "TARGET_FILES = $(TARGET_FILES)"
-	@echo ""
+	@echo
+	@echo "SUITE_MODULES = $(SUITE_MODULES)"
+	@echo "SUITES        = $(SUITES)"
+	@echo
 
 help:
-	@echo ""
-	@echo "This Makefile controls the test of the $(APPLICATION) application. "
-	@echo ""
+	@echo
+	@echo "This Makefile controls the test of the $(APPLICATION) application."
+	@echo
 	@echo "There are two separate ways to perform the test of $(APPLICATION)."
-	@echo ""
+	@echo
 	@echo "  a) Run the official OTP test-server (which we do not describe here)"
-	@echo ""
-	@echo "  b) Run the test-server provided with this application. "
+	@echo
+	@echo "  b) Run the test-server provided with this application."
 	@echo "     There are a number of targets to run the entire or parts"
 	@echo "     of this applications ($(APPLICATION)) test-suite"
-	@echo ""
+	@echo
 	@echo "Targets:"
-	@echo ""
+	@echo
 	@echo "    help"
 	@echo "      Print this info"
-	@echo ""
+	@echo
 	@echo "    info"
-	@echo "      Prints various environment variables. "
-	@echo "      May be useful when debugging the Makefile. "
-	@echo ""
+	@echo "      Prints various environment variables."
+	@echo "      May be useful when debugging the Makefile."
+	@echo
 	@echo "    tests | debug | opt "
-	@echo "      Compile all test-code. "
-	@echo ""
+	@echo "      Compile all test-code."
+	@echo
 	@echo "    clean "
-	@echo "      Remove all targets. "
-	@echo ""
+	@echo "      Remove all targets."
+	@echo
 	@echo "    test"
-	@echo "      Run the entire $(APPLICATION) test-suite. "
-	@echo ""
-	@echo "    app"
-	@echo "      Run the $(APPLICATION) application sub-test-suite. "
-	@echo ""
-	@echo "    compiler"
-	@echo "      Run the $(APPLICATION) compiler sub-test-suite(s). "
-	@echo ""
-	@echo "    conf"
-	@echo "      Run the $(APPLICATION) config sub-test-suite. "
-	@echo "      Checks various aspects of the $(APPLICATION) configuration. "
-	@echo ""
-	@echo "    sync"
-	@echo "      Run the $(APPLICATION) sync sub-test-suite. "
-	@echo ""
-	@echo "    stats"
-	@echo "      Run the $(APPLICATION) stats sub-test-suite. "
-	@echo ""
-	@echo "    reg"
-	@echo "      Run the $(APPLICATION) reg sub-test-suite. "
-	@echo ""
-	@echo "    peer"
-	@echo "      Run the $(APPLICATION) peer sub-test-suite"
-	@echo ""
-	@echo "    ptab"
-	@echo "      Run the $(APPLICATION) persistent-table sub-test-suite"
-	@echo ""
-	@echo "    tcp"
-	@echo "      Run the $(APPLICATION) tcp sub-test-suite"
-	@echo ""
-	@echo ""
+	@echo "      Run all test suites."
+	@echo
+	@echo "    $(SUITES)"
+	@echo "      Run a specific test suite."
+	@echo
 
 
 # ----------------------------------------------------
 # Special Targets
 # ----------------------------------------------------
 
-all: make
-	@echo "make sure epmd is new"
-	@epmd -kill > /dev/null
-	@echo "Running all sub-suites separatelly"
-	@for i in $(ALL_CASES); do \
-            echo "SUITE: $$i"; \
-            clearmake -V $$i > $$i.log; \
-        done
-
-aall: make
-	@echo "make sure epmd is new"
-	@epmd -kill > /dev/null
-	@echo "Running all app sub-suites separatelly"
-	@for i in $(APP_CASES); do \
-            echo "SUITE: $$i"; \
-            clearmake -V $$i > $$i.log; \
-        done
-	echo "done"
-
-tall: make
-	@echo "make sure epmd is new"
-	@epmd -kill > /dev/null
-	@echo "Running all transport sub-suites separatelly"
-	@for i in $(TRANSPORT_CASES); do \
-            echo "SUITE: $$i"; \
-            clearmake -V $$i > $$i.log; \
-        done
-
-make: targets
-
-test: make
-	$(MERL) $(ARGS) -sname diameter_test $(ERL_PATH) \
-            -s $(DIAMETER_TEST_SERVER) t $(SUITE) \
-            $(MAYBE_ESTOP)
+all: $(SUITES)
+
+beam: targets
 
 log:
 	mkdir $@
@@ -303,31 +187,19 @@ log:
 
 # This assumes GNU sed to exit 1 if the output looks to indicate failure.
 # diameter_ct:run/1 itself can't tell (it seems).
-app codec dict reg stats sync: log make
-	$(MERL) $(ARGS) \
+$(SUITES): log beam
+	$(ERL) -noshell \
+	        -pa ../ebin \
 	        -sname diameter_test_$@ \
 	        -s diameter_ct run $@ \
 	        -s init stop \
 	| sed '/ FAILED /h; p; $$!d; x; /./!d; Q 1'
 
-compiler conf peer tcp: make
-	$(MERL) $(ARGS) -sname diameter_$@ $(ERL_PATH) \
-            -s $(DIAMETER_TEST_SERVER) t diameter_$@_test \
-            $(ESTOP)
-
-node:
-	$(MERL) -sname diameter $(ERL_PATH)
-
-
 # ----------------------------------------------------
 # Release Targets
 # ---------------------------------------------------- 
 
-ifneq ($(ERL_TOP),)
-include $(ERL_TOP)/make/otp_release_targets.mk
-else
-include $(DIAMETER_TOP)/make/release_targets.mk
-endif
+include $(TOP)/make/otp_release_targets.mk
 
 release_spec: 
 
@@ -340,5 +212,4 @@ release_tests_spec: tests
 #                        $(HRL_FILES) $(ERL_FILES) \
 #                        $(RELSYSDIR)
 #
-	chmod -R u+w $(RELSYSDIR)
-
+	chmod -f -R u+w $(RELSYSDIR)
diff --git a/lib/diameter/test/diameter_ct.hrl b/lib/diameter/test/diameter_ct.hrl
new file mode 100644
index 0000000000..b6bd2ca9da
--- /dev/null
+++ b/lib/diameter/test/diameter_ct.hrl
@@ -0,0 +1,21 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+-define(APP, diameter).
+-define(ERROR(T), erlang:error({?MODULE, ?LINE, T})).
diff --git a/lib/diameter/test/diameter_peer_test.erl b/lib/diameter/test/diameter_peer_test.erl
deleted file mode 100644
index 27e75e26ef..0000000000
--- a/lib/diameter/test/diameter_peer_test.erl
+++ /dev/null
@@ -1,104 +0,0 @@
-%%
-%% %CopyrightBegin%
-%%
-%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
-%%
-%% The contents of this file are subject to the Erlang Public License,
-%% Version 1.1, (the "License"); you may not use this file except in
-%% compliance with the License. You should have received a copy of the
-%% Erlang Public License along with this software. If not, it can be
-%% retrieved online at http://www.erlang.org/.
-%%
-%% Software distributed under the License is distributed on an "AS IS"
-%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
-%% the License for the specific language governing rights and limitations
-%% under the License.
-%%
-%% %CopyrightEnd%
-%%
-
-%%
-%%----------------------------------------------------------------------
-%% Purpose: Verify the peer component of the Diameter application
-%%----------------------------------------------------------------------
-%% 
--module(diameter_peer_test).
-
--export([
-	 init_per_testcase/2, fin_per_testcase/2, 
-
-	 all/0,
-	 groups/0, 
-	 init_per_suite/1, end_per_suite/1, 
-	 suite_init/1, suite_fin/1, 
-	 init_per_group/2, end_per_group/2
-
-	 %% foo/1
-	]).
-
--export([t/0, t/1]).
-
--include("diameter_test_lib.hrl").
-
-
-t()     -> diameter_test_server:t(?MODULE).
-t(Case) -> diameter_test_server:t({?MODULE, Case}).
-
-
-%% Test server callbacks
-init_per_testcase(Case, Config) ->
-    diameter_test_server:init_per_testcase(Case, Config).
-
-fin_per_testcase(Case, Config) ->
-    diameter_test_server:fin_per_testcase(Case, Config).
-
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-all() ->
-    [].
-
-groups() ->
-    [].
-
-init_per_group(_GroupName, Config) ->
-    Config.
-
-end_per_group(_GroupName, Config) ->
-    Config.
-
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-suite_init(X) -> init_per_suite(X).
-
-init_per_suite(suite) -> [];
-init_per_suite(doc) -> [];
-init_per_suite(Config) when is_list(Config) ->
-    Config.
-
-
-suite_fin(X) -> end_per_suite(X).
-
-end_per_suite(suite) -> [];
-end_per_suite(doc) -> [];
-end_per_suite(Config) when is_list(Config) ->
-    Config.
-
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%% 
-%% Test case example
-%% 
-
-%% foo(suite) ->
-%%     [];
-%% foo(doc) ->
-%%     [];
-%% foo(Config) when is_list(Config) ->
-%%     ok.
-
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-
diff --git a/lib/diameter/test/diameter_watchdog_SUITE.erl b/lib/diameter/test/diameter_watchdog_SUITE.erl
new file mode 100644
index 0000000000..dec307529a
--- /dev/null
+++ b/lib/diameter/test/diameter_watchdog_SUITE.erl
@@ -0,0 +1,540 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2010-2011. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%
+%% Tests of the RFC3539 watchdog state machine as implemented by
+%% module diameter_watchdog.
+%%
+
+-module(diameter_watchdog_SUITE).
+
+-export([suite/0,
+         all/0,
+         init_per_suite/1,
+         end_per_suite/1]).
+
+%% testcases
+-export([reopen/1, reopen/4]).
+
+-export([start/3, %% diameter_transport callback
+         id/1,    %% jitter callback
+         run/1]).
+
+-include_lib("diameter/include/diameter.hrl").
+-include("diameter_ct.hrl").
+
+%% ===========================================================================
+
+-define(util, diameter_util).
+
+-define(BASE,    diameter_gen_base_rfc3588).
+-define(APPL_ID, diameter_gen_base_rfc3588:id()).
+-define(SUCCESS, 2001).  %% DIAMETER_SUCCESS
+
+%% Addresses for the local and remote diameter nodes. The values don't
+%% matter since we're faking transport.
+-define(LOCALHOST, {127,0,0,1}).
+-define(REMOTEHOST, {10,0,0,1}).
+
+-define(CAPS, #diameter_caps{origin_host = "node.innan.com",
+                             origin_realm = "innan.com",
+                             host_ip_address = [?LOCALHOST],
+                             vendor_id = 1022,
+                             product_name = "remote",
+                             auth_application_id = [?APPL_ID]}).
+
+-define(APPL, #diameter_app{alias = ?MODULE,
+                            dictionary = ?BASE,
+                            module = [?MODULE],
+                            init_state = now(),
+                            id = ?APPL_ID,
+                            mutable = false}).
+
+%% Service record maintained by our faked service process.
+-define(SERVICE, #diameter_service{pid = self(),
+                                   capabilities = ?CAPS,
+                                   applications = [?APPL]}).
+
+%% Watchdog timer as a callback.
+-define(WD(T), {?MODULE, id, [T]}).
+
+%% Watchdog timers used by the testcases. Note that the short timeout
+%% with random jitter is excluded since the reopen/1 isn't smart
+%% enough to deal with it: see ONE_WD below.
+-define(WD_TIMERS, [?WD(6000)
+                    | [F_(T_) || T_ <- [10000, 20000, 30000],
+                                 F_ <- [fun(T__) -> T__ end,
+                                        fun(T__) -> ?WD(T__) end]]]).
+
+%% Transport types.
+-define(TRANSPORTS, [connect, accept]).
+
+%% Message over the transport interface.
+-define(TMSG(T), {diameter, T}).
+
+%% Receive a message within a specified time.
+-define(RECV(T, Timeout),
+        receive T -> now()
+        after Timeout -> ?ERROR({timeout, Timeout})
+        end).
+
+%% Receive a message in a given number of watchdogs, plus or minus
+%% half. Note that the call to now_diff assumes left to right
+%% evaluation order.
+-define(RECV(T, N, WdL, WdH),
+        [?ERROR({received, _Elapsed_, _LowerBound_, N, WdL})
+         || _UpperBound_ <- [(N)*(WdH) + (WdH) div 2],
+            _Elapsed_    <- [now_diff(now(), ?RECV(T, _UpperBound_))],
+            _LowerBound_ <- [(N)*(WdL) - (WdL) div 2],
+            _Elapsed_ =< _LowerBound_*1000]).
+
+%% A timeout that ensures one watchdog. The ensure only one watchdog
+%% requires (Wd + 2000) + 1000 < 2*(Wd - 2000) ==> 7000 < Wd for the
+%% case with random jitter.
+-define(ONE_WD(Wd), jitter(Wd,2000) + 1000).
+
+%% ===========================================================================
+
+suite() ->
+    [{timetrap, {minutes, 6}}].%% enough for 11 watchdogs @ 30 sec plus jitter
+
+all() ->
+    [reopen].
+
+init_per_suite(Config) ->
+    ok = diameter:start(),
+    Config.
+
+end_per_suite(_Config) ->
+    ok = diameter:stop().
+
+%% ===========================================================================
+%% # reopen/1
+%% ===========================================================================
+
+%% Test the watchdog state machine for the required failover, failback
+%% and reopen behaviour. Do this by having the testcase replace
+%% diameter_service and start watchdogs, and having this module
+%% implement a transport process that plays the role of the peer
+%% Diameter node.
+
+reopen(_) ->
+    [] = ?util:run([{?MODULE, [run, [reopen, Wd, T, N, M]]}
+                    || Wd <- ?WD_TIMERS,
+                       T <- ?TRANSPORTS,
+                       N <- [0,1,2],
+                       M <- ['DWR', 'DWA', other]]).
+
+reopen(Wd, Type, N, What) ->
+    Ref = make_ref(),
+
+    %% The maker of transport processes.
+    TPid = start({N, Wd, What, Ref}),
+
+    %% Act like diameter_service and start the watchdog process, which
+    %% in turn starts a peer_fsm process, which in turn starts a
+    %% transport process by way of start/3. Messages received by the
+    %% testcase are those sent by diameter_watchdog to the service
+    %% process (= process starting the watchdog).
+    WPid1 = watchdog(Type, Ref, TPid, Wd),
+
+    %% Low/high watchdog timeouts.
+    WdL = jitter(Wd, -2000),
+    WdH = jitter(Wd, 2000),
+
+    %% Connection should come up immediately as a consequence of
+    %% starting the watchdog process. In the accepting case this
+    %% results in a new watchdog on a transport waiting for a new
+    %% connection.
+    ?RECV({connection_up, WPid1, _}, 1000),
+
+    WPid2 = case Type of
+                connect ->
+                    WPid1;
+                accept ->
+                    watchdog(Type, Ref, TPid, Wd)
+            end,
+
+    %%   OKAY          Timer expires &      Failover()
+    %%                 Pending              SetWatchdog()        SUSPECT
+    %%
+    %% Since our transport is replying to N DWR's before becoming
+    %% silent, we should go down after N+2 watchdog_timer expirations:
+    %% that is, after the first unanswered DWR. Knowing the min/max
+    %% watchdog timeout values gives the time interval in which the
+    %% down message is expected.
+    ?RECV({connection_down, WPid1}, N+2, WdL, WdH),
+
+    %%   SUSPECT       Receive DWA          Pending = FALSE
+    %%                                      Failback()
+    %%                                      SetWatchdog()        OKAY
+    %%
+    %%   SUSPECT       Receive non-DWA      Failback()
+    %%                                      SetWatchdog()        OKAY
+    %%
+    %% The transport receives a message before the expiry of another
+    %% watchdog to induce failback.
+    ?RECV({connection_up, WPid1}, WdH),
+
+    %%   OKAY          Timer expires &      SendWatchdog()
+    %%                 !Pending             SetWatchdog()
+    %%                                      Pending = TRUE       OKAY
+    %%
+    %%   OKAY          Timer expires &      Failover()
+    %%                 Pending              SetWatchdog()        SUSPECT
+    %%
+    %% The transport is still not responding to watchdogs so the
+    %% connection should go back down after either one or two watchdog
+    %% expiries, depending on whether or not DWA restored the connection.
+    F = choose(What == 'DWA', 2, 1),
+    ?RECV({connection_down, WPid1}, F, WdL, WdH),
+
+    %%   SUSPECT       Timer expires        CloseConnection()
+    %%                                      SetWatchdog()        DOWN
+    %%
+    %%   DOWN          Timer expires        AttemptOpen()
+    %%                                      SetWatchdog()        DOWN
+    %%
+    %% Our transport tells us when the fake connection is
+    %% reestablished, which should happen after another couple of
+    %% watchdog expiries, the first bringing the watchdog to state
+    %% DOWN, the second triggering an attempt to reopen the
+    %% connection.
+    ?RECV({reopen, Ref}, 2, WdL, WdH),
+
+    %%   DOWN          Connection up        NumDWA = 0
+    %%                                      SendWatchdog()
+    %%                                      SetWatchdog()
+    %%                                      Pending = TRUE       REOPEN
+    %%
+    %%   REOPEN        Receive DWA &        Pending = FALSE
+    %%                 NumDWA < 2           NumDWA++             REOPEN
+    %%
+    %%   REOPEN        Receive DWA &        Pending = FALSE
+    %%                 NumDWA == 2          NumDWA++
+    %%                                      Failback()           OKAY
+    %%
+    %% Now the watchdog should require three received DWA's before
+    %% taking the connection back up. The first DWR is sent directly
+    %% after capabilities exchange so it should take no more than two
+    %% watchdog expiries.
+    ?RECV({connection_up, WPid2, _}, 2, WdL, WdH).
+
+%% ===========================================================================
+
+%% Start the fake transport process. From diameter's point of view
+%% it's started when diameter calls start/3. We start it before this
+%% happens since we use the same fake transport each time diameter
+%% calls start/3. The process lives and dies with the test case.
+start(Config) ->
+    Pid = self(),
+    spawn(fun() -> loop(init(Pid, Config)) end).
+
+%% Transport start from diameter. This may be called multiple times
+%% depending on the testcase.
+start({Type, _Ref}, #diameter_service{}, Pid) ->
+    Ref = make_ref(),
+    MRef = erlang:monitor(process, Pid),
+    Pid ! {start, self(), Type, Ref},
+    {Ref, TPid} = receive
+                      {Ref, _} = T ->
+                          T;
+                      {'DOWN', MRef, process, _, _} = T ->
+                          T
+                  end,
+    erlang:demonitor(MRef, [flush]),
+    {ok, TPid}.
+
+%% id/1
+
+id(T) ->
+    T.
+
+%% ===========================================================================
+
+choose(true, X, _)  -> X;
+choose(false, _, X) -> X.
+
+%% run/1
+%%
+%% A more useful badmatch in case of failure.
+
+run([F|A]) ->
+    ok = try
+             apply(?MODULE, F, A),
+             ok
+         catch
+             E:R ->
+                 {A, E, R, erlang:get_stacktrace()}
+         end.
+
+%% now_diff/2
+
+now_diff(T1, T2) ->
+    timer:now_diff(T2, T1).
+
+%% jitter/2
+
+jitter(?WD(T), _) ->
+    T;
+jitter(T,D) ->
+    T+D.
+
+%% watchdog/4
+%%
+%% Fake the call from diameter_service. The watchdog process will send
+%% messages to the calling "service" process so our tests are that the
+%% watchdog responds as expected.
+
+watchdog(Type, Ref, TPid, Wd) ->
+    Opts = [{transport_module, ?MODULE},
+            {transport_config, TPid},
+            {watchdog_timer, Wd}],
+    monitor(diameter_watchdog:start({Type, Ref},
+                                    {false, Opts, false, ?SERVICE})).
+
+monitor(Pid) ->
+    erlang:monitor(process, Pid),
+    Pid.
+
+%% ===========================================================================
+
+%% Transport process implmentation. Fakes reception of messages by
+%% sending fakes to the parent (peer fsm) process that called start/3.
+
+-record(transport,
+        {type,    %% connect | accept | manager
+         parent,  %% pid() of peer_fsm/ervice process
+         open = false,  %% done with capabilities exchange?
+         config}).%% testcase-specific config
+
+%% init/2
+
+%% Testcase starting the manager.
+init(SvcPid, {_,_,_,_} = Config) ->
+    putr(peer, [{'Origin-Host', hostname() ++ ".utan.com"},
+                {'Origin-Realm', "utan.com"}]),
+    #transport{type = manager,
+               parent = monitor(SvcPid),
+               config = Config};
+
+%% Manager starting a transport.
+init(_, {Type, ParentPid, SvcPid, TwinPid, Peer, {N,_,_,_} = Config}) ->
+    putr(peer, Peer),
+    putr(service, SvcPid),
+    putr(count, init(Type, ParentPid, TwinPid, N)),%% number of DWR's to answer
+    #transport{type = Type,
+               parent = monitor(ParentPid),
+               config = Config}.
+
+init(Type, ParentPid, undefined, N) ->
+    connected(ParentPid, Type),
+    N;
+init(_, _, TPid, _) ->
+    monitor(TPid),
+    3.
+
+%% Generate a unique hostname for the faked peer.
+hostname() ->
+    lists:flatten(io_lib:format("~p-~p-~p", tuple_to_list(now()))).
+
+%% loop/1
+
+loop(S) ->
+    loop(msg(receive T -> T end, S)).
+
+msg(T,S) ->
+    case transition(T,S) of
+        ok ->
+            S;
+        #transport{} = NS ->
+            NS;
+        {stop, Reason} ->
+            x(Reason)
+    end.
+
+x(Reason) ->
+    exit(Reason).
+
+%% transition/2
+
+%% Manager is being asked for a new transport process.
+transition({start, Pid, Type, Ref}, #transport{type = manager,
+                                               parent = SvcPid,
+                                               config = Config}) ->
+    TPid = start({Type, Pid, SvcPid, getr(transport), getr(peer), Config}),
+    Pid ! {Ref, TPid},
+    putr(transport, TPid),
+    ok;
+
+%% Peer fsm or testcase process has died.
+transition({'DOWN', _, process, Pid, _} = T, #transport{parent = Pid}) ->
+    {stop, T};
+
+%% Twin transport process has gone down. In the connect case, the
+%% transport isn't started until this happens in the first place so
+%% connect immediately. In the accept case, fake the peer reconnecting
+%% only after another watchdog expiry.
+transition({'DOWN', _, process, _, _}, #transport{type = Type,
+                                                  config = {_, Wd, _, _}}) ->
+    Tmo = case Type of
+              connect ->
+                  0;
+              accept ->
+                  ?ONE_WD(Wd)
+          end,
+    erlang:send_after(Tmo, self(), reconnect),
+    ok;
+
+transition(reconnect, #transport{type = Type,
+                                 parent = Pid,
+                                 config = {_,_,_,Ref}}) ->
+    getr(service) ! {reopen, Ref},
+    connected(Pid, Type),
+    ok;
+
+%% Peer fsm process is sending CER: fake the peer's CEA.
+transition(?TMSG({send, Bin}), #transport{type = connect,
+                                          open = false,
+                                          parent = Pid}
+                               = S) ->
+    {Code, Flags, _} = ?BASE:msg_header('CER'),
+    <<_:32, Flags:8, Code:24, _:96, _/binary>> = Bin,
+    Hdr = make_header(Bin),
+    recv(Pid, {Hdr, make_cea()}),
+    S#transport{open = true};
+
+%% Peer fsm process is sending CEA.
+transition(?TMSG({send, Bin}), #transport{type = accept,
+                                          open = false}
+                               = S) ->
+    {Code, Flags, _} = ?BASE:msg_header('CEA'),
+    <<_:32, Flags:8, Code:24, _:96, _/binary>> = Bin,
+    S#transport{open = true};
+
+%% Watchdog is sending DWR or DWA.
+transition(?TMSG({send, Bin}), #transport{open = true} = S) ->
+    {Code, _, _} = ?BASE:msg_header('DWR'),
+    {Code, _, _} = ?BASE:msg_header('DWA'),
+    <<_:32, R:1, 0:7, Code:24, _:96, _/binary>> = Bin,
+    Hdr = make_header(Bin),
+    dwa(1 == R, S, Hdr),
+    ok;
+
+%% We're telling ourselves to fake a received message.
+transition({recv, Msg}, #transport{parent = Pid}) ->
+    recv(Pid, Msg),
+    ok;
+
+%% We're telling ourselves to receive a message to induce failback.
+transition(failback = T, #transport{parent = Pid}) ->
+    recv(Pid, eraser(T)),
+    ok.
+
+make_header(Bin) ->
+    #diameter_header{end_to_end_id = E,
+                     hop_by_hop_id = H}
+        = diameter_codec:decode_header(Bin),
+    #diameter_header{end_to_end_id = E,
+                     hop_by_hop_id = H}.
+
+recv(Pid, Msg) ->
+    Pid ! ?TMSG({recv, encode(Msg)}).
+
+%% Replace the end-to-end/hop-by-hop identifiers with those from an
+%% incoming request to which we're constructing a reply.
+encode({Hdr, [_|_] = Msg}) ->
+    #diameter_header{hop_by_hop_id = HBH,
+                     end_to_end_id = E2E}
+        = Hdr,
+    #diameter_packet{bin = Bin} = diameter_codec:encode(?BASE, Msg),
+    <<H:12/binary, _:64, T/binary>> = Bin,
+    <<H/binary, HBH:32, E2E:32, T/binary>>;
+
+encode([_|_] = Msg) ->
+    #diameter_packet{bin = Bin} = diameter_codec:encode(?BASE, Msg),
+    Bin.
+
+connected(Pid, connect) ->
+    Pid ! ?TMSG({self(), connected, make_ref()});
+connected(Pid, accept) ->
+    Pid ! ?TMSG({self(), connected}),
+    recv(Pid, make_cer()).
+
+make_cer() ->
+    ['CER' | getr(peer)] ++ [{'Host-IP-Address', [?REMOTEHOST]},
+                             {'Vendor-Id', 1028},
+                             {'Product-Name', "Utan"},
+                             {'Auth-Application-Id', [?APPL_ID]}].
+
+make_cea() ->
+    ['CER' | Rest] = make_cer(),
+    ['CEA', {'Result-Code', ?SUCCESS} | Rest].
+
+make_dwr() ->
+    ['DWR' | getr(peer)].
+
+make_dwa() ->
+    ['DWR' | Rest] = make_dwr(),
+    ['DWA', {'Result-Code', ?SUCCESS} | Rest].
+
+dwa(false, _, _) ->  %% outgoing was DWA ...
+    ok;
+dwa(true, S, Hdr) -> %% ... or DWR
+    dwa(getr(count), Hdr, S);
+
+%% React to the DWR only after another watchdog expiry. We shouldn't
+%% get another DWR while the answer is pending.
+dwa(0, Hdr, #transport{config = {_, Wd, What, _}}) ->
+    erlang:send_after(?ONE_WD(Wd), self(), failback),
+    putr(failback, make_msg(What, Hdr)),
+    eraser(count);
+
+dwa(undefined, _, _) ->
+    undefined = getr(failback),  %% ensure this is after failback
+    ok;
+
+%% Reply with DWA.
+dwa(N, Hdr, #transport{parent = Pid}) ->
+    putr(count, N-1),
+    recv(Pid, {Hdr, make_dwa()}).
+
+%% Answer to received DWR.
+make_msg('DWA', Hdr) ->
+    {Hdr, make_dwa()};
+
+%% DWR from peer.
+make_msg('DWR', _) ->
+    make_dwr();
+
+%% An unexpected answer is discarded after passing through the
+%% watchdog state machine.
+make_msg(other, _) ->
+    ['RAA', {'Session-Id', diameter:session_id("abc")},
+            {'Result-Code', 2001}
+          | getr(peer)].
+
+putr(Key, Val) ->
+    put({?MODULE, Key}, Val).
+
+getr(Key) ->
+    get({?MODULE, Key}).
+
+eraser(Key) ->
+    erase({?MODULE, Key}).
diff --git a/lib/diameter/test/modules.mk b/lib/diameter/test/modules.mk
index d27d64bb98..daf6ef64ec 100644
--- a/lib/diameter/test/modules.mk
+++ b/lib/diameter/test/modules.mk
@@ -17,8 +17,7 @@
 #
 # %CopyrightEnd%
 
-TEST_SPEC_FILE = diameter.spec
-
+TEST_SPEC_FILE  = diameter.spec
 COVER_SPEC_FILE = diameter.cover
 
 MODULES = \
@@ -32,13 +31,7 @@ MODULES = \
 	diameter_reg_SUITE \
 	diameter_sync_SUITE \
 	diameter_stats_SUITE \
-	\
-	diameter_compiler_test \
-	diameter_config_test \
-	diameter_peer_test \
-	diameter_tcp_test \
-	diameter_test_lib \
-	diameter_test_server
+	diameter_watchdog_SUITE
 
 INTERNAL_HRL_FILES = \
-	diameter_test_lib.hrl
+	diameter_ct.hrl