From 26be0fbf3434a4c4a4f9d9915e18b7435aa2ad8f Mon Sep 17 00:00:00 2001
From: Siri Hansen
Date: Tue, 20 May 2014 11:46:30 +0200
Subject: [ct] Add test of cover support when merge_tests=false
---
lib/common_test/test/Makefile | 3 +-
lib/common_test/test/ct_cover_nomerge_SUITE.erl | 235 +++++++++++++++++++++
.../cover_nomerge_local_SUITE.erl | 63 ++++++
.../cover_nomerge_remote_SUITE.erl | 75 +++++++
.../cover_nomerge_remote_nostop_SUITE.erl | 68 ++++++
.../ct_cover_nomerge_SUITE_data/cover_test_mod.erl | 4 +
.../test/ct_cover_nomerge_SUITE_data/local.spec | 6 +
.../test/ct_cover_nomerge_SUITE_data/remote.spec | 6 +
.../ct_cover_nomerge_SUITE_data/remote_nostop.spec | 6 +
9 files changed, 465 insertions(+), 1 deletion(-)
create mode 100644 lib/common_test/test/ct_cover_nomerge_SUITE.erl
create mode 100644 lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_local_SUITE.erl
create mode 100644 lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_SUITE.erl
create mode 100644 lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_nostop_SUITE.erl
create mode 100644 lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_test_mod.erl
create mode 100644 lib/common_test/test/ct_cover_nomerge_SUITE_data/local.spec
create mode 100644 lib/common_test/test/ct_cover_nomerge_SUITE_data/remote.spec
create mode 100644 lib/common_test/test/ct_cover_nomerge_SUITE_data/remote_nostop.spec
(limited to 'lib/common_test')
diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index 085f19d023..a0ac47f12a 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -1,7 +1,7 @@
#
# %CopyrightBegin%
#
-# Copyright Ericsson AB 2008-2013. All Rights Reserved.
+# Copyright Ericsson AB 2008-2014. All Rights Reserved.
#
# The contents of this file are subject to the Erlang Public License,
# Version 1.1, (the "License"); you may not use this file except in
@@ -61,6 +61,7 @@ MODULES= \
ct_snmp_SUITE \
ct_group_leader_SUITE \
ct_cover_SUITE \
+ ct_cover_nomerge_SUITE \
ct_groups_search_SUITE \
ct_surefire_SUITE \
ct_telnet_SUITE
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE.erl
new file mode 100644
index 0000000000..4344fc30df
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE.erl
@@ -0,0 +1,235 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2014. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_cover_nomerge_SUITE
+%%%
+%%% Description:
+%%% Test code cover analysis support when merge_tests=false
+%%%
+%%%-------------------------------------------------------------------
+-module(ct_cover_nomerge_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+-define(mod, cover_test_mod).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ case test_server:is_cover() of
+ true ->
+ {skip,"Test server is running cover already - skipping"};
+ false ->
+ ct_test_support:init_per_suite(Config)
+ end.
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ try apply(?MODULE,TestCase,[cleanup,Config])
+ catch error:undef -> ok
+ end,
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [
+ local,
+ remote,
+ remote_nostop
+ ].
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+local(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Spec = filename:join(DataDir, "local.spec"),
+ PrivDir = ?config(priv_dir,Config),
+ ExportFile = filename:join(PrivDir,"local.coverdata"),
+ CoverSpec = [{incl_mods,[?mod]},
+ {export, ExportFile}],
+ CoverFile = create_cover_file(local,CoverSpec,Config),
+ {Opts,ERPid} = setup([{spec,Spec},{label,local},{cover,CoverFile}], Config),
+ {ok,Events} = execute(local, local, Opts, ERPid, Config),
+ false = check_cover(Config),
+ check_calls(Events,[{t1,1},{t2,2}]),
+ ok.
+
+remote(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Spec = filename:join(DataDir, "remote.spec"),
+ %% extending some timers for slow test hosts
+ {ok,_Node} = ct_slave:start(ct_nomerge,[{boot_timeout,15},
+ {init_timeout,15},
+ {startup_timeout,15}]),
+
+ PrivDir = ?config(priv_dir,Config),
+ ExportFile = filename:join(PrivDir,"remote.coverdata"),
+ CoverSpec = [{incl_mods,[?mod]},
+ {export, ExportFile}],
+ CoverFile = create_cover_file(remote,CoverSpec,Config),
+ {Opts,ERPid} = setup([{spec,Spec},{label,remote},{cover,CoverFile}], Config),
+ {ok,Events} = execute(remote, remote, Opts, ERPid, Config),
+ false = check_cover(Config),
+ check_calls(Events,[{t1,1},{t2,2}]),
+ ok.
+remote(cleanup,_Config) ->
+ {ok,_} = ct_slave:stop(ct_nomerge),
+ ok.
+
+remote_nostop(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Spec = filename:join(DataDir, "remote_nostop.spec"),
+ %% extending some timers for slow test hosts
+ {ok,Node} = ct_slave:start(ct_nomerge,[{boot_timeout,15},
+ {init_timeout,15},
+ {startup_timeout,15}]),
+
+ PrivDir = ?config(priv_dir,Config),
+ ExportFile = filename:join(PrivDir,"remote_nostop.coverdata"),
+ CoverSpec = [{nodes,[Node]},
+ {incl_mods,[?mod]},
+ {export,ExportFile}],
+ CoverFile = create_cover_file(remote_nostop,CoverSpec,Config),
+ {Opts,ERPid} = setup([{spec,Spec},{label,remote_nostop},
+ {cover,CoverFile},{cover_stop,false}],
+ Config),
+ {ok,Events} = execute(remote_nostop, remote_nostop, Opts, ERPid, Config),
+ {true,[Node],[cover_test_mod]} = check_cover(Config),
+ check_calls(Events,[{t1,1},{t2,2}]),
+ ok.
+remote_nostop(cleanup,Config) ->
+ CtNode = ?config(ct_node,Config),
+ ok = rpc:call(CtNode,cover,stop,[]),
+ {ok,_} = ct_slave:stop(ct_nomerge),
+ ok.
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+execute(Name, Testcase, Opts, ERPid, Config) ->
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(Name,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+ TestEvents = events_to_check(Testcase),
+ R = ct_test_support:verify_events(TestEvents, Events, Config),
+ {R,Events}.
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+
+events_to_check(local) ->
+ events_to_check1(cover_nomerge_local_SUITE);
+events_to_check(remote) ->
+ events_to_check1(cover_nomerge_remote_SUITE);
+events_to_check(remote_nostop) ->
+ events_to_check1(cover_nomerge_remote_nostop_SUITE).
+events_to_check1(Suite) ->
+ OneTest =
+ [{?eh,start_logging,{'DEF','RUNDIR'}}] ++
+ [{?eh,tc_done,{Suite,t1,ok}}] ++
+ [{?eh,tc_done,{Suite,t2,ok}}] ++
+ [{?eh,stop_logging,[]}],
+
+ %% 2 tests (ct:run_test + script_start) is default
+ OneTest ++ OneTest.
+
+check_cover(Config) when is_list(Config) ->
+ CTNode = proplists:get_value(ct_node, Config),
+ check_cover(CTNode);
+check_cover(Node) when is_atom(Node) ->
+ case rpc:call(Node,test_server,is_cover,[]) of
+ true ->
+ {true,
+ rpc:call(Node,cover,which_nodes,[]),
+ rpc:call(Node,cover,modules,[])};
+ false ->
+ false
+ end.
+
+%% Get the log dir "run." for all tests
+get_run_dirs(Events,Testcase) ->
+ [filename:dirname(TCLog) ||
+ {ct_test_support_eh,
+ {event,tc_logfile,_Node,
+ {{_Suite,TC},TCLog}}} <- Events,
+ TC==Testcase].
+
+%% Check that each coverlog includes N calls to ?mod:foo/0
+check_calls(Events,N) ->
+ check_calls(Events,{?mod,foo,0},N).
+check_calls(Events,MFA,[{Testcase,N}|Expected]) ->
+ CoverLogs =
+ [filename:join(D,"all.coverdata") || D <- get_run_dirs(Events,Testcase)],
+ ok = do_check_logs(CoverLogs,MFA,N),
+ check_calls(Events,MFA,Expected);
+check_calls(_,_,[]) ->
+ ok.
+
+do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) ->
+ {ok,_} = cover:start(),
+ ok = cover:import(CoverLog),
+ {ok,Calls} = cover:analyse(Mod,calls,function),
+ ok = cover:stop(),
+ {MFA,N} = lists:keyfind(MFA,1,Calls),
+ do_check_logs(CoverLogs,MFA,N);
+do_check_logs([],_,_) ->
+ ok.
+
+create_cover_file(Filename,Terms,Config) ->
+ PrivDir = ?config(priv_dir,Config),
+ File = filename:join(PrivDir,Filename) ++ ".cover",
+ {ok,Fd} = file:open(File,[write]),
+ lists:foreach(fun(Term) ->
+ file:write(Fd,io_lib:format("~p.~n",[Term]))
+ end,Terms),
+ ok = file:close(Fd),
+ File.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_local_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_local_SUITE.erl
new file mode 100644
index 0000000000..e1fe3b5fc9
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_local_SUITE.erl
@@ -0,0 +1,63 @@
+%%--------------------------------------------------------------------
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2014. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%----------------------------------------------------------------------
+-module(cover_nomerge_local_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+%% Default timetrap timeout (set in init_per_testcase).
+-define(default_timeout, ?t:minutes(1)).
+
+suite() ->
+ [].
+
+all() ->
+ [t1,t2].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ Config.
+
+init_per_testcase(_Case, Config) ->
+ Dog = test_server:timetrap(?default_timeout),
+ [{watchdog, Dog}|Config].
+
+end_per_testcase(Case, Config) ->
+ Dog=?config(watchdog, Config),
+ test_server:timetrap_cancel(Dog),
+ ok.
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+break(_Config) ->
+ test_server:break(""),
+ ok.
+
+t1(_Config) ->
+ cover_compiled = code:which(cover_test_mod),
+ ok = cover_test_mod:foo(),
+ ok.
+
+t2(_Config) ->
+ cover_compiled = code:which(cover_test_mod),
+ ok = cover_test_mod:foo(),
+ ok.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_SUITE.erl
new file mode 100644
index 0000000000..a77ae0c2db
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_SUITE.erl
@@ -0,0 +1,75 @@
+%%--------------------------------------------------------------------
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2014. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%----------------------------------------------------------------------
+-module(cover_nomerge_remote_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+%% Default timetrap timeout (set in init_per_testcase).
+-define(default_timeout, ?t:minutes(1)).
+
+suite() ->
+ [].
+
+all() ->
+ [t1,t2].
+
+init_per_suite(Config) ->
+ {ok,Host} = inet:gethostname(),
+ Node = list_to_atom("ct_nomerge@"++Host),
+ pong = net_adm:ping(Node),
+
+%% Include this row, and exclude the equivalent row in end_per_suite =>
+%% fails every now and then with missing data. Why?
+%% ct_cover:remove_nodes([Node]),
+ ct_cover:add_nodes([Node]),
+ [{node,Node}|Config].
+
+end_per_suite(Config) ->
+ Node = ?config(node,Config),
+ ct_cover:remove_nodes([Node]),
+ Config.
+
+init_per_testcase(_Case, Config) ->
+ Dog = test_server:timetrap(?default_timeout),
+ [{watchdog, Dog}|Config].
+
+end_per_testcase(Case, Config) ->
+ Dog=?config(watchdog, Config),
+ test_server:timetrap_cancel(Dog),
+ ok.
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+break(_Config) ->
+ test_server:break(""),
+ ok.
+
+t1(Config) ->
+ Node = ?config(node,Config),
+ cover_compiled = rpc:call(Node, code, which, [cover_test_mod]),
+ ok = rpc:call(Node, cover_test_mod, foo, []),
+ ok.
+
+t2(Config) ->
+ Node = ?config(node,Config),
+ cover_compiled = rpc:call(Node, code, which, [cover_test_mod]),
+ ok = rpc:call(Node, cover_test_mod, foo, []),
+ ok.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_nostop_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_nostop_SUITE.erl
new file mode 100644
index 0000000000..0b3159f2c3
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_nostop_SUITE.erl
@@ -0,0 +1,68 @@
+%%--------------------------------------------------------------------
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2014. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%----------------------------------------------------------------------
+-module(cover_nomerge_remote_nostop_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+%% Default timetrap timeout (set in init_per_testcase).
+-define(default_timeout, ?t:minutes(1)).
+
+suite() ->
+ [].
+
+all() ->
+ [t1,t2].
+
+init_per_suite(Config) ->
+ {ok,Host} = inet:gethostname(),
+ Node = list_to_atom("ct_nomerge@"++Host),
+ pong = net_adm:ping(Node),
+ [{node,Node}|Config].
+
+end_per_suite(Config) ->
+ Config.
+
+init_per_testcase(_Case, Config) ->
+ Dog = test_server:timetrap(?default_timeout),
+ [{watchdog, Dog}|Config].
+
+end_per_testcase(Case, Config) ->
+ Dog=?config(watchdog, Config),
+ test_server:timetrap_cancel(Dog),
+ ok.
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+break(_Config) ->
+ test_server:break(""),
+ ok.
+
+t1(Config) ->
+ Node = ?config(node,Config),
+ cover_compiled = rpc:call(Node, code, which, [cover_test_mod]),
+ ok = rpc:call(Node, cover_test_mod, foo, []),
+ ok.
+
+t2(Config) ->
+ Node = ?config(node,Config),
+ cover_compiled = rpc:call(Node, code, which, [cover_test_mod]),
+ ok = rpc:call(Node, cover_test_mod, foo, []),
+ ok.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_test_mod.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_test_mod.erl
new file mode 100644
index 0000000000..d4f69452c3
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_test_mod.erl
@@ -0,0 +1,4 @@
+-module(cover_test_mod).
+-compile(export_all).
+foo() ->
+ ok.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/local.spec b/lib/common_test/test/ct_cover_nomerge_SUITE_data/local.spec
new file mode 100644
index 0000000000..893c48b010
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/local.spec
@@ -0,0 +1,6 @@
+{merge_tests,false}.
+
+{alias,dir,"."}.
+
+{cases, dir, cover_nomerge_local_SUITE, [t1]}.
+{cases, dir, cover_nomerge_local_SUITE, [t2]}.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote.spec b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote.spec
new file mode 100644
index 0000000000..78c4332270
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote.spec
@@ -0,0 +1,6 @@
+{merge_tests,false}.
+
+{alias,dir,"."}.
+
+{cases, dir, cover_nomerge_remote_SUITE, [t1]}.
+{cases, dir, cover_nomerge_remote_SUITE, [t2]}.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote_nostop.spec b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote_nostop.spec
new file mode 100644
index 0000000000..049f586c72
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote_nostop.spec
@@ -0,0 +1,6 @@
+{merge_tests,false}.
+
+{alias,dir,"."}.
+
+{cases, dir, cover_nomerge_remote_nostop_SUITE, [t1]}.
+{cases, dir, cover_nomerge_remote_nostop_SUITE, [t2]}.
--
cgit v1.2.3
From 5a3c4668908254ee930c8db2e5cf9741945f9b2b Mon Sep 17 00:00:00 2001
From: Siri Hansen
Date: Wed, 21 May 2014 16:39:03 +0200
Subject: Improve cover analysis via common_test
This addresses several bugs in common_test (ct) when using the cover
analysis mechanism:
In a ct test run, one can give a cover spec file which indicates that
cover analysis shall be run, including a number of modules and a
number of nodes. During the ct test run, multiple jobs may be started
in test_server, and when the cover option is used, test_server would
cover compile and analyse all given modules for each job. This commit
instead allows the compilation and analysis to be explicitly ordered
by ct for each test run. This way each module will only be cover
compiled and analysed once. The cover log will be located in the
common_test log directory (ct_run.), and the "Coverage log"
link in each suite.log.html will point to this file. A new button is
also added in the top level ct index file, which points to the cover
log.
This change also reduces the need of using the 'export' and 'import'
options, since there is no longer any need to accumulate cover data
over multiple test_server jobs. However, these options may still be
used for importing and exporting cover data in order to store or
accumulate data from multiple ct test runs.
The 'nodes' option was earlier only used by ct to start cover on the
given nodes before starting the first test_server job. After this job
was completed, test_server would stop cover completely and then start
it again for the next job without any knowledge of the 'nodes'
options. For the next test_server jobs cover would therefore no longer
be running on these nodes. Explcit calls to ct_cover:add_nodes had to
be done in order to collect data through all test_server jobs. This
bug has now been solved, since cover is no longer stopped between each
test_server job.
Finally, ct no longer stores cover data using
ct_util:set_testdata. This was earlier used by ct_cover:add_nodes to
make sure no node was added twice.This did, however, cause some
problems when ct and cover were out of sync. ct could belive that a
node was running cover and thus reject adding this node, while in
reality cover had been stopped on the node (e.g. by test_server) so no
cover data was collected. ct_cover:add_nodes will now instead use
cover:which_nodes to check if a node is already running.
---
lib/common_test/doc/src/cover_chapter.xml | 25 ++--
lib/common_test/src/ct_cover.erl | 30 ++--
lib/common_test/src/ct_framework.erl | 33 +---
lib/common_test/src/ct_logs.erl | 20 ++-
lib/common_test/src/ct_run.erl | 191 ++++++++++++++----------
lib/common_test/test/ct_cover_SUITE.erl | 15 +-
lib/common_test/test/ct_cover_nomerge_SUITE.erl | 44 ++----
7 files changed, 184 insertions(+), 174 deletions(-)
(limited to 'lib/common_test')
diff --git a/lib/common_test/doc/src/cover_chapter.xml b/lib/common_test/doc/src/cover_chapter.xml
index a215c8c2f3..accb94e1a9 100644
--- a/lib/common_test/doc/src/cover_chapter.xml
+++ b/lib/common_test/doc/src/cover_chapter.xml
@@ -4,7 +4,7 @@
- 20062013
+ 20062014
Ericsson AB. All Rights Reserved.
@@ -81,10 +81,7 @@
specify that previously exported data should be imported and
included in the analysis for a test (you can specify multiple
import files). This way it is possible to analyse total code coverage
- without necessarily running all tests at once. Note that even if
- you run separate tests in one test run, code coverage data will
- not be passed on from one test to another unless you specify an
- export file for Common Test to use for this purpose.
+ without necessarily running all tests at once.
To activate the code coverage support, you simply specify the
name of the cover specification file as you start Common Test.
@@ -266,10 +263,20 @@ ct_cover:cross_cover_analyse(Level, [{s1,S1LogDir},{s2,S2LogDir}]).
Logging
- To view the result of a code coverage test, follow the
- "Coverage log" link on the test suite results page. This
- takes you to the code coverage overview page. If you have
- successfully performed a detailed coverage analysis, you
+
To view the result of a code coverage test, click the button
+ labled "COVER LOG" in the top level index page for the test run.
+
+ Prior to Erlang/OTP 17.1, if your test run consisted of
+ multiple tests, cover would be started and stopped for each test
+ within the test run. Separate logs would be available via the
+ "Coverage log" link on the test suite result pages. These links
+ are still available, but now they all point to the same page as
+ the button on the top level index page. The log contains the
+ accumulated results for the complete test run. See the release
+ notes for more information about this change.
+
+ The buttonc takes you to the code coverage overview page. If you
+ have successfully performed a detailed coverage analysis, you
find links to each individual module coverage page here.
If cross cover analysis has been performed, and there are
diff --git a/lib/common_test/src/ct_cover.erl b/lib/common_test/src/ct_cover.erl
index ae671c750a..cf2860ae25 100644
--- a/lib/common_test/src/ct_cover.erl
+++ b/lib/common_test/src/ct_cover.erl
@@ -47,18 +47,21 @@ add_nodes(Nodes) ->
undefined ->
{error,cover_not_running};
_ ->
- {File,Nodes0,Import,Export,AppInfo} = ct_util:get_testdata(cover),
+ Nodes0 = cover:which_nodes(),
Nodes1 = [Node || Node <- Nodes,
lists:member(Node,Nodes0) == false],
ct_logs:log("COVER INFO",
"Adding nodes to cover test: ~w", [Nodes1]),
case cover:start(Nodes1) of
- Result = {ok,_} ->
- ct_util:set_testdata({cover,{File,Nodes1++Nodes0,
- Import,Export,AppInfo}}),
-
+ Result = {ok,StartedNodes} ->
+ ct_logs:log("COVER INFO",
+ "Successfully added nodes to cover test: ~w",
+ [StartedNodes]),
Result;
Error ->
+ ct_logs:log("COVER INFO",
+ "Failed to add nodes to cover test: ~tp",
+ [Error]),
Error
end
end.
@@ -81,19 +84,20 @@ remove_nodes(Nodes) ->
undefined ->
{error,cover_not_running};
_ ->
- {File,Nodes0,Import,Export,AppInfo} = ct_util:get_testdata(cover),
+ Nodes0 = cover:which_nodes(),
ToRemove = [Node || Node <- Nodes, lists:member(Node,Nodes0)],
ct_logs:log("COVER INFO",
- "Removing nodes from cover test: ~w", [ToRemove]),
+ "Removing nodes from cover test: ~w", [ToRemove]),
case cover:stop(ToRemove) of
ok ->
- Nodes1 = lists:foldl(fun(N,Deleted) ->
- lists:delete(N,Deleted)
- end, Nodes0, ToRemove),
- ct_util:set_testdata({cover,{File,Nodes1,
- Import,Export,AppInfo}}),
+ ct_logs:log("COVER INFO",
+ "Successfully removed nodes from cover test.",
+ []),
ok;
Error ->
+ ct_logs:log("COVER INFO",
+ "Failed to remove nodes from cover test: ~tp",
+ [Error]),
Error
end
end.
@@ -149,7 +153,7 @@ get_spec_test(File) ->
{value,{_,[Exp]}} ->
filename:absname(Exp);
_ ->
- []
+ undefined
end,
Nodes =
case lists:keysearch(nodes, 1, Terms) of
diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl
index 20903607dc..e8ea7992b4 100644
--- a/lib/common_test/src/ct_framework.erl
+++ b/lib/common_test/src/ct_framework.erl
@@ -1244,38 +1244,7 @@ report(What,Data) ->
ct_logs:make_all_suites_index({TestName,RunDir}),
ok;
tests_start ->
- case ct_util:get_testdata(cover) of
- undefined ->
- ok;
- {_CovFile,_CovNodes,CovImport,CovExport,_CovAppData} ->
- %% Always import cover data from files specified by CovImport
- %% if no CovExport defined. If CovExport is defined, only
- %% import from CovImport files initially, then use CovExport
- %% to pass coverdata between proceeding tests (in the same run).
- Imps =
- case CovExport of
- [] -> % don't export data between tests
- CovImport;
- _ ->
- case filelib:is_file(CovExport) of
- true ->
- [CovExport];
- false ->
- CovImport
- end
- end,
- lists:foreach(
- fun(Imp) ->
- case cover:import(Imp) of
- ok ->
- ok;
- {error,Reason} ->
- ct_logs:log("COVER INFO",
- "Importing cover data from: ~ts fails! "
- "Reason: ~p", [Imp,Reason])
- end
- end, Imps)
- end;
+ ok;
tests_done ->
ok;
severe_error ->
diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl
index a4ad65c0a4..32c8773ca5 100644
--- a/lib/common_test/src/ct_logs.erl
+++ b/lib/common_test/src/ct_logs.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2003-2013. All Rights Reserved.
+%% Copyright Ericsson AB 2003-2014. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -62,6 +62,7 @@
-define(totals_name, "totals.info").
-define(log_cache_name, "ct_log_cache").
-define(misc_io_log, "misc_io.log.html").
+-define(coverlog_name, "cover.html"). % must be same as in test_server_ctrl
-define(table_color1,"#ADD8E6").
-define(table_color2,"#E4F0FE").
@@ -1368,6 +1369,19 @@ index_header(Label, StartTime) ->
format_time(StartTime),
{[],[1],[2,3,4,5]})
end,
+ Cover =
+ case filelib:is_regular(?abs(?coverlog_name)) of
+ true ->
+ xhtml(["
Cover Log
\n"],
+ ["
"
+ "
"]);
+ false ->
+ xhtml("
\n", "
\n")
+ end,
+
[Head |
["\n",
xhtml(["
["
"
""]),
- xhtml("
\n", "
\n"),
+ "\">COMMON TEST FRAMEWORK LOG\n
\n"]),
+ Cover,
xhtml(["
\n"],
["\n",
diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl
index 03cf06abed..00d0aab507 100644
--- a/lib/common_test/src/ct_run.erl
+++ b/lib/common_test/src/ct_run.erl
@@ -1,7 +1,7 @@
%%
%% %CopyrightBegin%
%%
-%% Copyright Ericsson AB 2004-2013. All Rights Reserved.
+%% Copyright Ericsson AB 2004-2014. All Rights Reserved.
%%
%% The contents of this file are subject to the Erlang Public License,
%% Version 1.1, (the "License"); you may not use this file except in
@@ -1646,7 +1646,7 @@ do_run(Tests, Misc, LogDir, LogOpts) when is_list(Misc),
do_run(Tests, [], Opts#opts{logdir = LogDir}, []);
do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) ->
- #opts{label = Label, profile = Profile, cover = Cover,
+ #opts{label = Label, profile = Profile,
verbosity = VLvls} = Opts,
%% label - used by ct_logs
TestLabel =
@@ -1670,22 +1670,6 @@ do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) ->
non_existing ->
{error,no_path_to_test_server};
_ ->
- Opts1 = if Cover == undefined ->
- Opts;
- true ->
- case ct_cover:get_spec(Cover) of
- {error,Reason} ->
- exit({error,Reason});
- CoverSpec ->
- CoverStop =
- case Opts#opts.cover_stop of
- undefined -> true;
- Stop -> Stop
- end,
- Opts#opts{coverspec = CoverSpec,
- cover_stop = CoverStop}
- end
- end,
%% This env variable is used by test_server to determine
%% which framework it runs under.
case os:getenv("TEST_SERVER_FRAMEWORK") of
@@ -1711,7 +1695,7 @@ do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) ->
_Pid ->
ct_util:set_testdata({starter,Opts#opts.starter}),
compile_and_run(Tests, Skip,
- Opts1#opts{verbosity=Verbosity}, Args)
+ Opts#opts{verbosity=Verbosity}, Args)
end
end.
@@ -2146,67 +2130,11 @@ check_and_add([{TestDir0,M,_} | Tests], Added, PA) ->
check_and_add([], _, PA) ->
{ok,PA}.
-do_run_test(Tests, Skip, Opts) ->
+do_run_test(Tests, Skip, Opts0) ->
case check_and_add(Tests, [], []) of
{ok,AddedToPath} ->
ct_util:set_testdata({stats,{0,0,{0,0}}}),
- ct_util:set_testdata({cover,undefined}),
test_server_ctrl:start_link(local),
- case Opts#opts.coverspec of
- CovData={CovFile,
- CovNodes,
- _CovImport,
- CovExport,
- #cover{app = CovApp,
- level = CovLevel,
- excl_mods = CovExcl,
- incl_mods = CovIncl,
- cross = CovCross,
- src = _CovSrc}} ->
- ct_logs:log("COVER INFO",
- "Using cover specification file: ~ts~n"
- "App: ~w~n"
- "Cross cover: ~w~n"
- "Including ~w modules~n"
- "Excluding ~w modules",
- [CovFile,CovApp,CovCross,
- length(CovIncl),length(CovExcl)]),
-
- %% cover export file will be used for export and import
- %% between tests so make sure it doesn't exist initially
- case filelib:is_file(CovExport) of
- true ->
- DelResult = file:delete(CovExport),
- ct_logs:log("COVER INFO",
- "Warning! "
- "Export file ~ts already exists. "
- "Deleting with result: ~p",
- [CovExport,DelResult]);
- false ->
- ok
- end,
-
- %% tell test_server which modules should be cover compiled
- %% note that actual compilation is done when tests start
- test_server_ctrl:cover(CovApp, CovFile, CovExcl, CovIncl,
- CovCross, CovExport, CovLevel,
- Opts#opts.cover_stop),
- %% save cover data (used e.g. to add nodes dynamically)
- ct_util:set_testdata({cover,CovData}),
- %% start cover on specified nodes
- if (CovNodes /= []) and (CovNodes /= undefined) ->
- ct_logs:log("COVER INFO",
- "Nodes included in cover "
- "session: ~w",
- [CovNodes]),
- cover:start(CovNodes);
- true ->
- ok
- end,
- true;
- _ ->
- false
- end,
%% let test_server expand the test tuples and count no of cases
{Suites,NoOfCases} = count_test_cases(Tests, Skip),
@@ -2231,24 +2159,31 @@ do_run_test(Tests, Skip, Opts) ->
end,
%% if the verbosity level is set lower than ?STD_IMPORTANCE, tell
%% test_server to ignore stdout printouts to the test case log file
- case proplists:get_value(default, Opts#opts.verbosity) of
+ case proplists:get_value(default, Opts0#opts.verbosity) of
VLvl when is_integer(VLvl), (?STD_IMPORTANCE < (100-VLvl)) ->
test_server_ctrl:reject_io_reqs(true);
_Lower ->
ok
end,
- test_server_ctrl:multiply_timetraps(Opts#opts.multiply_timetraps),
- test_server_ctrl:scale_timetraps(Opts#opts.scale_timetraps),
+ test_server_ctrl:multiply_timetraps(Opts0#opts.multiply_timetraps),
+ test_server_ctrl:scale_timetraps(Opts0#opts.scale_timetraps),
test_server_ctrl:create_priv_dir(choose_val(
- Opts#opts.create_priv_dir,
+ Opts0#opts.create_priv_dir,
auto_per_run)),
+
+ {ok,LogDir} = ct_logs:get_log_dir(true),
+ {TsCoverInfo,Opts} = maybe_start_cover(Opts0, LogDir),
+
ct_event:notify(#event{name=start_info,
node=node(),
data={NoOfTests,NoOfSuites,NoOfCases}}),
CleanUp = add_jobs(Tests, Skip, Opts, []),
unlink(whereis(test_server_ctrl)),
catch test_server_ctrl:wait_finish(),
+
+ maybe_stop_cover(Opts, TsCoverInfo, LogDir),
+
%% check if last testcase has left a "dead" trace window
%% behind, and if so, kill it
case ct_util:get_testdata(interpret) of
@@ -2281,6 +2216,102 @@ do_run_test(Tests, Skip, Opts) ->
exit(Error)
end.
+maybe_start_cover(Opts=#opts{cover=Cover,cover_stop=CoverStop0},LogDir) ->
+ if Cover == undefined ->
+ {undefined,Opts};
+ true ->
+ case ct_cover:get_spec(Cover) of
+ {error,Reason} ->
+ exit({error,Reason});
+ CoverSpec ->
+ CoverStop =
+ case CoverStop0 of
+ undefined -> true;
+ Stop -> Stop
+ end,
+ start_cover(Opts#opts{coverspec=CoverSpec,
+ cover_stop=CoverStop},
+ LogDir)
+ end
+ end.
+
+start_cover(Opts=#opts{coverspec=CovData,cover_stop=CovStop},LogDir) ->
+ {CovFile,
+ CovNodes,
+ CovImport,
+ _CovExport,
+ #cover{app = CovApp,
+ level = CovLevel,
+ excl_mods = CovExcl,
+ incl_mods = CovIncl,
+ cross = CovCross,
+ src = _CovSrc}} = CovData,
+ ct_logs:log("COVER INFO",
+ "Using cover specification file: ~ts~n"
+ "App: ~w~n"
+ "Cross cover: ~w~n"
+ "Including ~w modules~n"
+ "Excluding ~w modules",
+ [CovFile,CovApp,CovCross,
+ length(CovIncl),length(CovExcl)]),
+
+ %% Tell test_server to print a link in its coverlog
+ %% pointing to the real coverlog which will be written in
+ %% maybe_stop_cover/2
+ test_server_ctrl:cover({log,LogDir}),
+
+ %% Cover compile all modules
+ {ok,TsCoverInfo} = test_server_ctrl:cover_compile(CovApp,CovFile,
+ CovExcl,CovIncl,
+ CovCross,CovLevel,
+ CovStop),
+ ct_logs:log("COVER INFO",
+ "Compilation completed - test_server cover info: ~tp",
+ [TsCoverInfo]),
+
+ %% start cover on specified nodes
+ if (CovNodes /= []) and (CovNodes /= undefined) ->
+ ct_logs:log("COVER INFO",
+ "Nodes included in cover "
+ "session: ~w",
+ [CovNodes]),
+ cover:start(CovNodes);
+ true ->
+ ok
+ end,
+ lists:foreach(
+ fun(Imp) ->
+ case cover:import(Imp) of
+ ok ->
+ ok;
+ {error,Reason} ->
+ ct_logs:log("COVER INFO",
+ "Importing cover data from: ~ts fails! "
+ "Reason: ~p", [Imp,Reason])
+ end
+ end, CovImport),
+ {TsCoverInfo,Opts}.
+
+maybe_stop_cover(_,undefined,_) ->
+ ok;
+maybe_stop_cover(#opts{coverspec=CovData},TsCoverInfo,LogDir) ->
+ {_CovFile,
+ _CovNodes,
+ _CovImport,
+ CovExport,
+ _AppData} = CovData,
+ case CovExport of
+ undefined -> ok;
+ _ ->
+ ct_logs:log("COVER INFO","Exporting cover data to ~tp",[CovExport]),
+ cover:export(CovExport)
+ end,
+ ct_logs:log("COVER INFO","Analysing cover data to ~tp",[LogDir]),
+ test_server_ctrl:cover_analyse(TsCoverInfo,LogDir),
+ ct_logs:log("COVER INFO","Analysis completed.",[]),
+ ok.
+
+
delete_dups([S | Suites]) ->
Suites1 = lists:delete(S, Suites),
[S | delete_dups(Suites1)];
diff --git a/lib/common_test/test/ct_cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE.erl
index ec2680f664..47080b5577 100644
--- a/lib/common_test/test/ct_cover_SUITE.erl
+++ b/lib/common_test/test/ct_cover_SUITE.erl
@@ -172,8 +172,8 @@ cross(Config) ->
check_calls(Events2,1),
%% Get the log dirs for each test and run cross cover analyse
- [D11,D12] = lists:sort(get_run_dirs(Events1)),
- [D21,D22] = lists:sort(get_run_dirs(Events2)),
+ [D11,D12] = lists:sort(get_log_dirs(Events1)),
+ [D21,D22] = lists:sort(get_log_dirs(Events2)),
ct_cover:cross_cover_analyse(details,[{cross1,D11},{cross2,D21}]),
ct_cover:cross_cover_analyse(details,[{cross1,D12},{cross2,D22}]),
@@ -267,18 +267,17 @@ check_cover(Node) when is_atom(Node) ->
false
end.
-%% Get the log dir "run." for all (both!) tests
-get_run_dirs(Events) ->
- [filename:dirname(TCLog) ||
+%% Get the log dir "ct_run." for all (both!) tests
+get_log_dirs(Events) ->
+ [LogDir ||
{ct_test_support_eh,
- {event,tc_logfile,_Node,
- {{?suite,init_per_suite},TCLog}}} <- Events].
+ {event,start_logging,_Node,LogDir}} <- Events].
%% Check that each coverlog includes N calls to ?mod:foo/0
check_calls(Events,N) ->
check_calls(Events,{?mod,foo,0},N).
check_calls(Events,MFA,N) ->
- CoverLogs = [filename:join(D,"all.coverdata") || D <- get_run_dirs(Events)],
+ CoverLogs = [filename:join(D,"all.coverdata") || D <- get_log_dirs(Events)],
do_check_logs(CoverLogs,MFA,N).
do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) ->
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE.erl
index 4344fc30df..8e2ee1b500 100644
--- a/lib/common_test/test/ct_cover_nomerge_SUITE.erl
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE.erl
@@ -79,34 +79,29 @@ all() ->
local(Config) ->
DataDir = ?config(data_dir, Config),
Spec = filename:join(DataDir, "local.spec"),
- PrivDir = ?config(priv_dir,Config),
- ExportFile = filename:join(PrivDir,"local.coverdata"),
- CoverSpec = [{incl_mods,[?mod]},
- {export, ExportFile}],
+ CoverSpec = [{incl_mods,[?mod]}],
CoverFile = create_cover_file(local,CoverSpec,Config),
{Opts,ERPid} = setup([{spec,Spec},{label,local},{cover,CoverFile}], Config),
{ok,Events} = execute(local, local, Opts, ERPid, Config),
false = check_cover(Config),
- check_calls(Events,[{t1,1},{t2,2}]),
+ check_calls(Events,2),
ok.
remote(Config) ->
DataDir = ?config(data_dir, Config),
Spec = filename:join(DataDir, "remote.spec"),
%% extending some timers for slow test hosts
- {ok,_Node} = ct_slave:start(ct_nomerge,[{boot_timeout,15},
+ {ok,Node} = ct_slave:start(ct_nomerge,[{boot_timeout,15},
{init_timeout,15},
{startup_timeout,15}]),
- PrivDir = ?config(priv_dir,Config),
- ExportFile = filename:join(PrivDir,"remote.coverdata"),
- CoverSpec = [{incl_mods,[?mod]},
- {export, ExportFile}],
+ CoverSpec = [{nodes,[Node]},
+ {incl_mods,[?mod]}],
CoverFile = create_cover_file(remote,CoverSpec,Config),
{Opts,ERPid} = setup([{spec,Spec},{label,remote},{cover,CoverFile}], Config),
{ok,Events} = execute(remote, remote, Opts, ERPid, Config),
false = check_cover(Config),
- check_calls(Events,[{t1,1},{t2,2}]),
+ check_calls(Events,2),
ok.
remote(cleanup,_Config) ->
{ok,_} = ct_slave:stop(ct_nomerge),
@@ -120,18 +115,15 @@ remote_nostop(Config) ->
{init_timeout,15},
{startup_timeout,15}]),
- PrivDir = ?config(priv_dir,Config),
- ExportFile = filename:join(PrivDir,"remote_nostop.coverdata"),
CoverSpec = [{nodes,[Node]},
- {incl_mods,[?mod]},
- {export,ExportFile}],
+ {incl_mods,[?mod]}],
CoverFile = create_cover_file(remote_nostop,CoverSpec,Config),
{Opts,ERPid} = setup([{spec,Spec},{label,remote_nostop},
{cover,CoverFile},{cover_stop,false}],
Config),
{ok,Events} = execute(remote_nostop, remote_nostop, Opts, ERPid, Config),
{true,[Node],[cover_test_mod]} = check_cover(Config),
- check_calls(Events,[{t1,1},{t2,2}]),
+ check_calls(Events,2),
ok.
remote_nostop(cleanup,Config) ->
CtNode = ?config(ct_node,Config),
@@ -195,24 +187,18 @@ check_cover(Node) when is_atom(Node) ->
false
end.
-%% Get the log dir "run." for all tests
-get_run_dirs(Events,Testcase) ->
- [filename:dirname(TCLog) ||
+%% Get the log dir "ct_run." for all (both!) tests
+get_log_dirs(Events) ->
+ [LogDir ||
{ct_test_support_eh,
- {event,tc_logfile,_Node,
- {{_Suite,TC},TCLog}}} <- Events,
- TC==Testcase].
+ {event,start_logging,_Node,LogDir}} <- Events].
%% Check that each coverlog includes N calls to ?mod:foo/0
check_calls(Events,N) ->
check_calls(Events,{?mod,foo,0},N).
-check_calls(Events,MFA,[{Testcase,N}|Expected]) ->
- CoverLogs =
- [filename:join(D,"all.coverdata") || D <- get_run_dirs(Events,Testcase)],
- ok = do_check_logs(CoverLogs,MFA,N),
- check_calls(Events,MFA,Expected);
-check_calls(_,_,[]) ->
- ok.
+check_calls(Events,MFA,N) ->
+ CoverLogs = [filename:join(D,"all.coverdata") || D <- get_log_dirs(Events)],
+ do_check_logs(CoverLogs,MFA,N).
do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) ->
{ok,_} = cover:start(),
--
cgit v1.2.3