aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSiri Hansen <[email protected]>2014-05-20 11:46:30 +0200
committerSiri Hansen <[email protected]>2014-05-20 11:50:56 +0200
commit26be0fbf3434a4c4a4f9d9915e18b7435aa2ad8f (patch)
tree6c261a0659f26bdde66839a9aa5e486a668fd95e
parent2ca034c782eb545eb327487df2ea3de270413b21 (diff)
downloadotp-26be0fbf3434a4c4a4f9d9915e18b7435aa2ad8f.tar.gz
otp-26be0fbf3434a4c4a4f9d9915e18b7435aa2ad8f.tar.bz2
otp-26be0fbf3434a4c4a4f9d9915e18b7435aa2ad8f.zip
[ct] Add test of cover support when merge_tests=false
-rw-r--r--lib/common_test/test/Makefile3
-rw-r--r--lib/common_test/test/ct_cover_nomerge_SUITE.erl235
-rw-r--r--lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_local_SUITE.erl63
-rw-r--r--lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_SUITE.erl75
-rw-r--r--lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_nostop_SUITE.erl68
-rw-r--r--lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_test_mod.erl4
-rw-r--r--lib/common_test/test/ct_cover_nomerge_SUITE_data/local.spec6
-rw-r--r--lib/common_test/test/ct_cover_nomerge_SUITE_data/remote.spec6
-rw-r--r--lib/common_test/test/ct_cover_nomerge_SUITE_data/remote_nostop.spec6
9 files changed, 465 insertions, 1 deletions
diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index 085f19d023..a0ac47f12a 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -1,7 +1,7 @@
#
# %CopyrightBegin%
#
-# Copyright Ericsson AB 2008-2013. All Rights Reserved.
+# Copyright Ericsson AB 2008-2014. All Rights Reserved.
#
# The contents of this file are subject to the Erlang Public License,
# Version 1.1, (the "License"); you may not use this file except in
@@ -61,6 +61,7 @@ MODULES= \
ct_snmp_SUITE \
ct_group_leader_SUITE \
ct_cover_SUITE \
+ ct_cover_nomerge_SUITE \
ct_groups_search_SUITE \
ct_surefire_SUITE \
ct_telnet_SUITE
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE.erl
new file mode 100644
index 0000000000..4344fc30df
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE.erl
@@ -0,0 +1,235 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2014. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_cover_nomerge_SUITE
+%%%
+%%% Description:
+%%% Test code cover analysis support when merge_tests=false
+%%%
+%%%-------------------------------------------------------------------
+-module(ct_cover_nomerge_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+-define(mod, cover_test_mod).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+ case test_server:is_cover() of
+ true ->
+ {skip,"Test server is running cover already - skipping"};
+ false ->
+ ct_test_support:init_per_suite(Config)
+ end.
+
+end_per_suite(Config) ->
+ ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+ ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+ try apply(?MODULE,TestCase,[cleanup,Config])
+ catch error:undef -> ok
+ end,
+ ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+ [
+ local,
+ remote,
+ remote_nostop
+ ].
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+local(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Spec = filename:join(DataDir, "local.spec"),
+ PrivDir = ?config(priv_dir,Config),
+ ExportFile = filename:join(PrivDir,"local.coverdata"),
+ CoverSpec = [{incl_mods,[?mod]},
+ {export, ExportFile}],
+ CoverFile = create_cover_file(local,CoverSpec,Config),
+ {Opts,ERPid} = setup([{spec,Spec},{label,local},{cover,CoverFile}], Config),
+ {ok,Events} = execute(local, local, Opts, ERPid, Config),
+ false = check_cover(Config),
+ check_calls(Events,[{t1,1},{t2,2}]),
+ ok.
+
+remote(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Spec = filename:join(DataDir, "remote.spec"),
+ %% extending some timers for slow test hosts
+ {ok,_Node} = ct_slave:start(ct_nomerge,[{boot_timeout,15},
+ {init_timeout,15},
+ {startup_timeout,15}]),
+
+ PrivDir = ?config(priv_dir,Config),
+ ExportFile = filename:join(PrivDir,"remote.coverdata"),
+ CoverSpec = [{incl_mods,[?mod]},
+ {export, ExportFile}],
+ CoverFile = create_cover_file(remote,CoverSpec,Config),
+ {Opts,ERPid} = setup([{spec,Spec},{label,remote},{cover,CoverFile}], Config),
+ {ok,Events} = execute(remote, remote, Opts, ERPid, Config),
+ false = check_cover(Config),
+ check_calls(Events,[{t1,1},{t2,2}]),
+ ok.
+remote(cleanup,_Config) ->
+ {ok,_} = ct_slave:stop(ct_nomerge),
+ ok.
+
+remote_nostop(Config) ->
+ DataDir = ?config(data_dir, Config),
+ Spec = filename:join(DataDir, "remote_nostop.spec"),
+ %% extending some timers for slow test hosts
+ {ok,Node} = ct_slave:start(ct_nomerge,[{boot_timeout,15},
+ {init_timeout,15},
+ {startup_timeout,15}]),
+
+ PrivDir = ?config(priv_dir,Config),
+ ExportFile = filename:join(PrivDir,"remote_nostop.coverdata"),
+ CoverSpec = [{nodes,[Node]},
+ {incl_mods,[?mod]},
+ {export,ExportFile}],
+ CoverFile = create_cover_file(remote_nostop,CoverSpec,Config),
+ {Opts,ERPid} = setup([{spec,Spec},{label,remote_nostop},
+ {cover,CoverFile},{cover_stop,false}],
+ Config),
+ {ok,Events} = execute(remote_nostop, remote_nostop, Opts, ERPid, Config),
+ {true,[Node],[cover_test_mod]} = check_cover(Config),
+ check_calls(Events,[{t1,1},{t2,2}]),
+ ok.
+remote_nostop(cleanup,Config) ->
+ CtNode = ?config(ct_node,Config),
+ ok = rpc:call(CtNode,cover,stop,[]),
+ {ok,_} = ct_slave:stop(ct_nomerge),
+ ok.
+
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+setup(Test, Config) ->
+ Opts0 = ct_test_support:get_opts(Config),
+ Level = ?config(trace_level, Config),
+ EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+ Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+ ERPid = ct_test_support:start_event_receiver(Config),
+ {Opts,ERPid}.
+
+execute(Name, Testcase, Opts, ERPid, Config) ->
+ ok = ct_test_support:run(Opts, Config),
+ Events = ct_test_support:get_events(ERPid, Config),
+
+ ct_test_support:log_events(Name,
+ reformat(Events, ?eh),
+ ?config(priv_dir, Config),
+ Opts),
+ TestEvents = events_to_check(Testcase),
+ R = ct_test_support:verify_events(TestEvents, Events, Config),
+ {R,Events}.
+
+reformat(Events, EH) ->
+ ct_test_support:reformat(Events, EH).
+
+events_to_check(local) ->
+ events_to_check1(cover_nomerge_local_SUITE);
+events_to_check(remote) ->
+ events_to_check1(cover_nomerge_remote_SUITE);
+events_to_check(remote_nostop) ->
+ events_to_check1(cover_nomerge_remote_nostop_SUITE).
+events_to_check1(Suite) ->
+ OneTest =
+ [{?eh,start_logging,{'DEF','RUNDIR'}}] ++
+ [{?eh,tc_done,{Suite,t1,ok}}] ++
+ [{?eh,tc_done,{Suite,t2,ok}}] ++
+ [{?eh,stop_logging,[]}],
+
+ %% 2 tests (ct:run_test + script_start) is default
+ OneTest ++ OneTest.
+
+check_cover(Config) when is_list(Config) ->
+ CTNode = proplists:get_value(ct_node, Config),
+ check_cover(CTNode);
+check_cover(Node) when is_atom(Node) ->
+ case rpc:call(Node,test_server,is_cover,[]) of
+ true ->
+ {true,
+ rpc:call(Node,cover,which_nodes,[]),
+ rpc:call(Node,cover,modules,[])};
+ false ->
+ false
+ end.
+
+%% Get the log dir "run.<timestamp>" for all tests
+get_run_dirs(Events,Testcase) ->
+ [filename:dirname(TCLog) ||
+ {ct_test_support_eh,
+ {event,tc_logfile,_Node,
+ {{_Suite,TC},TCLog}}} <- Events,
+ TC==Testcase].
+
+%% Check that each coverlog includes N calls to ?mod:foo/0
+check_calls(Events,N) ->
+ check_calls(Events,{?mod,foo,0},N).
+check_calls(Events,MFA,[{Testcase,N}|Expected]) ->
+ CoverLogs =
+ [filename:join(D,"all.coverdata") || D <- get_run_dirs(Events,Testcase)],
+ ok = do_check_logs(CoverLogs,MFA,N),
+ check_calls(Events,MFA,Expected);
+check_calls(_,_,[]) ->
+ ok.
+
+do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) ->
+ {ok,_} = cover:start(),
+ ok = cover:import(CoverLog),
+ {ok,Calls} = cover:analyse(Mod,calls,function),
+ ok = cover:stop(),
+ {MFA,N} = lists:keyfind(MFA,1,Calls),
+ do_check_logs(CoverLogs,MFA,N);
+do_check_logs([],_,_) ->
+ ok.
+
+create_cover_file(Filename,Terms,Config) ->
+ PrivDir = ?config(priv_dir,Config),
+ File = filename:join(PrivDir,Filename) ++ ".cover",
+ {ok,Fd} = file:open(File,[write]),
+ lists:foreach(fun(Term) ->
+ file:write(Fd,io_lib:format("~p.~n",[Term]))
+ end,Terms),
+ ok = file:close(Fd),
+ File.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_local_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_local_SUITE.erl
new file mode 100644
index 0000000000..e1fe3b5fc9
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_local_SUITE.erl
@@ -0,0 +1,63 @@
+%%--------------------------------------------------------------------
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2014. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%----------------------------------------------------------------------
+-module(cover_nomerge_local_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+%% Default timetrap timeout (set in init_per_testcase).
+-define(default_timeout, ?t:minutes(1)).
+
+suite() ->
+ [].
+
+all() ->
+ [t1,t2].
+
+init_per_suite(Config) ->
+ Config.
+
+end_per_suite(Config) ->
+ Config.
+
+init_per_testcase(_Case, Config) ->
+ Dog = test_server:timetrap(?default_timeout),
+ [{watchdog, Dog}|Config].
+
+end_per_testcase(Case, Config) ->
+ Dog=?config(watchdog, Config),
+ test_server:timetrap_cancel(Dog),
+ ok.
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+break(_Config) ->
+ test_server:break(""),
+ ok.
+
+t1(_Config) ->
+ cover_compiled = code:which(cover_test_mod),
+ ok = cover_test_mod:foo(),
+ ok.
+
+t2(_Config) ->
+ cover_compiled = code:which(cover_test_mod),
+ ok = cover_test_mod:foo(),
+ ok.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_SUITE.erl
new file mode 100644
index 0000000000..a77ae0c2db
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_SUITE.erl
@@ -0,0 +1,75 @@
+%%--------------------------------------------------------------------
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2014. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%----------------------------------------------------------------------
+-module(cover_nomerge_remote_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+%% Default timetrap timeout (set in init_per_testcase).
+-define(default_timeout, ?t:minutes(1)).
+
+suite() ->
+ [].
+
+all() ->
+ [t1,t2].
+
+init_per_suite(Config) ->
+ {ok,Host} = inet:gethostname(),
+ Node = list_to_atom("ct_nomerge@"++Host),
+ pong = net_adm:ping(Node),
+
+%% Include this row, and exclude the equivalent row in end_per_suite =>
+%% fails every now and then with missing data. Why?
+%% ct_cover:remove_nodes([Node]),
+ ct_cover:add_nodes([Node]),
+ [{node,Node}|Config].
+
+end_per_suite(Config) ->
+ Node = ?config(node,Config),
+ ct_cover:remove_nodes([Node]),
+ Config.
+
+init_per_testcase(_Case, Config) ->
+ Dog = test_server:timetrap(?default_timeout),
+ [{watchdog, Dog}|Config].
+
+end_per_testcase(Case, Config) ->
+ Dog=?config(watchdog, Config),
+ test_server:timetrap_cancel(Dog),
+ ok.
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+break(_Config) ->
+ test_server:break(""),
+ ok.
+
+t1(Config) ->
+ Node = ?config(node,Config),
+ cover_compiled = rpc:call(Node, code, which, [cover_test_mod]),
+ ok = rpc:call(Node, cover_test_mod, foo, []),
+ ok.
+
+t2(Config) ->
+ Node = ?config(node,Config),
+ cover_compiled = rpc:call(Node, code, which, [cover_test_mod]),
+ ok = rpc:call(Node, cover_test_mod, foo, []),
+ ok.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_nostop_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_nostop_SUITE.erl
new file mode 100644
index 0000000000..0b3159f2c3
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_nostop_SUITE.erl
@@ -0,0 +1,68 @@
+%%--------------------------------------------------------------------
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2014. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%----------------------------------------------------------------------
+-module(cover_nomerge_remote_nostop_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+%% Default timetrap timeout (set in init_per_testcase).
+-define(default_timeout, ?t:minutes(1)).
+
+suite() ->
+ [].
+
+all() ->
+ [t1,t2].
+
+init_per_suite(Config) ->
+ {ok,Host} = inet:gethostname(),
+ Node = list_to_atom("ct_nomerge@"++Host),
+ pong = net_adm:ping(Node),
+ [{node,Node}|Config].
+
+end_per_suite(Config) ->
+ Config.
+
+init_per_testcase(_Case, Config) ->
+ Dog = test_server:timetrap(?default_timeout),
+ [{watchdog, Dog}|Config].
+
+end_per_testcase(Case, Config) ->
+ Dog=?config(watchdog, Config),
+ test_server:timetrap_cancel(Dog),
+ ok.
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+break(_Config) ->
+ test_server:break(""),
+ ok.
+
+t1(Config) ->
+ Node = ?config(node,Config),
+ cover_compiled = rpc:call(Node, code, which, [cover_test_mod]),
+ ok = rpc:call(Node, cover_test_mod, foo, []),
+ ok.
+
+t2(Config) ->
+ Node = ?config(node,Config),
+ cover_compiled = rpc:call(Node, code, which, [cover_test_mod]),
+ ok = rpc:call(Node, cover_test_mod, foo, []),
+ ok.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_test_mod.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_test_mod.erl
new file mode 100644
index 0000000000..d4f69452c3
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_test_mod.erl
@@ -0,0 +1,4 @@
+-module(cover_test_mod).
+-compile(export_all).
+foo() ->
+ ok.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/local.spec b/lib/common_test/test/ct_cover_nomerge_SUITE_data/local.spec
new file mode 100644
index 0000000000..893c48b010
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/local.spec
@@ -0,0 +1,6 @@
+{merge_tests,false}.
+
+{alias,dir,"."}.
+
+{cases, dir, cover_nomerge_local_SUITE, [t1]}.
+{cases, dir, cover_nomerge_local_SUITE, [t2]}.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote.spec b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote.spec
new file mode 100644
index 0000000000..78c4332270
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote.spec
@@ -0,0 +1,6 @@
+{merge_tests,false}.
+
+{alias,dir,"."}.
+
+{cases, dir, cover_nomerge_remote_SUITE, [t1]}.
+{cases, dir, cover_nomerge_remote_SUITE, [t2]}.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote_nostop.spec b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote_nostop.spec
new file mode 100644
index 0000000000..049f586c72
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote_nostop.spec
@@ -0,0 +1,6 @@
+{merge_tests,false}.
+
+{alias,dir,"."}.
+
+{cases, dir, cover_nomerge_remote_nostop_SUITE, [t1]}.
+{cases, dir, cover_nomerge_remote_nostop_SUITE, [t2]}.