[ct] Add test of cover support when merge_tests=false

9 files changed, 465 insertions, 1 deletions

diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile
index 085f19d023..a0ac47f12a 100644
--- a/lib/common_test/test/Makefile
+++ b/lib/common_test/test/Makefile
@@ -1,7 +1,7 @@
 #
 # %CopyrightBegin%
 #
-# Copyright Ericsson AB 2008-2013. All Rights Reserved.
+# Copyright Ericsson AB 2008-2014. All Rights Reserved.
 #
 # The contents of this file are subject to the Erlang Public License,
 # Version 1.1, (the "License"); you may not use this file except in
@@ -61,6 +61,7 @@ MODULES= \
 	ct_snmp_SUITE \
 	ct_group_leader_SUITE \
 	ct_cover_SUITE \
+	ct_cover_nomerge_SUITE \
 	ct_groups_search_SUITE \
 	ct_surefire_SUITE \
 	ct_telnet_SUITE
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE.erl
new file mode 100644
index 0000000000..4344fc30df
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE.erl
@@ -0,0 +1,235 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2014. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%%-------------------------------------------------------------------
+%%% File: ct_cover_nomerge_SUITE
+%%%
+%%% Description:
+%%% Test code cover analysis support when merge_tests=false
+%%%
+%%%-------------------------------------------------------------------
+-module(ct_cover_nomerge_SUITE).
+
+-compile(export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("common_test/include/ct_event.hrl").
+
+-define(eh, ct_test_support_eh).
+-define(mod, cover_test_mod).
+
+%%--------------------------------------------------------------------
+%% TEST SERVER CALLBACK FUNCTIONS
+%%--------------------------------------------------------------------
+
+%%--------------------------------------------------------------------
+%% Description: Since Common Test starts another Test Server
+%% instance, the tests need to be performed on a separate node (or
+%% there will be clashes with logging processes etc).
+%%--------------------------------------------------------------------
+init_per_suite(Config) ->
+    case test_server:is_cover() of
+	true ->
+	    {skip,"Test server is running cover already - skipping"};
+	false ->
+	    ct_test_support:init_per_suite(Config)
+    end.
+
+end_per_suite(Config) ->
+    ct_test_support:end_per_suite(Config).
+
+init_per_testcase(TestCase, Config) ->
+    ct_test_support:init_per_testcase(TestCase, Config).
+
+end_per_testcase(TestCase, Config) ->
+    try apply(?MODULE,TestCase,[cleanup,Config])
+    catch error:undef -> ok
+    end,
+    ct_test_support:end_per_testcase(TestCase, Config).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+    [
+     local,
+     remote,
+     remote_nostop
+    ].
+
+%%--------------------------------------------------------------------
+%% TEST CASES
+%%--------------------------------------------------------------------
+
+local(Config) ->
+    DataDir = ?config(data_dir, Config),
+    Spec = filename:join(DataDir, "local.spec"),
+    PrivDir = ?config(priv_dir,Config),
+    ExportFile = filename:join(PrivDir,"local.coverdata"),
+    CoverSpec = [{incl_mods,[?mod]},
+		 {export, ExportFile}],
+    CoverFile = create_cover_file(local,CoverSpec,Config),
+    {Opts,ERPid} = setup([{spec,Spec},{label,local},{cover,CoverFile}], Config),
+    {ok,Events} = execute(local, local, Opts, ERPid, Config),
+    false = check_cover(Config),
+    check_calls(Events,[{t1,1},{t2,2}]),
+    ok.
+
+remote(Config) ->
+    DataDir = ?config(data_dir, Config),
+    Spec = filename:join(DataDir, "remote.spec"),
+    %% extending some timers for slow test hosts
+    {ok,_Node} = ct_slave:start(ct_nomerge,[{boot_timeout,15},
+					   {init_timeout,15},
+					   {startup_timeout,15}]),
+    
+    PrivDir = ?config(priv_dir,Config),
+    ExportFile = filename:join(PrivDir,"remote.coverdata"),
+    CoverSpec = [{incl_mods,[?mod]},
+		 {export, ExportFile}],
+    CoverFile = create_cover_file(remote,CoverSpec,Config),
+    {Opts,ERPid} = setup([{spec,Spec},{label,remote},{cover,CoverFile}], Config),
+    {ok,Events} = execute(remote, remote, Opts, ERPid, Config),
+    false = check_cover(Config),
+    check_calls(Events,[{t1,1},{t2,2}]),
+    ok.
+remote(cleanup,_Config) ->
+    {ok,_} = ct_slave:stop(ct_nomerge),
+    ok.
+
+remote_nostop(Config) ->
+    DataDir = ?config(data_dir, Config),
+    Spec = filename:join(DataDir, "remote_nostop.spec"),
+    %% extending some timers for slow test hosts
+    {ok,Node} = ct_slave:start(ct_nomerge,[{boot_timeout,15},
+					   {init_timeout,15},
+					   {startup_timeout,15}]),
+    
+    PrivDir = ?config(priv_dir,Config),
+    ExportFile = filename:join(PrivDir,"remote_nostop.coverdata"),
+    CoverSpec = [{nodes,[Node]},
+		 {incl_mods,[?mod]},
+		 {export,ExportFile}],
+    CoverFile = create_cover_file(remote_nostop,CoverSpec,Config),
+    {Opts,ERPid} = setup([{spec,Spec},{label,remote_nostop},
+			  {cover,CoverFile},{cover_stop,false}],
+			 Config),
+    {ok,Events} = execute(remote_nostop, remote_nostop, Opts, ERPid, Config),
+    {true,[Node],[cover_test_mod]} = check_cover(Config),
+    check_calls(Events,[{t1,1},{t2,2}]),
+    ok.
+remote_nostop(cleanup,Config) ->
+    CtNode = ?config(ct_node,Config),
+    ok = rpc:call(CtNode,cover,stop,[]),
+    {ok,_} = ct_slave:stop(ct_nomerge),
+    ok.
+    
+
+%%%-----------------------------------------------------------------
+%%% HELP FUNCTIONS
+%%%-----------------------------------------------------------------
+setup(Test, Config) ->
+    Opts0 = ct_test_support:get_opts(Config),
+    Level = ?config(trace_level, Config),
+    EvHArgs = [{cbm,ct_test_support},{trace_level,Level}],
+    Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test],
+    ERPid = ct_test_support:start_event_receiver(Config),
+    {Opts,ERPid}.
+
+execute(Name, Testcase, Opts, ERPid, Config) ->
+    ok = ct_test_support:run(Opts, Config),
+    Events = ct_test_support:get_events(ERPid, Config),
+
+    ct_test_support:log_events(Name,
+			       reformat(Events, ?eh),
+			       ?config(priv_dir, Config),
+			       Opts),
+    TestEvents = events_to_check(Testcase),
+    R = ct_test_support:verify_events(TestEvents, Events, Config),
+    {R,Events}.
+
+reformat(Events, EH) ->
+    ct_test_support:reformat(Events, EH).
+
+events_to_check(local) ->
+    events_to_check1(cover_nomerge_local_SUITE);
+events_to_check(remote) ->
+    events_to_check1(cover_nomerge_remote_SUITE);
+events_to_check(remote_nostop) ->
+    events_to_check1(cover_nomerge_remote_nostop_SUITE).
+events_to_check1(Suite) ->
+    OneTest =
+	[{?eh,start_logging,{'DEF','RUNDIR'}}] ++
+	[{?eh,tc_done,{Suite,t1,ok}}] ++
+	[{?eh,tc_done,{Suite,t2,ok}}] ++
+	[{?eh,stop_logging,[]}],
+
+    %% 2 tests (ct:run_test + script_start) is default
+    OneTest ++ OneTest.
+
+check_cover(Config) when is_list(Config) ->
+    CTNode = proplists:get_value(ct_node, Config),
+    check_cover(CTNode);
+check_cover(Node) when is_atom(Node) ->
+    case rpc:call(Node,test_server,is_cover,[]) of
+	true ->
+	    {true,
+	     rpc:call(Node,cover,which_nodes,[]),
+	     rpc:call(Node,cover,modules,[])};
+	false ->
+	    false
+    end.
+
+%% Get the log dir "run.<timestamp>" for all tests
+get_run_dirs(Events,Testcase) ->
+    [filename:dirname(TCLog) ||
+	{ct_test_support_eh,
+	 {event,tc_logfile,_Node,
+	  {{_Suite,TC},TCLog}}} <- Events,
+	TC==Testcase].
+
+%% Check that each coverlog includes N calls to ?mod:foo/0
+check_calls(Events,N) ->
+    check_calls(Events,{?mod,foo,0},N).
+check_calls(Events,MFA,[{Testcase,N}|Expected]) ->
+    CoverLogs =
+	[filename:join(D,"all.coverdata") || D <- get_run_dirs(Events,Testcase)],
+    ok = do_check_logs(CoverLogs,MFA,N),
+    check_calls(Events,MFA,Expected);
+check_calls(_,_,[]) ->
+    ok.
+
+do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) ->
+    {ok,_} = cover:start(),
+    ok = cover:import(CoverLog),
+    {ok,Calls} = cover:analyse(Mod,calls,function),
+    ok = cover:stop(),
+    {MFA,N} = lists:keyfind(MFA,1,Calls),
+    do_check_logs(CoverLogs,MFA,N);
+do_check_logs([],_,_) ->
+    ok.
+
+create_cover_file(Filename,Terms,Config) ->
+    PrivDir = ?config(priv_dir,Config),
+    File = filename:join(PrivDir,Filename) ++ ".cover",
+    {ok,Fd} = file:open(File,[write]),
+    lists:foreach(fun(Term) ->
+			  file:write(Fd,io_lib:format("~p.~n",[Term]))
+		  end,Terms),
+    ok = file:close(Fd),
+    File.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_local_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_local_SUITE.erl
new file mode 100644
index 0000000000..e1fe3b5fc9
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_local_SUITE.erl
@@ -0,0 +1,63 @@
+%%--------------------------------------------------------------------
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2014. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%----------------------------------------------------------------------
+-module(cover_nomerge_local_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+%% Default timetrap timeout (set in init_per_testcase).
+-define(default_timeout, ?t:minutes(1)).
+
+suite() ->
+    [].
+
+all() ->
+    [t1,t2].
+
+init_per_suite(Config) ->
+    Config.
+
+end_per_suite(Config) ->
+    Config.
+
+init_per_testcase(_Case, Config) ->
+    Dog = test_server:timetrap(?default_timeout),
+    [{watchdog, Dog}|Config].
+
+end_per_testcase(Case, Config) ->
+    Dog=?config(watchdog, Config),
+    test_server:timetrap_cancel(Dog),
+    ok.
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+break(_Config) ->
+    test_server:break(""),
+    ok.
+
+t1(_Config) ->
+    cover_compiled = code:which(cover_test_mod),
+    ok = cover_test_mod:foo(),
+    ok.
+
+t2(_Config) ->
+    cover_compiled = code:which(cover_test_mod),
+    ok = cover_test_mod:foo(),
+    ok.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_SUITE.erl
new file mode 100644
index 0000000000..a77ae0c2db
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_SUITE.erl
@@ -0,0 +1,75 @@
+%%--------------------------------------------------------------------
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2014. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%----------------------------------------------------------------------
+-module(cover_nomerge_remote_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+%% Default timetrap timeout (set in init_per_testcase).
+-define(default_timeout, ?t:minutes(1)).
+
+suite() ->
+    [].
+
+all() ->
+    [t1,t2].
+
+init_per_suite(Config) ->
+    {ok,Host} = inet:gethostname(),
+    Node = list_to_atom("ct_nomerge@"++Host),
+    pong = net_adm:ping(Node),
+
+%% Include this row, and exclude the equivalent row in end_per_suite =>
+%% fails every now and then with missing data. Why?
+%%    ct_cover:remove_nodes([Node]),
+    ct_cover:add_nodes([Node]),
+    [{node,Node}|Config].
+
+end_per_suite(Config) ->
+    Node = ?config(node,Config),
+    ct_cover:remove_nodes([Node]),
+    Config.
+
+init_per_testcase(_Case, Config) ->
+    Dog = test_server:timetrap(?default_timeout),
+    [{watchdog, Dog}|Config].
+
+end_per_testcase(Case, Config) ->
+    Dog=?config(watchdog, Config),
+    test_server:timetrap_cancel(Dog),
+    ok.
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+break(_Config) ->
+    test_server:break(""),
+    ok.
+
+t1(Config) ->
+    Node = ?config(node,Config),
+    cover_compiled = rpc:call(Node, code, which, [cover_test_mod]),
+    ok = rpc:call(Node, cover_test_mod, foo, []),
+    ok.
+
+t2(Config) ->
+    Node = ?config(node,Config),
+    cover_compiled = rpc:call(Node, code, which, [cover_test_mod]),
+    ok = rpc:call(Node, cover_test_mod, foo, []),
+    ok.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_nostop_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_nostop_SUITE.erl
new file mode 100644
index 0000000000..0b3159f2c3
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_nostop_SUITE.erl
@@ -0,0 +1,68 @@
+%%--------------------------------------------------------------------
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2014. All Rights Reserved.
+%%
+%% The contents of this file are subject to the Erlang Public License,
+%% Version 1.1, (the "License"); you may not use this file except in
+%% compliance with the License. You should have received a copy of the
+%% Erlang Public License along with this software. If not, it can be
+%% retrieved online at http://www.erlang.org/.
+%%
+%% Software distributed under the License is distributed on an "AS IS"
+%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+%% the License for the specific language governing rights and limitations
+%% under the License.
+%%
+%% %CopyrightEnd%
+%%
+%%----------------------------------------------------------------------
+-module(cover_nomerge_remote_nostop_SUITE).
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+%% Default timetrap timeout (set in init_per_testcase).
+-define(default_timeout, ?t:minutes(1)).
+
+suite() ->
+    [].
+
+all() ->
+    [t1,t2].
+
+init_per_suite(Config) ->
+    {ok,Host} = inet:gethostname(),
+    Node = list_to_atom("ct_nomerge@"++Host),
+    pong = net_adm:ping(Node),
+    [{node,Node}|Config].
+
+end_per_suite(Config) ->
+    Config.
+
+init_per_testcase(_Case, Config) ->
+    Dog = test_server:timetrap(?default_timeout),
+    [{watchdog, Dog}|Config].
+
+end_per_testcase(Case, Config) ->
+    Dog=?config(watchdog, Config),
+    test_server:timetrap_cancel(Dog),
+    ok.
+
+%%%-----------------------------------------------------------------
+%%% Test cases
+break(_Config) ->
+    test_server:break(""),
+    ok.
+
+t1(Config) ->
+    Node = ?config(node,Config),
+    cover_compiled = rpc:call(Node, code, which, [cover_test_mod]),
+    ok = rpc:call(Node, cover_test_mod, foo, []),
+    ok.
+
+t2(Config) ->
+    Node = ?config(node,Config),
+    cover_compiled = rpc:call(Node, code, which, [cover_test_mod]),
+    ok = rpc:call(Node, cover_test_mod, foo, []),
+    ok.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_test_mod.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_test_mod.erl
new file mode 100644
index 0000000000..d4f69452c3
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_test_mod.erl
@@ -0,0 +1,4 @@
+-module(cover_test_mod).
+-compile(export_all).
+foo() ->
+    ok.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/local.spec b/lib/common_test/test/ct_cover_nomerge_SUITE_data/local.spec
new file mode 100644
index 0000000000..893c48b010
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/local.spec
@@ -0,0 +1,6 @@
+{merge_tests,false}.
+
+{alias,dir,"."}.
+
+{cases, dir, cover_nomerge_local_SUITE, [t1]}.
+{cases, dir, cover_nomerge_local_SUITE, [t2]}.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote.spec b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote.spec
new file mode 100644
index 0000000000..78c4332270
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote.spec
@@ -0,0 +1,6 @@
+{merge_tests,false}.
+
+{alias,dir,"."}.
+
+{cases, dir, cover_nomerge_remote_SUITE, [t1]}.
+{cases, dir, cover_nomerge_remote_SUITE, [t2]}.
diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote_nostop.spec b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote_nostop.spec
new file mode 100644
index 0000000000..049f586c72
--- /dev/null
+++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote_nostop.spec
@@ -0,0 +1,6 @@
+{merge_tests,false}.
+
+{alias,dir,"."}.
+
+{cases, dir, cover_nomerge_remote_nostop_SUITE, [t1]}.
+{cases, dir, cover_nomerge_remote_nostop_SUITE, [t2]}.