Improve cover analysis via common_test

This addresses several bugs in common_test (ct) when using the cover
analysis mechanism:

In a ct test run, one can give a cover spec file which indicates that
cover analysis shall be run, including a number of modules and a
number of nodes. During the ct test run, multiple jobs may be started
in test_server, and when the cover option is used, test_server would
cover compile and analyse all given modules for each job. This commit
instead allows the compilation and analysis to be explicitly ordered
by ct for each test run. This way each module will only be cover
compiled and analysed once. The cover log will be located in the
common_test log directory (ct_run.<timestamp>), and the "Coverage log"
link in each suite.log.html will point to this file. A new button is
also added in the top level ct index file, which points to the cover
log.

This change also reduces the need of using the 'export' and 'import'
options, since there is no longer any need to accumulate cover data
over multiple test_server jobs. However, these options may still be
used for importing and exporting cover data in order to store or
accumulate data from multiple ct test runs.

The 'nodes' option was earlier only used by ct to start cover on the
given nodes before starting the first test_server job. After this job
was completed, test_server would stop cover completely and then start
it again for the next job without any knowledge of the 'nodes'
options. For the next test_server jobs cover would therefore no longer
be running on these nodes. Explcit calls to ct_cover:add_nodes had to
be done in order to collect data through all test_server jobs. This
bug has now been solved, since cover is no longer stopped between each
test_server job.

Finally, ct no longer stores cover data using
ct_util:set_testdata. This was earlier used by ct_cover:add_nodes to
make sure no node was added twice.This did, however, cause some
problems when ct and cover were out of sync. ct could belive that a
node was running cover and thus reject adding this node, while in
reality cover had been stopped on the node (e.g. by test_server) so no
cover data was collected. ct_cover:add_nodes will now instead use
cover:which_nodes to check if a node is already running.

4 files changed, 159 insertions, 86 deletions

diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl
index 97e1247d85..f63bfdec05 100644
--- a/lib/test_server/src/test_server.erl
+++ b/lib/test_server/src/test_server.erl
@@ -22,7 +22,7 @@
 
 %%% TEST_SERVER_CTRL INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 -export([run_test_case_apply/1,init_target_info/0,init_purify/0]).
--export([cover_compile/1,cover_analyse/4]).
+-export([cover_compile/1,cover_analyse/2]).
 
 %%% TEST_SERVER_SUP INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 -export([get_loc/1,set_tc_state/1]).
@@ -80,8 +80,8 @@ init_purify() ->
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%% cover_compile({App,Include,Exclude,Cross}) ->
-%%                                          {ok,AnalyseModules} | {error,Reason}
+%% cover_compile(#cover{app=App,incl=Include,excl=Exclude,cross=Cross}) ->
+%%        {ok,#cover{mods=AnalyseModules}} | {error,Reason}
 %%
 %% App = atom() , name of application to be compiled
 %% Exclude = [atom()], list of modules to exclude
@@ -90,33 +90,35 @@ init_purify() ->
 %% Cross = [atoms()], list of modules outside of App shat should be included
 %%                 in the cover compilation, but that shall not be part of
 %%                 the cover analysis for this application.
+%% AnalyseModules = [atom()], list of successfully compiled modules
 %%
-%% Cover compile the given application. Return {ok,AnalyseMods} if application
-%% is found, else {error,application_not_found}.
+%% Cover compile the given application. Return {ok,CoverInfo} if
+%% compilation succeeds, else (if application is not found and there
+%% are no modules to compile) {error,application_not_found}.
 
-cover_compile({none,_Exclude,Include,Cross}) ->
+cover_compile(CoverInfo=#cover{app=none,incl=Include,cross=Cross}) ->
     CrossMods = lists:flatmap(fun({_,M}) -> M end,Cross),
     CompileMods = Include++CrossMods,
     case length(CompileMods) of
 	0 ->
 	    io:fwrite("WARNING: No modules to cover compile!\n\n",[]),
 	    cover:start(),			% start cover server anyway
-	    {ok,[]};
+	    {ok,CoverInfo#cover{mods=[]}};
 	N ->
 	    io:fwrite("Cover compiling ~w modules - "
 		      "this may take some time... ",[N]),
 	    do_cover_compile(CompileMods),
 	    io:fwrite("done\n\n",[]),
-	    {ok,Include}
+	    {ok,CoverInfo#cover{mods=Include}}
     end;
-cover_compile({App,all,Include,Cross}) ->
+cover_compile(CoverInfo=#cover{app=App,excl=all,incl=Include,cross=Cross}) ->
     CrossMods = lists:flatmap(fun({_,M}) -> M end,Cross),
     CompileMods = Include++CrossMods,
     case length(CompileMods) of
 	0 ->
 	    io:fwrite("WARNING: No modules to cover compile!\n\n",[]),
 	    cover:start(),			% start cover server anyway
-	    {ok,[]};
+	    {ok,CoverInfo#cover{mods=[]}};
 	N ->
 	    io:fwrite("Cover compiling '~w' (~w files) - "
 		      "this may take some time... ",[App,N]),
@@ -126,9 +128,9 @@ cover_compile({App,all,Include,Cross}) ->
 		      "~tp\n", [App,CompileMods]),
 	    do_cover_compile(CompileMods),
 	    io:fwrite("done\n\n",[]),
-	    {ok,Include}
+	    {ok,CoverInfo=#cover{mods=Include}}
     end;
-cover_compile({App,Exclude,Include,Cross}) ->
+cover_compile(CoverInfo=#cover{app=App,excl=Exclude,incl=Include,cross=Cross}) ->
     CrossMods = lists:flatmap(fun({_,M}) -> M end,Cross),
     case code:lib_dir(App) of
 	{error,bad_name} ->
@@ -146,7 +148,7 @@ cover_compile({App,Exclude,Include,Cross}) ->
 			      "~tp\n", [App,Include]),
 		    do_cover_compile(CompileMods),
 		    io:fwrite("done\n\n",[]),
-		    {ok,Include}
+		    {ok,CoverInfo#cover{mods=Include}}
 	    end;
 	LibDir ->
 	    EbinDir = filename:join([LibDir,"ebin"]),
@@ -158,13 +160,13 @@ cover_compile({App,Exclude,Include,Cross}) ->
 		0 ->
 		    io:fwrite("WARNING: No modules to cover compile!\n\n",[]),
 		    cover:start(),		% start cover server anyway
-		    {ok,[]};
+		    {ok,CoverInfo#cover{mods=[]}};
 		N ->
 		    io:fwrite("Cover compiling '~w' (~w files) - "
 			      "this may take some time... ",[App,N]),
 		    do_cover_compile(CompileMods),
 		    io:fwrite("done\n\n",[]),
-		    {ok,AnalyseMods}
+		    {ok,CoverInfo#cover{mods=AnalyseMods}}
 	    end
     end.
 
@@ -174,9 +176,11 @@ module_names(Beams) ->
 
 
 do_cover_compile(Modules) ->
-    do_cover_compile1(lists:usort(Modules)). % remove duplicates
+    cover:start(),
+    pmap1(fun(M) -> do_cover_compile1(M) end,lists:usort(Modules)),
+    ok.
 
-do_cover_compile1([M|Rest]) ->
+do_cover_compile1(M) ->
     case {code:is_sticky(M),code:is_loaded(M)} of
 	{true,_} ->
 	    code:unstick_mod(M),
@@ -187,15 +191,13 @@ do_cover_compile1([M|Rest]) ->
 		    io:fwrite("\nWARNING: Could not cover compile ~w: ~p\n",
 			      [M,Error])
 	    end,
-	    code:stick_mod(M),
-	    do_cover_compile1(Rest);
+	    code:stick_mod(M);
 	{false,false} ->
 	    case code:load_file(M) of
 		{module,_} ->
-		    do_cover_compile1([M|Rest]);
+		    do_cover_compile1(M);
 		Error ->
-		    io:fwrite("\nWARNING: Could not load ~w: ~p\n",[M,Error]),
-		    do_cover_compile1(Rest)
+		    io:fwrite("\nWARNING: Could not load ~w: ~p\n",[M,Error])
 	    end;
 	{false,_} ->
 	    case cover:compile_beam(M) of
@@ -204,24 +206,52 @@ do_cover_compile1([M|Rest]) ->
 		Error ->
 		    io:fwrite("\nWARNING: Could not cover compile ~w: ~p\n",
 			      [M,Error])
-	    end,
-	    do_cover_compile1(Rest)
-    end;
-do_cover_compile1([]) ->
-    ok.
+	    end
+    end.
+
+pmap1(Fun,List) ->
+    NTot = length(List),
+    NProcs = erlang:system_info(schedulers) * 2,
+    NPerProc = (NTot div NProcs) + 1,
+
+    {[],Pids} =
+	lists:foldr(
+	  fun(_,{L,Ps}) ->
+		  {L1,L2} = if length(L)>=NPerProc -> lists:split(NPerProc,L);
+			       true -> {L,[]} % last chunk
+			    end,
+		  {P,_Ref} =
+		      spawn_monitor(fun() ->
+					    exit(lists:map(Fun,L1))
+				    end),
+		  {L2,[P|Ps]}
+	  end,
+	  {List,[]},
+	  lists:seq(1,NProcs)),
+    collect(Pids,[]).
+
+collect([],Acc) ->
+    lists:append(Acc);
+collect([Pid|Pids],Acc) ->
+    receive
+	{'DOWN', _Ref, process, Pid, Result} ->
+	    %% collect(lists:delete(Pid,Pids),[Result|Acc])
+	    collect(Pids,[Result|Acc])
+    end.
+
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-%% cover_analyse(Dir,Analyse,Modules,Stop) -> [{M,{Cov,NotCov,Details}}]
+%% cover_analyse(Dir,#cover{level=Analyse,mods=Modules,stop=Stop) ->
+%%            [{M,{Cov,NotCov,Details}}]
 %%
 %% Dir = string()
 %% Analyse = details | overview
 %% Modules = [atom()], the modules to analyse
 %%
-%% Cover analysis. If Analyse=={details,Dir} analyse_to_file is used.
+%% Cover analysis. If Analyse==details analyse_to_file is used.
 %%
-%% If Analyse=={overview,Dir} analyse_to_file is not used, only an
-%% overview containing the number of covered/not covered lines in each
-%% module.
+%% If Analyse==overview analyse_to_file is not used, only an overview
+%% containing the number of covered/not covered lines in each module.
 %%
 %% Also, cover data will be exported to a file called all.coverdata in
 %% the given directory.
@@ -236,8 +266,8 @@ do_cover_compile1([]) ->
 %% which means that the modules will stay cover compiled. Note that
 %% this is only recommended if the erlang node is being terminated
 %% after the test is completed.
-cover_analyse(Dir,Analyse,Modules,Stop) ->
-    print(stdout, "Cover analysing...\n", []),
+cover_analyse(Dir,#cover{level=Analyse,mods=Modules,stop=Stop}) ->
+    io:fwrite(user, "Cover analysing... ", []),
     DetailsFun =
 	case Analyse of
 	    details ->
@@ -265,17 +295,19 @@ cover_analyse(Dir,Analyse,Modules,Stop) ->
 			fun(_) -> Error end
 		end
 	end,
-    R = pmap(
+    R = pmap2(
 	  fun(M) ->
 		  case cover:analyse(M,module) of
 		      {ok,{M,{Cov,NotCov}}} ->
 			  {M,{Cov,NotCov,DetailsFun(M)}};
 		      Err ->
-			  io:fwrite("WARNING: Analysis failed for ~w. Reason: ~p\n",
+			  io:fwrite(user,
+				    "\nWARNING: Analysis failed for ~w. Reason: ~p\n",
 				    [M,Err]),
 			  {M,Err}
 		  end
 	  end, Modules),
+    io:fwrite(user, "done\n\n", []),
 
     case Stop of
 	true ->
@@ -287,12 +319,12 @@ cover_analyse(Dir,Analyse,Modules,Stop) ->
     end,
     R.
 
-pmap(Fun,List) ->
+pmap2(Fun,List) ->
     Collector = self(),
     Pids = lists:map(fun(E) ->
 			     spawn(fun() ->
-					   Collector ! {res,self(),Fun(E)} 
-				   end) 
+					   Collector ! {res,self(),Fun(E)}
+				   end)
 		     end, List),
     lists:map(fun(Pid) ->
 		      receive
@@ -301,7 +333,6 @@ pmap(Fun,List) ->
 		      end
 	      end, Pids).
 
-
 do_cover_for_node(Node,CoverFunc) ->
     do_cover_for_node(Node,CoverFunc,true).
 do_cover_for_node(Node,CoverFunc,StickUnstick) ->
diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl
index 30787081be..c3550d4533 100644
--- a/lib/test_server/src/test_server_ctrl.erl
+++ b/lib/test_server/src/test_server_ctrl.erl
@@ -52,7 +52,9 @@
 -export([reject_io_reqs/1, get_levels/0, set_levels/3]).
 -export([multiply_timetraps/1, scale_timetraps/1, get_timetrap_parameters/0]).
 -export([create_priv_dir/1]).
--export([cover/2, cover/3, cover/8, cross_cover_analyse/2, trc/1, stop_trace/0]).
+-export([cover/1, cover/2, cover/3,
+	 cover_compile/7, cover_analyse/2, cross_cover_analyse/2,
+	 trc/1, stop_trace/0]).
 -export([testcase_callback/1]).
 -export([set_random_seed/1]).
 -export([kill_slavenodes/0]).
@@ -409,11 +411,26 @@ cover(App, Analyse) when is_atom(App) ->
 cover(CoverFile, Analyse) ->
     cover(none, CoverFile, Analyse).
 cover(App, CoverFile, Analyse) ->
-    controller_call({cover,{{App,CoverFile},Analyse,true}}).
-cover(App, CoverFile, Exclude, Include, Cross, Export, Analyse, Stop) ->
-    controller_call({cover,
-		     {{App,{CoverFile,Exclude,Include,Cross,Export}},
-		      Analyse,Stop}}).
+    {Excl,Incl,Cross} = read_cover_file(CoverFile),
+    CoverInfo = #cover{app=App,
+		       file=CoverFile,
+		       excl=Excl,
+		       incl=Incl,
+		       cross=Cross,
+		       level=Analyse},
+    controller_call({cover,CoverInfo}).
+
+cover(CoverInfo) ->
+    controller_call({cover,CoverInfo}).
+
+cover_compile(App,File,Excl,Incl,Cross,Analyse,Stop) ->
+    cover_compile(#cover{app=App,
+			 file=File,
+			 excl=Excl,
+			 incl=Incl,
+			 cross=Cross,
+			 level=Analyse,
+			 stop=Stop}).
 
 testcase_callback(ModFunc) ->
     controller_call({testcase_callback,ModFunc}).
@@ -1204,7 +1221,7 @@ elapsed_time(Before, After) ->
 start_extra_tools(ExtraTools) ->
     start_extra_tools(ExtraTools, []).
 start_extra_tools([{cover,CoverInfo} | ExtraTools], Started) ->
-    case cover_compile(CoverInfo) of
+    case start_cover(CoverInfo) of
 	{ok,NewCoverInfo} ->
 	    start_extra_tools(ExtraTools,[{cover,NewCoverInfo}|Started]);
 	{error,_} ->
@@ -1225,8 +1242,8 @@ stop_extra_tools(ExtraTools) ->
     end,
     stop_extra_tools(ExtraTools, TestDir).
 
-stop_extra_tools([{cover,{App,Analyse,AnalyseMods,Stop}}|ExtraTools], TestDir) ->
-    cover_analyse(App, Analyse, AnalyseMods, Stop, TestDir),
+stop_extra_tools([{cover,CoverInfo}|ExtraTools], TestDir) ->
+    stop_cover(CoverInfo,TestDir),
     stop_extra_tools(ExtraTools, TestDir);
 %%stop_extra_tools([_ | ExtraTools], TestDir) ->
 %%    stop_extra_tools(ExtraTools, TestDir);
@@ -1568,13 +1585,20 @@ do_test_cases(TopCases, SkipCases,
 			    ok
 		    end
 	    end,
-	    
+	    CoverLog =
+		case get(test_server_cover_log_dir) of
+		    undefined ->
+			?coverlog_name;
+		    AbsLogDir ->
+			AbsLog = filename:join(AbsLogDir,?coverlog_name),
+			make_relative(AbsLog, TestDir)
+		end,
 	    print(html,
 		  "<p><ul>\n"
 		  "<li><a href=\"~ts\">Full textual log</a></li>\n"
 		  "<li><a href=\"~ts\">Coverage log</a></li>\n"
 		  "<li><a href=\"~ts\">Unexpected I/O log</a></li>\n</ul></p>\n",
-		  [?suitelog_name,?coverlog_name,?unexpected_io_log]),
+		  [?suitelog_name,CoverLog,?unexpected_io_log]),
 	    print(html,
 		  "<p>~ts</p>\n" ++
 		  xhtml("<table bgcolor=\"white\" border=\"3\" cellpadding=\"5\">",
@@ -5086,23 +5110,15 @@ pinfo(P) ->
 
 %% Cover compilation
 %% The compilation is executed on the target node
-cover_compile({AppInfo,Analyse,Stop}) ->
-    case cover_compile1(AppInfo) of
-	{ok,AnalyseMods} ->
-	    {ok,{AppInfo,Analyse,AnalyseMods,Stop}};
-	Error ->
-	    Error
-    end.
-
-cover_compile1({App,{_File,Exclude,Include,Cross,_Export}}) ->
-    cover_compile2({App,Exclude,Include,Cross});
-
-cover_compile1({App,CoverFile}) ->
-    {Exclude,Include,Cross} = read_cover_file(CoverFile),
-    cover_compile2({App,Exclude,Include,Cross}).
+start_cover(#cover{}=CoverInfo) ->
+    cover_compile(CoverInfo);
+start_cover({log,CoverLogDir}=CoverInfo) ->
+    %% Cover is controlled by the framework - here's the log
+    put(test_server_cover_log_dir,CoverLogDir),
+    {ok,CoverInfo}.
 
-cover_compile2(What) ->
-    test_server:cover_compile(What).
+cover_compile(CoverInfo) ->
+    test_server:cover_compile(CoverInfo).
 
 %% Read the coverfile for an application and return a list of modules
 %% that are members of the application but shall not be compiled
@@ -5170,25 +5186,45 @@ check_cross([]) ->
 %%
 %% This per application analysis writes the file cover.html in the
 %% application's run.<timestamp> directory.
-cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, Stop, TestDir) ->
+stop_cover(#cover{}=CoverInfo, TestDir) ->
+    cover_analyse(CoverInfo, TestDir);
+stop_cover(_CoverInfo, _TestDir) ->
+    %% Cover is probably controlled by the framework
+    ok.
+
+make_relative(AbsDir, VsDir) ->
+    DirTokens = filename:split(AbsDir),
+    VsTokens = filename:split(VsDir),
+    filename:join(make_relative1(DirTokens, VsTokens)).
+
+make_relative1([T | DirTs], [T | VsTs]) ->
+    make_relative1(DirTs, VsTs);
+make_relative1(Last = [_File], []) ->
+    Last;
+make_relative1(Last = [_File], VsTs) ->
+    Ups = ["../" || _ <- VsTs],
+    Ups ++ Last;
+make_relative1(DirTs, []) ->
+    DirTs;
+make_relative1(DirTs, VsTs) ->
+    Ups = ["../" || _ <- VsTs],
+    Ups ++ DirTs.
+
+
+cover_analyse(CoverInfo, TestDir) ->
     write_default_cross_coverlog(TestDir),
 
     {ok,CoverLog} = open_html_file(filename:join(TestDir, ?coverlog_name)),
     write_coverlog_header(CoverLog),
+    #cover{app=App,
+	   file=CoverFile,
+	   excl=Excluded,
+	   cross=Cross} = CoverInfo,
     io:fwrite(CoverLog, "<h1>Coverage for application '~w'</h1>\n", [App]),
     io:fwrite(CoverLog,
 	      "<p><a href=\"~ts\">Coverdata collected over all tests</a></p>",
 	      [?cross_coverlog_name]),
 
-    {CoverFile,_Included,Excluded,Cross} =
-	case CoverInfo of
-	    {File,Excl,Incl,Cr,Export} ->
-		cover:export(Export),
-		{File,Incl,Excl,Cr};
-	    File ->
-		{Excl,Incl,Cr} = read_cover_file(File),
-		{File,Incl,Excl,Cr}
-	end,
     io:fwrite(CoverLog, "<p>CoverFile: <code>~tp</code>\n", [CoverFile]),
     write_cross_cover_info(TestDir,Cross),
 
@@ -5203,7 +5239,7 @@ cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, Stop, TestDir) ->
 
     io:fwrite(CoverLog, "<p>Excluded module(s): <code>~tp</code>\n", [Excluded]),
 
-    Coverage = cover_analyse(TestDir, Analyse, AnalyseMods, Stop),
+    Coverage = test_server:cover_analyse(TestDir, CoverInfo),
     write_binary_file(filename:join(TestDir,?raw_coverlog_name),
 		      term_to_binary(Coverage)),
 
@@ -5222,10 +5258,6 @@ cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, Stop, TestDir) ->
     write_binary_file(filename:join(TestDir, ?cover_total),
 		      term_to_binary(TotPercent)).
 
-cover_analyse(TestDir, Analyse, AnalyseMods, Stop) ->
-    test_server:cover_analyse(TestDir, Analyse, AnalyseMods, Stop).
-
-
 %% Cover analysis - accumulated over multiple tests
 %% This can be executed on any node after all tests are finished.
 %% Analyse = overview | details
diff --git a/lib/test_server/src/test_server_internal.hrl b/lib/test_server/src/test_server_internal.hrl
index 4e734a330b..bafeeaabfe 100644
--- a/lib/test_server/src/test_server_internal.hrl
+++ b/lib/test_server/src/test_server_internal.hrl
@@ -49,3 +49,12 @@
 	      master,
 	      cookie}).
 	      
+
+-record(cover, {app,    % application; Name | none
+		file,   % cover spec file
+		incl,   % explicitly include modules
+		excl,   % explicitly exclude modules
+		level,  % analyse level; details | overview
+		mods,   % actually cover compiled modules
+		stop=true,   % stop cover after analyse; boolean()
+		cross}).% cross cover analyse info
diff --git a/lib/test_server/src/ts.erl b/lib/test_server/src/ts.erl
index bc7d244c7c..d6d2e865e2 100644
--- a/lib/test_server/src/ts.erl
+++ b/lib/test_server/src/ts.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 1997-2013. All Rights Reserved.
+%% Copyright Ericsson AB 1997-2014. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -641,16 +641,17 @@ get_last_app_tests([Dir|Dirs],RE,Acc) ->
     NewAcc =
 	case re:run(Dir,RE,[{capture,all,list}]) of
 	    {match,[Dir,AppStr]} ->
+		Dir1 = filename:dirname(Dir), % cover logs in ct_run.<t> dir
 		App = list_to_atom(AppStr),
 		case lists:keytake(App,1,Acc) of
 		    {value,{App,LastDir},Rest} ->
-			if Dir > LastDir ->
-				[{App,Dir}|Rest];
+			if Dir1 > LastDir ->
+				[{App,Dir1}|Rest];
 			   true ->
 				Acc
 			end;
 		    false ->
-			[{App,Dir} | Acc]
+			[{App,Dir1} | Acc]
 		end;
 	    _ ->
 		Acc