Merge branch 'siri/cross-cover/OTP-9870'

* siri/cross-cover/OTP-9870:
  [test_server] Don't use print/3 in cross_cover_analyse
  [common_test] Add documentation for cross cover analysis
  [common_test] Add test case for cross cover mechanism
  [test_server] Update documentation about cross cover
  [test_server] Add test of code cover mechanism
  [test_server,common_test] Fix cross cover mechansim

4 files changed, 166 insertions, 22 deletions

diff --git a/lib/common_test/doc/src/cover_chapter.xml b/lib/common_test/doc/src/cover_chapter.xml
index b2e64bfff0..4fa92d5583 100644
--- a/lib/common_test/doc/src/cover_chapter.xml
+++ b/lib/common_test/doc/src/cover_chapter.xml
@@ -108,8 +108,8 @@
       specifications</seealso>).</p>
   </section>
 
+  <marker id="cover_stop"></marker>
   <section>
-    <marker id="cover_stop"></marker>
     <title>Stopping the cover tool when tests are completed</title>
     <p>By default the Cover tool is automatically stopped when the
       tests are completed. This causes the original (non cover
@@ -175,6 +175,11 @@
 
       %% Specific modules to exclude in cover.
       {excl_mods, Mods}.
+
+      %% Cross cover compilation
+      %% Tag = atom(), an identifier for a test run
+      %% Mod = [atom()], modules to compile for accumulated analysis
+      {cross,[{Tag,Mods}]}.
     </pre>
 
     <p>The <c>incl_dirs_r</c> and <c>excl_dirs_r</c> terms tell Common
@@ -190,6 +195,81 @@
        specification file for Common Test).</p>
   </section>
 
+  <marker id="cross_cover"/>
+  <section>
+    <title>Cross cover analysis</title>
+    <p>The cross cover mechanism allows cover analysis of modules
+    across multiple tests. It is useful if some code, e.g. a library
+    module, is used by many different tests and the accumulated cover
+    result is desirable.</p>
+
+    <p>This can of course also be achieved in a more customized way by
+    using the <c>export</c> parameter in the cover specification and
+    analysing the result off line, but the cross cover mechanism is a
+    build in solution which also provides the logging.</p>
+
+    <p>The mechanism is easiest explained via an example:</p>
+
+    <p>Let's say that there are two systems, <c>s1</c> and <c>s2</c>,
+    which are tested in separate test runs. System <c>s1</c> contains
+    a library module <c>m1</c> which is tested by the <c>s1</c> test
+    run and is included in <c>s1</c>'s cover specification:</p>
+
+<code type="none">
+s1.cover:
+  {incl_mods,[m1]}.</code>
+
+    <p>When analysing code coverage, the result for <c>m1</c> can be
+    seen in the cover log in the <c>s1</c> test result.</p>
+
+    <p>Now, let's imagine that since <c>m1</c> is a library module, it
+    is also used quite a bit by system <c>s2</c>. The <c>s2</c> test
+    run does not specifically test <c>m1</c>, but it might still be
+    interesting to see which parts of <c>m1</c> is actually covered by
+    the <c>s2</c> tests. To do this, <c>m1</c> could be included also
+    in <c>s2</c>'s cover specification:</p>
+
+<code type="none">
+s2.cover:
+  {incl_mods,[m1]}.</code>
+
+    <p>This would give an entry for <c>m1</c> also in the cover log
+    for the <c>s2</c> test run. The problem is that this would only
+    reflect the coverage by <c>s2</c> tests, not the accumulated
+    result over <c>s1</c> and <c>s2</c>. And this is where the cross
+    cover mechanism comes in handy.</p>
+
+    <p>If instead the cover specification for <c>s2</c> was like
+    this:</p>
+
+<code type="none">
+s2.cover:
+  {cross,[{s1,[m1]}]}.</code>
+
+    <p>then <c>m1</c> would be cover compiled in the <c>s2</c> test
+    run, but not shown in the coverage log. Instead, if
+    <c>ct_cover:cross_cover_analyse/2</c> is called after both
+    <c>s1</c> and <c>s2</c> test runs are completed, the accumulated
+    result for <c>m1</c> would be available in the cross cover log for
+    the <c>s1</c> test run.</p>
+
+    <p>The call to the analyse function must be like this:</p>
+
+<code type="none">
+ct_cover:cross_cover_analyse(Level, [{s1,S1LogDir},{s2,S2LogDir}]).</code>
+
+    <p>where <c>S1LogDir</c> and <c>S2LogDir</c> are the directories
+    named <c>&lt;TestName&gt;.logs</c> for each test respectively.</p>
+
+    <p>Note the tags <c>s1</c> and <c>s2</c> which are used in the
+    cover specification file and in the call to
+    <c>ct_cover:cross_cover_analyse/2</c>. The point of these are only
+    to map the modules specified in the cover specification to the log
+    directory specified in the call to the analyse function. The name
+    of the tag has no meaning beyond this.</p>
+
+  </section>
+
   <section>
     <title>Logging</title>
     <p>To view the result of a code coverage test, follow the
@@ -197,6 +277,11 @@
       takes you to the code coverage overview page. If you have 
       successfully performed a detailed coverage analysis, you 
       find links to each individual module coverage page here.</p>
+
+    <p>If cross cover analysis has been performed, and there are
+      accumulated coverage results for the current test, then the -
+      "Coverdata collected over all tests" link will take you to these
+      results.</p>
   </section>
 
 </chapter>
diff --git a/lib/common_test/src/ct_cover.erl b/lib/common_test/src/ct_cover.erl
index d39f50ba00..ae671c750a 100644
--- a/lib/common_test/src/ct_cover.erl
+++ b/lib/common_test/src/ct_cover.erl
@@ -1,7 +1,7 @@
 %%
 %% %CopyrightBegin%
 %% 
-%% Copyright Ericsson AB 2006-2009. All Rights Reserved.
+%% Copyright Ericsson AB 2006-2012. All Rights Reserved.
 %% 
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -24,7 +24,7 @@
 
 -module(ct_cover).
 
--export([get_spec/1, add_nodes/1, remove_nodes/1]).
+-export([get_spec/1, add_nodes/1, remove_nodes/1, cross_cover_analyse/2]).
 
 -include("ct_util.hrl").
 
@@ -100,6 +100,22 @@ remove_nodes(Nodes) ->
     
     
 %%%-----------------------------------------------------------------
+%%% @spec cross_cover_analyse(Level,Tests) -> ok
+%%%    Level = overview | details
+%%%    Tests = [{Tag,Dir}]
+%%%    Tag = atom()
+%%%    Dir = string()
+%%%
+%%% @doc Accumulate cover results over multiple tests.
+%%%      See the chapter about <seealso
+%%%      marker="cover_chapter#cross_cover">cross cover
+%%%      analysis</seealso> in the users's guide.
+%%%
+cross_cover_analyse(Level,Tests) ->
+    test_server_ctrl:cross_cover_analyse(Level,Tests).
+
+
+%%%-----------------------------------------------------------------
 %%% @hidden 
 
 %% Read cover specification file and return the parsed info.
@@ -249,9 +265,11 @@ get_app_info(App=#cover{app=Name}, [{excl_mods,Name,Mods1}|Terms]) ->
     Mods = App#cover.excl_mods,
     get_app_info(App#cover{excl_mods=Mods++Mods1},Terms);
 
-get_app_info(App=#cover{app=Name}, [{cross_apps,Name,AppMods1}|Terms]) ->
-    AppMods = App#cover.cross,
-    get_app_info(App#cover{cross=AppMods++AppMods1},Terms);
+get_app_info(App=#cover{app=none}, [{cross,Cross}|Terms]) ->
+    get_app_info(App, [{cross,none,Cross}|Terms]);
+get_app_info(App=#cover{app=Name}, [{cross,Name,Cross1}|Terms]) ->
+    Cross = App#cover.cross,
+    get_app_info(App#cover{cross=Cross++Cross1},Terms);
 
 get_app_info(App=#cover{app=none}, [{src_dirs,Dirs}|Terms]) ->
     get_app_info(App, [{src_dirs,none,Dirs}|Terms]);
@@ -354,10 +372,10 @@ remove_excludes_and_dups(CoverData=#cover{excl_mods=Excl,incl_mods=Incl}) ->
     
 files2mods(Info=#cover{excl_mods=ExclFs,
 		       incl_mods=InclFs,
-		       cross=CrossFs}) ->
+		       cross=Cross}) ->
     Info#cover{excl_mods=files2mods1(ExclFs),
 	       incl_mods=files2mods1(InclFs),
-	       cross=files2mods1(CrossFs)}.
+	       cross=[{Tag,files2mods1(Fs)} || {Tag,Fs} <- Cross]}.
 
 files2mods1([M|Fs]) when is_atom(M) ->
     [M|files2mods1(Fs)];
diff --git a/lib/common_test/test/common_test.cover b/lib/common_test/test/common_test.cover
index 66697854ea..3aa49623e7 100644
--- a/lib/common_test/test/common_test.cover
+++ b/lib/common_test/test/common_test.cover
@@ -1,10 +1,10 @@
 %% -*- erlang -*-
 {incl_app,common_test,details}.
-{cross_apps,common_test,[erl2html2,
-			 test_server,
-			 test_server_ctrl,
-			 test_server_gl,
-			 test_server_h,
-			 test_server_io,
-			 test_server_node,
-			 test_server_sup]}.
+{cross,common_test,[{test_server,[erl2html2,
+				  test_server,
+				  test_server_ctrl,
+				  test_server_gl,
+				  test_server_h,
+				  test_server_io,
+				  test_server_node,
+				  test_server_sup]}]}.
diff --git a/lib/common_test/test/ct_cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE.erl
index bebfce70d0..cb49dc423f 100644
--- a/lib/common_test/test/ct_cover_SUITE.erl
+++ b/lib/common_test/test/ct_cover_SUITE.erl
@@ -77,7 +77,8 @@ all() ->
      slave_start_slave,
      cover_node_option,
      ct_cover_add_remove_nodes,
-     otp_9956
+     otp_9956,
+     cross
     ].
 
 %%--------------------------------------------------------------------
@@ -161,6 +162,43 @@ otp_9956(Config) ->
     check_calls(Events,{?suite,otp_9956,1},1),
     ok.
 
+%% Test cross cover mechanism
+cross(Config) ->
+    {ok,Events1} = run_test(cross1,Config),
+    check_calls(Events1,1),
+
+    CoverFile2 = create_cover_file(cross1,[{cross,[{cross1,[?mod]}]}],Config),
+    {ok,Events2} = run_test(cross2,[{cover,CoverFile2}],Config),
+    check_calls(Events2,1),
+
+    %% Get the log dirs for each test and run cross cover analyse
+    [D11,D12] = lists:sort(get_run_dirs(Events1)),
+    [D21,D22] = lists:sort(get_run_dirs(Events2)),
+
+    ct_cover:cross_cover_analyse(details,[{cross1,D11},{cross2,D21}]),
+    ct_cover:cross_cover_analyse(details,[{cross1,D12},{cross2,D22}]),
+
+    %% Get the cross cover logs and read for each test
+    [C11,C12,C21,C22] =
+	[filename:join(D,"cross_cover.html") || D <- [D11,D12,D21,D22]],
+
+    {ok,CrossData} = file:read_file(C11),
+    {ok,CrossData} = file:read_file(C12),
+
+    {ok,Def} = file:read_file(C21),
+    {ok,Def} = file:read_file(C22),
+
+    %% A simple test: just check that the test module exists in the
+    %% log from cross1 test, and that it does not exist in the log
+    %% from cross2 test.
+    TestMod = list_to_binary(atom_to_list(?mod)),
+    {_,_} = binary:match(CrossData,TestMod),
+    nomatch = binary:match(Def,TestMod),
+    {_,_} = binary:match(Def,
+			 <<"No cross cover modules exist for this application">>),
+
+    ok.
+
 
 %%%-----------------------------------------------------------------
 %%% HELP FUNCTIONS
@@ -229,15 +267,18 @@ check_cover(Node) when is_atom(Node) ->
 	    false
     end.
 
+%% Get the log dir "run.<timestamp>" for all (both!) tests
+get_run_dirs(Events) ->
+    [filename:dirname(TCLog) ||
+	{ct_test_support_eh,
+	 {event,tc_logfile,_Node,
+	  {{?suite,init_per_suite},TCLog}}} <- Events].
+
 %% Check that each coverlog includes N calls to ?mod:foo/0
 check_calls(Events,N) ->
     check_calls(Events,{?mod,foo,0},N).
 check_calls(Events,MFA,N) ->
-    CoverLogs =
-	[filename:join(filename:dirname(TCLog),"all.coverdata") ||
-	    {ct_test_support_eh,
-	     {event,tc_logfile,ct@falco,
-	      {{?suite,init_per_suite},TCLog}}} <- Events],
+    CoverLogs = [filename:join(D,"all.coverdata") || D <- get_run_dirs(Events)],
     do_check_logs(CoverLogs,MFA,N).
 
 do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) ->