From bd67967b55a172b46c29a503439feb1e7c58ab06 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Wed, 21 Nov 2012 11:44:19 +0100 Subject: [test_server,common_test] Fix cross cover mechansim Update the interface for cross cover analysis (collection of cover data over multiple tests) so it can be used via common_test and directly with test server. The concept of 'application' in the cross cover interface is removed and replaced with an arbitrary Tag=atom() which identifies a test run. --- lib/common_test/src/ct_cover.erl | 32 +++++++++++++++++++++++++------- lib/common_test/test/common_test.cover | 16 ++++++++-------- 2 files changed, 33 insertions(+), 15 deletions(-) (limited to 'lib/common_test') diff --git a/lib/common_test/src/ct_cover.erl b/lib/common_test/src/ct_cover.erl index d39f50ba00..ae671c750a 100644 --- a/lib/common_test/src/ct_cover.erl +++ b/lib/common_test/src/ct_cover.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2009. All Rights Reserved. +%% Copyright Ericsson AB 2006-2012. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -24,7 +24,7 @@ -module(ct_cover). --export([get_spec/1, add_nodes/1, remove_nodes/1]). +-export([get_spec/1, add_nodes/1, remove_nodes/1, cross_cover_analyse/2]). -include("ct_util.hrl"). @@ -99,6 +99,22 @@ remove_nodes(Nodes) -> end. +%%%----------------------------------------------------------------- +%%% @spec cross_cover_analyse(Level,Tests) -> ok +%%% Level = overview | details +%%% Tests = [{Tag,Dir}] +%%% Tag = atom() +%%% Dir = string() +%%% +%%% @doc Accumulate cover results over multiple tests. +%%% See the chapter about cross cover +%%% analysis in the users's guide. +%%% +cross_cover_analyse(Level,Tests) -> + test_server_ctrl:cross_cover_analyse(Level,Tests). + + %%%----------------------------------------------------------------- %%% @hidden @@ -249,9 +265,11 @@ get_app_info(App=#cover{app=Name}, [{excl_mods,Name,Mods1}|Terms]) -> Mods = App#cover.excl_mods, get_app_info(App#cover{excl_mods=Mods++Mods1},Terms); -get_app_info(App=#cover{app=Name}, [{cross_apps,Name,AppMods1}|Terms]) -> - AppMods = App#cover.cross, - get_app_info(App#cover{cross=AppMods++AppMods1},Terms); +get_app_info(App=#cover{app=none}, [{cross,Cross}|Terms]) -> + get_app_info(App, [{cross,none,Cross}|Terms]); +get_app_info(App=#cover{app=Name}, [{cross,Name,Cross1}|Terms]) -> + Cross = App#cover.cross, + get_app_info(App#cover{cross=Cross++Cross1},Terms); get_app_info(App=#cover{app=none}, [{src_dirs,Dirs}|Terms]) -> get_app_info(App, [{src_dirs,none,Dirs}|Terms]); @@ -354,10 +372,10 @@ remove_excludes_and_dups(CoverData=#cover{excl_mods=Excl,incl_mods=Incl}) -> files2mods(Info=#cover{excl_mods=ExclFs, incl_mods=InclFs, - cross=CrossFs}) -> + cross=Cross}) -> Info#cover{excl_mods=files2mods1(ExclFs), incl_mods=files2mods1(InclFs), - cross=files2mods1(CrossFs)}. + cross=[{Tag,files2mods1(Fs)} || {Tag,Fs} <- Cross]}. files2mods1([M|Fs]) when is_atom(M) -> [M|files2mods1(Fs)]; diff --git a/lib/common_test/test/common_test.cover b/lib/common_test/test/common_test.cover index 66697854ea..3aa49623e7 100644 --- a/lib/common_test/test/common_test.cover +++ b/lib/common_test/test/common_test.cover @@ -1,10 +1,10 @@ %% -*- erlang -*- {incl_app,common_test,details}. -{cross_apps,common_test,[erl2html2, - test_server, - test_server_ctrl, - test_server_gl, - test_server_h, - test_server_io, - test_server_node, - test_server_sup]}. +{cross,common_test,[{test_server,[erl2html2, + test_server, + test_server_ctrl, + test_server_gl, + test_server_h, + test_server_io, + test_server_node, + test_server_sup]}]}. -- cgit v1.2.3 From f7430935c631fb94db15843656dff6937de88a71 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Mon, 3 Dec 2012 18:53:35 +0100 Subject: [common_test] Add test case for cross cover mechanism --- lib/common_test/test/ct_cover_SUITE.erl | 53 +++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 6 deletions(-) (limited to 'lib/common_test') diff --git a/lib/common_test/test/ct_cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE.erl index bebfce70d0..cb49dc423f 100644 --- a/lib/common_test/test/ct_cover_SUITE.erl +++ b/lib/common_test/test/ct_cover_SUITE.erl @@ -77,7 +77,8 @@ all() -> slave_start_slave, cover_node_option, ct_cover_add_remove_nodes, - otp_9956 + otp_9956, + cross ]. %%-------------------------------------------------------------------- @@ -161,6 +162,43 @@ otp_9956(Config) -> check_calls(Events,{?suite,otp_9956,1},1), ok. +%% Test cross cover mechanism +cross(Config) -> + {ok,Events1} = run_test(cross1,Config), + check_calls(Events1,1), + + CoverFile2 = create_cover_file(cross1,[{cross,[{cross1,[?mod]}]}],Config), + {ok,Events2} = run_test(cross2,[{cover,CoverFile2}],Config), + check_calls(Events2,1), + + %% Get the log dirs for each test and run cross cover analyse + [D11,D12] = lists:sort(get_run_dirs(Events1)), + [D21,D22] = lists:sort(get_run_dirs(Events2)), + + ct_cover:cross_cover_analyse(details,[{cross1,D11},{cross2,D21}]), + ct_cover:cross_cover_analyse(details,[{cross1,D12},{cross2,D22}]), + + %% Get the cross cover logs and read for each test + [C11,C12,C21,C22] = + [filename:join(D,"cross_cover.html") || D <- [D11,D12,D21,D22]], + + {ok,CrossData} = file:read_file(C11), + {ok,CrossData} = file:read_file(C12), + + {ok,Def} = file:read_file(C21), + {ok,Def} = file:read_file(C22), + + %% A simple test: just check that the test module exists in the + %% log from cross1 test, and that it does not exist in the log + %% from cross2 test. + TestMod = list_to_binary(atom_to_list(?mod)), + {_,_} = binary:match(CrossData,TestMod), + nomatch = binary:match(Def,TestMod), + {_,_} = binary:match(Def, + <<"No cross cover modules exist for this application">>), + + ok. + %%%----------------------------------------------------------------- %%% HELP FUNCTIONS @@ -229,15 +267,18 @@ check_cover(Node) when is_atom(Node) -> false end. +%% Get the log dir "run." for all (both!) tests +get_run_dirs(Events) -> + [filename:dirname(TCLog) || + {ct_test_support_eh, + {event,tc_logfile,_Node, + {{?suite,init_per_suite},TCLog}}} <- Events]. + %% Check that each coverlog includes N calls to ?mod:foo/0 check_calls(Events,N) -> check_calls(Events,{?mod,foo,0},N). check_calls(Events,MFA,N) -> - CoverLogs = - [filename:join(filename:dirname(TCLog),"all.coverdata") || - {ct_test_support_eh, - {event,tc_logfile,ct@falco, - {{?suite,init_per_suite},TCLog}}} <- Events], + CoverLogs = [filename:join(D,"all.coverdata") || D <- get_run_dirs(Events)], do_check_logs(CoverLogs,MFA,N). do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) -> -- cgit v1.2.3 From 3f1db1cd7d1ba9cf77a10d767b2d82f2542e01a0 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Tue, 4 Dec 2012 11:34:09 +0100 Subject: [common_test] Add documentation for cross cover analysis --- lib/common_test/doc/src/cover_chapter.xml | 87 ++++++++++++++++++++++++++++++- 1 file changed, 86 insertions(+), 1 deletion(-) (limited to 'lib/common_test') diff --git a/lib/common_test/doc/src/cover_chapter.xml b/lib/common_test/doc/src/cover_chapter.xml index b2e64bfff0..4fa92d5583 100644 --- a/lib/common_test/doc/src/cover_chapter.xml +++ b/lib/common_test/doc/src/cover_chapter.xml @@ -108,8 +108,8 @@ specifications).

+
- Stopping the cover tool when tests are completed

By default the Cover tool is automatically stopped when the tests are completed. This causes the original (non cover @@ -175,6 +175,11 @@ %% Specific modules to exclude in cover. {excl_mods, Mods}. + + %% Cross cover compilation + %% Tag = atom(), an identifier for a test run + %% Mod = [atom()], modules to compile for accumulated analysis + {cross,[{Tag,Mods}]}.

The incl_dirs_r and excl_dirs_r terms tell Common @@ -190,6 +195,81 @@ specification file for Common Test).

+ +
+ Cross cover analysis +

The cross cover mechanism allows cover analysis of modules + across multiple tests. It is useful if some code, e.g. a library + module, is used by many different tests and the accumulated cover + result is desirable.

+ +

This can of course also be achieved in a more customized way by + using the export parameter in the cover specification and + analysing the result off line, but the cross cover mechanism is a + build in solution which also provides the logging.

+ +

The mechanism is easiest explained via an example:

+ +

Let's say that there are two systems, s1 and s2, + which are tested in separate test runs. System s1 contains + a library module m1 which is tested by the s1 test + run and is included in s1's cover specification:

+ + +s1.cover: + {incl_mods,[m1]}. + +

When analysing code coverage, the result for m1 can be + seen in the cover log in the s1 test result.

+ +

Now, let's imagine that since m1 is a library module, it + is also used quite a bit by system s2. The s2 test + run does not specifically test m1, but it might still be + interesting to see which parts of m1 is actually covered by + the s2 tests. To do this, m1 could be included also + in s2's cover specification:

+ + +s2.cover: + {incl_mods,[m1]}. + +

This would give an entry for m1 also in the cover log + for the s2 test run. The problem is that this would only + reflect the coverage by s2 tests, not the accumulated + result over s1 and s2. And this is where the cross + cover mechanism comes in handy.

+ +

If instead the cover specification for s2 was like + this:

+ + +s2.cover: + {cross,[{s1,[m1]}]}. + +

then m1 would be cover compiled in the s2 test + run, but not shown in the coverage log. Instead, if + ct_cover:cross_cover_analyse/2 is called after both + s1 and s2 test runs are completed, the accumulated + result for m1 would be available in the cross cover log for + the s1 test run.

+ +

The call to the analyse function must be like this:

+ + +ct_cover:cross_cover_analyse(Level, [{s1,S1LogDir},{s2,S2LogDir}]). + +

where S1LogDir and S2LogDir are the directories + named <TestName>.logs for each test respectively.

+ +

Note the tags s1 and s2 which are used in the + cover specification file and in the call to + ct_cover:cross_cover_analyse/2. The point of these are only + to map the modules specified in the cover specification to the log + directory specified in the call to the analyse function. The name + of the tag has no meaning beyond this.

+ +
+
Logging

To view the result of a code coverage test, follow the @@ -197,6 +277,11 @@ takes you to the code coverage overview page. If you have successfully performed a detailed coverage analysis, you find links to each individual module coverage page here.

+ +

If cross cover analysis has been performed, and there are + accumulated coverage results for the current test, then the - + "Coverdata collected over all tests" link will take you to these + results.

-- cgit v1.2.3