From bd67967b55a172b46c29a503439feb1e7c58ab06 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Wed, 21 Nov 2012 11:44:19 +0100 Subject: [test_server,common_test] Fix cross cover mechansim Update the interface for cross cover analysis (collection of cover data over multiple tests) so it can be used via common_test and directly with test server. The concept of 'application' in the cross cover interface is removed and replaced with an arbitrary Tag=atom() which identifies a test run. --- lib/test_server/src/Makefile | 3 +- lib/test_server/src/test_server.erl | 11 +- lib/test_server/src/test_server_ctrl.erl | 285 +++++++++++++++++-------------- lib/test_server/src/ts.erl | 31 +--- 4 files changed, 171 insertions(+), 159 deletions(-) (limited to 'lib/test_server') diff --git a/lib/test_server/src/Makefile b/lib/test_server/src/Makefile index 20e7a5942c..3261936472 100644 --- a/lib/test_server/src/Makefile +++ b/lib/test_server/src/Makefile @@ -69,7 +69,6 @@ INTERNAL_HRL_FILES = test_server_internal.hrl TS_HRL_FILES= ts.hrl C_FILES = AUTOCONF_FILES = configure.in conf_vars.in -COVER_FILES = cross.cover PROGRAMS = configure config.sub config.guess install-sh CONFIG = ts.config ts.unix.config ts.win32.config @@ -137,7 +136,7 @@ release_tests_spec: opt $(INSTALL_DATA) $(ERL_FILES) $(TS_ERL_FILES) \ $(HRL_FILES) $(INTERNAL_HRL_FILES) $(TS_HRL_FILES) \ $(TS_TARGET_FILES) \ - $(AUTOCONF_FILES) $(C_FILES) $(COVER_FILES) $(CONFIG) \ + $(AUTOCONF_FILES) $(C_FILES) $(CONFIG) \ "$(RELEASE_PATH)/test_server" $(INSTALL_SCRIPT) $(PROGRAMS) "$(RELEASE_PATH)/test_server" diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl index 14cdfd391a..72bc7bb0d7 100644 --- a/lib/test_server/src/test_server.erl +++ b/lib/test_server/src/test_server.erl @@ -97,7 +97,8 @@ init_purify() -> %% is found, else {error,application_not_found}. cover_compile({none,_Exclude,Include,Cross}) -> - CompileMods = Include++Cross, + CrossMods = lists:flatmap(fun({_,M}) -> M end,Cross), + CompileMods = Include++CrossMods, case length(CompileMods) of 0 -> io:fwrite("WARNING: No modules to cover compile!\n\n",[]), @@ -111,7 +112,8 @@ cover_compile({none,_Exclude,Include,Cross}) -> {ok,Include} end; cover_compile({App,all,Include,Cross}) -> - CompileMods = Include++Cross, + CrossMods = lists:flatmap(fun({_,M}) -> M end,Cross), + CompileMods = Include++CrossMods, case length(CompileMods) of 0 -> io:fwrite("WARNING: No modules to cover compile!\n\n",[]), @@ -129,9 +131,10 @@ cover_compile({App,all,Include,Cross}) -> {ok,Include} end; cover_compile({App,Exclude,Include,Cross}) -> + CrossMods = lists:flatmap(fun({_,M}) -> M end,Cross), case code:lib_dir(App) of {error,bad_name} -> - case Include++Cross of + case Include++CrossMods of [] -> io:format("\nWARNING: Can't find lib_dir for \'~w\'\n" "Not cover compiling!\n\n",[App]), @@ -152,7 +155,7 @@ cover_compile({App,Exclude,Include,Cross}) -> WC = filename:join(EbinDir,"*.beam"), AllMods = module_names(filelib:wildcard(WC)), AnalyseMods = (AllMods ++ Include) -- Exclude, - CompileMods = AnalyseMods ++ Cross, + CompileMods = AnalyseMods ++ CrossMods, case length(CompileMods) of 0 -> io:fwrite("WARNING: No modules to cover compile!\n\n",[]), diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index bc08c12089..10cba3fbd4 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -53,8 +53,7 @@ -export([reject_io_reqs/1, get_levels/0, set_levels/3]). -export([multiply_timetraps/1, scale_timetraps/1, get_timetrap_parameters/0]). -export([create_priv_dir/1]). --export([cover/2, cover/3, cover/8, - cross_cover_analyse/2, cross_cover_analyse/3, trc/1, stop_trace/0]). +-export([cover/2, cover/3, cover/8, cross_cover_analyse/2, trc/1, stop_trace/0]). -export([testcase_callback/1]). -export([set_random_seed/1]). -export([kill_slavenodes/0]). @@ -88,14 +87,16 @@ -define(data_dir_suffix, "_data/"). -define(suitelog_name, "suite.log"). -define(coverlog_name, "cover.html"). +-define(raw_coverlog_name, "cover.log"). -define(cross_coverlog_name, "cross_cover.html"). +-define(raw_cross_coverlog_name, "cross_cover.log"). +-define(cross_cover_info, "cross_cover.info"). -define(cover_total, "total_cover.log"). -define(unexpected_io_log, "unexpected_io.log"). -define(last_file, "last_name"). -define(last_link, "last_link"). -define(last_test, "last_test"). -define(html_ext, ".html"). --define(cross_cover_file, "cross.cover"). -define(now, erlang:now()). -define(pl2a(M), test_server_sup:package_atom(M)). @@ -409,7 +410,9 @@ cover(CoverFile, Analyse) -> cover(App, CoverFile, Analyse) -> controller_call({cover,{App,CoverFile},Analyse,true}). cover(App, CoverFile, Exclude, Include, Cross, Export, Analyse, Stop) -> - controller_call({cover,{App,{CoverFile,Exclude,Include,Cross,Export}},Analyse,Stop}). + controller_call({cover, + {App,{CoverFile,Exclude,Include,Cross,Export}}, + Analyse,Stop}). testcase_callback(ModFunc) -> controller_call({testcase_callback,ModFunc}). @@ -4898,33 +4901,52 @@ pinfo(P) -> %% - it does not belong to the application, but is listed in the %% {include,List} part of the App.cover file %% - it does not belong to the application, but is listed in the -%% cross.cover file (in the test_server application) under 'all' -%% or under the tested application. -%% -%% The modules listed in the cross.cover file are modules that are -%% hevily used by other applications than the one they belong -%% to. After all tests are completed, these modules can be analysed -%% with coverage data from all tests - see cross_cover_analyse/1. The -%% result is stored in a file called cross_cover.html in the -%% run. directory of the application the modules belong -%% to. -%% -%% For example, the lists module is listed in cross.cover to be -%% included in all tests. lists belongs to the stdlib -%% application. cross_cover_analyse/1 will create a file named -%% cross_cover.html under the newest stdlib.logs/run.xxx directory, -%% where the coverage result for the lists module from all tests is -%% presented. -%% -%% The lists module is also presented in the normal coverage log -%% for stdlib, but that only includes the coverage achieved by -%% the stdlib tests themselves. -%% -%% The Cross cover file cross.cover contains elements like this: -%% {App,Modules}. -%% where App can be an application name or the atom all. The -%% application (or all applications) shall cover compile the listed -%% Modules. +%% {cross,[{Tag,List}]} part of the App.cover file +%% +%% The modules listed in the 'cross' part of the cover file are +%% modules that are heavily used by other tests than the one where +%% they are explicitly tested. They should then be listed as 'cross' +%% in the cover file for the test where they are used but do not +%% belong. +%% +%% After all tests are completed, the these modules can be analysed +%% with coverage data from all tests where they are compiled - see +%% cross_cover_analyse/2. The result is stored in a file called +%% cross_cover.html in the run. directory of the +%% test the modules belong to. +%% +%% Example: +%% If the module m1 belongs to system s1 but is heavily used also in +%% the tests for another system s2, then the cover files for the two +%% systems could be like this: +%% +%% s1.cover: +%% {include,[m1]}. +%% +%% s2.cover: +%% {include,[....]}. % modules belonging to system s2 +%% {cross,[{s1,[m1]}]}. +%% +%% When the tests for both s1 and s2 are completed, run +%% cross_cover_analyse(Level,[{s1,S1LogDir},{s2,S2LogDir}]), and +%% the accumulated cover data for m1 will be written to +%% S1LogDir/[run./]cross_cover.html +%% +%% S1LogDir and S2LogDir are either the run. directories +%% for the two tests, or the parent directory of these, in which case +%% the latest run. directory will be chosen. +%% +%% Note that the m1 module will also be presented in the normal +%% coverage log for s1 (due to the include statement in s1.cover), but +%% that only includes the coverage achieved by the s1 test itself. +%% +%% The Tag in the 'cross' statement in the cover file has no other +%% purpose than mapping the list of modules ([m1] in the example +%% above) to the correct log directory where it should be included in +%% the cross_cover.html file (S1LogDir in the example above). +%% I.e. the value of the Tag has no meaning, it could be foo as well +%% as s1 above, as long as the same Tag is used in the cover file and +%% in the call to cross_cover_analyse/2. %% Cover compilation @@ -4933,8 +4955,7 @@ cover_compile({App,{_File,Exclude,Include,Cross,_Export}}) -> cover_compile1({App,Exclude,Include,Cross}); cover_compile({App,CoverFile}) -> - Cross = get_cross_modules(App), - {Exclude,Include} = read_cover_file(CoverFile), + {Exclude,Include,Cross} = read_cover_file(CoverFile), cover_compile1({App,Exclude,Include,Cross}). cover_compile1(What) -> @@ -4945,41 +4966,57 @@ cover_compile1(What) -> %% (Exclude), and a list of modules that are not members of the %% application but shall be compiled (Include). read_cover_file(none) -> - {[],[]}; + {[],[],[]}; read_cover_file(CoverFile) -> case file:consult(CoverFile) of {ok,List} -> - case check_cover_file(List, [], []) of - {ok,Exclude,Include} -> {Exclude,Include}; + case check_cover_file(List, [], [], []) of + {ok,Exclude,Include,Cross} -> {Exclude,Include,Cross}; error -> io:fwrite("Faulty format of CoverFile ~p\n", [CoverFile]), - {[],[]} + {[],[],[]} end; {error,Reason} -> io:fwrite("Can't read CoverFile ~p\nReason: ~p\n", [CoverFile,Reason]), - {[],[]} + {[],[],[]} end. -check_cover_file([{exclude,all}|Rest], _, Include) -> - check_cover_file(Rest, all, Include); -check_cover_file([{exclude,Exclude}|Rest], _, Include) -> +check_cover_file([{exclude,all}|Rest], _, Include, Cross) -> + check_cover_file(Rest, all, Include, Cross); +check_cover_file([{exclude,Exclude}|Rest], _, Include, Cross) -> case lists:all(fun(M) -> is_atom(M) end, Exclude) of true -> - check_cover_file(Rest, Exclude, Include); + check_cover_file(Rest, Exclude, Include, Cross); false -> error end; -check_cover_file([{include,Include}|Rest], Exclude, _) -> +check_cover_file([{include,Include}|Rest], Exclude, _, Cross) -> case lists:all(fun(M) -> is_atom(M) end, Include) of true -> - check_cover_file(Rest, Exclude, Include); + check_cover_file(Rest, Exclude, Include, Cross); false -> error end; -check_cover_file([], Exclude, Include) -> - {ok,Exclude,Include}. +check_cover_file([{cross,Cross}|Rest], Exclude, Include, _) -> + case check_cross(Cross) of + true -> + check_cover_file(Rest, Exclude, Include, Cross); + false -> + error + end; +check_cover_file([], Exclude, Include, Cross) -> + {ok,Exclude,Include,Cross}. +check_cross([{Tag,Modules}|Rest]) -> + case lists:all(fun(M) -> is_atom(M) end, [Tag|Modules]) of + true -> + check_cross(Rest); + false -> + false + end; +check_cross([]) -> + true. %% Cover analysis, per application @@ -5000,16 +5037,17 @@ cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, Stop, TestDir) -> "

Coverdata collected over all tests

", [?cross_coverlog_name]), - {CoverFile,_Included,Excluded} = + {CoverFile,_Included,Excluded,Cross} = case CoverInfo of - {File,Excl,Incl,_Cross,Export} -> + {File,Excl,Incl,Cr,Export} -> cover:export(Export), - {File,Incl,Excl}; + {File,Incl,Excl,Cr}; File -> - {Excl,Incl} = read_cover_file(File), - {File,Incl,Excl} + {Excl,Incl,Cr} = read_cover_file(File), + {File,Incl,Excl,Cr} end, io:fwrite(CoverLog, "

CoverFile: ~p\n", [CoverFile]), + write_cross_cover_info(TestDir,Cross), case length(cover:imported_modules()) of Imps when Imps > 0 -> @@ -5022,6 +5060,8 @@ cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, Stop, TestDir) -> io:fwrite(CoverLog, "

Excluded module(s): ~p\n", [Excluded]), Coverage = cover_analyse(Analyse, AnalyseMods, Stop), + file:write_file(filename:join(TestDir,?raw_coverlog_name), + term_to_binary(Coverage)), case lists:filter(fun({_M,{_,_,_}}) -> false; (_) -> true @@ -5043,22 +5083,22 @@ cover_analyse(Analyse, AnalyseMods, Stop) -> test_server:cover_analyse({Analyse,TestDir}, AnalyseMods, Stop). -%% Cover analysis, cross application +%% Cover analysis - accumulated over multiple tests %% This can be executed on any node after all tests are finished. -%% Apps = [{App,Dir}] -%% App = atom(), application name -%% Dir = string(), the log directory for App, normally where -%% run. is found. -%% Modules = [atom()], modules that have been cover compiled during tests -%% of other apps than the one they belong to. -cross_cover_analyse(Analyse, Apps) -> - cross_cover_analyse(Analyse, Apps, get_cross_modules()). -cross_cover_analyse(Analyse, Apps, Modules) -> - Apps1 = get_latest_run_dirs(Apps), - Apps2 = add_cross_modules(Modules,Apps1), - CoverdataFiles = get_coverdata_files(Apps2), +%% Analyse = overview | details +%% TagDirs = [{Tag,Dir}] +%% Tag = atom(), identifier +%% Dir = string(), the log directory for Tag, it can be a +%% run. directory or the parent directory of +%% such (in which case the latest run. directory +%% is used) +cross_cover_analyse(Analyse, TagDirs0) -> + TagDirs = get_latest_run_dirs(TagDirs0), + TagMods = get_all_cross_info(TagDirs,[]), + TagDirMods = add_cross_modules(TagMods,TagDirs), + CoverdataFiles = get_coverdata_files(TagDirMods), lists:foreach(fun(CDF) -> cover:import(CDF) end, CoverdataFiles), - io:fwrite("Cover analysing...\n", []), + print(stdout, "Cover analysing...\n", []), DetailsFun = case Analyse of details -> @@ -5066,39 +5106,52 @@ cross_cover_analyse(Analyse, Apps, Modules) -> OutFile = filename:join(Dir, atom_to_list(M) ++ ".CROSS_COVER.html"), - cover:analyse_to_file(M, OutFile, [html]), - {file,OutFile} + case cover:analyse_to_file(M, OutFile, [html]) of + {ok,_} -> + {file,OutFile}; + Error -> + Error + end end; _ -> fun(_,_) -> undefined end end, - Coverage = analyse_apps(Apps2, DetailsFun, []), + Coverage = analyse_tests(TagDirMods, DetailsFun, []), cover:stop(), - write_cross_cover_logs(Coverage,Apps2). + write_cross_cover_logs(Coverage,TagDirMods). -%% For each application from which there are cross cover analysed +write_cross_cover_info(_Dir,[]) -> + ok; +write_cross_cover_info(Dir,Cross) -> + {ok,Fd} = file:open(filename:join(Dir,?cross_cover_info),[write]), + lists:foreach(fun(C) -> io:format(Fd,"~p.~n",[C]) end, Cross), + file:close(Fd). + +%% For each test from which there are cross cover analysed %% modules, write a cross cover log (cross_cover.html). -write_cross_cover_logs([{App,Coverage}|T],Apps) -> - case lists:keyfind(App,1,Apps) of +write_cross_cover_logs([{Tag,Coverage}|T],TagDirMods) -> + case lists:keyfind(Tag,1,TagDirMods) of {_,Dir,Mods} when Mods=/=[] -> + file:write_file(filename:join(Dir,?raw_cross_coverlog_name), + term_to_binary(Coverage)), CoverLogName = filename:join(Dir,?cross_coverlog_name), {ok,CoverLog} = file:open(CoverLogName, [write]), write_coverlog_header(CoverLog), io:fwrite(CoverLog, "

Coverage results for \'~w\' from all tests

\n", - [App]), + [Tag]), write_cover_result_table(CoverLog, Coverage), io:fwrite("Written file ~p\n", [CoverLogName]); _ -> ok end, - write_cross_cover_logs(T,Apps); + write_cross_cover_logs(T,TagDirMods); write_cross_cover_logs([],_) -> io:fwrite("done\n", []). %% Get the latest run. directories -get_latest_run_dirs([{App,Dir}|Apps]) -> - [{App,get_latest_run_dir(Dir)} | get_latest_run_dirs(Apps)]; +get_latest_run_dirs([{Tag,Dir}|Rest]) -> + [{Tag,get_latest_run_dir(Dir)} | get_latest_run_dirs(Rest)]; get_latest_run_dirs([]) -> []. @@ -5117,44 +5170,47 @@ get_latest_dir([_|T],Latest) -> get_latest_dir([],Latest) -> Latest. -%% Associate the cross cover modules with their applications. -add_cross_modules(Mods,Apps)-> - do_add_cross_modules(Mods,[{App,Dir,[]} || {App,Dir} <- Apps]). -do_add_cross_modules([Mod|Mods],Apps)-> - App = get_app(Mod), - NewApps = - case lists:keytake(App,1,Apps) of - {value,{App,Dir,AppMods},Rest} -> - [{App,Dir,lists:umerge([Mod],AppMods)}|Rest]; +get_all_cross_info([{_Tag,Dir}|Rest],Acc) -> + case file:consult(filename:join(Dir,?cross_cover_info)) of + {ok,TagMods} -> + get_all_cross_info(Rest,TagMods++Acc); + _ -> + get_all_cross_info(Rest,Acc) + end; +get_all_cross_info([],Acc) -> + Acc. + +%% Associate the cross cover modules with their log directories +add_cross_modules(TagMods,TagDirs)-> + do_add_cross_modules(TagMods,[{Tag,Dir,[]} || {Tag,Dir} <- TagDirs]). +do_add_cross_modules([{Tag,Mods1}|TagMods],TagDirMods)-> + NewTagDirMods = + case lists:keytake(Tag,1,TagDirMods) of + {value,{Tag,Dir,Mods},Rest} -> + [{Tag,Dir,lists:umerge(lists:sort(Mods1),Mods)}|Rest]; false -> - Apps + TagDirMods end, - do_add_cross_modules(Mods,NewApps); -do_add_cross_modules([],Apps) -> - %% Just to get the modules in the same order as app-only cover log - [{App,Dir,lists:reverse(Mods)} || {App,Dir,Mods} <- Apps]. - -get_app(Module) -> - Beam = code:which(Module), - AppDir = filename:basename(filename:dirname(filename:dirname(Beam))), - [AppStr|_] = string:tokens(AppDir,"-"), - list_to_atom(AppStr). + do_add_cross_modules(TagMods,NewTagDirMods); +do_add_cross_modules([],TagDirMods) -> + %% Just to get the modules in the same order as in the normal cover log + [{Tag,Dir,lists:reverse(Mods)} || {Tag,Dir,Mods} <- TagDirMods]. %% Find all exported coverdata files. -get_coverdata_files(Apps) -> +get_coverdata_files(TagDirMods) -> lists:flatmap( - fun({_,LatestAppDir,_}) -> - filelib:wildcard(filename:join(LatestAppDir,"all.coverdata")) + fun({_,LatestDir,_}) -> + filelib:wildcard(filename:join(LatestDir,"all.coverdata")) end, - Apps). + TagDirMods). -%% For each application, analyse all modules +%% For each test, analyse all modules %% Used for cross cover analysis. -analyse_apps([{App,LastTest,Modules}|T], DetailsFun, Acc) -> +analyse_tests([{Tag,LastTest,Modules}|T], DetailsFun, Acc) -> Cov = analyse_modules(LastTest, Modules, DetailsFun, []), - analyse_apps(T, DetailsFun, [{App,Cov}|Acc]); -analyse_apps([], _DetailsFun, Acc) -> + analyse_tests(T, DetailsFun, [{Tag,Cov}|Acc]); +analyse_tests([], _DetailsFun, Acc) -> Acc. %% Analyse each module @@ -5167,27 +5223,6 @@ analyse_modules(_Dir, [], _DetailsFun, Acc) -> Acc. -%% Read the cross cover file (cross.cover) -get_cross_modules() -> - get_cross_modules(all). -get_cross_modules(App) -> - case file:consult(?cross_cover_file) of - {ok,List} -> - get_cross_modules(App, List, []); - _X -> - [] - end. - -get_cross_modules(App, [{_To,Modules}|T], Acc) when App==all-> - get_cross_modules(App, T, Acc ++ Modules); -get_cross_modules(App, [{To,Modules}|T], Acc) when To==App; To==all-> - get_cross_modules(App, T, Acc ++ Modules); -get_cross_modules(App, [_H|T], Acc) -> - get_cross_modules(App, T, Acc); -get_cross_modules(_App, [], Acc) -> - Acc. - - %% Support functions for writing the cover logs (both cross and normal) write_coverlog_header(CoverLog) -> case catch diff --git a/lib/test_server/src/ts.erl b/lib/test_server/src/ts.erl index 3ddc58fdbc..bb6fb991e0 100644 --- a/lib/test_server/src/ts.erl +++ b/lib/test_server/src/ts.erl @@ -160,8 +160,8 @@ help(installed) -> " the given run options\n", " ts:cross_cover_analyse(Level)\n" " - Used after ts:run with option cover or \n" - " cover_details. Analyses modules specified in\n" - " cross.cover.\n" + " cover_details. Analyses modules specified with\n" + " a 'cross' statement in the cover spec file.\n" " Level can be 'overview' or 'details'.\n", " ts:compile_testcases()~n" " ts:compile_testcases(Apps)~n" @@ -506,8 +506,7 @@ cross_cover_analyse([Level]) -> cross_cover_analyse(Level); cross_cover_analyse(Level) -> Apps = get_last_app_tests(), - Modules = get_cross_modules(Apps,[]), - test_server_ctrl:cross_cover_analyse(Level,Apps,Modules). + test_server_ctrl:cross_cover_analyse(Level,Apps). get_last_app_tests() -> AllTests = filelib:wildcard(filename:join(["*","*_test.logs"])), @@ -536,30 +535,6 @@ get_last_app_tests([Dir|Dirs],RE,Acc) -> get_last_app_tests([],_,Acc) -> Acc. -get_cross_modules([{App,_}|Apps],Acc) -> - Mods = cross_modules(App), - get_cross_modules(Apps,lists:umerge(Mods,Acc)); -get_cross_modules([],Acc) -> - Acc. - -cross_modules(App) -> - case default_coverfile(App) of - none -> - []; - File -> - case catch file:consult(File) of - {ok,CoverSpec} -> - case lists:keyfind(cross_apps,1,CoverSpec) of - false -> - []; - {cross_apps,App,Modules} -> - lists:usort(Modules) - end; - _ -> - [] - end - end. - %%% Implementation. check_and_run(Fun) -> -- cgit v1.2.3 From a71ec754bb995f72f0c7fa244ceb21383d717cfc Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Tue, 27 Nov 2012 16:19:17 +0100 Subject: [test_server] Add test of code cover mechanism --- lib/test_server/test/test_server_SUITE.erl | 100 +++++++++++++++++---- .../test/test_server_SUITE_data/Makefile.src | 4 +- .../test_server_break_SUITE.erl | 10 +-- .../test_server_cover_SUITE.erl | 58 ++++++++++++ .../test_server_cover_SUITE_data/cover_helper.erl | 4 + 5 files changed, 152 insertions(+), 24 deletions(-) create mode 100644 lib/test_server/test/test_server_SUITE_data/test_server_cover_SUITE.erl create mode 100644 lib/test_server/test/test_server_SUITE_data/test_server_cover_SUITE_data/cover_helper.erl (limited to 'lib/test_server') diff --git a/lib/test_server/test/test_server_SUITE.erl b/lib/test_server/test/test_server_SUITE.erl index 95a3423fef..fb82a87fd0 100644 --- a/lib/test_server/test/test_server_SUITE.erl +++ b/lib/test_server/test/test_server_SUITE.erl @@ -80,7 +80,7 @@ all() -> [test_server_SUITE, test_server_parallel01_SUITE, test_server_conf02_SUITE, test_server_conf01_SUITE, test_server_skip_SUITE, test_server_shuffle01_SUITE, - test_server_break_SUITE]. + test_server_break_SUITE, test_server_cover_SUITE]. %%-------------------------------------------------------------------- @@ -93,37 +93,95 @@ test_server_SUITE(Config) -> % rpc:call(Node,dbg, tracer,[]), % rpc:call(Node,dbg, p,[all,c]), % rpc:call(Node,dbg, tpl,[test_server_ctrl,x]), - run_test_server_tests("test_server_SUITE", 38, 1, 30, - 19, 9, 1, 11, 2, 25, Config). + run_test_server_tests("test_server_SUITE", + [{test_server_SUITE,skip_case7,"SKIPPED!"}], + 38, 1, 30, 19, 9, 1, 11, 2, 25, Config). test_server_parallel01_SUITE(Config) -> - run_test_server_tests("test_server_parallel01_SUITE", 37, 0, 19, - 19, 0, 0, 0, 0, 37, Config). + run_test_server_tests("test_server_parallel01_SUITE", [], + 37, 0, 19, 19, 0, 0, 0, 0, 37, Config). test_server_shuffle01_SUITE(Config) -> - run_test_server_tests("test_server_shuffle01_SUITE", 130, 0, 0, - 76, 0, 0, 0, 0, 130, Config). + run_test_server_tests("test_server_shuffle01_SUITE", [], + 130, 0, 0, 76, 0, 0, 0, 0, 130, Config). test_server_skip_SUITE(Config) -> - run_test_server_tests("test_server_skip_SUITE", 3, 0, 1, - 0, 0, 1, 3, 0, 0, Config). + run_test_server_tests("test_server_skip_SUITE", [], + 3, 0, 1, 0, 0, 1, 3, 0, 0, Config). test_server_conf01_SUITE(Config) -> - run_test_server_tests("test_server_conf01_SUITE", 24, 0, 12, - 12, 0, 0, 0, 0, 24, Config). + run_test_server_tests("test_server_conf01_SUITE", [], + 24, 0, 12, 12, 0, 0, 0, 0, 24, Config). test_server_conf02_SUITE(Config) -> - run_test_server_tests("test_server_conf02_SUITE", 26, 0, 12, - 12, 0, 0, 0, 0, 26, Config). + run_test_server_tests("test_server_conf02_SUITE", [], + 26, 0, 12, 12, 0, 0, 0, 0, 26, Config). test_server_break_SUITE(Config) -> - D = run_test_server_tests("test_server_break_SUITE", 8, 2, 6, - 4, 0, 0, 0, 2, 6, Config), - D. + run_test_server_tests("test_server_break_SUITE", [], + 8, 2, 6, 4, 0, 0, 0, 2, 6, Config). -run_test_server_tests(SuiteName, NCases, NFail, NExpected, NSucc, +test_server_cover_SUITE(Config) -> + case test_server:is_cover() of + true -> + {skip, "Cover already running"}; + false -> + PrivDir = ?config(priv_dir,Config), + + %% Test suite has two test cases + %% tc1 calls cover_helper:foo/0 + %% tc2 calls cover_helper:bar/0 + %% Each function in cover_helper is one line. + %% + %% First test run skips tc2, so only cover_helper:foo/0 is executed. + %% Cover file specifies to include cover_helper in this test run. + CoverFile1 = filename:join(PrivDir,"t1.cover"), + CoverSpec1 = {include,[cover_helper]}, + file:write_file(CoverFile1,io_lib:format("~p.~n",[CoverSpec1])), + run_test_server_tests("test_server_cover_SUITE", + [{test_server_cover_SUITE,tc2,"SKIPPED!"}], + 4, 0, 2, 1, 1, 0, 1, 0, 3, + CoverFile1, Config), + + %% Next test run skips tc1, so only cover_helper:bar/0 is executed. + %% Cover file specifies cross compilation of cover_helper + CoverFile2 = filename:join(PrivDir,"t2.cover"), + CoverSpec2 = {cross,[{t1,[cover_helper]}]}, + file:write_file(CoverFile2,io_lib:format("~p.~n",[CoverSpec2])), + run_test_server_tests("test_server_cover_SUITE", + [{test_server_cover_SUITE,tc1,"SKIPPED!"}], + 4, 0, 2, 1, 1, 0, 1, 0, 3, CoverFile2, Config), + + %% Cross cover analyse + WorkDir = ?config(work_dir,Config), + WC = filename:join([WorkDir,"test_server_cover_SUITE.logs","run.*"]), + [D2,D1|_] = lists:reverse(lists:sort(filelib:wildcard(WC))), + TagDirs = [{t1,D1},{t2,D2}], + test_server_ctrl:cross_cover_analyse(details,TagDirs), + + %% Check that cover log shows only what is really included + %% in the test and cross cover log show the accumulated + %% result. + {ok,Cover1} = file:read_file(filename:join(D1,"cover.log")), + [{cover_helper,{1,1,_}}] = binary_to_term(Cover1), + {ok,Cover2} = file:read_file(filename:join(D2,"cover.log")), + [] = binary_to_term(Cover2), + {ok,Cross} = file:read_file(filename:join(D1,"cross_cover.log")), + [{cover_helper,{2,0,_}}] = binary_to_term(Cross), + ok + end. + + +run_test_server_tests(SuiteName, Skip, NCases, NFail, NExpected, NSucc, NUsrSkip, NAutoSkip, NActualSkip, NActualFail, NActualSucc, Config) -> + run_test_server_tests(SuiteName, Skip, NCases, NFail, NExpected, NSucc, + NUsrSkip, NAutoSkip, + NActualSkip, NActualFail, NActualSucc, false, Config). + +run_test_server_tests(SuiteName, Skip, NCases, NFail, NExpected, NSucc, + NUsrSkip, NAutoSkip, + NActualSkip, NActualFail, NActualSucc, Cover, Config) -> WorkDir = proplists:get_value(work_dir, Config), ct:log("Test case log files\n", @@ -131,11 +189,17 @@ run_test_server_tests(SuiteName, NCases, NFail, NExpected, NSucc, Node = proplists:get_value(node, Config), {ok,_Pid} = rpc:call(Node,test_server_ctrl, start, []), + case Cover of + false -> + ok; + _ -> + rpc:call(Node,test_server_ctrl,cover,[Cover,details]) + end, rpc:call(Node, test_server_ctrl,add_dir_with_skip, [SuiteName, [proplists:get_value(data_dir,Config)],SuiteName, - [{test_server_SUITE,skip_case7,"SKIPPED!"}]]), + Skip]), until(fun() -> rpc:call(Node,test_server_ctrl,jobs,[]) =:= [] diff --git a/lib/test_server/test/test_server_SUITE_data/Makefile.src b/lib/test_server/test/test_server_SUITE_data/Makefile.src index ec8ddd78b0..c770627f04 100644 --- a/lib/test_server/test/test_server_SUITE_data/Makefile.src +++ b/lib/test_server/test/test_server_SUITE_data/Makefile.src @@ -5,4 +5,6 @@ all: erlc test_server_shuffle01_SUITE.erl erlc test_server_conf02_SUITE.erl erlc test_server_skip_SUITE.erl - erlc test_server_break_SUITE.erl \ No newline at end of file + erlc test_server_break_SUITE.erl + erlc test_server_cover_SUITE.erl + erlc +debug_info test_server_cover_SUITE_data/cover_helper.erl \ No newline at end of file diff --git a/lib/test_server/test/test_server_SUITE_data/test_server_break_SUITE.erl b/lib/test_server/test/test_server_SUITE_data/test_server_break_SUITE.erl index 70e30a3334..d9f009679a 100644 --- a/lib/test_server/test/test_server_SUITE_data/test_server_break_SUITE.erl +++ b/lib/test_server/test/test_server_SUITE_data/test_server_break_SUITE.erl @@ -41,7 +41,7 @@ init_per_suite(Config) -> spawn(fun break_and_continue_sup/0), Config. -end_per_suite(Config) -> +end_per_suite(_Config) -> ok. init_per_testcase(Case,Config) when Case==break_in_init_tc -> @@ -90,19 +90,19 @@ break_in_end_tc_after_fail(Config) when is_list(Config) -> break_in_end_tc_after_abort(Config) when is_list(Config) -> ?t:adjusted_sleep(2000). % will cause a timetrap timeout -%%%----------------------------------------------------------------- -%%% Internal functions - %% This test case checks that all breaks in previous test cases was %% also continued, and that the break lasted as long as expected. %% The reason for this is that some of the breaks above are in %% end_per_testcase, and failures there will only produce a warning, %% not an error - so this is to catch the error for real. -check_all_breaks(Config) -> +check_all_breaks(Config) when is_list(Config) -> break_and_continue_sup ! {done,self()}, receive {Breaks,Continued} -> check_all_breaks(Breaks,Continued) end. +%%%----------------------------------------------------------------- +%%% Internal functions + check_all_breaks([{From,Case,T,Start}|Breaks],[{From,End}|Continued]) -> Diff = timer:now_diff(End,Start), diff --git a/lib/test_server/test/test_server_SUITE_data/test_server_cover_SUITE.erl b/lib/test_server/test/test_server_SUITE_data/test_server_cover_SUITE.erl new file mode 100644 index 0000000000..b1ae70a302 --- /dev/null +++ b/lib/test_server/test/test_server_SUITE_data/test_server_cover_SUITE.erl @@ -0,0 +1,58 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2012. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(test_server_cover_SUITE). + +-export([all/1, init_per_suite/1, end_per_suite/1]). +-export([init_per_testcase/2, end_per_testcase/2]). +-export([tc1/1, tc2/1]). + +-include_lib("test_server/include/test_server.hrl"). + +all(suite) -> + [tc1,tc2]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(_Case,Config) -> + Dog = ?t:timetrap({minutes,10}), + [{watchdog, Dog}|Config]. + +end_per_testcase(_Case,Config) -> + Dog=?config(watchdog, Config), + ?t:timetrap_cancel(Dog), + ok. + + +%%%----------------------------------------------------------------- +%%% Test cases +tc1(Config) when is_list(Config) -> + cover_helper:foo(), + ok. + +tc2(Config) when is_list(Config) -> + cover_helper:bar(), + ok. + +%%%----------------------------------------------------------------- +%%% Internal functions + diff --git a/lib/test_server/test/test_server_SUITE_data/test_server_cover_SUITE_data/cover_helper.erl b/lib/test_server/test/test_server_SUITE_data/test_server_cover_SUITE_data/cover_helper.erl new file mode 100644 index 0000000000..6c74eb4e8a --- /dev/null +++ b/lib/test_server/test/test_server_SUITE_data/test_server_cover_SUITE_data/cover_helper.erl @@ -0,0 +1,4 @@ +-module(cover_helper). +-compile(export_all). +foo() -> ok. +bar() -> ok. -- cgit v1.2.3 From a6142c21052ab16c080314aa05d1a30e6631a64c Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Tue, 27 Nov 2012 18:41:30 +0100 Subject: [test_server] Update documentation about cross cover Add info about cross statement in cover file: {cross,[{Tag,Modules}]}. Add parameter Tests = [{Tag,Dir}] to test_server_ctrl:cross_cover_analyse --- lib/test_server/doc/src/test_server_ctrl.xml | 93 +++++++++++++++++++++------- lib/test_server/doc/src/ts.xml | 4 +- 2 files changed, 74 insertions(+), 23 deletions(-) (limited to 'lib/test_server') diff --git a/lib/test_server/doc/src/test_server_ctrl.xml b/lib/test_server/doc/src/test_server_ctrl.xml index 41bc0bcc75..af96f1fe7e 100644 --- a/lib/test_server/doc/src/test_server_ctrl.xml +++ b/lib/test_server/doc/src/test_server_ctrl.xml @@ -427,11 +427,21 @@ Optional, if not given the test server controller node

A CoverFile can have the following entries:

{exclude, all | ExcludeModuleList}. -{include, IncludeModuleList}. +{include, IncludeModuleList}. +{cross, CrossCoverInfo}.

Note that each line must end with a full stop. ExcludeModuleList and IncludeModuleList are lists of atoms, where each atom is a module name.

+ +

CrossCoverInfo is used when collecting cover data + over multiple tests. Modules listed here are compiled, but + they will not be analysed when the test is finished. See + cross_cover_analyse/2 + for more information about the cross cover mechanism and the + format of CrossCoverInfo. +

If both an Application and a CoverFile is given, all modules in the application are cover compiled, except for the modules listed in ExcludeModuleList. The @@ -467,30 +477,71 @@ Optional, if not given the test server controller node - cross_cover_analyse(Level) -> ok - Analyse cover data collected from all tests + cross_cover_analyse(Level, Tests) -> ok + Analyse cover data collected from multiple tests Level = details | overview + Tests = [{Tag,LogDir}] + Tag = atom() + Test identifier. + LogDir = string() + Log directory for the test identified by Tag. This + can either be the run.<timestamp> directory or + the parent directory of this (in which case the latest + run.<timestamp> directory is chosen. -

Analyse cover data collected from all tests. The modules - analysed are the ones listed in the cross cover file - cross.cover in the current directory of the test - server.

-

The modules listed in the cross.cover file are - modules that are heavily used by other applications than the - one they belong to. This function should be run after all - tests are completed, and the result will be stored in a file - called cross_cover.html in the run.<timestamp> - directory of the application the modules belong to. -

-

The cross.cover file contains elements like this:

- 
-{App,Modules}.
-

where App can be an application name or the atom - all. The application (or all applications) will cover - compile the listed Modules. -

+

Analyse cover data collected from multiple tests. The modules + analysed are the ones listed in cross statements in + the cover files. These are modules that are heavily used by + other tests than the one where they belong or are explicitly + tested. They should then be listed as cross modules in the + cover file for the test where they are used but do not + belong. Se example below.

+

This function should be run after all tests are completed, + and the result will be stored in a file called + cross_cover.html in the run.<timestamp> + directory of the test the modules belong to.

+

Note that the function can be executed on any node, and it + does not require test_server_ctrl to be started first.

+

The cross statement in the cover file must be like this:

+ +{cross,[{Tag,Modules}]}. +

where Tag is the same as Tag in the + Tests parameter to this function and Modules is a + list of module names (atoms).

+

Example:

+

If the module m1 belongs to system s1 but is + heavily used also in the tests for another system s2, + then the cover files for the two systems' tests could be like + this:

+ +s1.cover: + {include,[m1]}. + +s2.cover: + {include,[....]}. % modules belonging to system s2 + {cross,[{s1,[m1]}]}. +

When the tests for both s1 and s2 are completed, run

+ +test_server_ctrl:cross_cover_analyse(Level,[{s1,S1LogDir},{s2,S2LogDir}]) + + +

and the accumulated cover data for m1 will be written to + S1LogDir/[run.<timestamp>/]cross_cover.html.

+

Note that the m1 module will also be presented in the + normal coverage log for s1 (due to the include statement in + s1.cover), but that only includes the coverage achieved by the + s1 test itself.

+

The Tag in the cross statement in the cover file has + no other purpose than mapping the list of modules + ([m1] in the example above) to the correct log + directory where it should be included in the + cross_cover.html file (S1LogDir in the example + above). I.e. the value of Tag has no meaning, it + could be foo as well as s1 above, as long as + the same Tag is used in the cover file and in the + call to this function.

diff --git a/lib/test_server/doc/src/ts.xml b/lib/test_server/doc/src/ts.xml index 4a2c536e96..82ba3a5017 100644 --- a/lib/test_server/doc/src/ts.xml +++ b/lib/test_server/doc/src/ts.xml @@ -5,7 +5,7 @@
2007 - 2011 + 2012 Ericsson AB, All Rights Reserved @@ -450,7 +450,7 @@ This option is mandatory for remote targets

Analyse cover data collected from all tests.

-

See test_server_ctrl:cross_cover_analyse/1 +

See test_server_ctrl:cross_cover_analyse/2

-- cgit v1.2.3 From 808ec6e48837ea6f13eda9ab403cc3da220137c8 Mon Sep 17 00:00:00 2001 From: Siri Hansen Date: Thu, 13 Dec 2012 11:54:18 +0100 Subject: [test_server] Don't use print/3 in cross_cover_analyse Cross cover analysis is executed outside the scope of test_server_ctrl, i.e. test_server_ctrl is not necessarily started. This may cause test_server_ctrl:print/3 to hang. Instead use io:fwrite directly (as it was before). --- lib/test_server/src/test_server_ctrl.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/test_server') diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 10cba3fbd4..3508623dd9 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -5098,7 +5098,7 @@ cross_cover_analyse(Analyse, TagDirs0) -> TagDirMods = add_cross_modules(TagMods,TagDirs), CoverdataFiles = get_coverdata_files(TagDirMods), lists:foreach(fun(CDF) -> cover:import(CDF) end, CoverdataFiles), - print(stdout, "Cover analysing...\n", []), + io:fwrite("Cover analysing...\n", []), DetailsFun = case Analyse of details -> -- cgit v1.2.3