diff options
Diffstat (limited to 'lib')
19 files changed, 793 insertions, 239 deletions
diff --git a/lib/common_test/doc/src/cover_chapter.xml b/lib/common_test/doc/src/cover_chapter.xml index a215c8c2f3..accb94e1a9 100644 --- a/lib/common_test/doc/src/cover_chapter.xml +++ b/lib/common_test/doc/src/cover_chapter.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>2006</year><year>2013</year> + <year>2006</year><year>2014</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -81,10 +81,7 @@ specify that previously exported data should be imported and included in the analysis for a test (you can specify multiple import files). This way it is possible to analyse total code coverage - without necessarily running all tests at once. Note that even if - you run separate tests in one test run, code coverage data will - not be passed on from one test to another unless you specify an - export file for Common Test to use for this purpose.</p> + without necessarily running all tests at once.</p> <p>To activate the code coverage support, you simply specify the name of the cover specification file as you start Common Test. @@ -266,10 +263,20 @@ ct_cover:cross_cover_analyse(Level, [{s1,S1LogDir},{s2,S2LogDir}]).</code> <section> <title>Logging</title> - <p>To view the result of a code coverage test, follow the - "Coverage log" link on the test suite results page. This - takes you to the code coverage overview page. If you have - successfully performed a detailed coverage analysis, you + <p>To view the result of a code coverage test, click the button + labled "COVER LOG" in the top level index page for the test run.</p> + + <p>Prior to Erlang/OTP 17.1, if your test run consisted of + multiple tests, cover would be started and stopped for each test + within the test run. Separate logs would be available via the + "Coverage log" link on the test suite result pages. These links + are still available, but now they all point to the same page as + the button on the top level index page. The log contains the + accumulated results for the complete test run. See the release + notes for more information about this change.</p> + + <p>The buttonc takes you to the code coverage overview page. If you + have successfully performed a detailed coverage analysis, you find links to each individual module coverage page here.</p> <p>If cross cover analysis has been performed, and there are diff --git a/lib/common_test/src/ct_cover.erl b/lib/common_test/src/ct_cover.erl index ae671c750a..cf2860ae25 100644 --- a/lib/common_test/src/ct_cover.erl +++ b/lib/common_test/src/ct_cover.erl @@ -47,18 +47,21 @@ add_nodes(Nodes) -> undefined -> {error,cover_not_running}; _ -> - {File,Nodes0,Import,Export,AppInfo} = ct_util:get_testdata(cover), + Nodes0 = cover:which_nodes(), Nodes1 = [Node || Node <- Nodes, lists:member(Node,Nodes0) == false], ct_logs:log("COVER INFO", "Adding nodes to cover test: ~w", [Nodes1]), case cover:start(Nodes1) of - Result = {ok,_} -> - ct_util:set_testdata({cover,{File,Nodes1++Nodes0, - Import,Export,AppInfo}}), - + Result = {ok,StartedNodes} -> + ct_logs:log("COVER INFO", + "Successfully added nodes to cover test: ~w", + [StartedNodes]), Result; Error -> + ct_logs:log("COVER INFO", + "Failed to add nodes to cover test: ~tp", + [Error]), Error end end. @@ -81,19 +84,20 @@ remove_nodes(Nodes) -> undefined -> {error,cover_not_running}; _ -> - {File,Nodes0,Import,Export,AppInfo} = ct_util:get_testdata(cover), + Nodes0 = cover:which_nodes(), ToRemove = [Node || Node <- Nodes, lists:member(Node,Nodes0)], ct_logs:log("COVER INFO", - "Removing nodes from cover test: ~w", [ToRemove]), + "Removing nodes from cover test: ~w", [ToRemove]), case cover:stop(ToRemove) of ok -> - Nodes1 = lists:foldl(fun(N,Deleted) -> - lists:delete(N,Deleted) - end, Nodes0, ToRemove), - ct_util:set_testdata({cover,{File,Nodes1, - Import,Export,AppInfo}}), + ct_logs:log("COVER INFO", + "Successfully removed nodes from cover test.", + []), ok; Error -> + ct_logs:log("COVER INFO", + "Failed to remove nodes from cover test: ~tp", + [Error]), Error end end. @@ -149,7 +153,7 @@ get_spec_test(File) -> {value,{_,[Exp]}} -> filename:absname(Exp); _ -> - [] + undefined end, Nodes = case lists:keysearch(nodes, 1, Terms) of diff --git a/lib/common_test/src/ct_framework.erl b/lib/common_test/src/ct_framework.erl index 20903607dc..e8ea7992b4 100644 --- a/lib/common_test/src/ct_framework.erl +++ b/lib/common_test/src/ct_framework.erl @@ -1244,38 +1244,7 @@ report(What,Data) -> ct_logs:make_all_suites_index({TestName,RunDir}), ok; tests_start -> - case ct_util:get_testdata(cover) of - undefined -> - ok; - {_CovFile,_CovNodes,CovImport,CovExport,_CovAppData} -> - %% Always import cover data from files specified by CovImport - %% if no CovExport defined. If CovExport is defined, only - %% import from CovImport files initially, then use CovExport - %% to pass coverdata between proceeding tests (in the same run). - Imps = - case CovExport of - [] -> % don't export data between tests - CovImport; - _ -> - case filelib:is_file(CovExport) of - true -> - [CovExport]; - false -> - CovImport - end - end, - lists:foreach( - fun(Imp) -> - case cover:import(Imp) of - ok -> - ok; - {error,Reason} -> - ct_logs:log("COVER INFO", - "Importing cover data from: ~ts fails! " - "Reason: ~p", [Imp,Reason]) - end - end, Imps) - end; + ok; tests_done -> ok; severe_error -> diff --git a/lib/common_test/src/ct_logs.erl b/lib/common_test/src/ct_logs.erl index a4ad65c0a4..32c8773ca5 100644 --- a/lib/common_test/src/ct_logs.erl +++ b/lib/common_test/src/ct_logs.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2013. All Rights Reserved. +%% Copyright Ericsson AB 2003-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -62,6 +62,7 @@ -define(totals_name, "totals.info"). -define(log_cache_name, "ct_log_cache"). -define(misc_io_log, "misc_io.log.html"). +-define(coverlog_name, "cover.html"). % must be same as in test_server_ctrl -define(table_color1,"#ADD8E6"). -define(table_color2,"#E4F0FE"). @@ -1368,6 +1369,19 @@ index_header(Label, StartTime) -> format_time(StartTime), {[],[1],[2,3,4,5]}) end, + Cover = + case filelib:is_regular(?abs(?coverlog_name)) of + true -> + xhtml(["<p><a href=\"",?coverlog_name, + "\">Cover Log</a></p><br>\n"], + ["<br />" + "<div id=\"button_holder\" class=\"btn\">\n" + "<a href=\"",?coverlog_name, + "\">COVER LOG</a>\n</div><br /><br />"]); + false -> + xhtml("<br>\n", "<br /><br /><br />\n") + end, + [Head | ["<center>\n", xhtml(["<p><a href=\"",?ct_log_name, @@ -1375,8 +1389,8 @@ index_header(Label, StartTime) -> ["<br />" "<div id=\"button_holder\" class=\"btn\">\n" "<a href=\"",?ct_log_name, - "\">COMMON TEST FRAMEWORK LOG</a>\n</div>"]), - xhtml("<br>\n", "<br /><br /><br />\n"), + "\">COMMON TEST FRAMEWORK LOG</a>\n</div><br>\n"]), + Cover, xhtml(["<table border=\"3\" cellpadding=\"5\" " "bgcolor=\"",?table_color3,"\">\n"], ["<table id=\"",?sortable_table_name,"\">\n", diff --git a/lib/common_test/src/ct_run.erl b/lib/common_test/src/ct_run.erl index 03cf06abed..00d0aab507 100644 --- a/lib/common_test/src/ct_run.erl +++ b/lib/common_test/src/ct_run.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2013. All Rights Reserved. +%% Copyright Ericsson AB 2004-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -1646,7 +1646,7 @@ do_run(Tests, Misc, LogDir, LogOpts) when is_list(Misc), do_run(Tests, [], Opts#opts{logdir = LogDir}, []); do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) -> - #opts{label = Label, profile = Profile, cover = Cover, + #opts{label = Label, profile = Profile, verbosity = VLvls} = Opts, %% label - used by ct_logs TestLabel = @@ -1670,22 +1670,6 @@ do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) -> non_existing -> {error,no_path_to_test_server}; _ -> - Opts1 = if Cover == undefined -> - Opts; - true -> - case ct_cover:get_spec(Cover) of - {error,Reason} -> - exit({error,Reason}); - CoverSpec -> - CoverStop = - case Opts#opts.cover_stop of - undefined -> true; - Stop -> Stop - end, - Opts#opts{coverspec = CoverSpec, - cover_stop = CoverStop} - end - end, %% This env variable is used by test_server to determine %% which framework it runs under. case os:getenv("TEST_SERVER_FRAMEWORK") of @@ -1711,7 +1695,7 @@ do_run(Tests, Skip, Opts, Args) when is_record(Opts, opts) -> _Pid -> ct_util:set_testdata({starter,Opts#opts.starter}), compile_and_run(Tests, Skip, - Opts1#opts{verbosity=Verbosity}, Args) + Opts#opts{verbosity=Verbosity}, Args) end end. @@ -2146,67 +2130,11 @@ check_and_add([{TestDir0,M,_} | Tests], Added, PA) -> check_and_add([], _, PA) -> {ok,PA}. -do_run_test(Tests, Skip, Opts) -> +do_run_test(Tests, Skip, Opts0) -> case check_and_add(Tests, [], []) of {ok,AddedToPath} -> ct_util:set_testdata({stats,{0,0,{0,0}}}), - ct_util:set_testdata({cover,undefined}), test_server_ctrl:start_link(local), - case Opts#opts.coverspec of - CovData={CovFile, - CovNodes, - _CovImport, - CovExport, - #cover{app = CovApp, - level = CovLevel, - excl_mods = CovExcl, - incl_mods = CovIncl, - cross = CovCross, - src = _CovSrc}} -> - ct_logs:log("COVER INFO", - "Using cover specification file: ~ts~n" - "App: ~w~n" - "Cross cover: ~w~n" - "Including ~w modules~n" - "Excluding ~w modules", - [CovFile,CovApp,CovCross, - length(CovIncl),length(CovExcl)]), - - %% cover export file will be used for export and import - %% between tests so make sure it doesn't exist initially - case filelib:is_file(CovExport) of - true -> - DelResult = file:delete(CovExport), - ct_logs:log("COVER INFO", - "Warning! " - "Export file ~ts already exists. " - "Deleting with result: ~p", - [CovExport,DelResult]); - false -> - ok - end, - - %% tell test_server which modules should be cover compiled - %% note that actual compilation is done when tests start - test_server_ctrl:cover(CovApp, CovFile, CovExcl, CovIncl, - CovCross, CovExport, CovLevel, - Opts#opts.cover_stop), - %% save cover data (used e.g. to add nodes dynamically) - ct_util:set_testdata({cover,CovData}), - %% start cover on specified nodes - if (CovNodes /= []) and (CovNodes /= undefined) -> - ct_logs:log("COVER INFO", - "Nodes included in cover " - "session: ~w", - [CovNodes]), - cover:start(CovNodes); - true -> - ok - end, - true; - _ -> - false - end, %% let test_server expand the test tuples and count no of cases {Suites,NoOfCases} = count_test_cases(Tests, Skip), @@ -2231,24 +2159,31 @@ do_run_test(Tests, Skip, Opts) -> end, %% if the verbosity level is set lower than ?STD_IMPORTANCE, tell %% test_server to ignore stdout printouts to the test case log file - case proplists:get_value(default, Opts#opts.verbosity) of + case proplists:get_value(default, Opts0#opts.verbosity) of VLvl when is_integer(VLvl), (?STD_IMPORTANCE < (100-VLvl)) -> test_server_ctrl:reject_io_reqs(true); _Lower -> ok end, - test_server_ctrl:multiply_timetraps(Opts#opts.multiply_timetraps), - test_server_ctrl:scale_timetraps(Opts#opts.scale_timetraps), + test_server_ctrl:multiply_timetraps(Opts0#opts.multiply_timetraps), + test_server_ctrl:scale_timetraps(Opts0#opts.scale_timetraps), test_server_ctrl:create_priv_dir(choose_val( - Opts#opts.create_priv_dir, + Opts0#opts.create_priv_dir, auto_per_run)), + + {ok,LogDir} = ct_logs:get_log_dir(true), + {TsCoverInfo,Opts} = maybe_start_cover(Opts0, LogDir), + ct_event:notify(#event{name=start_info, node=node(), data={NoOfTests,NoOfSuites,NoOfCases}}), CleanUp = add_jobs(Tests, Skip, Opts, []), unlink(whereis(test_server_ctrl)), catch test_server_ctrl:wait_finish(), + + maybe_stop_cover(Opts, TsCoverInfo, LogDir), + %% check if last testcase has left a "dead" trace window %% behind, and if so, kill it case ct_util:get_testdata(interpret) of @@ -2281,6 +2216,102 @@ do_run_test(Tests, Skip, Opts) -> exit(Error) end. +maybe_start_cover(Opts=#opts{cover=Cover,cover_stop=CoverStop0},LogDir) -> + if Cover == undefined -> + {undefined,Opts}; + true -> + case ct_cover:get_spec(Cover) of + {error,Reason} -> + exit({error,Reason}); + CoverSpec -> + CoverStop = + case CoverStop0 of + undefined -> true; + Stop -> Stop + end, + start_cover(Opts#opts{coverspec=CoverSpec, + cover_stop=CoverStop}, + LogDir) + end + end. + +start_cover(Opts=#opts{coverspec=CovData,cover_stop=CovStop},LogDir) -> + {CovFile, + CovNodes, + CovImport, + _CovExport, + #cover{app = CovApp, + level = CovLevel, + excl_mods = CovExcl, + incl_mods = CovIncl, + cross = CovCross, + src = _CovSrc}} = CovData, + ct_logs:log("COVER INFO", + "Using cover specification file: ~ts~n" + "App: ~w~n" + "Cross cover: ~w~n" + "Including ~w modules~n" + "Excluding ~w modules", + [CovFile,CovApp,CovCross, + length(CovIncl),length(CovExcl)]), + + %% Tell test_server to print a link in its coverlog + %% pointing to the real coverlog which will be written in + %% maybe_stop_cover/2 + test_server_ctrl:cover({log,LogDir}), + + %% Cover compile all modules + {ok,TsCoverInfo} = test_server_ctrl:cover_compile(CovApp,CovFile, + CovExcl,CovIncl, + CovCross,CovLevel, + CovStop), + ct_logs:log("COVER INFO", + "Compilation completed - test_server cover info: ~tp", + [TsCoverInfo]), + + %% start cover on specified nodes + if (CovNodes /= []) and (CovNodes /= undefined) -> + ct_logs:log("COVER INFO", + "Nodes included in cover " + "session: ~w", + [CovNodes]), + cover:start(CovNodes); + true -> + ok + end, + lists:foreach( + fun(Imp) -> + case cover:import(Imp) of + ok -> + ok; + {error,Reason} -> + ct_logs:log("COVER INFO", + "Importing cover data from: ~ts fails! " + "Reason: ~p", [Imp,Reason]) + end + end, CovImport), + {TsCoverInfo,Opts}. + +maybe_stop_cover(_,undefined,_) -> + ok; +maybe_stop_cover(#opts{coverspec=CovData},TsCoverInfo,LogDir) -> + {_CovFile, + _CovNodes, + _CovImport, + CovExport, + _AppData} = CovData, + case CovExport of + undefined -> ok; + _ -> + ct_logs:log("COVER INFO","Exporting cover data to ~tp",[CovExport]), + cover:export(CovExport) + end, + ct_logs:log("COVER INFO","Analysing cover data to ~tp",[LogDir]), + test_server_ctrl:cover_analyse(TsCoverInfo,LogDir), + ct_logs:log("COVER INFO","Analysis completed.",[]), + ok. + + delete_dups([S | Suites]) -> Suites1 = lists:delete(S, Suites), [S | delete_dups(Suites1)]; diff --git a/lib/common_test/test/Makefile b/lib/common_test/test/Makefile index 085f19d023..a0ac47f12a 100644 --- a/lib/common_test/test/Makefile +++ b/lib/common_test/test/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2008-2013. All Rights Reserved. +# Copyright Ericsson AB 2008-2014. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -61,6 +61,7 @@ MODULES= \ ct_snmp_SUITE \ ct_group_leader_SUITE \ ct_cover_SUITE \ + ct_cover_nomerge_SUITE \ ct_groups_search_SUITE \ ct_surefire_SUITE \ ct_telnet_SUITE diff --git a/lib/common_test/test/ct_cover_SUITE.erl b/lib/common_test/test/ct_cover_SUITE.erl index ec2680f664..47080b5577 100644 --- a/lib/common_test/test/ct_cover_SUITE.erl +++ b/lib/common_test/test/ct_cover_SUITE.erl @@ -172,8 +172,8 @@ cross(Config) -> check_calls(Events2,1), %% Get the log dirs for each test and run cross cover analyse - [D11,D12] = lists:sort(get_run_dirs(Events1)), - [D21,D22] = lists:sort(get_run_dirs(Events2)), + [D11,D12] = lists:sort(get_log_dirs(Events1)), + [D21,D22] = lists:sort(get_log_dirs(Events2)), ct_cover:cross_cover_analyse(details,[{cross1,D11},{cross2,D21}]), ct_cover:cross_cover_analyse(details,[{cross1,D12},{cross2,D22}]), @@ -267,18 +267,17 @@ check_cover(Node) when is_atom(Node) -> false end. -%% Get the log dir "run.<timestamp>" for all (both!) tests -get_run_dirs(Events) -> - [filename:dirname(TCLog) || +%% Get the log dir "ct_run.<timestamp>" for all (both!) tests +get_log_dirs(Events) -> + [LogDir || {ct_test_support_eh, - {event,tc_logfile,_Node, - {{?suite,init_per_suite},TCLog}}} <- Events]. + {event,start_logging,_Node,LogDir}} <- Events]. %% Check that each coverlog includes N calls to ?mod:foo/0 check_calls(Events,N) -> check_calls(Events,{?mod,foo,0},N). check_calls(Events,MFA,N) -> - CoverLogs = [filename:join(D,"all.coverdata") || D <- get_run_dirs(Events)], + CoverLogs = [filename:join(D,"all.coverdata") || D <- get_log_dirs(Events)], do_check_logs(CoverLogs,MFA,N). do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) -> diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE.erl new file mode 100644 index 0000000000..8e2ee1b500 --- /dev/null +++ b/lib/common_test/test/ct_cover_nomerge_SUITE.erl @@ -0,0 +1,221 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2014. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File: ct_cover_nomerge_SUITE +%%% +%%% Description: +%%% Test code cover analysis support when merge_tests=false +%%% +%%%------------------------------------------------------------------- +-module(ct_cover_nomerge_SUITE). + +-compile(export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("common_test/include/ct_event.hrl"). + +-define(eh, ct_test_support_eh). +-define(mod, cover_test_mod). + +%%-------------------------------------------------------------------- +%% TEST SERVER CALLBACK FUNCTIONS +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% Description: Since Common Test starts another Test Server +%% instance, the tests need to be performed on a separate node (or +%% there will be clashes with logging processes etc). +%%-------------------------------------------------------------------- +init_per_suite(Config) -> + case test_server:is_cover() of + true -> + {skip,"Test server is running cover already - skipping"}; + false -> + ct_test_support:init_per_suite(Config) + end. + +end_per_suite(Config) -> + ct_test_support:end_per_suite(Config). + +init_per_testcase(TestCase, Config) -> + ct_test_support:init_per_testcase(TestCase, Config). + +end_per_testcase(TestCase, Config) -> + try apply(?MODULE,TestCase,[cleanup,Config]) + catch error:undef -> ok + end, + ct_test_support:end_per_testcase(TestCase, Config). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ + local, + remote, + remote_nostop + ]. + +%%-------------------------------------------------------------------- +%% TEST CASES +%%-------------------------------------------------------------------- + +local(Config) -> + DataDir = ?config(data_dir, Config), + Spec = filename:join(DataDir, "local.spec"), + CoverSpec = [{incl_mods,[?mod]}], + CoverFile = create_cover_file(local,CoverSpec,Config), + {Opts,ERPid} = setup([{spec,Spec},{label,local},{cover,CoverFile}], Config), + {ok,Events} = execute(local, local, Opts, ERPid, Config), + false = check_cover(Config), + check_calls(Events,2), + ok. + +remote(Config) -> + DataDir = ?config(data_dir, Config), + Spec = filename:join(DataDir, "remote.spec"), + %% extending some timers for slow test hosts + {ok,Node} = ct_slave:start(ct_nomerge,[{boot_timeout,15}, + {init_timeout,15}, + {startup_timeout,15}]), + + CoverSpec = [{nodes,[Node]}, + {incl_mods,[?mod]}], + CoverFile = create_cover_file(remote,CoverSpec,Config), + {Opts,ERPid} = setup([{spec,Spec},{label,remote},{cover,CoverFile}], Config), + {ok,Events} = execute(remote, remote, Opts, ERPid, Config), + false = check_cover(Config), + check_calls(Events,2), + ok. +remote(cleanup,_Config) -> + {ok,_} = ct_slave:stop(ct_nomerge), + ok. + +remote_nostop(Config) -> + DataDir = ?config(data_dir, Config), + Spec = filename:join(DataDir, "remote_nostop.spec"), + %% extending some timers for slow test hosts + {ok,Node} = ct_slave:start(ct_nomerge,[{boot_timeout,15}, + {init_timeout,15}, + {startup_timeout,15}]), + + CoverSpec = [{nodes,[Node]}, + {incl_mods,[?mod]}], + CoverFile = create_cover_file(remote_nostop,CoverSpec,Config), + {Opts,ERPid} = setup([{spec,Spec},{label,remote_nostop}, + {cover,CoverFile},{cover_stop,false}], + Config), + {ok,Events} = execute(remote_nostop, remote_nostop, Opts, ERPid, Config), + {true,[Node],[cover_test_mod]} = check_cover(Config), + check_calls(Events,2), + ok. +remote_nostop(cleanup,Config) -> + CtNode = ?config(ct_node,Config), + ok = rpc:call(CtNode,cover,stop,[]), + {ok,_} = ct_slave:stop(ct_nomerge), + ok. + + +%%%----------------------------------------------------------------- +%%% HELP FUNCTIONS +%%%----------------------------------------------------------------- +setup(Test, Config) -> + Opts0 = ct_test_support:get_opts(Config), + Level = ?config(trace_level, Config), + EvHArgs = [{cbm,ct_test_support},{trace_level,Level}], + Opts = Opts0 ++ [{event_handler,{?eh,EvHArgs}}|Test], + ERPid = ct_test_support:start_event_receiver(Config), + {Opts,ERPid}. + +execute(Name, Testcase, Opts, ERPid, Config) -> + ok = ct_test_support:run(Opts, Config), + Events = ct_test_support:get_events(ERPid, Config), + + ct_test_support:log_events(Name, + reformat(Events, ?eh), + ?config(priv_dir, Config), + Opts), + TestEvents = events_to_check(Testcase), + R = ct_test_support:verify_events(TestEvents, Events, Config), + {R,Events}. + +reformat(Events, EH) -> + ct_test_support:reformat(Events, EH). + +events_to_check(local) -> + events_to_check1(cover_nomerge_local_SUITE); +events_to_check(remote) -> + events_to_check1(cover_nomerge_remote_SUITE); +events_to_check(remote_nostop) -> + events_to_check1(cover_nomerge_remote_nostop_SUITE). +events_to_check1(Suite) -> + OneTest = + [{?eh,start_logging,{'DEF','RUNDIR'}}] ++ + [{?eh,tc_done,{Suite,t1,ok}}] ++ + [{?eh,tc_done,{Suite,t2,ok}}] ++ + [{?eh,stop_logging,[]}], + + %% 2 tests (ct:run_test + script_start) is default + OneTest ++ OneTest. + +check_cover(Config) when is_list(Config) -> + CTNode = proplists:get_value(ct_node, Config), + check_cover(CTNode); +check_cover(Node) when is_atom(Node) -> + case rpc:call(Node,test_server,is_cover,[]) of + true -> + {true, + rpc:call(Node,cover,which_nodes,[]), + rpc:call(Node,cover,modules,[])}; + false -> + false + end. + +%% Get the log dir "ct_run.<timestamp>" for all (both!) tests +get_log_dirs(Events) -> + [LogDir || + {ct_test_support_eh, + {event,start_logging,_Node,LogDir}} <- Events]. + +%% Check that each coverlog includes N calls to ?mod:foo/0 +check_calls(Events,N) -> + check_calls(Events,{?mod,foo,0},N). +check_calls(Events,MFA,N) -> + CoverLogs = [filename:join(D,"all.coverdata") || D <- get_log_dirs(Events)], + do_check_logs(CoverLogs,MFA,N). + +do_check_logs([CoverLog|CoverLogs],{Mod,_,_} = MFA,N) -> + {ok,_} = cover:start(), + ok = cover:import(CoverLog), + {ok,Calls} = cover:analyse(Mod,calls,function), + ok = cover:stop(), + {MFA,N} = lists:keyfind(MFA,1,Calls), + do_check_logs(CoverLogs,MFA,N); +do_check_logs([],_,_) -> + ok. + +create_cover_file(Filename,Terms,Config) -> + PrivDir = ?config(priv_dir,Config), + File = filename:join(PrivDir,Filename) ++ ".cover", + {ok,Fd} = file:open(File,[write]), + lists:foreach(fun(Term) -> + file:write(Fd,io_lib:format("~p.~n",[Term])) + end,Terms), + ok = file:close(Fd), + File. diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_local_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_local_SUITE.erl new file mode 100644 index 0000000000..e1fe3b5fc9 --- /dev/null +++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_local_SUITE.erl @@ -0,0 +1,63 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2014. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%%---------------------------------------------------------------------- +-module(cover_nomerge_local_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +%% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +suite() -> + []. + +all() -> + [t1,t2]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + Config. + +init_per_testcase(_Case, Config) -> + Dog = test_server:timetrap(?default_timeout), + [{watchdog, Dog}|Config]. + +end_per_testcase(Case, Config) -> + Dog=?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +%%%----------------------------------------------------------------- +%%% Test cases +break(_Config) -> + test_server:break(""), + ok. + +t1(_Config) -> + cover_compiled = code:which(cover_test_mod), + ok = cover_test_mod:foo(), + ok. + +t2(_Config) -> + cover_compiled = code:which(cover_test_mod), + ok = cover_test_mod:foo(), + ok. diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_SUITE.erl new file mode 100644 index 0000000000..a77ae0c2db --- /dev/null +++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_SUITE.erl @@ -0,0 +1,75 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2014. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%%---------------------------------------------------------------------- +-module(cover_nomerge_remote_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +%% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +suite() -> + []. + +all() -> + [t1,t2]. + +init_per_suite(Config) -> + {ok,Host} = inet:gethostname(), + Node = list_to_atom("ct_nomerge@"++Host), + pong = net_adm:ping(Node), + +%% Include this row, and exclude the equivalent row in end_per_suite => +%% fails every now and then with missing data. Why? +%% ct_cover:remove_nodes([Node]), + ct_cover:add_nodes([Node]), + [{node,Node}|Config]. + +end_per_suite(Config) -> + Node = ?config(node,Config), + ct_cover:remove_nodes([Node]), + Config. + +init_per_testcase(_Case, Config) -> + Dog = test_server:timetrap(?default_timeout), + [{watchdog, Dog}|Config]. + +end_per_testcase(Case, Config) -> + Dog=?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +%%%----------------------------------------------------------------- +%%% Test cases +break(_Config) -> + test_server:break(""), + ok. + +t1(Config) -> + Node = ?config(node,Config), + cover_compiled = rpc:call(Node, code, which, [cover_test_mod]), + ok = rpc:call(Node, cover_test_mod, foo, []), + ok. + +t2(Config) -> + Node = ?config(node,Config), + cover_compiled = rpc:call(Node, code, which, [cover_test_mod]), + ok = rpc:call(Node, cover_test_mod, foo, []), + ok. diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_nostop_SUITE.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_nostop_SUITE.erl new file mode 100644 index 0000000000..0b3159f2c3 --- /dev/null +++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_nomerge_remote_nostop_SUITE.erl @@ -0,0 +1,68 @@ +%%-------------------------------------------------------------------- +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2014. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%%---------------------------------------------------------------------- +-module(cover_nomerge_remote_nostop_SUITE). +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +%% Default timetrap timeout (set in init_per_testcase). +-define(default_timeout, ?t:minutes(1)). + +suite() -> + []. + +all() -> + [t1,t2]. + +init_per_suite(Config) -> + {ok,Host} = inet:gethostname(), + Node = list_to_atom("ct_nomerge@"++Host), + pong = net_adm:ping(Node), + [{node,Node}|Config]. + +end_per_suite(Config) -> + Config. + +init_per_testcase(_Case, Config) -> + Dog = test_server:timetrap(?default_timeout), + [{watchdog, Dog}|Config]. + +end_per_testcase(Case, Config) -> + Dog=?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +%%%----------------------------------------------------------------- +%%% Test cases +break(_Config) -> + test_server:break(""), + ok. + +t1(Config) -> + Node = ?config(node,Config), + cover_compiled = rpc:call(Node, code, which, [cover_test_mod]), + ok = rpc:call(Node, cover_test_mod, foo, []), + ok. + +t2(Config) -> + Node = ?config(node,Config), + cover_compiled = rpc:call(Node, code, which, [cover_test_mod]), + ok = rpc:call(Node, cover_test_mod, foo, []), + ok. diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_test_mod.erl b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_test_mod.erl new file mode 100644 index 0000000000..d4f69452c3 --- /dev/null +++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/cover_test_mod.erl @@ -0,0 +1,4 @@ +-module(cover_test_mod). +-compile(export_all). +foo() -> + ok. diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/local.spec b/lib/common_test/test/ct_cover_nomerge_SUITE_data/local.spec new file mode 100644 index 0000000000..893c48b010 --- /dev/null +++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/local.spec @@ -0,0 +1,6 @@ +{merge_tests,false}. + +{alias,dir,"."}. + +{cases, dir, cover_nomerge_local_SUITE, [t1]}. +{cases, dir, cover_nomerge_local_SUITE, [t2]}. diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote.spec b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote.spec new file mode 100644 index 0000000000..78c4332270 --- /dev/null +++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote.spec @@ -0,0 +1,6 @@ +{merge_tests,false}. + +{alias,dir,"."}. + +{cases, dir, cover_nomerge_remote_SUITE, [t1]}. +{cases, dir, cover_nomerge_remote_SUITE, [t2]}. diff --git a/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote_nostop.spec b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote_nostop.spec new file mode 100644 index 0000000000..049f586c72 --- /dev/null +++ b/lib/common_test/test/ct_cover_nomerge_SUITE_data/remote_nostop.spec @@ -0,0 +1,6 @@ +{merge_tests,false}. + +{alias,dir,"."}. + +{cases, dir, cover_nomerge_remote_nostop_SUITE, [t1]}. +{cases, dir, cover_nomerge_remote_nostop_SUITE, [t2]}. diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl index 70dc7a1441..f63bfdec05 100644 --- a/lib/test_server/src/test_server.erl +++ b/lib/test_server/src/test_server.erl @@ -22,7 +22,7 @@ %%% TEST_SERVER_CTRL INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -export([run_test_case_apply/1,init_target_info/0,init_purify/0]). --export([cover_compile/1,cover_analyse/3]). +-export([cover_compile/1,cover_analyse/2]). %%% TEST_SERVER_SUP INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -export([get_loc/1,set_tc_state/1]). @@ -80,8 +80,8 @@ init_purify() -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% cover_compile({App,Include,Exclude,Cross}) -> -%% {ok,AnalyseModules} | {error,Reason} +%% cover_compile(#cover{app=App,incl=Include,excl=Exclude,cross=Cross}) -> +%% {ok,#cover{mods=AnalyseModules}} | {error,Reason} %% %% App = atom() , name of application to be compiled %% Exclude = [atom()], list of modules to exclude @@ -90,33 +90,35 @@ init_purify() -> %% Cross = [atoms()], list of modules outside of App shat should be included %% in the cover compilation, but that shall not be part of %% the cover analysis for this application. +%% AnalyseModules = [atom()], list of successfully compiled modules %% -%% Cover compile the given application. Return {ok,AnalyseMods} if application -%% is found, else {error,application_not_found}. +%% Cover compile the given application. Return {ok,CoverInfo} if +%% compilation succeeds, else (if application is not found and there +%% are no modules to compile) {error,application_not_found}. -cover_compile({none,_Exclude,Include,Cross}) -> +cover_compile(CoverInfo=#cover{app=none,incl=Include,cross=Cross}) -> CrossMods = lists:flatmap(fun({_,M}) -> M end,Cross), CompileMods = Include++CrossMods, case length(CompileMods) of 0 -> io:fwrite("WARNING: No modules to cover compile!\n\n",[]), cover:start(), % start cover server anyway - {ok,[]}; + {ok,CoverInfo#cover{mods=[]}}; N -> io:fwrite("Cover compiling ~w modules - " "this may take some time... ",[N]), do_cover_compile(CompileMods), io:fwrite("done\n\n",[]), - {ok,Include} + {ok,CoverInfo#cover{mods=Include}} end; -cover_compile({App,all,Include,Cross}) -> +cover_compile(CoverInfo=#cover{app=App,excl=all,incl=Include,cross=Cross}) -> CrossMods = lists:flatmap(fun({_,M}) -> M end,Cross), CompileMods = Include++CrossMods, case length(CompileMods) of 0 -> io:fwrite("WARNING: No modules to cover compile!\n\n",[]), cover:start(), % start cover server anyway - {ok,[]}; + {ok,CoverInfo#cover{mods=[]}}; N -> io:fwrite("Cover compiling '~w' (~w files) - " "this may take some time... ",[App,N]), @@ -126,9 +128,9 @@ cover_compile({App,all,Include,Cross}) -> "~tp\n", [App,CompileMods]), do_cover_compile(CompileMods), io:fwrite("done\n\n",[]), - {ok,Include} + {ok,CoverInfo=#cover{mods=Include}} end; -cover_compile({App,Exclude,Include,Cross}) -> +cover_compile(CoverInfo=#cover{app=App,excl=Exclude,incl=Include,cross=Cross}) -> CrossMods = lists:flatmap(fun({_,M}) -> M end,Cross), case code:lib_dir(App) of {error,bad_name} -> @@ -146,7 +148,7 @@ cover_compile({App,Exclude,Include,Cross}) -> "~tp\n", [App,Include]), do_cover_compile(CompileMods), io:fwrite("done\n\n",[]), - {ok,Include} + {ok,CoverInfo#cover{mods=Include}} end; LibDir -> EbinDir = filename:join([LibDir,"ebin"]), @@ -158,13 +160,13 @@ cover_compile({App,Exclude,Include,Cross}) -> 0 -> io:fwrite("WARNING: No modules to cover compile!\n\n",[]), cover:start(), % start cover server anyway - {ok,[]}; + {ok,CoverInfo#cover{mods=[]}}; N -> io:fwrite("Cover compiling '~w' (~w files) - " "this may take some time... ",[App,N]), do_cover_compile(CompileMods), io:fwrite("done\n\n",[]), - {ok,AnalyseMods} + {ok,CoverInfo#cover{mods=AnalyseMods}} end end. @@ -174,9 +176,11 @@ module_names(Beams) -> do_cover_compile(Modules) -> - do_cover_compile1(lists:usort(Modules)). % remove duplicates + cover:start(), + pmap1(fun(M) -> do_cover_compile1(M) end,lists:usort(Modules)), + ok. -do_cover_compile1([M|Rest]) -> +do_cover_compile1(M) -> case {code:is_sticky(M),code:is_loaded(M)} of {true,_} -> code:unstick_mod(M), @@ -187,15 +191,13 @@ do_cover_compile1([M|Rest]) -> io:fwrite("\nWARNING: Could not cover compile ~w: ~p\n", [M,Error]) end, - code:stick_mod(M), - do_cover_compile1(Rest); + code:stick_mod(M); {false,false} -> case code:load_file(M) of {module,_} -> - do_cover_compile1([M|Rest]); + do_cover_compile1(M); Error -> - io:fwrite("\nWARNING: Could not load ~w: ~p\n",[M,Error]), - do_cover_compile1(Rest) + io:fwrite("\nWARNING: Could not load ~w: ~p\n",[M,Error]) end; {false,_} -> case cover:compile_beam(M) of @@ -204,23 +206,52 @@ do_cover_compile1([M|Rest]) -> Error -> io:fwrite("\nWARNING: Could not cover compile ~w: ~p\n", [M,Error]) - end, - do_cover_compile1(Rest) - end; -do_cover_compile1([]) -> - ok. + end + end. + +pmap1(Fun,List) -> + NTot = length(List), + NProcs = erlang:system_info(schedulers) * 2, + NPerProc = (NTot div NProcs) + 1, + + {[],Pids} = + lists:foldr( + fun(_,{L,Ps}) -> + {L1,L2} = if length(L)>=NPerProc -> lists:split(NPerProc,L); + true -> {L,[]} % last chunk + end, + {P,_Ref} = + spawn_monitor(fun() -> + exit(lists:map(Fun,L1)) + end), + {L2,[P|Ps]} + end, + {List,[]}, + lists:seq(1,NProcs)), + collect(Pids,[]). + +collect([],Acc) -> + lists:append(Acc); +collect([Pid|Pids],Acc) -> + receive + {'DOWN', _Ref, process, Pid, Result} -> + %% collect(lists:delete(Pid,Pids),[Result|Acc]) + collect(Pids,[Result|Acc]) + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% cover_analyse(Analyse,Modules,Stop) -> [{M,{Cov,NotCov,Details}}] +%% cover_analyse(Dir,#cover{level=Analyse,mods=Modules,stop=Stop) -> +%% [{M,{Cov,NotCov,Details}}] %% -%% Analyse = {details,Dir} | details | {overview,void()} | overview +%% Dir = string() +%% Analyse = details | overview %% Modules = [atom()], the modules to analyse %% -%% Cover analysis. If Analyse=={details,Dir} analyse_to_file is used. +%% Cover analysis. If Analyse==details analyse_to_file is used. %% -%% If Analyse=={overview,Dir} analyse_to_file is not used, only an -%% overview containing the number of covered/not covered lines in each -%% module. +%% If Analyse==overview analyse_to_file is not used, only an overview +%% containing the number of covered/not covered lines in each module. %% %% Also, cover data will be exported to a file called all.coverdata in %% the given directory. @@ -235,11 +266,11 @@ do_cover_compile1([]) -> %% which means that the modules will stay cover compiled. Note that %% this is only recommended if the erlang node is being terminated %% after the test is completed. -cover_analyse(Analyse,Modules,Stop) -> - print(stdout, "Cover analysing...\n", []), +cover_analyse(Dir,#cover{level=Analyse,mods=Modules,stop=Stop}) -> + io:fwrite(user, "Cover analysing... ", []), DetailsFun = case Analyse of - {details,Dir} -> + details -> case cover:export(filename:join(Dir,"all.coverdata")) of ok -> fun(M) -> @@ -256,7 +287,7 @@ cover_analyse(Analyse,Modules,Stop) -> Error -> fun(_) -> Error end end; - {overview,Dir} -> + overview -> case cover:export(filename:join(Dir,"all.coverdata")) of ok -> fun(_) -> undefined end; @@ -264,17 +295,19 @@ cover_analyse(Analyse,Modules,Stop) -> fun(_) -> Error end end end, - R = pmap( + R = pmap2( fun(M) -> case cover:analyse(M,module) of {ok,{M,{Cov,NotCov}}} -> {M,{Cov,NotCov,DetailsFun(M)}}; Err -> - io:fwrite("WARNING: Analysis failed for ~w. Reason: ~p\n", + io:fwrite(user, + "\nWARNING: Analysis failed for ~w. Reason: ~p\n", [M,Err]), {M,Err} end end, Modules), + io:fwrite(user, "done\n\n", []), case Stop of true -> @@ -286,12 +319,12 @@ cover_analyse(Analyse,Modules,Stop) -> end, R. -pmap(Fun,List) -> +pmap2(Fun,List) -> Collector = self(), Pids = lists:map(fun(E) -> spawn(fun() -> - Collector ! {res,self(),Fun(E)} - end) + Collector ! {res,self(),Fun(E)} + end) end, List), lists:map(fun(Pid) -> receive @@ -300,7 +333,6 @@ pmap(Fun,List) -> end end, Pids). - do_cover_for_node(Node,CoverFunc) -> do_cover_for_node(Node,CoverFunc,true). do_cover_for_node(Node,CoverFunc,StickUnstick) -> diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 5fbc47a813..c3550d4533 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2013. All Rights Reserved. +%% Copyright Ericsson AB 2002-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -52,7 +52,9 @@ -export([reject_io_reqs/1, get_levels/0, set_levels/3]). -export([multiply_timetraps/1, scale_timetraps/1, get_timetrap_parameters/0]). -export([create_priv_dir/1]). --export([cover/2, cover/3, cover/8, cross_cover_analyse/2, trc/1, stop_trace/0]). +-export([cover/1, cover/2, cover/3, + cover_compile/7, cover_analyse/2, cross_cover_analyse/2, + trc/1, stop_trace/0]). -export([testcase_callback/1]). -export([set_random_seed/1]). -export([kill_slavenodes/0]). @@ -409,11 +411,26 @@ cover(App, Analyse) when is_atom(App) -> cover(CoverFile, Analyse) -> cover(none, CoverFile, Analyse). cover(App, CoverFile, Analyse) -> - controller_call({cover,{App,CoverFile},Analyse,true}). -cover(App, CoverFile, Exclude, Include, Cross, Export, Analyse, Stop) -> - controller_call({cover, - {App,{CoverFile,Exclude,Include,Cross,Export}}, - Analyse,Stop}). + {Excl,Incl,Cross} = read_cover_file(CoverFile), + CoverInfo = #cover{app=App, + file=CoverFile, + excl=Excl, + incl=Incl, + cross=Cross, + level=Analyse}, + controller_call({cover,CoverInfo}). + +cover(CoverInfo) -> + controller_call({cover,CoverInfo}). + +cover_compile(App,File,Excl,Incl,Cross,Analyse,Stop) -> + cover_compile(#cover{app=App, + file=File, + excl=Excl, + incl=Incl, + cross=Cross, + level=Analyse, + stop=Stop}). testcase_callback(ModFunc) -> controller_call({testcase_callback,ModFunc}). @@ -563,7 +580,7 @@ handle_call({add_job,Dir,Name,TopCase,Skip}, _From, State) -> ExtraTools = case State#state.cover of false -> []; - {App,Analyse,Stop} -> [{cover,App,Analyse,Stop}] + CoverInfo -> [{cover,CoverInfo}] end, ExtraTools1 = case State#state.random_seed of @@ -816,13 +833,13 @@ handle_call(stop_trace, _From, State) -> {reply,R,State#state{trc=false}}; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% handle_call({cover,App,Analyse,Stop}, _, State) -> ok | {error,Reason} +%% handle_call({cover,CoverInfo}, _, State) -> ok | {error,Reason} %% -%% All modules inn application App are cover compiled -%% Analyse indicates on which level the coverage should be analysed +%% Set specification of cover analysis to be used when running tests +%% (see start_extra_tools/1 and stop_extra_tools/1) -handle_call({cover,App,Analyse,Stop}, _From, State) -> - {reply,ok,State#state{cover={App,Analyse,Stop}}}; +handle_call({cover,CoverInfo}, _From, State) -> + {reply,ok,State#state{cover=CoverInfo}}; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% handle_call({create_priv_dir,Value}, _, State) -> ok | {error,Reason} @@ -1203,11 +1220,10 @@ elapsed_time(Before, After) -> start_extra_tools(ExtraTools) -> start_extra_tools(ExtraTools, []). -start_extra_tools([{cover,App,Analyse,Stop} | ExtraTools], Started) -> - case cover_compile(App) of - {ok,AnalyseMods} -> - start_extra_tools(ExtraTools, - [{cover,App,Analyse,AnalyseMods,Stop}|Started]); +start_extra_tools([{cover,CoverInfo} | ExtraTools], Started) -> + case start_cover(CoverInfo) of + {ok,NewCoverInfo} -> + start_extra_tools(ExtraTools,[{cover,NewCoverInfo}|Started]); {error,_} -> start_extra_tools(ExtraTools, Started) end; @@ -1226,8 +1242,8 @@ stop_extra_tools(ExtraTools) -> end, stop_extra_tools(ExtraTools, TestDir). -stop_extra_tools([{cover,App,Analyse,AnalyseMods,Stop}|ExtraTools], TestDir) -> - cover_analyse(App, Analyse, AnalyseMods, Stop, TestDir), +stop_extra_tools([{cover,CoverInfo}|ExtraTools], TestDir) -> + stop_cover(CoverInfo,TestDir), stop_extra_tools(ExtraTools, TestDir); %%stop_extra_tools([_ | ExtraTools], TestDir) -> %% stop_extra_tools(ExtraTools, TestDir); @@ -1569,13 +1585,20 @@ do_test_cases(TopCases, SkipCases, ok end end, - + CoverLog = + case get(test_server_cover_log_dir) of + undefined -> + ?coverlog_name; + AbsLogDir -> + AbsLog = filename:join(AbsLogDir,?coverlog_name), + make_relative(AbsLog, TestDir) + end, print(html, "<p><ul>\n" "<li><a href=\"~ts\">Full textual log</a></li>\n" "<li><a href=\"~ts\">Coverage log</a></li>\n" "<li><a href=\"~ts\">Unexpected I/O log</a></li>\n</ul></p>\n", - [?suitelog_name,?coverlog_name,?unexpected_io_log]), + [?suitelog_name,CoverLog,?unexpected_io_log]), print(html, "<p>~ts</p>\n" ++ xhtml("<table bgcolor=\"white\" border=\"3\" cellpadding=\"5\">", @@ -5087,15 +5110,15 @@ pinfo(P) -> %% Cover compilation %% The compilation is executed on the target node -cover_compile({App,{_File,Exclude,Include,Cross,_Export}}) -> - cover_compile1({App,Exclude,Include,Cross}); +start_cover(#cover{}=CoverInfo) -> + cover_compile(CoverInfo); +start_cover({log,CoverLogDir}=CoverInfo) -> + %% Cover is controlled by the framework - here's the log + put(test_server_cover_log_dir,CoverLogDir), + {ok,CoverInfo}. -cover_compile({App,CoverFile}) -> - {Exclude,Include,Cross} = read_cover_file(CoverFile), - cover_compile1({App,Exclude,Include,Cross}). - -cover_compile1(What) -> - test_server:cover_compile(What). +cover_compile(CoverInfo) -> + test_server:cover_compile(CoverInfo). %% Read the coverfile for an application and return a list of modules %% that are members of the application but shall not be compiled @@ -5163,25 +5186,45 @@ check_cross([]) -> %% %% This per application analysis writes the file cover.html in the %% application's run.<timestamp> directory. -cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, Stop, TestDir) -> +stop_cover(#cover{}=CoverInfo, TestDir) -> + cover_analyse(CoverInfo, TestDir); +stop_cover(_CoverInfo, _TestDir) -> + %% Cover is probably controlled by the framework + ok. + +make_relative(AbsDir, VsDir) -> + DirTokens = filename:split(AbsDir), + VsTokens = filename:split(VsDir), + filename:join(make_relative1(DirTokens, VsTokens)). + +make_relative1([T | DirTs], [T | VsTs]) -> + make_relative1(DirTs, VsTs); +make_relative1(Last = [_File], []) -> + Last; +make_relative1(Last = [_File], VsTs) -> + Ups = ["../" || _ <- VsTs], + Ups ++ Last; +make_relative1(DirTs, []) -> + DirTs; +make_relative1(DirTs, VsTs) -> + Ups = ["../" || _ <- VsTs], + Ups ++ DirTs. + + +cover_analyse(CoverInfo, TestDir) -> write_default_cross_coverlog(TestDir), {ok,CoverLog} = open_html_file(filename:join(TestDir, ?coverlog_name)), write_coverlog_header(CoverLog), + #cover{app=App, + file=CoverFile, + excl=Excluded, + cross=Cross} = CoverInfo, io:fwrite(CoverLog, "<h1>Coverage for application '~w'</h1>\n", [App]), io:fwrite(CoverLog, "<p><a href=\"~ts\">Coverdata collected over all tests</a></p>", [?cross_coverlog_name]), - {CoverFile,_Included,Excluded,Cross} = - case CoverInfo of - {File,Excl,Incl,Cr,Export} -> - cover:export(Export), - {File,Incl,Excl,Cr}; - File -> - {Excl,Incl,Cr} = read_cover_file(File), - {File,Incl,Excl,Cr} - end, io:fwrite(CoverLog, "<p>CoverFile: <code>~tp</code>\n", [CoverFile]), write_cross_cover_info(TestDir,Cross), @@ -5196,7 +5239,7 @@ cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, Stop, TestDir) -> io:fwrite(CoverLog, "<p>Excluded module(s): <code>~tp</code>\n", [Excluded]), - Coverage = cover_analyse(Analyse, AnalyseMods, Stop), + Coverage = test_server:cover_analyse(TestDir, CoverInfo), write_binary_file(filename:join(TestDir,?raw_coverlog_name), term_to_binary(Coverage)), @@ -5215,11 +5258,6 @@ cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, Stop, TestDir) -> write_binary_file(filename:join(TestDir, ?cover_total), term_to_binary(TotPercent)). -cover_analyse(Analyse, AnalyseMods, Stop) -> - TestDir = get(test_server_log_dir_base), - test_server:cover_analyse({Analyse,TestDir}, AnalyseMods, Stop). - - %% Cover analysis - accumulated over multiple tests %% This can be executed on any node after all tests are finished. %% Analyse = overview | details diff --git a/lib/test_server/src/test_server_internal.hrl b/lib/test_server/src/test_server_internal.hrl index 4e734a330b..bafeeaabfe 100644 --- a/lib/test_server/src/test_server_internal.hrl +++ b/lib/test_server/src/test_server_internal.hrl @@ -49,3 +49,12 @@ master, cookie}). + +-record(cover, {app, % application; Name | none + file, % cover spec file + incl, % explicitly include modules + excl, % explicitly exclude modules + level, % analyse level; details | overview + mods, % actually cover compiled modules + stop=true, % stop cover after analyse; boolean() + cross}).% cross cover analyse info diff --git a/lib/test_server/src/ts.erl b/lib/test_server/src/ts.erl index bc7d244c7c..d6d2e865e2 100644 --- a/lib/test_server/src/ts.erl +++ b/lib/test_server/src/ts.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2014. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -641,16 +641,17 @@ get_last_app_tests([Dir|Dirs],RE,Acc) -> NewAcc = case re:run(Dir,RE,[{capture,all,list}]) of {match,[Dir,AppStr]} -> + Dir1 = filename:dirname(Dir), % cover logs in ct_run.<t> dir App = list_to_atom(AppStr), case lists:keytake(App,1,Acc) of {value,{App,LastDir},Rest} -> - if Dir > LastDir -> - [{App,Dir}|Rest]; + if Dir1 > LastDir -> + [{App,Dir1}|Rest]; true -> Acc end; false -> - [{App,Dir} | Acc] + [{App,Dir1} | Acc] end; _ -> Acc |