diff options
author | Siri Hansen <[email protected]> | 2012-10-31 16:51:15 +0100 |
---|---|---|
committer | Siri Hansen <[email protected]> | 2012-10-31 16:51:30 +0100 |
commit | a6d90a806eeac9a75ffbadbe1f5322d1b3a2710d (patch) | |
tree | e4e03916764ad79af1d05eeb87e5b029b3dc9e5f /lib/test_server/src | |
parent | c16c80532af30922cb222f75971cd2aecdb9e36c (diff) | |
parent | c3d34e536e9516ab54a9f5e9a30e21043eda565b (diff) | |
download | otp-a6d90a806eeac9a75ffbadbe1f5322d1b3a2710d.tar.gz otp-a6d90a806eeac9a75ffbadbe1f5322d1b3a2710d.tar.bz2 otp-a6d90a806eeac9a75ffbadbe1f5322d1b3a2710d.zip |
Merge branch 'siri/cover-tests'
* siri/cover-tests: (21 commits)
[common_test] Extend timer for flushing error logger
[cover] Allow reconnection if node has been disconnected or down
[cover] Don't kill remote nodes when connection to main node is lost
[test_server] Add option {start_cover,false} to test_server:start_node
Use code:lib_dir instead of code:which to get application directory
[common_test] Add test for OTP-9956
Include all kernel modules in code coverage analysis
[common_test] Add test suite for code coverage support
[common_test, test_server] Don't flush cover if cover is not running
[common_test] Add option cover_stop
[test_server] Allow cross cover analysis when testing through common_test
[test_server] Start cover in test_server:wait_for_node
[test_server] Multiply timers with timetrap_scale_factor when starting nodes
Include all stdlib modules in code coverage analysis
[test_server] Include all test_server modules in code coverage analysis
Skip epp_SUITE:otp_8911 if cover is running
[common_test] Start cover on slave nodes if running cover tests
[common_test] Don't stop cover before stopping slave node
[test_server] Don't stop cover after test is finished
[cover] Add support for test_server
...
OTP-10427
Diffstat (limited to 'lib/test_server/src')
-rw-r--r-- | lib/test_server/src/test_server.erl | 83 | ||||
-rw-r--r-- | lib/test_server/src/test_server_ctrl.erl | 175 | ||||
-rw-r--r-- | lib/test_server/src/ts.erl | 56 | ||||
-rw-r--r-- | lib/test_server/src/ts_run.erl | 2 |
4 files changed, 208 insertions, 108 deletions
diff --git a/lib/test_server/src/test_server.erl b/lib/test_server/src/test_server.erl index 57834120f7..988dc7a760 100644 --- a/lib/test_server/src/test_server.erl +++ b/lib/test_server/src/test_server.erl @@ -25,7 +25,7 @@ %%% TEST_SERVER_CTRL INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -export([run_test_case_apply/1,init_target_info/0,init_purify/0]). --export([cover_compile/1,cover_analyse/2]). +-export([cover_compile/1,cover_analyse/3]). %%% TEST_SERVER_SUP INTERFACE %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -export([get_loc/1,set_tc_state/2]). @@ -377,9 +377,7 @@ module_names(Beams) -> do_cover_compile(Modules) -> do_cover_compile1(lists:usort(Modules)). % remove duplicates -do_cover_compile1([Dont|Rest]) when Dont=:=cover; - Dont=:=test_server; - Dont=:=test_server_ctrl -> +do_cover_compile1([Dont|Rest]) when Dont=:=cover -> do_cover_compile1(Rest); do_cover_compile1([M|Rest]) -> case {code:is_sticky(M),code:is_loaded(M)} of @@ -416,7 +414,7 @@ do_cover_compile1([]) -> ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% cover_analyse(Analyse,Modules) -> [{M,{Cov,NotCov,Details}}] +%% cover_analyse(Analyse,Modules,Stop) -> [{M,{Cov,NotCov,Details}}] %% %% Analyse = {details,Dir} | details | {overview,void()} | overview %% Modules = [atom()], the modules to analyse @@ -432,7 +430,18 @@ do_cover_compile1([]) -> %% %% Also, if a Dir exists, cover data will be exported to a file called %% all.coverdata in that directory. -cover_analyse(Analyse,Modules) -> +%% +%% Finally, if Stop==true, then cover will be stopped after the +%% analysis is completed. Stopping cover causes the original (non +%% cover compiled) modules to be loaded back in. If a process at this +%% point is still running old code of any of the cover compiled +%% modules, meaning that is has not done any fully qualified function +%% call after the cover compilation, the process will now be +%% killed. To avoid this scenario, it is possible to set Stop=false, +%% which means that the modules will stay cover compiled. Note that +%% this is only recommended if the erlang node is being terminated +%% after the test is completed. +cover_analyse(Analyse,Modules,Stop) -> io:fwrite("Cover analysing...\n",[]), DetailsFun = case Analyse of @@ -483,9 +492,15 @@ cover_analyse(Analyse,Modules) -> {M,Err} end end, Modules), - Sticky = unstick_all_sticky(node()), - cover:stop(), - stick_all_sticky(node(),Sticky), + + case Stop of + true -> + Sticky = unstick_all_sticky(node()), + cover:stop(), + stick_all_sticky(node(),Sticky); + false -> + ok + end, R. pmap(Fun,List) -> @@ -502,7 +517,20 @@ pmap(Fun,List) -> end end, Pids). + +do_cover_for_node(Node,CoverFunc) -> + %% In case a slave node is starting another slave node! I.e. this + %% function is executed on a slave node - then the cover function + %% must be executed on the master node. This is for instance the + %% case in test_server's own tests. + MainCoverNode = cover:get_main_node(), + Sticky = unstick_all_sticky(MainCoverNode,Node), + rpc:call(MainCoverNode,cover,CoverFunc,[Node]), + stick_all_sticky(Node,Sticky). + unstick_all_sticky(Node) -> + unstick_all_sticky(node(),Node). +unstick_all_sticky(MainCoverNode,Node) -> lists:filter( fun(M) -> case code:is_sticky(M) of @@ -513,7 +541,7 @@ unstick_all_sticky(Node) -> false end end, - cover:modules()). + rpc:call(MainCoverNode,cover,modules,[])). stick_all_sticky(Node,Sticky) -> lists:foreach( @@ -2381,7 +2409,10 @@ start_node(Name, Type, Options) -> %% by a shielded node. Cover = case is_cover() of true -> - not is_shielded(Name) andalso same_version(Node); + not is_shielded(Name) + andalso same_version(Node) + andalso proplists:get_value(start_cover,Options, + true); false -> false end, @@ -2389,9 +2420,7 @@ start_node(Name, Type, Options) -> net_adm:ping(Node), case Cover of true -> - Sticky = unstick_all_sticky(Node), - cover:start(Node), - stick_all_sticky(Node,Sticky); + do_cover_for_node(Node,start); _ -> ok end, @@ -2419,7 +2448,27 @@ wait_for_node(Slave) -> group_leader() ! {sync_apply, self(), {test_server_ctrl,wait_for_node,[Slave]}}, - receive {sync_result,R} -> R end. + Result = receive {sync_result,R} -> R end, + case Result of + ok -> + Cover = case is_cover() of + true -> + not is_shielded(Slave) andalso same_version(Slave); + false -> + false + end, + + net_adm:ping(Slave), + case Cover of + true -> + do_cover_for_node(Slave,start); + _ -> + ok + end; + _ -> + ok + end, + Result. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -2431,9 +2480,7 @@ stop_node(Slave) -> Nocover = is_shielded(Slave) orelse not same_version(Slave), case is_cover() of true when not Nocover -> - Sticky = unstick_all_sticky(Slave), - cover:stop(Slave), - stick_all_sticky(Slave,Sticky); + do_cover_for_node(Slave,flush); _ -> ok end, diff --git a/lib/test_server/src/test_server_ctrl.erl b/lib/test_server/src/test_server_ctrl.erl index 7f04e2eb23..50208410a9 100644 --- a/lib/test_server/src/test_server_ctrl.erl +++ b/lib/test_server/src/test_server_ctrl.erl @@ -165,8 +165,8 @@ -export([reject_io_reqs/1, get_levels/0, set_levels/3]). -export([multiply_timetraps/1, scale_timetraps/1, get_timetrap_parameters/0]). -export([create_priv_dir/1]). --export([cover/2, cover/3, cover/7, - cross_cover_analyse/1, cross_cover_analyse/2, trc/1, stop_trace/0]). +-export([cover/2, cover/3, cover/8, + cross_cover_analyse/2, cross_cover_analyse/3, trc/1, stop_trace/0]). -export([testcase_callback/1]). -export([set_random_seed/1]). -export([kill_slavenodes/0]). @@ -521,9 +521,9 @@ cover(App, Analyse) when is_atom(App) -> cover(CoverFile, Analyse) -> cover(none, CoverFile, Analyse). cover(App, CoverFile, Analyse) -> - controller_call({cover,{App,CoverFile},Analyse}). -cover(App, CoverFile, Exclude, Include, Cross, Export, Analyse) -> - controller_call({cover,{App,{CoverFile,Exclude,Include,Cross,Export}},Analyse}). + controller_call({cover,{App,CoverFile},Analyse,true}). +cover(App, CoverFile, Exclude, Include, Cross, Export, Analyse, Stop) -> + controller_call({cover,{App,{CoverFile,Exclude,Include,Cross,Export}},Analyse,Stop}). testcase_callback(ModFunc) -> controller_call({testcase_callback,ModFunc}). @@ -796,7 +796,7 @@ handle_call({add_job,Dir,Name,TopCase,Skip}, _From, State) -> ExtraTools = case State#state.cover of false -> []; - {App,Analyse} -> [{cover,App,Analyse}] + {App,Analyse,Stop} -> [{cover,App,Analyse,Stop}] end, ExtraTools1 = case State#state.random_seed of @@ -1052,13 +1052,13 @@ handle_call(stop_trace, _From, State) -> {reply,R,State#state{trc=false}}; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% handle_call({cover,App,Analyse}, _, State) -> ok | {error,Reason} +%% handle_call({cover,App,Analyse,Stop}, _, State) -> ok | {error,Reason} %% %% All modules inn application App are cover compiled %% Analyse indicates on which level the coverage should be analysed -handle_call({cover,App,Analyse}, _From, State) -> - {reply,ok,State#state{cover={App,Analyse}}}; +handle_call({cover,App,Analyse,Stop}, _From, State) -> + {reply,ok,State#state{cover={App,Analyse,Stop}}}; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% handle_call({create_priv_dir,Value}, _, State) -> ok | {error,Reason} @@ -1465,11 +1465,11 @@ elapsed_time(Before, After) -> start_extra_tools(ExtraTools) -> start_extra_tools(ExtraTools, []). -start_extra_tools([{cover,App,Analyse} | ExtraTools], Started) -> +start_extra_tools([{cover,App,Analyse,Stop} | ExtraTools], Started) -> case cover_compile(App) of {ok,AnalyseMods} -> start_extra_tools(ExtraTools, - [{cover,App,Analyse,AnalyseMods}|Started]); + [{cover,App,Analyse,AnalyseMods,Stop}|Started]); {error,_} -> start_extra_tools(ExtraTools, Started) end; @@ -1488,8 +1488,8 @@ stop_extra_tools(ExtraTools) -> end, stop_extra_tools(ExtraTools, TestDir). -stop_extra_tools([{cover,App,Analyse,AnalyseMods}|ExtraTools], TestDir) -> - cover_analyse(App, Analyse, AnalyseMods, TestDir), +stop_extra_tools([{cover,App,Analyse,AnalyseMods,Stop}|ExtraTools], TestDir) -> + cover_analyse(App, Analyse, AnalyseMods, Stop, TestDir), stop_extra_tools(ExtraTools, TestDir); %%stop_extra_tools([_ | ExtraTools], TestDir) -> %% stop_extra_tools(ExtraTools, TestDir); @@ -5119,7 +5119,7 @@ get_target_info() -> %% Called by test_server. See test_server:start_node/3 for details start_node(Name, Type, Options) -> - T = 10 * ?ACCEPT_TIMEOUT, % give some extra time + T = 10 * ?ACCEPT_TIMEOUT * test_server:timetrap_scale_factor(), format(minor, "Attempt to start ~w node ~p with options ~p", [Type, Name, Options]), case controller_call({start_node,Name,Type,Options}, T) of @@ -5164,7 +5164,8 @@ start_node(Name, Type, Options) -> %% when the new node has contacted test_server_ctrl again wait_for_node(Slave) -> - case catch controller_call({wait_for_node,Slave},10000) of + T = 10000 * test_server:timetrap_scale_factor(), + case catch controller_call({wait_for_node,Slave},T) of {'EXIT',{timeout,_}} -> {error,timeout}; ok -> ok end. @@ -5447,7 +5448,7 @@ check_cover_file([], Exclude, Include) -> %% %% This per application analysis writes the file cover.html in the %% application's run.<timestamp> directory. -cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, TestDir) -> +cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, Stop, TestDir) -> write_default_cross_coverlog(TestDir), {ok,CoverLog} = file:open(filename:join(TestDir, ?coverlog_name), [write]), @@ -5478,7 +5479,7 @@ cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, TestDir) -> io:fwrite(CoverLog, "<p>Excluded module(s): <code>~p</code>\n", [Excluded]), - Coverage = cover_analyse(Analyse, AnalyseMods), + Coverage = cover_analyse(Analyse, AnalyseMods, Stop), case lists:filter(fun({_M,{_,_,_}}) -> false; (_) -> true @@ -5495,32 +5496,37 @@ cover_analyse({App,CoverInfo}, Analyse, AnalyseMods, TestDir) -> file:write_file(filename:join(TestDir, ?cover_total), term_to_binary(TotPercent)). -cover_analyse(Analyse, AnalyseMods) -> +cover_analyse(Analyse, AnalyseMods, Stop) -> TestDir = get(test_server_log_dir_base), case get(test_server_ctrl_job_sock) of undefined -> %% local target - test_server:cover_analyse({Analyse,TestDir}, AnalyseMods); + test_server:cover_analyse({Analyse,TestDir}, AnalyseMods, Stop); JobSock -> %% remote target request(JobSock, {sync_apply,{test_server, cover_analyse, - [Analyse,AnalyseMods]}}), + [Analyse,AnalyseMods, Stop]}}), read_job_sock_loop(JobSock) end. %% Cover analysis, cross application %% This can be executed on any node after all tests are finished. -%% The node's current directory must be the same as when the tests -%% were run. -cross_cover_analyse(Analyse) -> - cross_cover_analyse(Analyse, undefined). - -cross_cover_analyse(Analyse, CrossModules) -> - CoverdataFiles = get_coverdata_files(), +%% Apps = [{App,Dir}] +%% App = atom(), application name +%% Dir = string(), the log directory for App, normally where +%% run.<timestamp> is found. +%% Modules = [atom()], modules that have been cover compiled during tests +%% of other apps than the one they belong to. +cross_cover_analyse(Analyse, Apps) -> + cross_cover_analyse(Analyse, Apps, get_cross_modules()). +cross_cover_analyse(Analyse, Apps, Modules) -> + Apps1 = get_latest_run_dirs(Apps), + Apps2 = add_cross_modules(Modules,Apps1), + CoverdataFiles = get_coverdata_files(Apps2), lists:foreach(fun(CDF) -> cover:import(CDF) end, CoverdataFiles), - io:fwrite("Cover analysing... ", []), + io:fwrite("Cover analysing...\n", []), DetailsFun = case Analyse of details -> @@ -5534,25 +5540,15 @@ cross_cover_analyse(Analyse, CrossModules) -> _ -> fun(_,_) -> undefined end end, - SortedModules = - case CrossModules of - undefined -> - sort_modules([Mod || Mod <- get_all_cross_modules(), - lists:member(Mod, cover:imported_modules())], []); - _ -> - sort_modules(CrossModules, []) - end, - Coverage = analyse_apps(SortedModules, DetailsFun, []), + Coverage = analyse_apps(Apps2, DetailsFun, []), cover:stop(), - write_cross_cover_logs(Coverage). + write_cross_cover_logs(Coverage,Apps2). -%% For each application from which there are modules listed in the -%% cross.cover, write a cross cover log (cross_cover.html). -write_cross_cover_logs([{App,Coverage}|T]) -> - case last_test_for_app(App) of - false -> - ok; - Dir -> +%% For each application from which there are cross cover analysed +%% modules, write a cross cover log (cross_cover.html). +write_cross_cover_logs([{App,Coverage}|T],Apps) -> + case lists:keyfind(App,1,Apps) of + {_,Dir,Mods} when Mods=/=[] -> CoverLogName = filename:join(Dir,?cross_coverlog_name), {ok,CoverLog} = file:open(CoverLogName, [write]), write_coverlog_header(CoverLog), @@ -5560,54 +5556,51 @@ write_cross_cover_logs([{App,Coverage}|T]) -> "<h1>Coverage results for \'~w\' from all tests</h1>\n", [App]), write_cover_result_table(CoverLog, Coverage), - io:fwrite("Written file ~p\n", [CoverLogName]) + io:fwrite("Written file ~p\n", [CoverLogName]); + _ -> + ok end, - write_cross_cover_logs(T); -write_cross_cover_logs([]) -> + write_cross_cover_logs(T,Apps); +write_cross_cover_logs([],_) -> io:fwrite("done\n", []). -%% Find all exported coverdata files. First find all the latest -%% run.<timestamp> directories, and the check if there is a file named -%% all.coverdata. -get_coverdata_files() -> - PossibleFiles = [last_coverdata_file(Dir) || - Dir <- filelib:wildcard([$*|?logdir_ext]), - filelib:is_dir(Dir)], - [File || File <- PossibleFiles, filelib:is_file(File)]. - -last_coverdata_file(Dir) -> - LastDir = last_test(filelib:wildcard(filename:join(Dir,"run.[1-2]*")),false), - filename:join(LastDir,"all.coverdata"). - - -%% Find the latest run.<timestamp> directory for the given application. -last_test_for_app(App) -> - AppLogDir = atom_to_list(App)++?logdir_ext, - last_test(filelib:wildcard(filename:join(AppLogDir,"run.[1-2]*")),false). - -last_test([Run|Rest], false) -> - last_test(Rest, Run); -last_test([Run|Rest], Latest) when Run > Latest -> - last_test(Rest, Run); -last_test([_|Rest], Latest) -> - last_test(Rest, Latest); -last_test([], Latest) -> +%% Get the latest run.<timestamp> directories +get_latest_run_dirs([{App,Dir}|Apps]) -> + [{App,get_latest_run_dir(Dir)} | get_latest_run_dirs(Apps)]; +get_latest_run_dirs([]) -> + []. + +get_latest_run_dir(Dir) -> + case filelib:wildcard(filename:join(Dir,"run.[1-2]*")) of + [] -> + Dir; + [H|T] -> + get_latest_dir(T,H) + end. + +get_latest_dir([H|T],Latest) when H>Latest -> + get_latest_dir(T,H); +get_latest_dir([_|T],Latest) -> + get_latest_dir(T,Latest); +get_latest_dir([],Latest) -> Latest. -%% Sort modules according to the application they belong to. -%% Return [{App,LastTestDir,ModuleList}] -sort_modules([M|Modules], Acc) -> - App = get_app(M), - Acc1 = - case lists:keysearch(App, 1, Acc) of - {value,{App,LastTest,List}} -> - lists:keyreplace(App, 1, Acc, {App,LastTest,[M|List]}); +%% Associate the cross cover modules with their applications. +add_cross_modules(Mods,Apps)-> + do_add_cross_modules(Mods,[{App,Dir,[]} || {App,Dir} <- Apps]). +do_add_cross_modules([Mod|Mods],Apps)-> + App = get_app(Mod), + NewApps = + case lists:keytake(App,1,Apps) of + {value,{App,Dir,AppMods},Rest} -> + [{App,Dir,lists:umerge([Mod],AppMods)}|Rest]; false -> - [{App,last_test_for_app(App),[M]}|Acc] + Apps end, - sort_modules(Modules, Acc1); -sort_modules([], Acc) -> - Acc. + do_add_cross_modules(Mods,NewApps); +do_add_cross_modules([],Apps) -> + %% Just to get the modules in the same order as app-only cover log + [{App,Dir,lists:reverse(Mods)} || {App,Dir,Mods} <- Apps]. get_app(Module) -> Beam = code:which(Module), @@ -5615,6 +5608,14 @@ get_app(Module) -> [AppStr|_] = string:tokens(AppDir,"-"), list_to_atom(AppStr). +%% Find all exported coverdata files. +get_coverdata_files(Apps) -> + lists:flatmap( + fun({_,LatestAppDir,_}) -> + filelib:wildcard(filename:join(LatestAppDir,"all.coverdata")) + end, + Apps). + %% For each application, analyse all modules %% Used for cross cover analysis. @@ -5635,7 +5636,7 @@ analyse_modules(_Dir, [], _DetailsFun, Acc) -> %% Read the cross cover file (cross.cover) -get_all_cross_modules() -> +get_cross_modules() -> get_cross_modules(all). get_cross_modules(App) -> case file:consult(?cross_cover_file) of diff --git a/lib/test_server/src/ts.erl b/lib/test_server/src/ts.erl index db16b6ecd2..1b2bd4296a 100644 --- a/lib/test_server/src/ts.erl +++ b/lib/test_server/src/ts.erl @@ -498,8 +498,60 @@ estone(Opts) when is_list(Opts) -> run(emulator,estone_SUITE,Opts). cross_cover_analyse([Level]) -> cross_cover_analyse(Level); cross_cover_analyse(Level) -> - test_server_ctrl:cross_cover_analyse(Level). - + Apps = get_last_app_tests(), + Modules = get_cross_modules(Apps,[]), + test_server_ctrl:cross_cover_analyse(Level,Apps,Modules). + +get_last_app_tests() -> + AllTests = filelib:wildcard(filename:join(["*","*_test.logs"])), + {ok,RE} = re:compile("^[^/]*/[^\.]*\.(.*)_test\.logs$"), + get_last_app_tests(AllTests,RE,[]). + +get_last_app_tests([Dir|Dirs],RE,Acc) -> + NewAcc = + case re:run(Dir,RE,[{capture,all,list}]) of + {match,[Dir,AppStr]} -> + App = list_to_atom(AppStr), + case lists:keytake(App,1,Acc) of + {value,{App,LastDir},Rest} -> + if Dir > LastDir -> + [{App,Dir}|Rest]; + true -> + Acc + end; + false -> + [{App,Dir} | Acc] + end; + _ -> + Acc + end, + get_last_app_tests(Dirs,RE,NewAcc); +get_last_app_tests([],_,Acc) -> + Acc. + +get_cross_modules([{App,_}|Apps],Acc) -> + Mods = cross_modules(App), + get_cross_modules(Apps,lists:umerge(Mods,Acc)); +get_cross_modules([],Acc) -> + Acc. + +cross_modules(App) -> + case default_coverfile(App) of + none -> + []; + File -> + case catch file:consult(File) of + {ok,CoverSpec} -> + case lists:keyfind(cross_apps,1,CoverSpec) of + false -> + []; + {cross_apps,App,Modules} -> + lists:usort(Modules) + end; + _ -> + [] + end + end. %%% Implementation. diff --git a/lib/test_server/src/ts_run.erl b/lib/test_server/src/ts_run.erl index 57d1b8806e..741dd483f5 100644 --- a/lib/test_server/src/ts_run.erl +++ b/lib/test_server/src/ts_run.erl @@ -368,7 +368,7 @@ make_common_test_args(Args0, Options0, _Vars) -> io:format("No cover file found for ~p~n",[App]), []; {value,{cover,_App,File,_Analyse}} -> - [{cover,to_list(File)}]; + [{cover,to_list(File)},{cover_stop,false}]; false -> [] end, |