diff options
Diffstat (limited to 'lib/tools/src')
| -rw-r--r-- | lib/tools/src/cover.erl | 851 |
1 files changed, 563 insertions, 288 deletions
diff --git a/lib/tools/src/cover.erl b/lib/tools/src/cover.erl index 31754015f7..6c32c47069 100644 --- a/lib/tools/src/cover.erl +++ b/lib/tools/src/cover.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2013. All Rights Reserved. +%% Copyright Ericsson AB 2001-2015. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -77,8 +77,11 @@ compile/1, compile/2, compile_module/1, compile_module/2, compile_directory/0, compile_directory/1, compile_directory/2, compile_beam/1, compile_beam_directory/0, compile_beam_directory/1, - analyse/1, analyse/2, analyse/3, analyze/1, analyze/2, analyze/3, + analyse/0, analyse/1, analyse/2, analyse/3, + analyze/0, analyze/1, analyze/2, analyze/3, + analyse_to_file/0, analyse_to_file/1, analyse_to_file/2, analyse_to_file/3, + analyze_to_file/0, analyze_to_file/1, analyze_to_file/2, analyze_to_file/3, async_analyse_to_file/1,async_analyse_to_file/2, async_analyse_to_file/3, async_analyze_to_file/1, @@ -109,6 +112,7 @@ line = '_' % integer() }). -define(BUMP_REC_NAME,bump). +-define(CHUNK_SIZE, 20000). -record(vars, {module, % atom() Module name @@ -181,10 +185,11 @@ start(Node) when is_atom(Node) -> start(Nodes) -> call({start_nodes,remove_myself(Nodes,[])}). -%% compile(ModFile) -> -%% compile(ModFile, Options) -> -%% compile_module(ModFile) -> Result -%% compile_module(ModFile, Options) -> Result +%% compile(ModFiles) -> +%% compile(ModFiles, Options) -> +%% compile_module(ModFiles) -> Result +%% compile_module(ModFiles, Options) -> Result +%% ModFiles = ModFile | [ModFile] %% ModFile = Module | File %% Module = atom() %% File = string() @@ -198,18 +203,27 @@ compile(ModFile, Options) -> compile_module(ModFile) when is_atom(ModFile); is_list(ModFile) -> compile_module(ModFile, []). -compile_module(Module, Options) when is_atom(Module), is_list(Options) -> - compile_module(atom_to_list(Module), Options); -compile_module(File, Options) when is_list(File), is_list(Options) -> - WithExt = case filename:extension(File) of - ".erl" -> - File; - _ -> - File++".erl" - end, - AbsFile = filename:absname(WithExt), - [R] = compile_modules([AbsFile], Options), - R. +compile_module(ModFile, Options) when is_atom(ModFile); + is_list(ModFile), is_integer(hd(ModFile)) -> + [R] = compile_module([ModFile], Options), + R; +compile_module(ModFiles, Options) when is_list(Options) -> + AbsFiles = + [begin + File = + case ModFile of + _ when is_atom(ModFile) -> atom_to_list(ModFile); + _ when is_list(ModFile) -> ModFile + end, + WithExt = case filename:extension(File) of + ".erl" -> + File; + _ -> + File++".erl" + end, + filename:absname(WithExt) + end || ModFile <- ModFiles], + compile_modules(AbsFiles, Options). %% compile_directory() -> %% compile_directory(Dir) -> @@ -240,13 +254,14 @@ compile_directory(Dir, Options) when is_list(Dir), is_list(Options) -> compile_modules(Files,Options) -> Options2 = filter_options(Options), - compile_modules(Files,Options2,[]). + %% compile_modules(Files,Options2,[]). + call({compile, Files, Options2}). -compile_modules([File|Files], Options, Result) -> - R = call({compile, File, Options}), - compile_modules(Files,Options,[R|Result]); -compile_modules([],_Opts,Result) -> - lists:reverse(Result). +%% compile_modules([File|Files], Options, Result) -> +%% R = call({compile, File, Options}), +%% compile_modules(Files,Options,[R|Result]); +%% compile_modules([],_Opts,Result) -> +%% lists:reverse(Result). filter_options(Options) -> lists:filter(fun(Option) -> @@ -264,30 +279,17 @@ filter_options(Options) -> %% ModFile - see compile/1 %% Result - see compile/1 %% Reason = non_existing | already_cover_compiled -compile_beam(Module) when is_atom(Module) -> - case code:which(Module) of - non_existing -> +compile_beam(ModFile0) when is_atom(ModFile0); + is_list(ModFile0), is_integer(hd(ModFile0)) -> + case compile_beams([ModFile0]) of + [{error,{non_existing,_}}] -> + %% Backwards compatibility {error,non_existing}; - ?TAG -> - compile_beam(Module,?TAG); - File -> - compile_beam(Module,File) + [Result] -> + Result end; -compile_beam(File) when is_list(File) -> - {WithExt,WithoutExt} - = case filename:rootname(File,".beam") of - File -> - {File++".beam",File}; - Rootname -> - {File,Rootname} - end, - AbsFile = filename:absname(WithExt), - Module = list_to_atom(filename:basename(WithoutExt)), - compile_beam(Module,AbsFile). - -compile_beam(Module,File) -> - call({compile_beam,Module,File}). - +compile_beam(ModFiles) when is_list(ModFiles) -> + compile_beams(ModFiles). %% compile_beam_directory(Dir) -> [Result] | {error,Reason} @@ -312,19 +314,52 @@ compile_beam_directory(Dir) when is_list(Dir) -> Error end. -compile_beams(Files) -> - compile_beams(Files,[]). -compile_beams([File|Files],Result) -> - R = compile_beam(File), - compile_beams(Files,[R|Result]); -compile_beams([],Result) -> - lists:reverse(Result). +compile_beams(ModFiles0) -> + ModFiles = get_mods_and_beams(ModFiles0,[]), + call({compile_beams,ModFiles}). - -%% analyse(Module) -> -%% analyse(Module, Analysis) -> -%% analyse(Module, Level) -> -%% analyse(Module, Analysis, Level) -> {ok,Answer} | {error,Error} +get_mods_and_beams([Module|ModFiles],Acc) when is_atom(Module) -> + case code:which(Module) of + non_existing -> + get_mods_and_beams(ModFiles,[{error,{non_existing,Module}}|Acc]); + File -> + get_mods_and_beams([{Module,File}|ModFiles],Acc) + end; +get_mods_and_beams([File|ModFiles],Acc) when is_list(File) -> + {WithExt,WithoutExt} + = case filename:rootname(File,".beam") of + File -> + {File++".beam",File}; + Rootname -> + {File,Rootname} + end, + AbsFile = filename:absname(WithExt), + Module = list_to_atom(filename:basename(WithoutExt)), + get_mods_and_beams([{Module,AbsFile}|ModFiles],Acc); +get_mods_and_beams([{Module,File}|ModFiles],Acc) -> + %% Check for duplicates + case lists:keyfind(Module,2,Acc) of + {ok,Module,File} -> + %% Duplicate, but same file so ignore + get_mods_and_beams(ModFiles,Acc); + {ok,Module,_OtherFile} -> + %% Duplicate and differnet file - error + get_mods_and_beams(ModFiles,[{error,{duplicate,Module}}|Acc]); + _ -> + get_mods_and_beams(ModFiles,[{ok,Module,File}|Acc]) + end; +get_mods_and_beams([],Acc) -> + lists:reverse(Acc). + + +%% analyse(Modules) -> +%% analyse(Analysis) -> +%% analyse(Level) -> +%% analyse(Modules, Analysis) -> +%% analyse(Modules, Level) -> +%% analyse(Analysis, Level) +%% analyse(Modules, Analysis, Level) -> {ok,Answer} | {error,Error} +%% Modules = Module | [Module] %% Module = atom() %% Analysis = coverage | calls %% Level = line | clause | function | module @@ -337,48 +372,74 @@ compile_beams([],Result) -> %% N = A = C = integer() %% Value = {Cov,NotCov} | Calls %% Cov = NotCov = Calls = integer() -%% Error = {not_cover_compiled,Module} +%% Error = {not_cover_compiled,Module} | not_main_node +-define(is_analysis(__A__), + (__A__=:=coverage orelse __A__=:=calls)). +-define(is_level(__L__), + (__L__=:=line orelse __L__=:=clause orelse + __L__=:=function orelse __L__=:=module)). +analyse() -> + analyse('_'). + +analyse(Analysis) when ?is_analysis(Analysis) -> + analyse('_', Analysis); +analyse(Level) when ?is_level(Level) -> + analyse('_', Level); analyse(Module) -> analyse(Module, coverage). -analyse(Module, Analysis) when Analysis=:=coverage; Analysis=:=calls -> + +analyse(Analysis, Level) when ?is_analysis(Analysis) andalso + ?is_level(Level) -> + analyse('_', Analysis, Level); +analyse(Module, Analysis) when ?is_analysis(Analysis) -> analyse(Module, Analysis, function); -analyse(Module, Level) when Level=:=line; Level=:=clause; Level=:=function; - Level=:=module -> +analyse(Module, Level) when ?is_level(Level) -> analyse(Module, coverage, Level). -analyse(Module, Analysis, Level) when is_atom(Module), - Analysis=:=coverage; Analysis=:=calls, - Level=:=line; Level=:=clause; - Level=:=function; Level=:=module -> + +analyse(Module, Analysis, Level) when ?is_analysis(Analysis), + ?is_level(Level) -> call({{analyse, Analysis, Level}, Module}). +analyze() -> analyse( ). analyze(Module) -> analyse(Module). analyze(Module, Analysis) -> analyse(Module, Analysis). analyze(Module, Analysis, Level) -> analyse(Module, Analysis, Level). -%% analyse_to_file(Module) -> -%% analyse_to_file(Module, Options) -> -%% analyse_to_file(Module, OutFile) -> -%% analyse_to_file(Module, OutFile, Options) -> {ok,OutFile} | {error,Error} +%% analyse_to_file() -> +%% analyse_to_file(Modules) -> +%% analyse_to_file(Modules, Options) -> +%% Modules = Module | [Module] %% Module = atom() %% OutFile = string() %% Options = [Option] -%% Option = html +%% Option = html | {outfile,filename()} | {outdir,dirname()} %% Error = {not_cover_compiled,Module} | no_source_code_found | %% {file,File,Reason} %% File = string() %% Reason = term() -analyse_to_file(Module) when is_atom(Module) -> - analyse_to_file(Module, outfilename(Module,[]), []). -analyse_to_file(Module, []) when is_atom(Module) -> - analyse_to_file(Module, outfilename(Module,[]), []); -analyse_to_file(Module, Options) when is_atom(Module), - is_list(Options), is_atom(hd(Options)) -> - analyse_to_file(Module, outfilename(Module,Options), Options); -analyse_to_file(Module, OutFile) when is_atom(Module), is_list(OutFile) -> - analyse_to_file(Module, OutFile, []). -analyse_to_file(Module, OutFile, Options) when is_atom(Module), is_list(OutFile) -> - call({{analyse_to_file, OutFile, Options}, Module}). - +%% +%% Kept for backwards compatibility: +%% analyse_to_file(Modules, OutFile) -> +%% analyse_to_file(Modules, OutFile, Options) -> {ok,OutFile} | {error,Error} +analyse_to_file() -> + analyse_to_file('_'). +analyse_to_file(Arg) -> + case is_options(Arg) of + true -> + analyse_to_file('_',Arg); + false -> + analyse_to_file(Arg,[]) + end. +analyse_to_file(Module, OutFile) when is_list(OutFile), is_integer(hd(OutFile)) -> + %% Kept for backwards compatibility + analyse_to_file(Module, [{outfile,OutFile}]); +analyse_to_file(Module, Options) when is_list(Options) -> + call({{analyse_to_file, Options}, Module}). +analyse_to_file(Module, OutFile, Options) when is_list(OutFile) -> + %% Kept for backwards compatibility + analyse_to_file(Module,[{outfile,OutFile}|Options]). + +analyze_to_file() -> analyse_to_file(). analyze_to_file(Module) -> analyse_to_file(Module). analyze_to_file(Module, OptOrOut) -> analyse_to_file(Module, OptOrOut). analyze_to_file(Module, OutFile, Options) -> @@ -391,6 +452,15 @@ async_analyse_to_file(Module, OutFileOrOpts) -> async_analyse_to_file(Module, OutFile, Options) -> do_spawn(?MODULE, analyse_to_file, [Module, OutFile, Options]). +is_options([html]) -> + true; % this is not 100% safe - could be a module named html... +is_options([html|Opts]) -> + is_options(Opts); +is_options([{Opt,_}|_]) when Opt==outfile; Opt==outdir -> + true; +is_options(_) -> + false. + do_spawn(M,F,A) -> spawn_link(fun() -> case apply(M,F,A) of @@ -408,13 +478,16 @@ async_analyze_to_file(Module, OutFileOrOpts) -> async_analyze_to_file(Module, OutFile, Options) -> async_analyse_to_file(Module, OutFile, Options). -outfilename(Module,Opts) -> - case lists:member(html,Opts) of - true -> - atom_to_list(Module)++".COVER.html"; - false -> - atom_to_list(Module)++".COVER.out" - end. +outfilename(undefined, Module, HTML) -> + outfilename(Module, HTML); +outfilename(OutDir, Module, HTML) -> + filename:join(OutDir, outfilename(Module, HTML)). + +outfilename(Module, true) -> + atom_to_list(Module)++".COVER.html"; +outfilename(Module, false) -> + atom_to_list(Module)++".COVER.out". + %% export(File) %% export(File,Module) -> ok | {error,Reason} @@ -559,7 +632,7 @@ init_main(Starter) -> ,{write_concurrency, true} ]), ets:new(?COVER_CLAUSE_TABLE, [set, public, named_table]), - ets:new(?BINARY_TABLE, [set, named_table]), + ets:new(?BINARY_TABLE, [set, public, named_table]), ets:new(?COLLECTION_TABLE, [set, public, named_table]), ets:new(?COLLECTION_CLAUSE_TABLE, [set, public, named_table]), net_kernel:monitor_nodes(true), @@ -573,55 +646,19 @@ main_process_loop(State) -> reply(From, {ok,StartedNodes}), main_process_loop(State1); - {From, {compile, File, Options}} -> - case do_compile(File, Options) of - {ok, Module} -> - remote_load_compiled(State#main_state.nodes,[{Module,File}]), - reply(From, {ok, Module}), - Compiled = add_compiled(Module, File, - State#main_state.compiled), - Imported = remove_imported(Module,State#main_state.imported), - NewState = State#main_state{compiled = Compiled, - imported = Imported}, - %% This module (cover) could have been reloaded. Make - %% sure we run the new code. - ?MODULE:main_process_loop(NewState); - error -> - reply(From, {error, File}), - main_process_loop(State) - end; + {From, {compile, Files, Options}} -> + {R,S} = do_compile(Files, Options, State), + reply(From,R), + %% This module (cover) could have been reloaded. Make + %% sure we run the new code. + ?MODULE:main_process_loop(S); - {From, {compile_beam, Module, BeamFile0}} -> - Compiled0 = State#main_state.compiled, - case get_beam_file(Module,BeamFile0,Compiled0) of - {ok,BeamFile} -> - UserOptions = get_compile_options(Module,BeamFile), - {Reply,Compiled} = - case do_compile_beam(Module,BeamFile,UserOptions) of - {ok, Module} -> - remote_load_compiled(State#main_state.nodes, - [{Module,BeamFile}]), - C = add_compiled(Module,BeamFile,Compiled0), - {{ok,Module},C}; - error -> - {{error, BeamFile}, Compiled0}; - {error,Reason} -> % no abstract code - {{error, {Reason, BeamFile}}, Compiled0} - end, - reply(From,Reply), - Imported = remove_imported(Module,State#main_state.imported), - NewState = State#main_state{compiled = Compiled, - imported = Imported}, - %% This module (cover) could have been reloaded. Make - %% sure we run the new code. - ?MODULE:main_process_loop(NewState); - {error,no_beam} -> - %% The module has first been compiled from .erl, and now - %% someone tries to compile it from .beam - reply(From, - {error,{already_cover_compiled,no_beam_found,Module}}), - main_process_loop(State) - end; + {From, {compile_beams, ModsAndFiles}} -> + {R,S} = do_compile_beams(ModsAndFiles,State), + reply(From,R), + %% This module (cover) could have been reloaded. Make + %% sure we run the new code. + ?MODULE:main_process_loop(S); {From, {export,OutFile,Module}} -> spawn(fun() -> @@ -706,6 +743,16 @@ main_process_loop(State) -> unregister(?SERVER), reply(From, ok); + {From, {{analyse, Analysis, Level}, '_'}} -> + R = analyse_all(Analysis, Level, State), + reply(From, R), + main_process_loop(State); + + {From, {{analyse, Analysis, Level}, Modules}} when is_list(Modules) -> + R = analyse_list(Modules, Analysis, Level, State), + reply(From, R), + main_process_loop(State); + {From, {{analyse, Analysis, Level}, Module}} -> S = try Loaded = is_loaded(Module, State), @@ -722,15 +769,23 @@ main_process_loop(State) -> end, main_process_loop(S); - {From, {{analyse_to_file, OutFile, Opts},Module}} -> + {From, {{analyse_to_file, Opts},'_'}} -> + R = analyse_all_to_file(Opts, State), + reply(From,R), + main_process_loop(State); + + {From, {{analyse_to_file, Opts},Modules}} when is_list(Modules) -> + R = analyse_list_to_file(Modules, Opts, State), + reply(From,R), + main_process_loop(State); + + {From, {{analyse_to_file, Opts},Module}} -> S = try Loaded = is_loaded(Module, State), spawn(fun() -> - ?SPAWN_DBG(analyse_to_file, - {Module,OutFile, Opts}), + ?SPAWN_DBG(analyse_to_file,{Module,Opts}), do_parallel_analysis_to_file( - Module, OutFile, Opts, - Loaded, From, State) + Module, Opts, Loaded, From, State) end), State catch throw:Reason -> @@ -848,11 +903,15 @@ remote_process_loop(State) -> {remote,collect,Module,CollectorPid} -> self() ! {remote,collect,Module,CollectorPid, ?SERVER}; - {remote,collect,Module,CollectorPid,From} -> + {remote,collect,Modules0,CollectorPid,From} -> + Modules = case Modules0 of + '_' -> [M || {M,_} <- State#remote_state.compiled]; + _ -> Modules0 + end, spawn(fun() -> ?SPAWN_DBG(remote_collect, - {Module, CollectorPid, From}), - do_collect(Module, CollectorPid, From) + {Modules, CollectorPid, From}), + do_collect(Modules, CollectorPid, From) end), remote_process_loop(State); @@ -893,39 +952,51 @@ remote_process_loop(State) -> end. -do_collect(Module, CollectorPid, From) -> - AllMods = - case Module of - '_' -> ets:tab2list(?COVER_CLAUSE_TABLE); - _ -> ets:lookup(?COVER_CLAUSE_TABLE, Module) - end, - - %% Sending clause by clause in order to avoid large lists +do_collect(Modules, CollectorPid, From) -> pmap( - fun({_Mod,Clauses}) -> - lists:map(fun(Clause) -> - send_collected_data(Clause, CollectorPid) - end,Clauses) - end,AllMods), + fun(Module) -> + Pattern = {#bump{module=Module, _='_'}, '$1'}, + MatchSpec = [{Pattern,[{'=/=','$1',0}],['$_']}], + Match = ets:select(?COVER_TABLE,MatchSpec,?CHUNK_SIZE), + send_chunks(Match, CollectorPid, []) + end,Modules), CollectorPid ! done, remote_reply(From, ok). -send_collected_data({M,F,A,C,_L}, CollectorPid) -> - Pattern = - {#bump{module=M, function=F, arity=A, clause=C}, '_'}, - Bumps = ets:match_object(?COVER_TABLE, Pattern), - %% Reset - lists:foreach(fun({Bump,_N}) -> - ets:insert(?COVER_TABLE, {Bump,0}) - end, - Bumps), - CollectorPid ! {chunk,Bumps}. +send_chunks('$end_of_table', _CollectorPid, Mons) -> + get_downs(Mons); +send_chunks({Chunk,Continuation}, CollectorPid, Mons) -> + Mon = spawn_monitor( + fun() -> + lists:foreach(fun({Bump,_N}) -> + ets:insert(?COVER_TABLE, {Bump,0}) + end, + Chunk) end), + send_chunk(CollectorPid,Chunk), + send_chunks(ets:select(Continuation), CollectorPid, [Mon|Mons]). + +send_chunk(CollectorPid,Chunk) -> + CollectorPid ! {chunk,Chunk,self()}, + receive continue -> ok end. + +get_downs([]) -> + ok; +get_downs(Mons) -> + receive + {'DOWN', Ref, _Type, Pid, _Reason} = Down -> + case lists:member({Pid,Ref},Mons) of + true -> + get_downs(lists:delete({Pid,Ref},Mons)); + false -> + %% This should be handled somewhere else + self() ! Down, + get_downs(Mons) + end + end. -reload_originals([{Module,_File}|Compiled]) -> - do_reload_original(Module), - reload_originals(Compiled); -reload_originals([]) -> - ok. +reload_originals(Compiled) -> + Modules = [M || {M,_} <- Compiled], + pmap(fun do_reload_original/1, Modules). do_reload_original(Module) -> case code:which(Module) of @@ -1068,15 +1139,40 @@ remote_load_compiled(_Nodes, [], [], _ModNum) -> ok; remote_load_compiled(Nodes, Compiled, Acc, ModNum) when Compiled == []; ModNum == ?MAX_MODS -> + RemoteLoadData = get_downs_r(Acc), lists:foreach( fun(Node) -> - remote_call(Node,{remote,load_compiled,Acc}) + remote_call(Node,{remote,load_compiled,RemoteLoadData}) end, Nodes), remote_load_compiled(Nodes, Compiled, [], 0); remote_load_compiled(Nodes, [MF | Rest], Acc, ModNum) -> remote_load_compiled( - Nodes, Rest, [get_data_for_remote_loading(MF) | Acc], ModNum + 1). + Nodes, Rest, + [spawn_job_r(fun() -> get_data_for_remote_loading(MF) end) | Acc], + ModNum + 1). + +spawn_job_r(Fun) -> + spawn_monitor(fun() -> exit(Fun()) end). + +get_downs_r([]) -> + []; +get_downs_r(Mons) -> + receive + {'DOWN', Ref, _Type, Pid, R={_,_,_,_}} -> + [R|get_downs_r(lists:delete({Pid,Ref},Mons))]; + {'DOWN', Ref, _Type, Pid, Reason} = Down -> + case lists:member({Pid,Ref},Mons) of + true -> + %% Something went really wrong - don't hang! + exit(Reason); + false -> + %% This should be handled somewhere else + self() ! Down, + get_downs_r(Mons) + end + end. + %% Read all data needed for loading a cover compiled module on a remote node %% Binary is the beam code for the module and InitialTable is the initial @@ -1113,11 +1209,11 @@ remote_reset(Module,Nodes) -> Nodes). %% Collect data from remote nodes - used for analyse or stop(Node) -remote_collect(Module,Nodes,Stop) -> +remote_collect(Modules,Nodes,Stop) -> pmap(fun(Node) -> ?SPAWN_DBG(remote_collect, - {Module, Nodes, Stop}), - do_collection(Node, Module, Stop) + {Modules, Nodes, Stop}), + do_collection(Node, Modules, Stop) end, Nodes). @@ -1138,8 +1234,9 @@ do_collection(Node, Module, Stop) -> collector_proc() -> ?SPAWN_DBG(collector_proc, []), receive - {chunk,Chunk} -> + {chunk,Chunk,From} -> insert_in_collection_table(Chunk), + From ! continue, collector_proc(); done -> ok @@ -1259,6 +1356,19 @@ add_compiled(Module, File, [H|Compiled]) -> add_compiled(Module, File, []) -> [{Module,File}]. +are_loaded([Module|Modules], State, Loaded, Imported, Error) -> + try is_loaded(Module,State) of + {loaded,File} -> + are_loaded(Modules, State, [{Module,File}|Loaded], Imported, Error); + {imported,File,_} -> + are_loaded(Modules, State, Loaded, [{Module,File}|Imported], Error) + catch throw:_ -> + are_loaded(Modules, State, Loaded, Imported, + [{not_cover_compiled,Module}|Error]) + end; +are_loaded([], _State, Loaded, Imported, Error) -> + {Loaded, Imported, Error}. + is_loaded(Module, State) -> case get_file(Module, State#main_state.compiled) of {ok, File} -> @@ -1333,18 +1443,75 @@ get_compiled_still_loaded(Nodes,Compiled0) -> %%%--Compilation--------------------------------------------------------- -%% do_compile(File, Options) -> {ok,Module} | {error,Error} -do_compile(File, UserOptions) -> +do_compile_beams(ModsAndFiles, State) -> + Result0 = pmap(fun({ok,Module,File}) -> + do_compile_beam(Module,File,State); + (Error) -> + Error + end, + ModsAndFiles), + Compiled = [{M,F} || {ok,M,F} <- Result0], + remote_load_compiled(State#main_state.nodes,Compiled), + fix_state_and_result(Result0,State,[]). + +do_compile_beam(Module,BeamFile0,State) -> + case get_beam_file(Module,BeamFile0,State#main_state.compiled) of + {ok,BeamFile} -> + UserOptions = get_compile_options(Module,BeamFile), + case do_compile_beam1(Module,BeamFile,UserOptions) of + {ok, Module} -> + {ok,Module,BeamFile}; + error -> + {error, BeamFile}; + {error,Reason} -> % no abstract code + {error, {Reason, BeamFile}} + end; + {error,no_beam} -> + %% The module has first been compiled from .erl, and now + %% someone tries to compile it from .beam + {error,{already_cover_compiled,no_beam_found,Module}} + end. + +fix_state_and_result([{ok,Module,BeamFile}|Rest],State,Acc) -> + Compiled = add_compiled(Module,BeamFile,State#main_state.compiled), + Imported = remove_imported(Module,State#main_state.imported), + NewState = State#main_state{compiled=Compiled,imported=Imported}, + fix_state_and_result(Rest,NewState,[{ok,Module}|Acc]); +fix_state_and_result([Error|Rest],State,Acc) -> + fix_state_and_result(Rest,State,[Error|Acc]); +fix_state_and_result([],State,Acc) -> + {lists:reverse(Acc),State}. + + +do_compile(Files, Options, State) -> + Result0 = pmap(fun(File) -> + do_compile(File, Options) + end, + Files), + Compiled = [{M,F} || {ok,M,F} <- Result0], + remote_load_compiled(State#main_state.nodes,Compiled), + fix_state_and_result(Result0,State,[]). + +do_compile(File, Options) -> + case do_compile1(File, Options) of + {ok, Module} -> + {ok,Module,File}; + error -> + {error,File} + end. + +%% do_compile1(File, Options) -> {ok,Module} | error +do_compile1(File, UserOptions) -> Options = [debug_info,binary,report_errors,report_warnings] ++ UserOptions, case compile:file(File, Options) of {ok, Module, Binary} -> - do_compile_beam(Module,Binary,UserOptions); + do_compile_beam1(Module,Binary,UserOptions); error -> error end. %% Beam is a binary or a .beam file name -do_compile_beam(Module,Beam,UserOptions) -> +do_compile_beam1(Module,Beam,UserOptions) -> %% Clear database do_clear(Module), @@ -1915,10 +2082,21 @@ common_elems(L1, L2) -> collect(Nodes) -> %% local node AllClauses = ets:tab2list(?COVER_CLAUSE_TABLE), - pmap(fun move_modules/1,AllClauses), - + Mon1 = spawn_monitor(fun() -> pmap(fun move_modules/1,AllClauses) end), + + %% remote nodes + Mon2 = spawn_monitor(fun() -> remote_collect('_',Nodes,false) end), + get_downs([Mon1,Mon2]). + +%% Collect data for a list of modules +collect(Modules,Nodes) -> + MS = [{{'$1','_'},[{'==','$1',M}],['$_']} || M <- Modules], + Clauses = ets:select(?COVER_CLAUSE_TABLE,MS), + Mon1 = spawn_monitor(fun() -> pmap(fun move_modules/1,Clauses) end), + %% remote nodes - remote_collect('_',Nodes,false). + Mon2 = spawn_monitor(fun() -> remote_collect('_',Nodes,false) end), + get_downs([Mon1,Mon2]). %% Collect data for one module collect(Module,Clauses,Nodes) -> @@ -1926,25 +2104,26 @@ collect(Module,Clauses,Nodes) -> move_modules({Module,Clauses}), %% remote nodes - remote_collect(Module,Nodes,false). + remote_collect([Module],Nodes,false). %% When analysing, the data from the local ?COVER_TABLE is moved to the %% ?COLLECTION_TABLE. Resetting data in ?COVER_TABLE move_modules({Module,Clauses}) -> ets:insert(?COLLECTION_CLAUSE_TABLE,{Module,Clauses}), - move_clauses(Clauses). + Pattern = {#bump{module=Module, _='_'}, '_'}, + MatchSpec = [{Pattern,[],['$_']}], + Match = ets:select(?COVER_TABLE,MatchSpec,?CHUNK_SIZE), + do_move_module(Match). -move_clauses([{M,F,A,C,_L}|Clauses]) -> - Pattern = {#bump{module=M, function=F, arity=A, clause=C}, '_'}, - Bumps = ets:match_object(?COVER_TABLE,Pattern), +do_move_module({Bumps,Continuation}) -> lists:foreach(fun({Key,Val}) -> ets:insert(?COVER_TABLE, {Key,0}), insert_in_collection_table(Key,Val) end, Bumps), - move_clauses(Clauses); -move_clauses([]) -> + do_move_module(ets:select(Continuation)); +do_move_module('$end_of_table') -> ok. %% Given a .beam file, find the .erl file. Look first in same directory as @@ -2002,6 +2181,26 @@ splice(BeamDir, SrcFile) -> revsplit(Path) -> lists:reverse(filename:split(Path)). +analyse_list(Modules, Analysis, Level, State) -> + {LoadedMF, ImportedMF, Error} = are_loaded(Modules, State, [], [], []), + Loaded = [M || {M,_} <- LoadedMF], + Imported = [M || {M,_} <- ImportedMF], + collect(Loaded, State#main_state.nodes), + MS = [{{'$1','_'},[{'==','$1',M}],['$_']} || M <- Loaded ++ Imported], + AllClauses = ets:select(?COLLECTION_CLAUSE_TABLE,MS), + Fun = fun({Module,Clauses}) -> + do_analyse(Module, Analysis, Level, Clauses) + end, + {result, lists:flatten(pmap(Fun, AllClauses)), Error}. + +analyse_all(Analysis, Level, State) -> + collect(State#main_state.nodes), + AllClauses = ets:tab2list(?COLLECTION_CLAUSE_TABLE), + Fun = fun({Module,Clauses}) -> + do_analyse(Module, Analysis, Level, Clauses) + end, + {result, lists:flatten(pmap(Fun, AllClauses)), []}. + do_parallel_analysis(Module, Analysis, Level, Loaded, From, State) -> analyse_info(Module,State#main_state.imported), C = case Loaded of @@ -2016,7 +2215,7 @@ do_parallel_analysis(Module, Analysis, Level, Loaded, From, State) -> Clauses end, R = do_analyse(Module, Analysis, Level, C), - reply(From, R). + reply(From, {ok,R}). %% do_analyse(Module, Analysis, Level, Clauses)-> {ok,Answer} | {error,Error} %% Clauses = [{Module,Function,Arity,Clause,Lines}] @@ -2035,37 +2234,44 @@ do_analyse(Module, Analysis, line, _Clauses) -> {{Module,L}, N} end end, - Answer = lists:keysort(1, lists:map(Fun, Bumps)), - {ok, Answer}; -do_analyse(_Module, Analysis, clause, Clauses) -> - Fun = case Analysis of - coverage -> - fun({M,F,A,C,Ls}) -> - Pattern = {#bump{module=M,function=F,arity=A, - clause=C},0}, - Bumps = ets:match_object(?COLLECTION_TABLE, Pattern), - NotCov = length(Bumps), - {{M,F,A,C}, {Ls-NotCov, NotCov}} - end; - calls -> - fun({M,F,A,C,_Ls}) -> - Pattern = {#bump{module=M,function=F,arity=A, - clause=C},'_'}, - Bumps = ets:match_object(?COLLECTION_TABLE, Pattern), - {_Bump, Calls} = hd(lists:keysort(1, Bumps)), - {{M,F,A,C}, Calls} - end - end, - Answer = lists:map(Fun, Clauses), - {ok, Answer}; + lists:keysort(1, lists:map(Fun, Bumps)); +do_analyse(Module, Analysis, clause, _Clauses) -> + Pattern = {#bump{module=Module},'_'}, + Bumps = lists:keysort(1,ets:match_object(?COLLECTION_TABLE, Pattern)), + analyse_clause(Analysis,Bumps); do_analyse(Module, Analysis, function, Clauses) -> - {ok, ClauseResult} = do_analyse(Module, Analysis, clause, Clauses), - Result = merge_clauses(ClauseResult, merge_fun(Analysis)), - {ok, Result}; + ClauseResult = do_analyse(Module, Analysis, clause, Clauses), + merge_clauses(ClauseResult, merge_fun(Analysis)); do_analyse(Module, Analysis, module, Clauses) -> - {ok, FunctionResult} = do_analyse(Module, Analysis, function, Clauses), + FunctionResult = do_analyse(Module, Analysis, function, Clauses), Result = merge_functions(FunctionResult, merge_fun(Analysis)), - {ok, {Module,Result}}. + {Module,Result}. + +analyse_clause(_,[]) -> + []; +analyse_clause(coverage, + [{#bump{module=M,function=F,arity=A,clause=C},_}|_]=Bumps) -> + analyse_clause_cov(Bumps,{M,F,A,C},0,0,[]); +analyse_clause(calls,Bumps) -> + analyse_clause_calls(Bumps,{x,x,x,x},[]). + +analyse_clause_cov([{#bump{module=M,function=F,arity=A,clause=C},N}|Bumps], + {M,F,A,C}=Clause,Ls,NotCov,Acc) -> + analyse_clause_cov(Bumps,Clause,Ls+1,if N==0->NotCov+1; true->NotCov end,Acc); +analyse_clause_cov([{#bump{module=M1,function=F1,arity=A1,clause=C1},_}|_]=Bumps, + Clause,Ls,NotCov,Acc) -> + analyse_clause_cov(Bumps,{M1,F1,A1,C1},0,0,[{Clause,{Ls-NotCov,NotCov}}|Acc]); +analyse_clause_cov([],Clause,Ls,NotCov,Acc) -> + lists:reverse(Acc,[{Clause,{Ls-NotCov,NotCov}}]). + +analyse_clause_calls([{#bump{module=M,function=F,arity=A,clause=C},_}|Bumps], + {M,F,A,C}=Clause,Acc) -> + analyse_clause_calls(Bumps,Clause,Acc); +analyse_clause_calls([{#bump{module=M1,function=F1,arity=A1,clause=C1},N}|Bumps], + _Clause,Acc) -> + analyse_clause_calls(Bumps,{M1,F1,A1,C1},[{{M1,F1,A1,C1},N}|Acc]); +analyse_clause_calls([],_Clause,Acc) -> + lists:reverse(Acc). merge_fun(coverage) -> fun({Cov1,NotCov1}, {Cov2,NotCov2}) -> @@ -2094,7 +2300,50 @@ merge_functions([{_MFA,R}|Functions], MFun, Result) -> merge_functions([], _MFun, Result) -> Result. -do_parallel_analysis_to_file(Module, OutFile, Opts, Loaded, From, State) -> +analyse_list_to_file(Modules, Opts, State) -> + {LoadedMF, ImportedMF, Error} = are_loaded(Modules, State, [], [], []), + collect([M || {M,_} <- LoadedMF], State#main_state.nodes), + OutDir = proplists:get_value(outdir,Opts), + HTML = lists:member(html,Opts), + Fun = fun({Module,File}) -> + OutFile = outfilename(OutDir,Module,HTML), + do_analyse_to_file(Module,File,OutFile,HTML,State) + end, + {Ok,Error1} = split_ok_error(pmap(Fun, LoadedMF++ImportedMF),[],[]), + {result,Ok,Error ++ Error1}. + +analyse_all_to_file(Opts, State) -> + collect(State#main_state.nodes), + AllModules = get_all_modules(State), + OutDir = proplists:get_value(outdir,Opts), + HTML = lists:member(html,Opts), + Fun = fun({Module,File}) -> + OutFile = outfilename(OutDir,Module,HTML), + do_analyse_to_file(Module,File,OutFile,HTML,State) + end, + {Ok,Error} = split_ok_error(pmap(Fun, AllModules),[],[]), + {result,Ok,Error}. + +get_all_modules(State) -> + get_all_modules(State#main_state.compiled ++ State#main_state.imported,[]). +get_all_modules([{Module,File}|Rest],Acc) -> + get_all_modules(Rest,[{Module,File}|Acc]); +get_all_modules([{Module,File,_}|Rest],Acc) -> + case lists:keymember(Module,1,Acc) of + true -> get_all_modules(Rest,Acc); + false -> get_all_modules(Rest,[{Module,File}|Acc]) + end; +get_all_modules([],Acc) -> + Acc. + +split_ok_error([{ok,R}|Result],Ok,Error) -> + split_ok_error(Result,[R|Ok],Error); +split_ok_error([{error,R}|Result],Ok,Error) -> + split_ok_error(Result,Ok,[R|Error]); +split_ok_error([],Ok,Error) -> + {Ok,Error}. + +do_parallel_analysis_to_file(Module, Opts, Loaded, From, State) -> File = case Loaded of {loaded, File0} -> [{Module,Clauses}] = @@ -2105,24 +2354,32 @@ do_parallel_analysis_to_file(Module, OutFile, Opts, Loaded, From, State) -> {imported, File0, _} -> File0 end, + HTML = lists:member(html,Opts), + OutFile = + case proplists:get_value(outfile,Opts) of + undefined -> + outfilename(proplists:get_value(outdir,Opts),Module,HTML); + F -> + F + end, + reply(From, do_analyse_to_file(Module,File,OutFile,HTML,State)). + +do_analyse_to_file(Module,File,OutFile,HTML,State) -> case find_source(Module, File) of {beam,_BeamFile} -> - reply(From, {error,no_source_code_found}); + {error,{no_source_code_found,Module}}; ErlFile -> analyse_info(Module,State#main_state.imported), - HTML = lists:member(html,Opts), - R = do_analyse_to_file(Module,OutFile, - ErlFile,HTML), - reply(From, R) + do_analyse_to_file1(Module,OutFile,ErlFile,HTML) end. -%% do_analyse_to_file(Module,OutFile,ErlFile) -> {ok,OutFile} | {error,Error} +%% do_analyse_to_file1(Module,OutFile,ErlFile) -> {ok,OutFile} | {error,Error} %% Module = atom() %% OutFile = ErlFile = string() -do_analyse_to_file(Module, OutFile, ErlFile, HTML) -> - case file:open(ErlFile, [read]) of +do_analyse_to_file1(Module, OutFile, ErlFile, HTML) -> + case file:open(ErlFile, [read,raw,read_ahead]) of {ok, InFd} -> - case file:open(OutFile, [write]) of + case file:open(OutFile, [write,raw,delayed_write]) of {ok, OutFd} -> if HTML -> Encoding = encoding(ErlFile), @@ -2160,9 +2417,14 @@ do_analyse_to_file(Module, OutFile, ErlFile, HTML) -> "**************************************" "\n\n"]), - print_lines(Module, InFd, OutFd, 1, HTML), + Pattern = {#bump{module=Module,line='$1',_='_'},'$2'}, + MS = [{Pattern,[],[{{'$1','$2'}}]}], + CovLines = lists:keysort(1,ets:select(?COLLECTION_TABLE, MS)), + print_lines(Module, CovLines, InFd, OutFd, 1, HTML), - if HTML -> io:format(OutFd,"</pre>\n</body>\n</html>\n",[]); + if + HTML -> + file:write(OutFd, "</pre>\n</body>\n</html>\n"); true -> ok end, @@ -2179,21 +2441,19 @@ do_analyse_to_file(Module, OutFile, ErlFile, HTML) -> {error, {file, ErlFile, Reason}} end. -print_lines(Module, InFd, OutFd, L, HTML) -> - case io:get_line(InFd, '') of + +print_lines(Module, CovLines, InFd, OutFd, L, HTML) -> + case file:read_line(InFd) of eof -> ignore; - "%"++_=Line -> %Comment line - not executed. - io:put_chars(OutFd, [tab(),escape_lt_and_gt(Line, HTML)]), - print_lines(Module, InFd, OutFd, L+1, HTML); - RawLine -> + {ok,"%"++_=Line} -> %Comment line - not executed. + file:write(OutFd, [tab(),escape_lt_and_gt(Line, HTML)]), + print_lines(Module, CovLines, InFd, OutFd, L+1, HTML); + {ok,RawLine} -> Line = escape_lt_and_gt(RawLine,HTML), - Pattern = {#bump{module=Module,line=L},'$1'}, - case ets:match(?COLLECTION_TABLE, Pattern) of - [] -> - io:put_chars(OutFd, [tab(),Line]); - Ns -> - N = lists:foldl(fun([Ni], Nacc) -> Nacc+Ni end, 0, Ns), + case CovLines of + [{L,N}|CovLines1] -> + %% N = lists:foldl(fun([Ni], Nacc) -> Nacc+Ni end, 0, Ns), if N=:=0, HTML=:=true -> LineNoNL = Line -- "\n", @@ -2201,19 +2461,22 @@ print_lines(Module, InFd, OutFd, L, HTML) -> %%Str = string:right("0", 6, 32), RedLine = ["<font color=red>",Str,fill1(), LineNoNL,"</font>\n"], - io:put_chars(OutFd, RedLine); + file:write(OutFd, RedLine); N<1000000 -> Str = string:right(integer_to_list(N), 6, 32), - io:put_chars(OutFd, [Str,fill1(),Line]); + file:write(OutFd, [Str,fill1(),Line]); N<10000000 -> Str = integer_to_list(N), - io:put_chars(OutFd, [Str,fill2(),Line]); + file:write(OutFd, [Str,fill2(),Line]); true -> Str = integer_to_list(N), - io:put_chars(OutFd, [Str,fill3(),Line]) - end - end, - print_lines(Module, InFd, OutFd, L+1, HTML) + file:write(OutFd, [Str,fill3(),Line]) + end, + print_lines(Module, CovLines1, InFd, OutFd, L+1, HTML); + _ -> + file:write(OutFd, [tab(),Line]), + print_lines(Module, CovLines, InFd, OutFd, L+1, HTML) + end end. tab() -> " | ". @@ -2223,7 +2486,7 @@ fill3() -> "| ". %%%--Export-------------------------------------------------------------- do_export(Module, OutFile, From, State) -> - case file:open(OutFile,[write,binary,raw]) of + case file:open(OutFile,[write,binary,raw,delayed_write]) of {ok,Fd} -> Reply = case Module of @@ -2362,21 +2625,21 @@ do_reset_collection_table(Module) -> ets:match_delete(?COLLECTION_TABLE, {#bump{module=Module},'_'}). %% do_reset(Module) -> ok -%% The reset is done on a per-clause basis to avoid building +%% The reset is done on ?CHUNK_SIZE number of bumps to avoid building %% long lists in the case of very large modules do_reset(Module) -> - [{Module,Clauses}] = ets:lookup(?COVER_CLAUSE_TABLE, Module), - do_reset2(Clauses). + Pattern = {#bump{module=Module, _='_'}, '$1'}, + MatchSpec = [{Pattern,[{'=/=','$1',0}],['$_']}], + Match = ets:select(?COVER_TABLE,MatchSpec,?CHUNK_SIZE), + do_reset2(Match). -do_reset2([{M,F,A,C,_L}|Clauses]) -> - Pattern = {#bump{module=M, function=F, arity=A, clause=C}, '_'}, - Bumps = ets:match_object(?COVER_TABLE, Pattern), +do_reset2({Bumps,Continuation}) -> lists:foreach(fun({Bump,_N}) -> ets:insert(?COVER_TABLE, {Bump,0}) end, Bumps), - do_reset2(Clauses); -do_reset2([]) -> + do_reset2(ets:select(Continuation)); +do_reset2('$end_of_table') -> ok. do_clear(Module) -> @@ -2419,31 +2682,43 @@ escape_lt_and_gt1([],Acc) -> escape_lt_and_gt1([H|T],Acc) -> escape_lt_and_gt1(T,[H|Acc]). -pmap(Fun, List) -> - pmap(Fun, List, 20). -pmap(Fun, List, Limit) -> - pmap(Fun, List, [], Limit, 0, []). -pmap(Fun, [E | Rest], Pids, Limit, Cnt, Acc) when Cnt < Limit -> - Collector = self(), - Pid = spawn_link(fun() -> - ?SPAWN_DBG(pmap,E), - Collector ! {res,self(),Fun(E)} - end), - erlang:monitor(process, Pid), - pmap(Fun, Rest, Pids ++ [Pid], Limit, Cnt + 1, Acc); -pmap(Fun, List, [Pid | Pids], Limit, Cnt, Acc) -> - receive - {'DOWN', _Ref, process, X, _} when is_pid(X) -> - pmap(Fun, List, [Pid | Pids], Limit, Cnt - 1, Acc); - {res, Pid, Res} -> - pmap(Fun, List, Pids, Limit, Cnt, [Res | Acc]) - end; -pmap(_Fun, [], [], _Limit, 0, Acc) -> - lists:reverse(Acc); -pmap(Fun, [], [], Limit, Cnt, Acc) -> +%%%--Internal functions for parallelization------------------------------ +pmap(Fun,List) -> + NTot = length(List), + NProcs = erlang:system_info(schedulers) * 2, + NPerProc = (NTot div NProcs) + 1, + Mons = pmap_spawn(Fun,NPerProc,List,[]), + pmap_collect(Mons,[]). + +pmap_spawn(_,_,[],Mons) -> + Mons; +pmap_spawn(Fun,NPerProc,List,Mons) -> + {L1,L2} = if length(List)>=NPerProc -> lists:split(NPerProc,List); + true -> {List,[]} % last chunk + end, + Mon = + spawn_monitor( + fun() -> + exit({pmap_done,lists:map(Fun,L1)}) + end), + pmap_spawn(Fun,NPerProc,L2,[Mon|Mons]). + +pmap_collect([],Acc) -> + lists:append(Acc); +pmap_collect(Mons,Acc) -> receive - {'DOWN', _Ref, process, X, _} when is_pid(X) -> - pmap(Fun, [], [], Limit, Cnt - 1, Acc) + {'DOWN', Ref, process, Pid, {pmap_done,Result}} -> + pmap_collect(lists:delete({Pid,Ref},Mons),[Result|Acc]); + {'DOWN', Ref, process, Pid, Reason} = Down -> + case lists:member({Pid,Ref},Mons) of + true -> + %% Something went really wrong - don't hang! + exit(Reason); + false -> + %% This should be handled somewhere else + self() ! Down, + pmap_collect(Mons,Acc) + end end. %%%----------------------------------------------------------------- |