From 5cd1994b89faa27657e2b0bb7ef026124acdff95 Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Fri, 13 Jan 2017 13:12:54 +0100 Subject: dialyzer: Do not keep the code server's type info on the heap The table used during translation of forms to types used to be a dictionary of the type info (-type(), -opaque(), and -record()) for all modules. Now it is reduced to the modules that are needed to handle references to types in other modules. The PLT:s type info is still kept on the heap. --- lib/dialyzer/src/dialyzer_analysis_callgraph.erl | 14 +-- lib/dialyzer/src/dialyzer_codeserver.erl | 45 ++++++--- lib/dialyzer/src/dialyzer_contracts.erl | 78 +++++++-------- lib/dialyzer/src/dialyzer_plt.erl | 7 ++ lib/dialyzer/src/dialyzer_utils.erl | 115 ++++++++++++++--------- 5 files changed, 151 insertions(+), 108 deletions(-) (limited to 'lib/dialyzer/src') diff --git a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl index ae1e4d8c38..a3bb2b858d 100644 --- a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl +++ b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl @@ -114,7 +114,6 @@ loop(#server_state{parent = Parent} = State, %% The Analysis %%-------------------------------------------------------------------- -%% Calls to erlang:garbage_collect() help to reduce the heap size. analysis_start(Parent, Analysis, LegalWarnings) -> CServer = dialyzer_codeserver:new(), Plt = Analysis#analysis.plt, @@ -136,11 +135,9 @@ analysis_start(Parent, Analysis, LegalWarnings) -> %% Remote type postprocessing NewCServer = try - NewRecords = dialyzer_codeserver:get_temp_records(TmpCServer0), + TmpCServer1 = dialyzer_utils:merge_types(TmpCServer0, Plt), NewExpTypes = dialyzer_codeserver:get_temp_exported_types(TmpCServer0), - OldRecords = dialyzer_plt:get_types(Plt), OldExpTypes0 = dialyzer_plt:get_exported_types(Plt), - MergedRecords = dialyzer_utils:merge_records(NewRecords, OldRecords), RemMods = [case Analysis#analysis.start_from of byte_code -> list_to_atom(filename:basename(F, ".beam")); @@ -148,10 +145,9 @@ analysis_start(Parent, Analysis, LegalWarnings) -> end || F <- Files], OldExpTypes1 = dialyzer_utils:sets_filter(RemMods, OldExpTypes0), MergedExpTypes = sets:union(NewExpTypes, OldExpTypes1), - TmpCServer1 = dialyzer_codeserver:set_temp_records(MergedRecords, TmpCServer0), TmpCServer2 = dialyzer_codeserver:finalize_exported_types(MergedExpTypes, TmpCServer1), - erlang:garbage_collect(), + erlang:garbage_collect(), % reduce heap size ?timing(State#analysis_state.timing_server, "remote", contracts_and_records(TmpCServer2)) catch @@ -200,11 +196,9 @@ contracts_and_records(CodeServer) -> contrs_and_recs(TmpCServer2) -> fun() -> Parent = receive {Pid, go} -> Pid end, - {TmpCServer3, RecordDict} = - dialyzer_utils:process_record_remote_types(TmpCServer2), + TmpCServer3 = dialyzer_utils:process_record_remote_types(TmpCServer2), TmpServer4 = - dialyzer_contracts:process_contract_remote_types(TmpCServer3, - RecordDict), + dialyzer_contracts:process_contract_remote_types(TmpCServer3), dialyzer_codeserver:give_away(TmpServer4, Parent), exit(TmpServer4) end. diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl index f53c713bfe..aa779e0e39 100644 --- a/lib/dialyzer/src/dialyzer_codeserver.erl +++ b/lib/dialyzer/src/dialyzer_codeserver.erl @@ -26,18 +26,20 @@ give_away/2, finalize_contracts/1, finalize_exported_types/2, - finalize_records/2, + finalize_records/1, get_contracts/1, get_callbacks/1, get_exported_types/1, get_exports/1, get_records/1, + get_records_table/1, get_next_core_label/1, get_temp_contracts/2, - contracts_modules/1, + all_temp_modules/1, store_contracts/4, get_temp_exported_types/1, - get_temp_records/1, + get_temp_records_table/1, + lookup_temp_mod_records/2, insert/3, insert_exports/2, insert_temp_exported_types/2, @@ -269,6 +271,11 @@ lookup_mod_records(Mod, #codeserver{records = RecDict}) when is_atom(Mod) -> {ok, Map} -> Map end. +-spec get_records_table(codeserver()) -> map_ets(). + +get_records_table(#codeserver{records = RecDict}) -> + RecDict. + -spec get_records(codeserver()) -> mod_records(). get_records(#codeserver{records = RecDict}) -> @@ -283,10 +290,18 @@ store_temp_records(Mod, Map, #codeserver{temp_records = TempRecDict} = CS) false -> CS#codeserver{temp_records = ets_map_store(Mod, Map, TempRecDict)} end. --spec get_temp_records(codeserver()) -> mod_records(). +-spec get_temp_records_table(codeserver()) -> map_ets(). + +get_temp_records_table(#codeserver{temp_records = TempRecDict}) -> + TempRecDict. + +-spec lookup_temp_mod_records(module(), codeserver()) -> types(). -get_temp_records(#codeserver{temp_records = TempRecDict}) -> - ets_dict_to_dict(TempRecDict). +lookup_temp_mod_records(Mod, #codeserver{temp_records = TempRecDict}) -> + case ets_dict_find(Mod, TempRecDict) of + error -> maps:new(); + {ok, Map} -> Map + end. -spec set_temp_records(mod_records(), codeserver()) -> codeserver(). @@ -296,13 +311,13 @@ set_temp_records(Dict, CS) -> true = ets_dict_store_dict(Dict, TempRecords), CS#codeserver{temp_records = TempRecords}. --spec finalize_records(mod_records(), codeserver()) -> codeserver(). +-spec finalize_records(codeserver()) -> codeserver(). -finalize_records(Dict, #codeserver{temp_records = TmpRecords, - records = Records} = CS) -> - true = ets:delete(TmpRecords), - true = ets_dict_store_dict(Dict, Records), - CS#codeserver{temp_records = clean}. +finalize_records(#codeserver{temp_records = TmpRecords, + records = Records} = CS) -> + true = ets:delete(Records), + ets:rename(TmpRecords, dialyzer_codeserver_records), + CS#codeserver{temp_records = clean, records = TmpRecords}. -spec lookup_mod_contracts(atom(), codeserver()) -> contracts(). @@ -348,12 +363,14 @@ store_temp_contracts(Mod, SpecMap, CallbackMap, #codeserver{temp_contracts = Cn, temp_callbacks = Cb} = CS) when is_atom(Mod) -> + %% Make sure Mod is stored even if there are not callbacks or + %% contracts. CS1 = CS#codeserver{temp_contracts = ets_map_store(Mod, SpecMap, Cn)}, CS1#codeserver{temp_callbacks = ets_map_store(Mod, CallbackMap, Cb)}. --spec contracts_modules(codeserver()) -> [module()]. +-spec all_temp_modules(codeserver()) -> [module()]. -contracts_modules(#codeserver{temp_contracts = TempContTable}) -> +all_temp_modules(#codeserver{temp_contracts = TempContTable}) -> ets:select(TempContTable, [{{'$1', '$2'}, [], ['$1']}]). -spec store_contracts(module(), contracts(), contracts(), codeserver()) -> diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl index 2078e58ce8..45f9099f4e 100644 --- a/lib/dialyzer/src/dialyzer_contracts.erl +++ b/lib/dialyzer/src/dialyzer_contracts.erl @@ -24,7 +24,7 @@ get_contract_return/2, %% get_contract_signature/1, is_overloaded/1, - process_contract_remote_types/2, + process_contract_remote_types/1, store_tmp_contract/5]). -export_type([file_contract/0, plt_contracts/0]). @@ -139,18 +139,18 @@ sequence([], _Delimiter) -> ""; sequence([H], _Delimiter) -> H; sequence([H|T], Delimiter) -> H ++ Delimiter ++ sequence(T, Delimiter). --spec process_contract_remote_types(dialyzer_codeserver:codeserver(), - erl_types:mod_records()) -> +-spec process_contract_remote_types(dialyzer_codeserver:codeserver()) -> dialyzer_codeserver:codeserver(). -process_contract_remote_types(CodeServer, RecordDict) -> - Mods = dialyzer_codeserver:contracts_modules(CodeServer), +process_contract_remote_types(CodeServer) -> + Mods = dialyzer_codeserver:all_temp_modules(CodeServer), + RecordTable = dialyzer_codeserver:get_records_table(CodeServer), ExpTypes = dialyzer_codeserver:get_exported_types(CodeServer), ContractFun = fun({{_M, _F, _A}=MFA, {File, TmpContract, Xtra}}, C0) -> #tmp_contract{contract_funs = CFuns, forms = Forms} = TmpContract, {NewCs, C2} = lists:mapfoldl(fun(CFun, C1) -> - CFun(ExpTypes, RecordDict, C1) + CFun(ExpTypes, RecordTable, C1) end, C0, CFuns), Args = general_domain(NewCs), Contract = #contract{contracts = NewCs, args = Args, forms = Forms}, @@ -451,10 +451,10 @@ contract_from_form(Forms, MFA, RecDict, FileLine) -> contract_from_form([{type, _, 'fun', [_, _]} = Form | Left], MFA, RecDict, FileLine, TypeAcc, FormAcc) -> TypeFun = - fun(ExpTypes, AllRecords, Cache) -> + fun(ExpTypes, RecordTable, Cache) -> {NewType, NewCache} = try - from_form_with_check(Form, ExpTypes, MFA, AllRecords, Cache) + from_form_with_check(Form, ExpTypes, MFA, RecordTable, Cache) catch throw:{error, Msg} -> {File, Line} = FileLine, @@ -472,12 +472,12 @@ contract_from_form([{type, _L1, bounded_fun, [{type, _L2, 'fun', [_, _]} = Form, Constr]}| Left], MFA, RecDict, FileLine, TypeAcc, FormAcc) -> TypeFun = - fun(ExpTypes, AllRecords, Cache) -> + fun(ExpTypes, RecordTable, Cache) -> {Constr1, VarTable, Cache1} = - process_constraints(Constr, MFA, RecDict, ExpTypes, AllRecords, + process_constraints(Constr, MFA, RecDict, ExpTypes, RecordTable, Cache), {NewType, NewCache} = - from_form_with_check(Form, ExpTypes, MFA, AllRecords, + from_form_with_check(Form, ExpTypes, MFA, RecordTable, VarTable, Cache1), NewTypeNoVars = erl_types:subst_all_vars_to_any(NewType), {{NewTypeNoVars, Constr1}, NewCache} @@ -488,28 +488,28 @@ contract_from_form([{type, _L1, bounded_fun, contract_from_form([], _MFA, _RecDict, _FileLine, TypeAcc, FormAcc) -> {lists:reverse(TypeAcc), lists:reverse(FormAcc)}. -process_constraints(Constrs, MFA, RecDict, ExpTypes, AllRecords, Cache) -> +process_constraints(Constrs, MFA, RecDict, ExpTypes, RecordTable, Cache) -> {Init0, NewCache} = initialize_constraints(Constrs, MFA, RecDict, ExpTypes, - AllRecords, Cache), + RecordTable, Cache), Init = remove_cycles(Init0), - constraints_fixpoint(Init, MFA, RecDict, ExpTypes, AllRecords, NewCache). + constraints_fixpoint(Init, MFA, RecDict, ExpTypes, RecordTable, NewCache). -initialize_constraints(Constrs, MFA, RecDict, ExpTypes, AllRecords, Cache) -> - initialize_constraints(Constrs, MFA, RecDict, ExpTypes, AllRecords, +initialize_constraints(Constrs, MFA, RecDict, ExpTypes, RecordTable, Cache) -> + initialize_constraints(Constrs, MFA, RecDict, ExpTypes, RecordTable, Cache, []). -initialize_constraints([], _MFA, _RecDict, _ExpTypes, _AllRecords, +initialize_constraints([], _MFA, _RecDict, _ExpTypes, _RecordTable, Cache, Acc) -> {Acc, Cache}; -initialize_constraints([Constr|Rest], MFA, RecDict, ExpTypes, AllRecords, +initialize_constraints([Constr|Rest], MFA, RecDict, ExpTypes, RecordTable, Cache, Acc) -> case Constr of {type, _, constraint, [{atom, _, is_subtype}, [Type1, Type2]]} -> VarTable = erl_types:var_table__new(), {T1, NewCache} = - final_form(Type1, ExpTypes, MFA, AllRecords, VarTable, Cache), + final_form(Type1, ExpTypes, MFA, RecordTable, VarTable, Cache), Entry = {T1, Type2}, - initialize_constraints(Rest, MFA, RecDict, ExpTypes, AllRecords, + initialize_constraints(Rest, MFA, RecDict, ExpTypes, RecordTable, NewCache, [Entry|Acc]); {type, _, constraint, [{atom,_,Name}, List]} -> N = length(List), @@ -517,18 +517,18 @@ initialize_constraints([Constr|Rest], MFA, RecDict, ExpTypes, AllRecords, io_lib:format("Unsupported type guard ~w/~w\n", [Name, N])}) end. -constraints_fixpoint(Constrs, MFA, RecDict, ExpTypes, AllRecords, Cache) -> +constraints_fixpoint(Constrs, MFA, RecDict, ExpTypes, RecordTable, Cache) -> VarTable = erl_types:var_table__new(), {VarTab, NewCache} = - constraints_to_dict(Constrs, MFA, RecDict, ExpTypes, AllRecords, + constraints_to_dict(Constrs, MFA, RecDict, ExpTypes, RecordTable, VarTable, Cache), constraints_fixpoint(VarTab, MFA, Constrs, RecDict, ExpTypes, - AllRecords, NewCache). + RecordTable, NewCache). constraints_fixpoint(OldVarTab, MFA, Constrs, RecDict, ExpTypes, - AllRecords, Cache) -> + RecordTable, Cache) -> {NewVarTab, NewCache} = - constraints_to_dict(Constrs, MFA, RecDict, ExpTypes, AllRecords, + constraints_to_dict(Constrs, MFA, RecDict, ExpTypes, RecordTable, OldVarTab, Cache), case NewVarTab of OldVarTab -> @@ -540,38 +540,38 @@ constraints_fixpoint(OldVarTab, MFA, Constrs, RecDict, ExpTypes, {FinalConstrs, NewVarTab, NewCache}; _Other -> constraints_fixpoint(NewVarTab, MFA, Constrs, RecDict, ExpTypes, - AllRecords, NewCache) + RecordTable, NewCache) end. -final_form(Form, ExpTypes, MFA, AllRecords, VarTable, Cache) -> - from_form_with_check(Form, ExpTypes, MFA, AllRecords, VarTable, Cache). +final_form(Form, ExpTypes, MFA, RecordTable, VarTable, Cache) -> + from_form_with_check(Form, ExpTypes, MFA, RecordTable, VarTable, Cache). -from_form_with_check(Form, ExpTypes, MFA, AllRecords, Cache) -> +from_form_with_check(Form, ExpTypes, MFA, RecordTable, Cache) -> VarTable = erl_types:var_table__new(), - from_form_with_check(Form, ExpTypes, MFA, AllRecords, VarTable, Cache). + from_form_with_check(Form, ExpTypes, MFA, RecordTable, VarTable, Cache). -from_form_with_check(Form, ExpTypes, MFA, AllRecords, VarTable, Cache) -> +from_form_with_check(Form, ExpTypes, MFA, RecordTable, VarTable, Cache) -> Site = {spec, MFA}, - C1 = erl_types:t_check_record_fields(Form, ExpTypes, Site, AllRecords, + C1 = erl_types:t_check_record_fields(Form, ExpTypes, Site, RecordTable, VarTable, Cache), - erl_types:t_from_form(Form, ExpTypes, Site, AllRecords, VarTable, C1). + erl_types:t_from_form(Form, ExpTypes, Site, RecordTable, VarTable, C1). -constraints_to_dict(Constrs, MFA, RecDict, ExpTypes, AllRecords, +constraints_to_dict(Constrs, MFA, RecDict, ExpTypes, RecordTable, VarTab, Cache) -> {Subtypes, NewCache} = - constraints_to_subs(Constrs, MFA, RecDict, ExpTypes, AllRecords, + constraints_to_subs(Constrs, MFA, RecDict, ExpTypes, RecordTable, VarTab, Cache, []), {insert_constraints(Subtypes), NewCache}. -constraints_to_subs([], _MFA, _RecDict, _ExpTypes, _AllRecords, +constraints_to_subs([], _MFA, _RecDict, _ExpTypes, _RecordTable, _VarTab, Cache, Acc) -> {Acc, Cache}; -constraints_to_subs([{T1, Form2}|Rest], MFA, RecDict, ExpTypes, AllRecords, +constraints_to_subs([{T1, Form2}|Rest], MFA, RecDict, ExpTypes, RecordTable, VarTab, Cache, Acc) -> {T2, NewCache} = - final_form(Form2, ExpTypes, MFA, AllRecords, VarTab, Cache), + final_form(Form2, ExpTypes, MFA, RecordTable, VarTab, Cache), NewAcc = [{subtype, T1, T2}|Acc], - constraints_to_subs(Rest, MFA, RecDict, ExpTypes, AllRecords, + constraints_to_subs(Rest, MFA, RecDict, ExpTypes, RecordTable, VarTab, NewCache, NewAcc). %% Replaces variables with '_' when necessary to break up cycles among diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl index 37c22fef48..1bf30cf83b 100644 --- a/lib/dialyzer/src/dialyzer_plt.erl +++ b/lib/dialyzer/src/dialyzer_plt.erl @@ -32,6 +32,7 @@ from_file/1, get_default_plt/0, get_types/1, + get_module_types/2, get_exported_types/1, %% insert/3, insert_list/2, @@ -199,6 +200,12 @@ insert_exported_types(PLT, Set) -> get_types(#plt{types = Types}) -> Types. +-spec get_module_types(plt(), atom()) -> + 'none' | {'value', erl_types:type_table()}. + +get_module_types(#plt{types = Types}, M) when is_atom(M) -> + table_lookup(Types, M). + -spec get_exported_types(plt()) -> sets:set(). get_exported_types(#plt{exported_types = ExpTypes}) -> diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl index 432d27571b..9eaf95c1a2 100644 --- a/lib/dialyzer/src/dialyzer_utils.erl +++ b/lib/dialyzer/src/dialyzer_utils.erl @@ -37,9 +37,9 @@ get_fun_meta_info/3, is_suppressed_fun/2, is_suppressed_tag/3, - merge_records/2, pp_hook/0, process_record_remote_types/1, + merge_types/2, sets_filter/2, src_compiler_opts/0, refold_pattern/1, @@ -188,7 +188,6 @@ get_core_from_abstract_code(AbstrCode, Opts) -> %% ============================================================================ -type type_table() :: erl_types:type_table(). --type mod_records() :: dict:dict(module(), type_table()). -spec get_record_and_type_info(abstract_code()) -> {'ok', type_table()} | {'error', string()}. @@ -289,18 +288,18 @@ get_record_fields([{record_field, _Line, Name, _Init}|Left], RecDict, Acc) -> get_record_fields([], _RecDict, Acc) -> lists:reverse(Acc). --spec process_record_remote_types(codeserver()) -> - {codeserver(), mod_records()}. +-spec process_record_remote_types(codeserver()) -> codeserver(). %% The field types are cached. Used during analysis when handling records. process_record_remote_types(CServer) -> - TempRecords = dialyzer_codeserver:get_temp_records(CServer), ExpTypes = dialyzer_codeserver:get_exported_types(CServer), - TempRecords1 = process_opaque_types0(TempRecords, ExpTypes), - %% A cache (not the field type cache) is used for speeding things up a bit. + Mods = dialyzer_codeserver:all_temp_modules(CServer), + process_opaque_types0(Mods, CServer, ExpTypes), VarTable = erl_types:var_table__new(), + RecordTable = dialyzer_codeserver:get_temp_records_table(CServer), ModuleFun = - fun({Module, Record}) -> + fun(Module) -> + RecordMap = dialyzer_codeserver:lookup_temp_mod_records(Module, CServer), RecordFun = fun({Key, Value}, C2) -> case Key of @@ -313,7 +312,7 @@ process_record_remote_types(CServer) -> {FieldT, C6} = erl_types:t_from_form (Field, ExpTypes, Site, - TempRecords1, VarTable, + RecordTable, VarTable, C5), {{FieldName, Field, FieldT}, C6} end, C4, Fields), @@ -328,30 +327,29 @@ process_record_remote_types(CServer) -> end, Cache = erl_types:cache__new(), {RecordList, _NewCache} = - lists:mapfoldl(RecordFun, Cache, maps:to_list(Record)), - {Module, maps:from_list(RecordList)} + lists:mapfoldl(RecordFun, Cache, maps:to_list(RecordMap)), + dialyzer_codeserver:store_temp_records(Module, + maps:from_list(RecordList), + CServer) end, - NewRecordsList = lists:map(ModuleFun, dict:to_list(TempRecords1)), - NewRecords = dict:from_list(NewRecordsList), - check_record_fields(NewRecords, ExpTypes), - {dialyzer_codeserver:finalize_records(NewRecords, CServer), NewRecords}. + lists:foreach(ModuleFun, Mods), + check_record_fields(Mods, CServer, ExpTypes), + dialyzer_codeserver:finalize_records(CServer). %% erl_types:t_from_form() substitutes the declaration of opaque types %% for the expanded type in some cases. To make sure the initial type, %% any(), is not used, the expansion is done twice. %% XXX: Recursive opaque types are not handled well. -process_opaque_types0(TempRecords0, TempExpTypes) -> - Cache = erl_types:cache__new(), - {TempRecords1, Cache1} = - process_opaque_types(TempRecords0, TempExpTypes, Cache), - {TempRecords, _NewCache} = - process_opaque_types(TempRecords1, TempExpTypes, Cache1), - TempRecords. - -process_opaque_types(TempRecords, TempExpTypes, Cache) -> +process_opaque_types0(AllModules, CServer, TempExpTypes) -> + process_opaque_types(AllModules, CServer, TempExpTypes), + process_opaque_types(AllModules, CServer, TempExpTypes). + +process_opaque_types(AllModules, CServer, TempExpTypes) -> VarTable = erl_types:var_table__new(), + RecordTable = dialyzer_codeserver:get_temp_records_table(CServer), ModuleFun = - fun({Module, Record}, C0) -> + fun(Module) -> + RecordMap = dialyzer_codeserver:lookup_temp_mod_records(Module, CServer), RecordFun = fun({Key, Value}, C2) -> case Key of @@ -360,32 +358,32 @@ process_opaque_types(TempRecords, TempExpTypes, Cache) -> Site = {type, {Module, Name, NArgs}}, {Type, C3} = erl_types:t_from_form(Form, TempExpTypes, Site, - TempRecords, VarTable, C2), + RecordTable, VarTable, C2), {{Key, {F, Type}}, C3}; _Other -> {{Key, Value}, C2} end end, - {RecordList, C1} = - lists:mapfoldl(RecordFun, C0, maps:to_list(Record)), - {{Module, maps:from_list(RecordList)}, C1} - %% dict:map(RecordFun, Record) + C0 = erl_types:cache__new(), + {RecordList, _NewCache} = + lists:mapfoldl(RecordFun, C0, maps:to_list(RecordMap)), + dialyzer_codeserver:store_temp_records(Module, + maps:from_list(RecordList), + CServer) end, - {TempRecordList, NewCache} = - lists:mapfoldl(ModuleFun, Cache, dict:to_list(TempRecords)), - {dict:from_list(TempRecordList), NewCache}. - %% dict:map(ModuleFun, TempRecords). + lists:foreach(ModuleFun, AllModules). -check_record_fields(Records, TempExpTypes) -> - Cache = erl_types:cache__new(), +check_record_fields(AllModules, CServer, TempExpTypes) -> VarTable = erl_types:var_table__new(), + RecordTable = dialyzer_codeserver:get_temp_records_table(CServer), CheckFun = - fun({Module, Element}, C0) -> + fun(Module) -> CheckForm = fun(Form, Site, C1) -> erl_types:t_check_record_fields(Form, TempExpTypes, - Site, Records, + Site, RecordTable, VarTable, C1) end, - ElemFun = + RecordMap = dialyzer_codeserver:lookup_temp_mod_records(Module, CServer), + RecordFun = fun({Key, Value}, C2) -> case Key of {record, Name} -> @@ -406,10 +404,10 @@ check_record_fields(Records, TempExpTypes) -> msg_with_position(Fun, FileLine) end end, - lists:foldl(ElemFun, C0, maps:to_list(Element)) + C0 = erl_types:cache__new(), + _ = lists:foldl(RecordFun, C0, maps:to_list(RecordMap)) end, - _NewCache = lists:foldl(CheckFun, Cache, dict:to_list(Records)), - ok. + lists:foreach(CheckFun, AllModules). msg_with_position(Fun, FileLine) -> try Fun() @@ -421,10 +419,37 @@ msg_with_position(Fun, FileLine) -> throw({error, NewMsg}) end. --spec merge_records(mod_records(), mod_records()) -> mod_records(). +-spec merge_types(codeserver(), dialyzer_plt:plt()) -> codeserver(). -merge_records(NewRecords, OldRecords) -> - dict:merge(fun(_Key, NewVal, _OldVal) -> NewVal end, NewRecords, OldRecords). +merge_types(CServer, Plt) -> + AllNewModules = dialyzer_codeserver:all_temp_modules(CServer), + AllNewModulesSet = sets:from_list(AllNewModules), + AllOldModulesSet = dialyzer_plt:all_modules(Plt), + AllModulesSet = sets:union(AllNewModulesSet, AllOldModulesSet), + ModuleFun = + fun(Module) -> + KeepOldFun = + fun() -> + case dialyzer_plt:get_module_types(Plt, Module) of + none -> no; + {value, OldRecords} -> + case sets:is_element(Module, AllNewModulesSet) of + true -> no; + false -> {yes, OldRecords} + end + end + end, + Records = + case KeepOldFun() of + no -> + dialyzer_codeserver:lookup_temp_mod_records(Module, CServer); + {yes, OldRecords} -> + OldRecords + end, + dialyzer_codeserver:store_temp_records(Module, Records, CServer) + end, + lists:foreach(ModuleFun, sets:to_list(AllModulesSet)), + CServer. %% ============================================================================ %% -- cgit v1.2.3 From e917ae0da957a8b151df54e2f1940fded8d21ad1 Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Mon, 16 Jan 2017 13:15:55 +0100 Subject: typer: Fix a bug regarding the -T option The -T option has not been working for a long time. --- lib/dialyzer/src/dialyzer_codeserver.erl | 8 +++++--- lib/dialyzer/src/dialyzer_contracts.erl | 4 ++-- lib/dialyzer/src/dialyzer_plt.erl | 4 ++++ 3 files changed, 11 insertions(+), 5 deletions(-) (limited to 'lib/dialyzer/src') diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl index aa779e0e39..e15569388e 100644 --- a/lib/dialyzer/src/dialyzer_codeserver.erl +++ b/lib/dialyzer/src/dialyzer_codeserver.erl @@ -72,7 +72,6 @@ -type mod_records() :: erl_types:mod_records(). -type contracts() :: #{mfa() => dialyzer_contracts:file_contract()}. --type mod_contracts() :: dict:dict(module(), contracts()). %% A property-list of data compiled from -compile and -dialyzer attributes. -type meta_info() :: [{{'nowarn_function' | dial_warn_tag()}, @@ -346,10 +345,13 @@ lookup_meta_info(MorMFA, #codeserver{fun_meta_info = FunMetaInfo}) -> {ok, PropList} -> PropList end. --spec get_contracts(codeserver()) -> mod_contracts(). +-spec get_contracts(codeserver()) -> + dict:dict(mfa(), dialyzer_contracts:file_contract()). get_contracts(#codeserver{contracts = ContDict}) -> - ets_dict_to_dict(ContDict). + dict:filter(fun({_M, _F, _A}, _) -> true; + (_, _) -> false + end, ets_dict_to_dict(ContDict)). -spec get_callbacks(codeserver()) -> list(). diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl index 45f9099f4e..5f24b5a668 100644 --- a/lib/dialyzer/src/dialyzer_contracts.erl +++ b/lib/dialyzer/src/dialyzer_contracts.erl @@ -177,7 +177,7 @@ process_contract_remote_types(CodeServer) -> -type fun_types() :: dict:dict(label(), erl_types:type_table()). --spec check_contracts([{mfa(), file_contract()}], +-spec check_contracts(orddict:orddict(mfa(), file_contract()), dialyzer_callgraph:callgraph(), fun_types(), opaques_fun()) -> plt_contracts(). @@ -206,7 +206,7 @@ check_contracts(Contracts, Callgraph, FunTypes, FindOpaques) -> error -> NewContracts end end, - dict:fold(FoldFun, [], FunTypes). + orddict:from_list(dict:fold(FoldFun, [], FunTypes)). %% Checks all components of a contract -spec check_contract(#contract{}, erl_types:erl_type()) -> 'ok' | {'error', term()}. diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl index 1bf30cf83b..511a17d577 100644 --- a/lib/dialyzer/src/dialyzer_plt.erl +++ b/lib/dialyzer/src/dialyzer_plt.erl @@ -144,6 +144,10 @@ delete_list(#plt{info = Info, types = Types, -spec insert_contract_list(plt(), dialyzer_contracts:plt_contracts()) -> plt(). +insert_contract_list(#plt{contracts = Contracts} = PLT, List) -> + NewContracts = dict:merge(fun(_MFA, _Old, New) -> New end, + Contracts, dict:from_list(List)), + PLT#plt{contracts = NewContracts}; insert_contract_list(#mini_plt{contracts = Contracts} = PLT, List) -> true = ets:insert(Contracts, List), PLT. -- cgit v1.2.3 From 2c577356c419b9420cae91a74d4b1602d584275a Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Tue, 17 Jan 2017 15:10:58 +0100 Subject: dialyzer: Use less memory for the PLT when analyzing The two tables for (record) types and exported types are no longer kept in memory (by the PLT) when analyzing, which reduces memory consumption. A first step towards removing (or replacing) the #plt{} record. --- lib/dialyzer/src/dialyzer_analysis_callgraph.erl | 29 ++++++---- lib/dialyzer/src/dialyzer_cl.erl | 16 +++++- lib/dialyzer/src/dialyzer_codeserver.erl | 68 ++++++++++++------------ lib/dialyzer/src/dialyzer_plt.erl | 21 +++----- 4 files changed, 73 insertions(+), 61 deletions(-) (limited to 'lib/dialyzer/src') diff --git a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl index a3bb2b858d..aeeb895a0c 100644 --- a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl +++ b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl @@ -153,16 +153,12 @@ analysis_start(Parent, Analysis, LegalWarnings) -> catch throw:{error, _ErrorMsg} = Error -> exit(Error) end, - NewPlt0 = dialyzer_plt:insert_types(Plt, dialyzer_codeserver:get_records(NewCServer)), - ExpTypes = dialyzer_codeserver:get_exported_types(NewCServer), - NewPlt1 = dialyzer_plt:insert_exported_types(NewPlt0, ExpTypes), - State0 = State#analysis_state{plt = NewPlt1}, - dump_callgraph(Callgraph, State0, Analysis), + dump_callgraph(Callgraph, State, Analysis), %% Remove all old versions of the files being analyzed AllNodes = dialyzer_callgraph:all_nodes(Callgraph), - Plt1_a = dialyzer_plt:delete_list(NewPlt1, AllNodes), + Plt1_a = dialyzer_plt:delete_list(Plt, AllNodes), Plt1 = dialyzer_plt:insert_callbacks(Plt1_a, NewCServer), - State1 = State0#analysis_state{codeserver = NewCServer, plt = Plt1}, + State1 = State#analysis_state{codeserver = NewCServer, plt = Plt1}, Exports = dialyzer_codeserver:get_exports(NewCServer), NonExports = sets:subtract(sets:from_list(AllNodes), Exports), NonExportsList = sets:to_list(NonExports), @@ -172,14 +168,17 @@ analysis_start(Parent, Analysis, LegalWarnings) -> false -> Callgraph end, State2 = analyze_callgraph(NewCallgraph, State1), - #analysis_state{plt = MiniPlt2, doc_plt = DocPlt} = State2, + #analysis_state{plt = MiniPlt2, + doc_plt = DocPlt, + codeserver = Codeserver0} = State2, + {Codeserver, MiniPlt3} = move_data(Codeserver0, MiniPlt2), dialyzer_callgraph:dispose_race_server(NewCallgraph), rcv_and_send_ext_types(Parent), %% Since the PLT is never used, a dummy is sent: DummyPlt = dialyzer_plt:new(), - send_codeserver_plt(Parent, CServer, DummyPlt), - MiniPlt3 = dialyzer_plt:delete_list(MiniPlt2, NonExportsList), - send_analysis_done(Parent, MiniPlt3, DocPlt). + send_codeserver_plt(Parent, Codeserver, DummyPlt), + MiniPlt4 = dialyzer_plt:delete_list(MiniPlt3, NonExportsList), + send_analysis_done(Parent, MiniPlt4, DocPlt). contracts_and_records(CodeServer) -> Fun = contrs_and_recs(CodeServer), @@ -203,6 +202,13 @@ contrs_and_recs(TmpCServer2) -> exit(TmpServer4) end. +move_data(CServer, MiniPlt) -> + {CServer1, Records} = dialyzer_codeserver:extract_records(CServer), + MiniPlt1 = dialyzer_plt:insert_types(MiniPlt, Records), + {NewCServer, ExpTypes} = dialyzer_codeserver:extract_exported_types(CServer1), + NewMiniPlt = dialyzer_plt:insert_exported_types(MiniPlt1, ExpTypes), + {NewCServer, NewMiniPlt}. + analyze_callgraph(Callgraph, #analysis_state{codeserver = Codeserver, doc_plt = DocPlt, plt = Plt, @@ -597,6 +603,7 @@ send_ext_types(Parent, ExtTypes) -> ok. send_codeserver_plt(Parent, CServer, Plt) -> + ok = dialyzer_codeserver:give_away(CServer, Parent), Parent ! {self(), cserver, CServer, Plt}, ok. diff --git a/lib/dialyzer/src/dialyzer_cl.erl b/lib/dialyzer/src/dialyzer_cl.erl index 158ee761af..93200f53d2 100644 --- a/lib/dialyzer/src/dialyzer_cl.erl +++ b/lib/dialyzer/src/dialyzer_cl.erl @@ -30,6 +30,8 @@ -record(cl_state, {backend_pid :: pid() | 'undefined', + code_server = none :: 'none' + | dialyzer_codeserver:codeserver(), erlang_mode = false :: boolean(), external_calls = [] :: [mfa()], external_types = [] :: [mfa()], @@ -630,6 +632,9 @@ cl_loop(State, LogCache) -> {BackendPid, warnings, Warnings} -> NewState = store_warnings(State, Warnings), cl_loop(NewState, LogCache); + {BackendPid, cserver, CodeServer, _Plt} -> % Plt is ignored + NewState = State#cl_state{code_server = CodeServer}, + cl_loop(NewState, LogCache); {BackendPid, done, NewMiniPlt, _NewDocPlt} -> return_value(State, NewMiniPlt); {BackendPid, ext_calls, ExtCalls} -> @@ -647,7 +652,6 @@ cl_loop(State, LogCache) -> cl_error(State, Msg); _Other -> %% io:format("Received ~p\n", [_Other]), - %% Note: {BackendPid, cserver, CodeServer, Plt} is ignored. cl_loop(State, LogCache) end. @@ -688,12 +692,20 @@ cl_error(State, Msg) -> maybe_close_output_file(State), throw({dialyzer_error, lists:flatten(Msg)}). -return_value(State = #cl_state{erlang_mode = ErlangMode, +return_value(State = #cl_state{code_server = CodeServer, + erlang_mode = ErlangMode, mod_deps = ModDeps, output_plt = OutputPlt, plt_info = PltInfo, stored_warnings = StoredWarnings}, MiniPlt) -> + %% Just for now: + case CodeServer =:= none of + true -> + ok; + false -> + dialyzer_codeserver:delete(CodeServer) + end, case OutputPlt =:= none of true -> dialyzer_plt:delete(MiniPlt); diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl index e15569388e..a1a7370eff 100644 --- a/lib/dialyzer/src/dialyzer_codeserver.erl +++ b/lib/dialyzer/src/dialyzer_codeserver.erl @@ -30,9 +30,10 @@ get_contracts/1, get_callbacks/1, get_exported_types/1, + extract_exported_types/1, get_exports/1, - get_records/1, get_records_table/1, + extract_records/1, get_next_core_label/1, get_temp_contracts/2, all_temp_modules/1, @@ -54,7 +55,6 @@ lookup_meta_info/2, new/0, set_next_core_label/2, - set_temp_records/2, store_temp_records/3, translate_fake_file/3]). @@ -69,7 +69,6 @@ -type set_ets() :: ets:tid(). -type types() :: erl_types:type_table(). --type mod_records() :: erl_types:mod_records(). -type contracts() :: #{mfa() => dialyzer_contracts:file_contract()}. @@ -81,8 +80,8 @@ -record(codeserver, {next_core_label = 0 :: label(), code :: dict_ets(), - exported_types :: set_ets(), % set(mfa()) - records :: map_ets(), + exported_types :: 'clean' | set_ets(), % set(mfa()) + records :: 'clean' | map_ets(), contracts :: map_ets(), callbacks :: map_ets(), fun_meta_info :: dict_ets(), % {mfa(), meta_info()} @@ -108,9 +107,6 @@ ets_map_store(Key, Element, Table) -> true = ets:insert(Table, {Key, Element}), Table. -ets_dict_store_dict(Dict, Table) -> - true = ets:insert(Table, dict:to_list(Dict)). - ets_dict_to_dict(Table) -> Fold = fun({Key,Value}, Dict) -> dict:store(Key, Value, Dict) end, ets:foldl(Fold, dict:new(), Table). @@ -165,11 +161,8 @@ new() -> -spec delete(codeserver()) -> 'ok'. -delete(#codeserver{code = Code, exported_types = ExportedTypes, - records = Records, contracts = Contracts, - callbacks = Callbacks}) -> - lists:foreach(fun ets:delete/1, - [Code, ExportedTypes, Records, Contracts, Callbacks]). +delete(CServer) -> + lists:foreach(fun(Table) -> true = ets:delete(Table) end, tables(CServer)). -spec insert(atom(), cerl:c_module(), codeserver()) -> codeserver(). @@ -223,6 +216,11 @@ is_exported(MFA, #codeserver{exports = Exports}) -> get_exported_types(#codeserver{exported_types = ExpTypes}) -> ets_set_to_set(ExpTypes). +-spec extract_exported_types(codeserver()) -> {codeserver(), set_ets()}. + +extract_exported_types(#codeserver{exported_types = ExpTypes} = CS) -> + {CS#codeserver{exported_types = 'clean'}, ExpTypes}. + -spec get_exports(codeserver()) -> sets:set(mfa()). get_exports(#codeserver{exports = Exports}) -> @@ -275,10 +273,10 @@ lookup_mod_records(Mod, #codeserver{records = RecDict}) when is_atom(Mod) -> get_records_table(#codeserver{records = RecDict}) -> RecDict. --spec get_records(codeserver()) -> mod_records(). +-spec extract_records(codeserver()) -> {codeserver(), map_ets()}. -get_records(#codeserver{records = RecDict}) -> - ets_dict_to_dict(RecDict). +extract_records(#codeserver{records = RecDict} = CS) -> + {CS#codeserver{records = clean}, RecDict}. -spec store_temp_records(module(), types(), codeserver()) -> codeserver(). @@ -302,14 +300,6 @@ lookup_temp_mod_records(Mod, #codeserver{temp_records = TempRecDict}) -> {ok, Map} -> Map end. --spec set_temp_records(mod_records(), codeserver()) -> codeserver(). - -set_temp_records(Dict, CS) -> - true = ets:delete(CS#codeserver.temp_records), - TempRecords = ets:new(dialyzer_codeserver_temp_records,[]), - true = ets_dict_store_dict(Dict, TempRecords), - CS#codeserver{temp_records = TempRecords}. - -spec finalize_records(codeserver()) -> codeserver(). finalize_records(#codeserver{temp_records = TmpRecords, @@ -399,17 +389,25 @@ get_temp_contracts(Mod, #codeserver{temp_contracts = TempContDict, -spec give_away(codeserver(), pid()) -> 'ok'. -give_away(#codeserver{temp_records = TempRecords, - temp_contracts = TempContracts, - temp_callbacks = TempCallbacks, - records = Records, - contracts = Contracts, - callbacks = Callbacks}, Pid) -> - _ = [true = ets:give_away(Table, Pid, any) || - Table <- [TempRecords, TempContracts, TempCallbacks, - Records, Contracts, Callbacks], - Table =/= clean], - ok. +give_away(CServer, Pid) -> + lists:foreach(fun(Table) -> true = ets:give_away(Table, Pid, any) + end, tables(CServer)). + +tables(#codeserver{code = Code, + fun_meta_info = FunMetaInfo, + exports = Exports, + temp_exported_types = TempExpTypes, + temp_records = TempRecords, + temp_contracts = TempContracts, + temp_callbacks = TempCallbacks, + exported_types = ExportedTypes, + records = Records, + contracts = Contracts, + callbacks = Callbacks}) -> + [Table || Table <- [Code, FunMetaInfo, Exports, TempExpTypes, + TempRecords, TempContracts, TempCallbacks, + ExportedTypes, Records, Contracts, Callbacks], + Table =/= clean]. -spec finalize_contracts(codeserver()) -> codeserver(). diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl index 511a17d577..26dfeac71e 100644 --- a/lib/dialyzer/src/dialyzer_plt.erl +++ b/lib/dialyzer/src/dialyzer_plt.erl @@ -31,10 +31,8 @@ included_files/1, from_file/1, get_default_plt/0, - get_types/1, get_module_types/2, get_exported_types/1, - %% insert/3, insert_list/2, insert_contract_list/2, insert_callbacks/2, @@ -189,20 +187,17 @@ lookup(Plt, Label) when is_integer(Label) -> lookup_1(#mini_plt{info = Info}, MFAorLabel) -> ets_table_lookup(Info, MFAorLabel). --spec insert_types(plt(), erl_types:mod_records()) -> plt(). +-spec insert_types(plt(), ets:tid()) -> plt(). -insert_types(PLT, Rec) -> - PLT#plt{types = Rec}. +insert_types(MiniPLT, Records) -> + ets:rename(Records, plt_types), + MiniPLT#mini_plt{types = Records}. --spec insert_exported_types(plt(), sets:set()) -> plt(). +-spec insert_exported_types(plt(), ets:tid()) -> plt(). -insert_exported_types(PLT, Set) -> - PLT#plt{exported_types = Set}. - --spec get_types(plt()) -> erl_types:mod_records(). - -get_types(#plt{types = Types}) -> - Types. +insert_exported_types(MiniPLT, ExpTypes) -> + ets:rename(ExpTypes, plt_exported_types), + MiniPLT#mini_plt{exported_types = ExpTypes}. -spec get_module_types(plt(), atom()) -> 'none' | {'value', erl_types:type_table()}. -- cgit v1.2.3 From a669fdb4bc19c508c8c5edbefa1c81ee647b3362 Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Thu, 19 Jan 2017 09:40:11 +0100 Subject: dialyzer: Check return value of digrahp:add_edge() --- lib/dialyzer/src/dialyzer_callgraph.erl | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'lib/dialyzer/src') diff --git a/lib/dialyzer/src/dialyzer_callgraph.erl b/lib/dialyzer/src/dialyzer_callgraph.erl index 68f3d7a240..8f6a3b99b3 100644 --- a/lib/dialyzer/src/dialyzer_callgraph.erl +++ b/lib/dialyzer/src/dialyzer_callgraph.erl @@ -285,7 +285,7 @@ module_postorder(#callgraph{digraph = DG}) -> Nodes = sets:from_list([M || {M,_F,_A} <- digraph_vertices(DG)]), MDG = digraph:new([acyclic]), digraph_confirm_vertices(sets:to_list(Nodes), MDG), - Foreach = fun({M1,M2}) -> digraph:add_edge(MDG, M1, M2) end, + Foreach = fun({M1,M2}) -> _ = digraph:add_edge(MDG, M1, M2) end, lists:foreach(Foreach, sets:to_list(Edges)), {digraph_utils:topsort(MDG), {'d', MDG}}. @@ -305,7 +305,7 @@ module_deps(#callgraph{digraph = DG}) -> Nodes = sets:from_list([M || {M,_F,_A} <- digraph_vertices(DG)]), MDG = digraph:new(), digraph_confirm_vertices(sets:to_list(Nodes), MDG), - Foreach = fun({M1,M2}) -> digraph:add_edge(MDG, M1, M2) end, + Foreach = fun({M1,M2}) -> check_add_edge(MDG, M1, M2) end, lists:foreach(Foreach, sets:to_list(Edges)), Deps = [{N, ordsets:from_list(digraph:in_neighbours(MDG, N))} || N <- sets:to_list(Nodes)], @@ -552,9 +552,21 @@ digraph_add_edge(From, To, DG) -> false -> digraph:add_vertex(DG, To); {To, _} -> ok end, - digraph:add_edge(DG, {From, To}, From, To, []), + check_add_edge(DG, {From, To}, From, To, []), ok. +check_add_edge(G, V1, V2) -> + case digraph:add_edge(G, V1, V2) of + {error, Error} -> exit({add_edge, V1, V2, Error}); + _Edge -> ok + end. + +check_add_edge(G, E, V1, V2, L) -> + case digraph:add_edge(G, E, V1, V2, L) of + {error, Error} -> exit({add_edge, E, V1, V2, L, Error}); + _Edge -> ok + end. + digraph_confirm_vertices([MFA|Left], DG) -> digraph:add_vertex(DG, MFA, confirmed), digraph_confirm_vertices(Left, DG); -- cgit v1.2.3 From 643b7dbdb7f208ff69265b4f6b14f79175865c2b Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Thu, 19 Jan 2017 16:19:11 +0100 Subject: dialyzer: Compress some PLT tables Compressing the signatures or the contract records would cost time. The contract records are still held in two places (code server and PLT) in many cases. --- lib/dialyzer/src/dialyzer_plt.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib/dialyzer/src') diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl index 26dfeac71e..eb63e9e695 100644 --- a/lib/dialyzer/src/dialyzer_plt.erl +++ b/lib/dialyzer/src/dialyzer_plt.erl @@ -526,10 +526,12 @@ get_mini_plt(#plt{info = Info, contracts = Contracts, callbacks = Callbacks, exported_types = ExpTypes}) -> - [ETSInfo, ETSTypes, ETSContracts, ETSCallbacks, ETSExpTypes] = + [ETSInfo, ETSContracts] = [ets:new(Name, [public]) || - Name <- [plt_info, plt_types, plt_contracts, plt_callbacks, - plt_exported_types]], + Name <- [plt_info, plt_contracts]], + [ETSTypes, ETSCallbacks, ETSExpTypes] = + [ets:new(Name, [compressed, public]) || + Name <- [plt_types, plt_callbacks, plt_exported_types]], CallbackList = dict:to_list(Callbacks), CallbacksByModule = [{M, [Cb || {{M1,_,_},_} = Cb <- CallbackList, M1 =:= M]} || -- cgit v1.2.3 From da2a7ff4d340db6d139c3c6abdcfa978dcadb4c2 Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Thu, 19 Jan 2017 16:22:06 +0100 Subject: dialyzer: Sort graphs topologically Although some variable names indicate that lists of SCCs and modules are sorted, that was not always the case. The effect on the execution time of sorting them doesn't seem to be significant. dialyzer_coordinator:sccs_to_pids() should now always return an empty second list (not yet started workers). Since the condensation of graphs often needs a lot of heap memory, it is run in a separate process. --- lib/dialyzer/src/dialyzer_callgraph.erl | 79 ++++++++++++++++++++++----------- 1 file changed, 53 insertions(+), 26 deletions(-) (limited to 'lib/dialyzer/src') diff --git a/lib/dialyzer/src/dialyzer_callgraph.erl b/lib/dialyzer/src/dialyzer_callgraph.erl index 8f6a3b99b3..27249440e9 100644 --- a/lib/dialyzer/src/dialyzer_callgraph.erl +++ b/lib/dialyzer/src/dialyzer_callgraph.erl @@ -287,7 +287,9 @@ module_postorder(#callgraph{digraph = DG}) -> digraph_confirm_vertices(sets:to_list(Nodes), MDG), Foreach = fun({M1,M2}) -> _ = digraph:add_edge(MDG, M1, M2) end, lists:foreach(Foreach, sets:to_list(Edges)), - {digraph_utils:topsort(MDG), {'d', MDG}}. + %% The out-neighbors of a vertex are the vertices called directly. + %% The used vertices are to occur *before* the calling vertex: + {lists:reverse(digraph_utils:topsort(MDG)), {'d', MDG}}. edge_fold({{M1,_,_},{M2,_,_}}, Set) -> case M1 =/= M2 of @@ -774,28 +776,53 @@ to_ps(#callgraph{} = CG, File, Args) -> ok. condensation(G) -> - SCCs = digraph_utils:strong_components(G), - %% Assign unique numbers to SCCs: - Ints = lists:seq(1, length(SCCs)), - IntToSCC = lists:zip(Ints, SCCs), - IntScc = sofs:relation(IntToSCC, [{int, scc}]), - %% Subsitute strong components for vertices in edges using the - %% unique numbers: - C2V = sofs:relation([{SC, V} || SC <- SCCs, V <- SC], [{scc, v}]), - I2V = sofs:relative_product(IntScc, C2V), % [{v, int}] - Es = sofs:relation(digraph:edges(G), [{v, v}]), - R1 = sofs:relative_product(I2V, Es), - R2 = sofs:relative_product(I2V, sofs:converse(R1)), - %% Create in- and out-neighbours: - In = sofs:relation_to_family(sofs:strict_relation(R2)), - R3 = sofs:converse(R2), - Out = sofs:relation_to_family(sofs:strict_relation(R3)), - [OutETS, InETS, MapsETS] = - [ets:new(Name,[{read_concurrency, true}]) || - Name <- [callgraph_deps_out, callgraph_deps_in, callgraph_scc_map]], - ets:insert(OutETS, sofs:to_external(Out)), - ets:insert(InETS, sofs:to_external(In)), - %% Create mappings from SCCs to unique integers, and the inverse: - ets:insert(MapsETS, lists:zip([{'scc', SCC} || SCC<- SCCs], Ints)), - ets:insert(MapsETS, IntToSCC), - {{'e', OutETS, InETS, MapsETS}, SCCs}. + erlang:garbage_collect(), % reduce heap size + {Pid, Ref} = erlang:spawn_monitor(do_condensation(G, self())), + receive {'DOWN', Ref, process, Pid, Result} -> + {SCCInts, OutETS, InETS, MapsETS} = Result, + NewSCCs = [ets:lookup_element(MapsETS, SCCInt, 2) || SCCInt <- SCCInts], + {{'e', OutETS, InETS, MapsETS}, NewSCCs} + end. + +-spec do_condensation(digraph:graph(), pid()) -> fun(() -> no_return()). + +do_condensation(G, Parent) -> + fun() -> + [OutETS, InETS, MapsETS] = + [ets:new(Name,[{read_concurrency, true}]) || + Name <- [callgraph_deps_out, callgraph_deps_in, callgraph_scc_map]], + SCCs = digraph_utils:strong_components(G), + %% Assign unique numbers to SCCs: + Ints = lists:seq(1, length(SCCs)), + IntToSCC = lists:zip(Ints, SCCs), + IntScc = sofs:relation(IntToSCC, [{int, scc}]), + %% Create mapping from unique integers to SCCs: + ets:insert(MapsETS, IntToSCC), + %% Subsitute strong components for vertices in edges using the + %% unique numbers: + C2V = sofs:relation([{SC, V} || SC <- SCCs, V <- SC], [{scc, v}]), + I2V = sofs:relative_product(IntScc, C2V), % [{v, int}] + Es = sofs:relation(digraph:edges(G), [{v, v}]), + R1 = sofs:relative_product(I2V, Es), + R2 = sofs:relative_product(I2V, sofs:converse(R1)), + R2Strict = sofs:strict_relation(R2), + %% Create out-neighbours: + Out = sofs:relation_to_family(sofs:converse(R2Strict)), + ets:insert(OutETS, sofs:to_external(Out)), + %% Sort the SCCs topologically: + DG = sofs:family_to_digraph(Out), + lists:foreach(fun(I) -> digraph:add_vertex(DG, I) end, Ints), + SCCInts0 = digraph_utils:topsort(DG), + digraph:delete(DG), + %% The out-neighbors of a vertex are the vertices called directly. + %% The used vertices are to occur *before* the calling vertex: + SCCInts = lists:reverse(SCCInts0), + %% Create in-neighbours: + In = sofs:relation_to_family(R2Strict), + ets:insert(InETS, sofs:to_external(In)), + %% Create mapping from SCCs to unique integers: + ets:insert(MapsETS, lists:zip([{'scc', SCC} || SCC<- SCCs], Ints)), + lists:foreach(fun(E) -> true = ets:give_away(E, Parent, any) + end, [OutETS, InETS, MapsETS]), + exit({SCCInts, OutETS, InETS, MapsETS}) + end. -- cgit v1.2.3 From 7e0a7a58eb3557bd0d5d372a3793119a5e9ffe61 Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Fri, 20 Jan 2017 13:04:31 +0100 Subject: dialyzer: Remove code for non-started workers Since SCCs and modules are now topologically sorted (relative the active graph), we can safely assume that workers have been started. --- lib/dialyzer/src/dialyzer_coordinator.erl | 14 ++++++-------- lib/dialyzer/src/dialyzer_worker.erl | 15 ++------------- 2 files changed, 8 insertions(+), 21 deletions(-) (limited to 'lib/dialyzer/src') diff --git a/lib/dialyzer/src/dialyzer_coordinator.erl b/lib/dialyzer/src/dialyzer_coordinator.erl index 99f95a4dca..de239f5c93 100644 --- a/lib/dialyzer/src/dialyzer_coordinator.erl +++ b/lib/dialyzer/src/dialyzer_coordinator.erl @@ -170,18 +170,16 @@ update_result(Mode, InitData, Job, Data, Result) -> end. -spec sccs_to_pids([scc() | module()], coordinator()) -> - {[dialyzer_worker:worker()], [scc() | module()]}. + [dialyzer_worker:worker()]. sccs_to_pids(SCCs, {_Collector, _Regulator, SCCtoPID}) -> Fold = - fun(SCC, {Pids, Unknown}) -> - try ets:lookup_element(SCCtoPID, SCC, 2) of - Result -> {[Result|Pids], Unknown} - catch - _:_ -> {Pids, [SCC|Unknown]} - end + fun(SCC, Pids) -> + %% The SCCs that SCC depends on have always been started. + Result = ets:lookup_element(SCCtoPID, SCC, 2), + [Result|Pids] end, - lists:foldl(Fold, {[], []}, SCCs). + lists:foldl(Fold, [], SCCs). -spec job_done(job(), job_result(), coordinator()) -> ok. diff --git a/lib/dialyzer/src/dialyzer_worker.erl b/lib/dialyzer/src/dialyzer_worker.erl index 418c9798b3..1a251b5307 100644 --- a/lib/dialyzer/src/dialyzer_worker.erl +++ b/lib/dialyzer/src/dialyzer_worker.erl @@ -110,25 +110,14 @@ waits_more_success_typings(#state{depends_on = Depends}) -> broadcast_done(#state{job = SCC, init_data = InitData, coordinator = Coordinator}) -> RequiredBy = dialyzer_succ_typings:find_required_by(SCC, InitData), - {Callers, Unknown} = - dialyzer_coordinator:sccs_to_pids(RequiredBy, Coordinator), - send_done(Callers, SCC), - continue_broadcast_done(Unknown, SCC, Coordinator). + Callers = dialyzer_coordinator:sccs_to_pids(RequiredBy, Coordinator), + send_done(Callers, SCC). send_done(Callers, SCC) -> ?debug("Sending ~p: ~p\n",[SCC, Callers]), SendSTFun = fun(PID) -> PID ! {done, SCC} end, lists:foreach(SendSTFun, Callers). -continue_broadcast_done([], _SCC, _Coordinator) -> ok; -continue_broadcast_done(Rest, SCC, Coordinator) -> - %% This time limit should be greater than the time required - %% by the coordinator to spawn all processes. - timer:sleep(500), - {Callers, Unknown} = dialyzer_coordinator:sccs_to_pids(Rest, Coordinator), - send_done(Callers, SCC), - continue_broadcast_done(Unknown, SCC, Coordinator). - wait_for_success_typings(#state{depends_on = DependsOn} = State) -> receive {done, SCC} -> -- cgit v1.2.3 From 416711a87dcc937d78990ada5da0b8df98a8e1ee Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Fri, 20 Jan 2017 15:28:33 +0100 Subject: dialyzer: Write PLT in subprocess At common case, which will otherwise leave a big heap. --- lib/dialyzer/src/dialyzer_cl.erl | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) (limited to 'lib/dialyzer/src') diff --git a/lib/dialyzer/src/dialyzer_cl.erl b/lib/dialyzer/src/dialyzer_cl.erl index 93200f53d2..8500c59ebe 100644 --- a/lib/dialyzer/src/dialyzer_cl.erl +++ b/lib/dialyzer/src/dialyzer_cl.erl @@ -710,8 +710,16 @@ return_value(State = #cl_state{code_server = CodeServer, true -> dialyzer_plt:delete(MiniPlt); false -> - Plt = dialyzer_plt:restore_full_plt(MiniPlt), - dialyzer_plt:to_file(OutputPlt, Plt, ModDeps, PltInfo) + Fun = to_file_fun(OutputPlt, MiniPlt, ModDeps, PltInfo), + {Pid, Ref} = erlang:spawn_monitor(Fun), + dialyzer_plt:give_away(MiniPlt, Pid), + Pid ! go, + receive {'DOWN', Ref, process, Pid, Result} -> + case Result of + ok -> ok; + Thrown -> throw(Thrown) + end + end end, UnknownWarnings = unknown_warnings(State), RetValue = @@ -732,6 +740,16 @@ return_value(State = #cl_state{code_server = CodeServer, {RetValue, set_warning_id(AllWarnings)} end. +-spec to_file_fun(_, _, _, _) -> fun(() -> no_return()). + +to_file_fun(Filename, MiniPlt, ModDeps, PltInfo) -> + fun() -> + receive go -> ok end, + Plt = dialyzer_plt:restore_full_plt(MiniPlt), + dialyzer_plt:to_file(Filename, Plt, ModDeps, PltInfo), + exit(ok) + end. + unknown_warnings(State = #cl_state{legal_warnings = LegalWarnings}) -> Unknown = case ordsets:is_element(?WARN_UNKNOWN, LegalWarnings) of true -> -- cgit v1.2.3 From c2a3a1cdf56009630532805f9bed830e2041427d Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Tue, 24 Jan 2017 08:46:45 +0100 Subject: dialyzer: Do not spawn all workers at once Spawning all worker processes at once has the potential to increase peak memory consumption. Therefore the implementation is now slightly modified: `20 * dialyzer_utils:parallelism()' processes are running in parallel. 20 i quite a big factor, but seems necessary to keep all schedulers busy according to the Observer application's Load Charts, with a 100 ms update interval. --- lib/dialyzer/src/dialyzer_callgraph.erl | 12 +++---- lib/dialyzer/src/dialyzer_coordinator.erl | 55 ++++++++++++++++++++++-------- lib/dialyzer/src/dialyzer_succ_typings.erl | 8 ++--- lib/dialyzer/src/dialyzer_worker.erl | 42 +++++++++-------------- 4 files changed, 67 insertions(+), 50 deletions(-) (limited to 'lib/dialyzer/src') diff --git a/lib/dialyzer/src/dialyzer_callgraph.erl b/lib/dialyzer/src/dialyzer_callgraph.erl index 27249440e9..bb02bc8c0d 100644 --- a/lib/dialyzer/src/dialyzer_callgraph.erl +++ b/lib/dialyzer/src/dialyzer_callgraph.erl @@ -40,7 +40,7 @@ module_postorder_from_funs/2, new/0, get_depends_on/2, - get_required_by/2, + %% get_required_by/2, in_neighbours/2, renew_race_info/4, renew_race_code/2, @@ -250,12 +250,12 @@ get_depends_on(SCC, #callgraph{active_digraph = {'e', Out, _In, Maps}}) -> get_depends_on(SCC, #callgraph{active_digraph = {'d', DG}}) -> digraph:out_neighbours(DG, SCC). --spec get_required_by(scc() | module(), callgraph()) -> [scc()]. +%% -spec get_required_by(scc() | module(), callgraph()) -> [scc()]. -get_required_by(SCC, #callgraph{active_digraph = {'e', _Out, In, Maps}}) -> - lookup_scc(SCC, In, Maps); -get_required_by(SCC, #callgraph{active_digraph = {'d', DG}}) -> - digraph:in_neighbours(DG, SCC). +%% get_required_by(SCC, #callgraph{active_digraph = {'e', _Out, In, Maps}}) -> +%% lookup_scc(SCC, In, Maps); +%% get_required_by(SCC, #callgraph{active_digraph = {'d', DG}}) -> +%% digraph:in_neighbours(DG, SCC). lookup_scc(SCC, Table, Maps) -> case ets_lookup_dict({'scc', SCC}, Maps) of diff --git a/lib/dialyzer/src/dialyzer_coordinator.erl b/lib/dialyzer/src/dialyzer_coordinator.erl index de239f5c93..7c1bc1de5a 100644 --- a/lib/dialyzer/src/dialyzer_coordinator.erl +++ b/lib/dialyzer/src/dialyzer_coordinator.erl @@ -76,6 +76,8 @@ active = 0 :: integer(), result :: result(), next_label = 0 :: integer(), + jobs :: [job()], + job_fun :: fun(), init_data :: init_data(), regulator :: regulator(), scc_to_pid :: scc_to_pid() @@ -108,16 +110,18 @@ spawn_jobs(Mode, Jobs, InitData, Timing) -> false -> unused end, Coordinator = {Collector, Regulator, SCCtoPID}, - Fold = - fun(Job, Count) -> - Pid = dialyzer_worker:launch(Mode, Job, InitData, Coordinator), - case TypesigOrDataflow of - true -> true = ets:insert(SCCtoPID, {Job, Pid}), ok; - false -> ok - end, - Count + 1 + JobFun = + fun(Job) -> + Pid = dialyzer_worker:launch(Mode, Job, InitData, Coordinator), + case TypesigOrDataflow of + true -> true = ets:insert(SCCtoPID, {Job, Pid}); + false -> true + end end, - JobCount = lists:foldl(Fold, 0, Jobs), + JobCount = length(Jobs), + NumberOfInitJobs = min(JobCount, 20 * dialyzer_utils:parallelism()), + {InitJobs, RestJobs} = lists:split(NumberOfInitJobs, Jobs), + lists:foreach(JobFun, InitJobs), Unit = case Mode of 'typesig' -> "SCCs"; @@ -129,11 +133,13 @@ spawn_jobs(Mode, Jobs, InitData, Timing) -> 'compile' -> dialyzer_analysis_callgraph:compile_init_result(); _ -> [] end, - #state{mode = Mode, active = JobCount, result = InitResult, next_label = 0, - init_data = InitData, regulator = Regulator, scc_to_pid = SCCtoPID}. + #state{mode = Mode, active = JobCount, result = InitResult, + next_label = 0, job_fun = JobFun, jobs = RestJobs, + init_data = InitData, regulator = Regulator, scc_to_pid = SCCtoPID}. collect_result(#state{mode = Mode, active = Active, result = Result, next_label = NextLabel, init_data = InitData, + jobs = JobsLeft, job_fun = JobFun, regulator = Regulator, scc_to_pid = SCCtoPID} = State) -> receive {next_label_request, Estimation, Pid} -> @@ -141,20 +147,35 @@ collect_result(#state{mode = Mode, active = Active, result = Result, collect_result(State#state{next_label = NextLabel + Estimation}); {done, Job, Data} -> NewResult = update_result(Mode, InitData, Job, Data, Result), + TypesigOrDataflow = (Mode =:= 'typesig') orelse (Mode =:= 'dataflow'), case Active of 1 -> kill_regulator(Regulator), case Mode of 'compile' -> {NewResult, NextLabel}; - X when X =:= 'typesig'; X =:= 'dataflow' -> + _ when TypesigOrDataflow -> ets:delete(SCCtoPID), NewResult; 'warnings' -> NewResult end; N -> - collect_result(State#state{result = NewResult, active = N - 1}) + case TypesigOrDataflow of + true -> true = ets:delete(SCCtoPID, Job); + false -> true + end, + NewJobsLeft = + case JobsLeft of + [] -> []; + [NewJob|JobsLeft1] -> + JobFun(NewJob), + JobsLeft1 + end, + NewState = State#state{result = NewResult, + jobs = NewJobsLeft, + active = N - 1}, + collect_result(NewState) end end. @@ -176,8 +197,12 @@ sccs_to_pids(SCCs, {_Collector, _Regulator, SCCtoPID}) -> Fold = fun(SCC, Pids) -> %% The SCCs that SCC depends on have always been started. - Result = ets:lookup_element(SCCtoPID, SCC, 2), - [Result|Pids] + try ets:lookup_element(SCCtoPID, SCC, 2) of + Pid when is_pid(Pid) -> + [Pid|Pids] + catch + _:_ -> Pids + end end, lists:foldl(Fold, [], SCCs). diff --git a/lib/dialyzer/src/dialyzer_succ_typings.erl b/lib/dialyzer/src/dialyzer_succ_typings.erl index 3c90f46e95..be685baf22 100644 --- a/lib/dialyzer/src/dialyzer_succ_typings.erl +++ b/lib/dialyzer/src/dialyzer_succ_typings.erl @@ -29,7 +29,7 @@ -export([ find_succ_types_for_scc/2, refine_one_module/2, - find_required_by/2, + %% find_required_by/2, find_depends_on/2, collect_warnings/2, lookup_names/2 @@ -236,10 +236,10 @@ refine_succ_typings(Modules, #st{codeserver = Codeserver, find_depends_on(SCC, {_Codeserver, Callgraph, _Plt, _Solvers}) -> dialyzer_callgraph:get_depends_on(SCC, Callgraph). --spec find_required_by(scc() | module(), fixpoint_init_data()) -> [scc()]. +%% -spec find_required_by(scc() | module(), fixpoint_init_data()) -> [scc()]. -find_required_by(SCC, {_Codeserver, Callgraph, _Plt, _Solvers}) -> - dialyzer_callgraph:get_required_by(SCC, Callgraph). +%% find_required_by(SCC, {_Codeserver, Callgraph, _Plt, _Solvers}) -> +%% dialyzer_callgraph:get_required_by(SCC, Callgraph). -spec lookup_names([label()], fixpoint_init_data()) -> [mfa_or_funlbl()]. diff --git a/lib/dialyzer/src/dialyzer_worker.erl b/lib/dialyzer/src/dialyzer_worker.erl index 1a251b5307..af0f2e9e08 100644 --- a/lib/dialyzer/src/dialyzer_worker.erl +++ b/lib/dialyzer/src/dialyzer_worker.erl @@ -56,10 +56,14 @@ launch(Mode, Job, InitData, Coordinator) -> %%-------------------------------------------------------------------- -init(#state{job = SCC, mode = Mode, init_data = InitData} = State) when +init(#state{job = SCC, mode = Mode, init_data = InitData, + coordinator = Coordinator} = State) when Mode =:= 'typesig'; Mode =:= 'dataflow' -> - DependsOn = dialyzer_succ_typings:find_depends_on(SCC, InitData), - ?debug("Deps ~p: ~p\n",[SCC, DependsOn]), + DependsOnSCCs = dialyzer_succ_typings:find_depends_on(SCC, InitData), + ?debug("~w: Deps ~p: ~p\n", [self(), SCC, DependsOnSCCs]), + Pids = dialyzer_coordinator:sccs_to_pids(DependsOnSCCs, Coordinator), + ?debug("~w: PidsDeps ~p\n", [self(), Pids]), + DependsOn = [{Pid, erlang:monitor(process, Pid)} || Pid <- Pids], loop(updating, State#state{depends_on = DependsOn}); init(#state{mode = Mode} = State) when Mode =:= 'compile'; Mode =:= 'warnings' -> @@ -67,7 +71,7 @@ init(#state{mode = Mode} = State) when loop(updating, #state{mode = Mode} = State) when Mode =:= 'typesig'; Mode =:= 'dataflow' -> - ?debug("Update: ~p\n",[State#state.job]), + ?debug("~w: Update: ~p\n", [self(), State#state.job]), NextStatus = case waits_more_success_typings(State) of true -> waiting; @@ -76,11 +80,11 @@ loop(updating, #state{mode = Mode} = State) when loop(NextStatus, State); loop(waiting, #state{mode = Mode} = State) when Mode =:= 'typesig'; Mode =:= 'dataflow' -> - ?debug("Wait: ~p\n",[State#state.job]), + ?debug("~w: Wait: ~p\n", [self(), State#state.job]), NewState = wait_for_success_typings(State), loop(updating, NewState); loop(running, #state{mode = 'compile'} = State) -> - dialyzer_coordinator:request_activation(State#state.coordinator), + request_activation(State), ?debug("Compile: ~s\n",[State#state.job]), Result = case start_compilation(State) of @@ -92,40 +96,28 @@ loop(running, #state{mode = 'compile'} = State) -> end, report_to_coordinator(Result, State); loop(running, #state{mode = 'warnings'} = State) -> - dialyzer_coordinator:request_activation(State#state.coordinator), + request_activation(State), ?debug("Warning: ~s\n",[State#state.job]), Result = collect_warnings(State), report_to_coordinator(Result, State); loop(running, #state{mode = Mode} = State) when Mode =:= 'typesig'; Mode =:= 'dataflow' -> request_activation(State), - ?debug("Run: ~p\n",[State#state.job]), + ?debug("~w: Run: ~p\n", [self(), State#state.job]), NotFixpoint = do_work(State), - ok = broadcast_done(State), report_to_coordinator(NotFixpoint, State). waits_more_success_typings(#state{depends_on = Depends}) -> Depends =/= []. -broadcast_done(#state{job = SCC, init_data = InitData, - coordinator = Coordinator}) -> - RequiredBy = dialyzer_succ_typings:find_required_by(SCC, InitData), - Callers = dialyzer_coordinator:sccs_to_pids(RequiredBy, Coordinator), - send_done(Callers, SCC). - -send_done(Callers, SCC) -> - ?debug("Sending ~p: ~p\n",[SCC, Callers]), - SendSTFun = fun(PID) -> PID ! {done, SCC} end, - lists:foreach(SendSTFun, Callers). - wait_for_success_typings(#state{depends_on = DependsOn} = State) -> receive - {done, SCC} -> - ?debug("GOT ~p: ~p\n",[State#state.job, SCC]), - State#state{depends_on = DependsOn -- [SCC]} + {'DOWN', Ref, process, Pid, _Info} -> + ?debug("~w: ~p got DOWN: ~p\n", [self(), State#state.job, Pid]), + State#state{depends_on = DependsOn -- [{Pid, Ref}]} after 5000 -> - ?debug("Still Waiting ~p: ~p\n",[State#state.job, DependsOn]), + ?debug("~w: Still Waiting ~p:\n ~p\n", [self(), State#state.job, DependsOn]), State end. @@ -139,7 +131,7 @@ do_work(#state{mode = Mode, job = Job, init_data = InitData}) -> end. report_to_coordinator(Result, #state{job = Job, coordinator = Coordinator}) -> - ?debug("Done: ~p\n",[Job]), + ?debug("~w: Done: ~p\n",[self(), Job]), dialyzer_coordinator:job_done(Job, Result, Coordinator). start_compilation(#state{job = Job, init_data = InitData}) -> -- cgit v1.2.3 From d8daf680e803b97869dcffe84798f728fc4918a6 Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Mon, 30 Jan 2017 15:37:31 +0100 Subject: dialyzer: Attempt to reduce memory consumption The 'deps' field is reset when no longer used, which could possibly save some memory. --- lib/dialyzer/src/dialyzer_typesig.erl | 58 +++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 19 deletions(-) (limited to 'lib/dialyzer/src') diff --git a/lib/dialyzer/src/dialyzer_typesig.erl b/lib/dialyzer/src/dialyzer_typesig.erl index 46cc17a230..76c1bfb774 100644 --- a/lib/dialyzer/src/dialyzer_typesig.erl +++ b/lib/dialyzer/src/dialyzer_typesig.erl @@ -81,7 +81,7 @@ -record(constraint_list, {type :: 'conj' | 'disj', list :: [constr()], deps :: deps(), - masks = maps:new() :: #{dep() => mask()}, + masks :: #{dep() => mask()} | 'undefined', id :: {'list', dep()} | 'undefined'}). -type constraint_list() :: #constraint_list{}. @@ -181,7 +181,6 @@ analyze_scc(SCC, NextLabel, CallGraph, CServer, Plt, PropTypes, Solvers0) -> M <- lists:usort([M || {M, _, _} <- SCC])], State2 = traverse_scc(SCC, CServer, DefSet, ModRecs, State1), State3 = state__finalize(State2), - erlang:garbage_collect(), Funs = state__scc(State3), pp_constrs_scc(Funs, State3), constraints_to_dot_scc(Funs, State3), @@ -3208,7 +3207,8 @@ order_fun_constraints(State) -> order_fun_constraints([#constraint_ref{id = Id}|Tail], State) -> Cs = state__get_cs(Id, State), - {[NewCs], State1} = order_fun_constraints([Cs], [], [], State), + {[Cs1], State1} = order_fun_constraints([Cs], [], [], State), + NewCs = Cs1#constraint_list{deps = Cs#constraint_list.deps}, NewState = state__store_constrs(Id, NewCs, State1), order_fun_constraints(Tail, NewState); order_fun_constraints([], State) -> @@ -3216,23 +3216,31 @@ order_fun_constraints([], State) -> order_fun_constraints([#constraint_ref{} = C|Tail], Funs, Acc, State) -> order_fun_constraints(Tail, [C|Funs], Acc, State); -order_fun_constraints([#constraint_list{list = List, type = Type} = C|Tail], +order_fun_constraints([#constraint_list{list = List, + type = Type, + masks = OldMasks} = C|Tail], Funs, Acc, State) -> - {NewList, NewState} = - case Type of - conj -> order_fun_constraints(List, [], [], State); - disj -> - FoldFun = fun(X, AccState) -> - {[NewX], NewAccState} = - order_fun_constraints([X], [], [], AccState), - {NewX, NewAccState} - end, - lists:mapfoldl(FoldFun, State, List) - end, - C1 = update_constraint_list(C, NewList), - Masks = calculate_masks(NewList, 1, []), - NewAcc = [update_masks(C1, Masks)|Acc], - order_fun_constraints(Tail, Funs, NewAcc, NewState); + case OldMasks of + undefined -> + {NewList, NewState} = + case Type of + conj -> order_fun_constraints(List, [], [], State); + disj -> + FoldFun = fun(X, AccState) -> + {[NewX], NewAccState} = + order_fun_constraints([X], [], [], AccState), + {NewX, NewAccState} + end, + lists:mapfoldl(FoldFun, State, List) + end, + NewList2 = reset_deps(NewList, State), + C1 = update_constraint_list(C, NewList2), + Masks = calculate_masks(NewList, 1, []), + NewAcc = [update_masks(C1, Masks)|Acc], + order_fun_constraints(Tail, Funs, NewAcc, NewState); + M when is_map(M) -> + order_fun_constraints(Tail, Funs, [C|Acc], State) + end; order_fun_constraints([#constraint{} = C|Tail], Funs, Acc, State) -> order_fun_constraints(Tail, Funs, [C|Acc], State); order_fun_constraints([], Funs, Acc, State) -> @@ -3242,6 +3250,18 @@ order_fun_constraints([], Funs, Acc, State) -> update_masks(C, Masks) -> C#constraint_list{masks = Masks}. +reset_deps(ConstrList, #state{solvers = Solvers}) -> + case lists:member(v1, Solvers) of + true -> + ConstrList; + false -> + [reset_deps(Constr) || Constr <- ConstrList] + end. + +reset_deps(#constraint{}=C) -> C#constraint{deps = []}; +reset_deps(#constraint_list{}=C) -> C#constraint_list{deps = []}; +reset_deps(#constraint_ref{}=C) -> C#constraint_ref{deps = []}. + calculate_masks([C|Cs], I, L0) -> calculate_masks(Cs, I+1, [{V, I} || V <- get_deps(C)] ++ L0); calculate_masks([], _I, L) -> -- cgit v1.2.3