From 12b3790003ca2c060b6ab143dffd0c23580b5476 Mon Sep 17 00:00:00 2001 From: Hans Bolinder Date: Sun, 1 Jan 2017 19:53:36 +0100 Subject: dialyzer: Try to reduce memory usage The translation from forms to types is done in a separate process in an attempt to reduce peak memory usage. Expect further optimizations as it is probably not feasible in the long run to keep all type information on the heap. --- lib/dialyzer/src/dialyzer_analysis_callgraph.erl | 32 +++++++++++---- lib/dialyzer/src/dialyzer_codeserver.erl | 51 ++++++++++++++++-------- lib/dialyzer/src/dialyzer_contracts.erl | 11 +++-- lib/dialyzer/src/dialyzer_utils.erl | 5 ++- lib/typer/src/typer.erl | 10 +++-- 5 files changed, 74 insertions(+), 35 deletions(-) (limited to 'lib') diff --git a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl index c8129cfbaa..fed9a7d1f2 100644 --- a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl +++ b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl @@ -160,12 +160,7 @@ analysis_start(Parent, Analysis, LegalWarnings) -> dialyzer_codeserver:finalize_exported_types(MergedExpTypes, TmpCServer1), erlang:garbage_collect(), ?timing(State#analysis_state.timing_server, "remote", - begin - TmpCServer3 = - dialyzer_utils:process_record_remote_types(TmpCServer2), - erlang:garbage_collect(), - dialyzer_contracts:process_contract_remote_types(TmpCServer3) - end) + contracts_and_records(TmpCServer2)) catch throw:{error, _ErrorMsg} = Error -> exit(Error) end, @@ -187,7 +182,6 @@ analysis_start(Parent, Analysis, LegalWarnings) -> true -> dialyzer_callgraph:put_race_detection(true, Callgraph); false -> Callgraph end, - erlang:garbage_collect(), State2 = analyze_callgraph(NewCallgraph, State1), #analysis_state{plt = MiniPlt2, doc_plt = DocPlt} = State2, dialyzer_callgraph:dispose_race_server(NewCallgraph), @@ -198,6 +192,30 @@ analysis_start(Parent, Analysis, LegalWarnings) -> MiniPlt3 = dialyzer_plt:delete_list(MiniPlt2, NonExportsList), send_analysis_done(Parent, MiniPlt3, DocPlt). +contracts_and_records(CodeServer) -> + Fun = contrs_and_recs(CodeServer), + {Pid, Ref} = erlang:spawn_monitor(Fun), + dialyzer_codeserver:give_away(CodeServer, Pid), + Pid ! {self(), go}, + receive {'DOWN', Ref, process, Pid, Return} -> + Return + end. + +-spec contrs_and_recs(dialyzer_codeserver:codeserver()) -> + fun(() -> no_return()). + +contrs_and_recs(TmpCServer2) -> + fun() -> + Parent = receive {Pid, go} -> Pid end, + {TmpCServer3, RecordDict} = + dialyzer_utils:process_record_remote_types(TmpCServer2), + TmpServer4 = + dialyzer_contracts:process_contract_remote_types(TmpCServer3, + RecordDict), + dialyzer_codeserver:give_away(TmpServer4, Parent), + exit(TmpServer4) + end. + analyze_callgraph(Callgraph, #analysis_state{codeserver = Codeserver, doc_plt = DocPlt, plt = Plt, diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl index ba278b627a..786ed229df 100644 --- a/lib/dialyzer/src/dialyzer_codeserver.erl +++ b/lib/dialyzer/src/dialyzer_codeserver.erl @@ -30,6 +30,7 @@ -export([delete/1, store_temp_contracts/4, + give_away/2, finalize_contracts/1, finalize_exported_types/2, finalize_records/2, @@ -85,8 +86,8 @@ -record(codeserver, {next_core_label = 0 :: label(), code :: dict_ets(), - exported_types :: set_ets() | 'undefined', % set(mfa()) - records :: map_ets() | 'undefined', + exported_types :: set_ets(), % set(mfa()) + records :: map_ets(), contracts :: map_ets(), callbacks :: map_ets(), fun_meta_info :: dict_ets(), % {mfa(), meta_info()} @@ -132,9 +133,6 @@ ets_set_to_set(Table) -> Fold = fun({E}, Set) -> sets:add_element(E, Set) end, ets:foldl(Fold, sets:new(), Table). -ets_read_concurrent_table(Name) -> - ets:new(Name, [compressed, {read_concurrency, true}]). - %%-------------------------------------------------------------------- -spec new() -> codeserver(). @@ -142,9 +140,14 @@ ets_read_concurrent_table(Name) -> new() -> CodeOptions = [compressed, public, {read_concurrency, true}], Code = ets:new(dialyzer_codeserver_code, CodeOptions), + ReadOptions = [compressed, {read_concurrency, true}], + [Contracts, Callbacks, Records, ExportedTypes] = + [ets:new(Name, ReadOptions) || + Name <- [dialyzer_codeserver_contracts, + dialyzer_codeserver_callbacks, + dialyzer_codeserver_records, + dialyzer_codeserver_exported_types]], TempOptions = [public, {write_concurrency, true}], - Contracts = ets_read_concurrent_table(dialyzer_codeserver_contracts), - Callbacks = ets_read_concurrent_table(dialyzer_codeserver_callbacks), [Exports, FunMetaInfo, TempExportedTypes, TempRecords, TempContracts, TempCallbacks] = [ets:new(Name, TempOptions) || @@ -156,6 +159,8 @@ new() -> #codeserver{code = Code, exports = Exports, fun_meta_info = FunMetaInfo, + exported_types = ExportedTypes, + records = Records, contracts = Contracts, callbacks = Callbacks, temp_exported_types = TempExportedTypes, @@ -228,12 +233,12 @@ get_exports(#codeserver{exports = Exports}) -> -spec finalize_exported_types(sets:set(mfa()), codeserver()) -> codeserver(). -finalize_exported_types(Set, CS) -> - ExportedTypes = ets_read_concurrent_table(dialyzer_codeserver_exported_types), +finalize_exported_types(Set, + #codeserver{exported_types = ExportedTypes, + temp_exported_types = TempETypes} = CS) -> true = ets_set_insert_set(Set, ExportedTypes), - TempExpTypes = CS#codeserver.temp_exported_types, - true = ets:delete(TempExpTypes), - CS#codeserver{exported_types = ExportedTypes, temp_exported_types = clean}. + true = ets:delete(TempETypes), + CS#codeserver{temp_exported_types = clean}. -spec lookup_mod_code(atom(), codeserver()) -> cerl:c_module(). @@ -297,11 +302,11 @@ set_temp_records(Dict, CS) -> -spec finalize_records(mod_records(), codeserver()) -> codeserver(). -finalize_records(Dict, CS) -> - true = ets:delete(CS#codeserver.temp_records), - Records = ets_read_concurrent_table(dialyzer_codeserver_records), +finalize_records(Dict, #codeserver{temp_records = TmpRecords, + records = Records} = CS) -> + true = ets:delete(TmpRecords), true = ets_dict_store_dict(Dict, Records), - CS#codeserver{records = Records, temp_records = clean}. + CS#codeserver{temp_records = clean}. -spec lookup_mod_contracts(atom(), codeserver()) -> contracts(). @@ -377,6 +382,20 @@ get_temp_contracts(Mod, #codeserver{temp_contracts = TempContDict, true = ets:delete(TempCallDict, Mod), {Contracts, Callbacks}. +-spec give_away(codeserver(), pid()) -> 'ok'. + +give_away(#codeserver{temp_records = TempRecords, + temp_contracts = TempContracts, + temp_callbacks = TempCallbacks, + records = Records, + contracts = Contracts, + callbacks = Callbacks}, Pid) -> + _ = [true = ets:give_away(Table, Pid, any) || + Table <- [TempRecords, TempContracts, TempCallbacks, + Records, Contracts, Callbacks], + Table =/= clean], + ok. + -spec finalize_contracts(codeserver()) -> codeserver(). finalize_contracts(#codeserver{temp_contracts = TempContDict, diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl index c0a4c6892c..f3fba68e84 100644 --- a/lib/dialyzer/src/dialyzer_contracts.erl +++ b/lib/dialyzer/src/dialyzer_contracts.erl @@ -31,7 +31,7 @@ get_contract_return/2, %% get_contract_signature/1, is_overloaded/1, - process_contract_remote_types/1, + process_contract_remote_types/2, store_tmp_contract/5]). -export_type([file_contract/0, plt_contracts/0]). @@ -146,13 +146,13 @@ sequence([], _Delimiter) -> ""; sequence([H], _Delimiter) -> H; sequence([H|T], Delimiter) -> H ++ Delimiter ++ sequence(T, Delimiter). --spec process_contract_remote_types(dialyzer_codeserver:codeserver()) -> - dialyzer_codeserver:codeserver(). +-spec process_contract_remote_types(dialyzer_codeserver:codeserver(), + erl_types:mod_records()) -> + dialyzer_codeserver:codeserver(). -process_contract_remote_types(CodeServer) -> +process_contract_remote_types(CodeServer, RecordDict) -> Mods = dialyzer_codeserver:contracts_modules(CodeServer), ExpTypes = dialyzer_codeserver:get_exported_types(CodeServer), - RecordDict = dialyzer_codeserver:get_records(CodeServer), ContractFun = fun({{_M, _F, _A}=MFA, {File, TmpContract, Xtra}}, C0) -> #tmp_contract{contract_funs = CFuns, forms = Forms} = TmpContract, @@ -178,7 +178,6 @@ process_contract_remote_types(CodeServer) -> CodeServer) end, lists:foreach(ModuleFun, Mods), - %% erlang:garbage_collect(), dialyzer_codeserver:finalize_contracts(CodeServer). -type opaques_fun() :: fun((module()) -> [erl_types:erl_type()]). diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl index e0a9628fb3..e71a953279 100644 --- a/lib/dialyzer/src/dialyzer_utils.erl +++ b/lib/dialyzer/src/dialyzer_utils.erl @@ -296,7 +296,8 @@ get_record_fields([{record_field, _Line, Name, _Init}|Left], RecDict, Acc) -> get_record_fields([], _RecDict, Acc) -> lists:reverse(Acc). --spec process_record_remote_types(codeserver()) -> codeserver(). +-spec process_record_remote_types(codeserver()) -> + {codeserver(), mod_records()}. %% The field types are cached. Used during analysis when handling records. process_record_remote_types(CServer) -> @@ -340,7 +341,7 @@ process_record_remote_types(CServer) -> NewRecordsList = lists:map(ModuleFun, dict:to_list(TempRecords1)), NewRecords = dict:from_list(NewRecordsList), check_record_fields(NewRecords, ExpTypes), - dialyzer_codeserver:finalize_records(NewRecords, CServer). + {dialyzer_codeserver:finalize_records(NewRecords, CServer), NewRecords}. %% erl_types:t_from_form() substitutes the declaration of opaque types %% for the expanded type in some cases. To make sure the initial type, diff --git a/lib/typer/src/typer.erl b/lib/typer/src/typer.erl index bf971ff152..cd67af41ed 100644 --- a/lib/typer/src/typer.erl +++ b/lib/typer/src/typer.erl @@ -143,8 +143,9 @@ extract(#analysis{macros = Macros, MergedRecords = dialyzer_utils:merge_records(NewRecords, OldRecords), CodeServer2 = dialyzer_codeserver:set_temp_records(MergedRecords, CodeServer1), CodeServer3 = dialyzer_codeserver:finalize_exported_types(NewExpTypes, CodeServer2), - CodeServer4 = dialyzer_utils:process_record_remote_types(CodeServer3), - dialyzer_contracts:process_contract_remote_types(CodeServer4) + {CodeServer4, RecordDict} = + dialyzer_utils:process_record_remote_types(CodeServer3), + dialyzer_contracts:process_contract_remote_types(CodeServer4, RecordDict) catch throw:{error, ErrorMsg} -> compile_error(ErrorMsg) @@ -851,8 +852,9 @@ collect_info(Analysis) -> TmpCServer1 = dialyzer_codeserver:set_temp_records(MergedRecords, TmpCServer), TmpCServer2 = dialyzer_codeserver:finalize_exported_types(MergedExpTypes, TmpCServer1), - TmpCServer3 = dialyzer_utils:process_record_remote_types(TmpCServer2), - dialyzer_contracts:process_contract_remote_types(TmpCServer3) + {TmpCServer3, RecordDict} = + dialyzer_utils:process_record_remote_types(TmpCServer2), + dialyzer_contracts:process_contract_remote_types(TmpCServer3, RecordDict) catch throw:{error, ErrorMsg} -> fatal_error(ErrorMsg) -- cgit v1.2.3