dialyzer: Try to reduce memory usage

The translation from forms to types is done in a separate process
in an attempt to reduce peak memory usage.

Expect further optimizations as it is probably not feasible
in the long run to keep all type information on the heap.

5 files changed, 74 insertions, 35 deletions

diff --git a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
index c8129cfbaa..fed9a7d1f2 100644
--- a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
+++ b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
@@ -160,12 +160,7 @@ analysis_start(Parent, Analysis, LegalWarnings) ->
         dialyzer_codeserver:finalize_exported_types(MergedExpTypes, TmpCServer1),
       erlang:garbage_collect(),
       ?timing(State#analysis_state.timing_server, "remote",
-              begin
-                TmpCServer3 =
-                  dialyzer_utils:process_record_remote_types(TmpCServer2),
-                erlang:garbage_collect(),
-                dialyzer_contracts:process_contract_remote_types(TmpCServer3)
-              end)
+              contracts_and_records(TmpCServer2))
     catch
       throw:{error, _ErrorMsg} = Error -> exit(Error)
     end,
@@ -187,7 +182,6 @@ analysis_start(Parent, Analysis, LegalWarnings) ->
       true -> dialyzer_callgraph:put_race_detection(true, Callgraph);
       false -> Callgraph
     end,
-  erlang:garbage_collect(),
   State2 = analyze_callgraph(NewCallgraph, State1),
   #analysis_state{plt = MiniPlt2, doc_plt = DocPlt} = State2,
   dialyzer_callgraph:dispose_race_server(NewCallgraph),
@@ -198,6 +192,30 @@ analysis_start(Parent, Analysis, LegalWarnings) ->
   MiniPlt3 = dialyzer_plt:delete_list(MiniPlt2, NonExportsList),
   send_analysis_done(Parent, MiniPlt3, DocPlt).
 
+contracts_and_records(CodeServer) ->
+  Fun = contrs_and_recs(CodeServer),
+  {Pid, Ref} = erlang:spawn_monitor(Fun),
+  dialyzer_codeserver:give_away(CodeServer, Pid),
+  Pid ! {self(), go},
+  receive {'DOWN', Ref, process, Pid, Return} ->
+      Return
+  end.
+
+-spec contrs_and_recs(dialyzer_codeserver:codeserver()) ->
+                         fun(() -> no_return()).
+
+contrs_and_recs(TmpCServer2) ->
+  fun() ->
+      Parent = receive {Pid, go} -> Pid end,
+      {TmpCServer3, RecordDict} =
+        dialyzer_utils:process_record_remote_types(TmpCServer2),
+      TmpServer4 =
+        dialyzer_contracts:process_contract_remote_types(TmpCServer3,
+                                                         RecordDict),
+      dialyzer_codeserver:give_away(TmpServer4, Parent),
+      exit(TmpServer4)
+  end.
+
 analyze_callgraph(Callgraph, #analysis_state{codeserver = Codeserver,
 					     doc_plt = DocPlt,
                                              plt = Plt,
diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl
index ba278b627a..786ed229df 100644
--- a/lib/dialyzer/src/dialyzer_codeserver.erl
+++ b/lib/dialyzer/src/dialyzer_codeserver.erl
@@ -30,6 +30,7 @@
 
 -export([delete/1,
 	 store_temp_contracts/4,
+         give_away/2,
 	 finalize_contracts/1,
          finalize_exported_types/2,
 	 finalize_records/2,
@@ -85,8 +86,8 @@
 
 -record(codeserver, {next_core_label = 0 :: label(),
 		     code		 :: dict_ets(),
-                     exported_types      :: set_ets() | 'undefined', % set(mfa())
-		     records             :: map_ets() | 'undefined',
+                     exported_types      :: set_ets(), % set(mfa())
+		     records             :: map_ets(),
 		     contracts           :: map_ets(),
 		     callbacks           :: map_ets(),
                      fun_meta_info       :: dict_ets(), % {mfa(), meta_info()}
@@ -132,9 +133,6 @@ ets_set_to_set(Table) ->
   Fold = fun({E}, Set) -> sets:add_element(E, Set) end,
   ets:foldl(Fold, sets:new(), Table).
 
-ets_read_concurrent_table(Name) ->
-  ets:new(Name, [compressed, {read_concurrency, true}]).
-
 %%--------------------------------------------------------------------
 
 -spec new() -> codeserver().
@@ -142,9 +140,14 @@ ets_read_concurrent_table(Name) ->
 new() ->
   CodeOptions = [compressed, public, {read_concurrency, true}],
   Code = ets:new(dialyzer_codeserver_code, CodeOptions),
+  ReadOptions = [compressed, {read_concurrency, true}],
+  [Contracts, Callbacks, Records, ExportedTypes] =
+    [ets:new(Name, ReadOptions) ||
+      Name <- [dialyzer_codeserver_contracts,
+               dialyzer_codeserver_callbacks,
+               dialyzer_codeserver_records,
+               dialyzer_codeserver_exported_types]],
   TempOptions = [public, {write_concurrency, true}],
-  Contracts = ets_read_concurrent_table(dialyzer_codeserver_contracts),
-  Callbacks = ets_read_concurrent_table(dialyzer_codeserver_callbacks),
   [Exports, FunMetaInfo, TempExportedTypes, TempRecords, TempContracts,
    TempCallbacks] =
     [ets:new(Name, TempOptions) ||
@@ -156,6 +159,8 @@ new() ->
   #codeserver{code                = Code,
 	      exports             = Exports,
               fun_meta_info       = FunMetaInfo,
+              exported_types      = ExportedTypes,
+              records             = Records,
               contracts           = Contracts,
               callbacks           = Callbacks,
 	      temp_exported_types = TempExportedTypes,
@@ -228,12 +233,12 @@ get_exports(#codeserver{exports = Exports}) ->
 
 -spec finalize_exported_types(sets:set(mfa()), codeserver()) -> codeserver().
 
-finalize_exported_types(Set, CS) ->
-  ExportedTypes = ets_read_concurrent_table(dialyzer_codeserver_exported_types),
+finalize_exported_types(Set,
+                        #codeserver{exported_types = ExportedTypes,
+                                    temp_exported_types = TempETypes} = CS) ->
   true = ets_set_insert_set(Set, ExportedTypes),
-  TempExpTypes = CS#codeserver.temp_exported_types,
-  true = ets:delete(TempExpTypes),
-  CS#codeserver{exported_types = ExportedTypes, temp_exported_types = clean}.
+  true = ets:delete(TempETypes),
+  CS#codeserver{temp_exported_types = clean}.
 
 -spec lookup_mod_code(atom(), codeserver()) -> cerl:c_module().
 
@@ -297,11 +302,11 @@ set_temp_records(Dict, CS) ->
 
 -spec finalize_records(mod_records(), codeserver()) -> codeserver().
 
-finalize_records(Dict, CS) ->
-  true = ets:delete(CS#codeserver.temp_records),
-  Records = ets_read_concurrent_table(dialyzer_codeserver_records),
+finalize_records(Dict, #codeserver{temp_records = TmpRecords,
+                                   records = Records} = CS) ->
+  true = ets:delete(TmpRecords),
   true = ets_dict_store_dict(Dict, Records),
-  CS#codeserver{records = Records, temp_records = clean}.
+  CS#codeserver{temp_records = clean}.
 
 -spec lookup_mod_contracts(atom(), codeserver()) -> contracts().
 
@@ -377,6 +382,20 @@ get_temp_contracts(Mod, #codeserver{temp_contracts = TempContDict,
   true = ets:delete(TempCallDict, Mod),
   {Contracts, Callbacks}.
 
+-spec give_away(codeserver(), pid()) -> 'ok'.
+
+give_away(#codeserver{temp_records = TempRecords,
+                      temp_contracts = TempContracts,
+                      temp_callbacks = TempCallbacks,
+                      records = Records,
+                      contracts = Contracts,
+                      callbacks = Callbacks}, Pid) ->
+  _ = [true = ets:give_away(Table, Pid, any) ||
+        Table <- [TempRecords, TempContracts, TempCallbacks,
+                  Records, Contracts, Callbacks],
+        Table =/= clean],
+  ok.
+
 -spec finalize_contracts(codeserver()) -> codeserver().
 
 finalize_contracts(#codeserver{temp_contracts = TempContDict,
diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl
index c0a4c6892c..f3fba68e84 100644
--- a/lib/dialyzer/src/dialyzer_contracts.erl
+++ b/lib/dialyzer/src/dialyzer_contracts.erl
@@ -31,7 +31,7 @@
 	 get_contract_return/2,
 	 %% get_contract_signature/1,
 	 is_overloaded/1,
-	 process_contract_remote_types/1,
+	 process_contract_remote_types/2,
 	 store_tmp_contract/5]).
 
 -export_type([file_contract/0, plt_contracts/0]).
@@ -146,13 +146,13 @@ sequence([], _Delimiter) -> "";
 sequence([H], _Delimiter) -> H;
 sequence([H|T], Delimiter) -> H ++ Delimiter ++ sequence(T, Delimiter).
 
--spec process_contract_remote_types(dialyzer_codeserver:codeserver()) ->
-	  dialyzer_codeserver:codeserver().
+-spec process_contract_remote_types(dialyzer_codeserver:codeserver(),
+                                    erl_types:mod_records()) ->
+                                       dialyzer_codeserver:codeserver().
 
-process_contract_remote_types(CodeServer) ->
+process_contract_remote_types(CodeServer, RecordDict) ->
   Mods = dialyzer_codeserver:contracts_modules(CodeServer),
   ExpTypes = dialyzer_codeserver:get_exported_types(CodeServer),
-  RecordDict = dialyzer_codeserver:get_records(CodeServer),
   ContractFun =
     fun({{_M, _F, _A}=MFA, {File, TmpContract, Xtra}}, C0) ->
         #tmp_contract{contract_funs = CFuns, forms = Forms} = TmpContract,
@@ -178,7 +178,6 @@ process_contract_remote_types(CodeServer) ->
                                             CodeServer)
     end,
   lists:foreach(ModuleFun, Mods),
-  %% erlang:garbage_collect(),
   dialyzer_codeserver:finalize_contracts(CodeServer).
 
 -type opaques_fun() :: fun((module()) -> [erl_types:erl_type()]).
diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl
index e0a9628fb3..e71a953279 100644
--- a/lib/dialyzer/src/dialyzer_utils.erl
+++ b/lib/dialyzer/src/dialyzer_utils.erl
@@ -296,7 +296,8 @@ get_record_fields([{record_field, _Line, Name, _Init}|Left], RecDict, Acc) ->
 get_record_fields([], _RecDict, Acc) ->
   lists:reverse(Acc).
 
--spec process_record_remote_types(codeserver()) -> codeserver().
+-spec process_record_remote_types(codeserver()) ->
+                                     {codeserver(), mod_records()}.
 
 %% The field types are cached. Used during analysis when handling records.
 process_record_remote_types(CServer) ->
@@ -340,7 +341,7 @@ process_record_remote_types(CServer) ->
   NewRecordsList = lists:map(ModuleFun, dict:to_list(TempRecords1)),
   NewRecords = dict:from_list(NewRecordsList),
   check_record_fields(NewRecords, ExpTypes),
-  dialyzer_codeserver:finalize_records(NewRecords, CServer).
+  {dialyzer_codeserver:finalize_records(NewRecords, CServer), NewRecords}.
 
 %% erl_types:t_from_form() substitutes the declaration of opaque types
 %% for the expanded type in some cases. To make sure the initial type,
diff --git a/lib/typer/src/typer.erl b/lib/typer/src/typer.erl
index bf971ff152..cd67af41ed 100644
--- a/lib/typer/src/typer.erl
+++ b/lib/typer/src/typer.erl
@@ -143,8 +143,9 @@ extract(#analysis{macros = Macros,
       MergedRecords = dialyzer_utils:merge_records(NewRecords, OldRecords),
       CodeServer2 = dialyzer_codeserver:set_temp_records(MergedRecords, CodeServer1),
       CodeServer3 = dialyzer_codeserver:finalize_exported_types(NewExpTypes, CodeServer2),
-      CodeServer4 = dialyzer_utils:process_record_remote_types(CodeServer3),
-      dialyzer_contracts:process_contract_remote_types(CodeServer4)
+      {CodeServer4, RecordDict} =
+        dialyzer_utils:process_record_remote_types(CodeServer3),
+      dialyzer_contracts:process_contract_remote_types(CodeServer4, RecordDict)
     catch
       throw:{error, ErrorMsg} ->
 	compile_error(ErrorMsg)
@@ -851,8 +852,9 @@ collect_info(Analysis) ->
       TmpCServer1 = dialyzer_codeserver:set_temp_records(MergedRecords, TmpCServer),
       TmpCServer2 =
         dialyzer_codeserver:finalize_exported_types(MergedExpTypes, TmpCServer1),
-      TmpCServer3 = dialyzer_utils:process_record_remote_types(TmpCServer2),
-      dialyzer_contracts:process_contract_remote_types(TmpCServer3)
+      {TmpCServer3, RecordDict} =
+        dialyzer_utils:process_record_remote_types(TmpCServer2),
+      dialyzer_contracts:process_contract_remote_types(TmpCServer3, RecordDict)
     catch
       throw:{error, ErrorMsg} ->
 	fatal_error(ErrorMsg)