dialyzer: Do not send full PLTs as messages

The mini PLT is extended to hold all data of the full PLT, and the
full PLT is restored when needed (for storing the PLT on file).

7 files changed, 161 insertions, 60 deletions

diff --git a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
index 08e55a78bd..8b3f70aa23 100644
--- a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
+++ b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
@@ -101,9 +101,9 @@ loop(#server_state{parent = Parent} = State,
     {AnalPid, cserver, CServer, Plt} ->
       send_codeserver_plt(Parent, CServer, Plt),
       loop(State, Analysis, ExtCalls);
-    {AnalPid, done, Plt, DocPlt} ->
+    {AnalPid, done, MiniPlt, DocPlt} ->
       send_ext_calls(Parent, ExtCalls),
-      send_analysis_done(Parent, Plt, DocPlt);
+      send_analysis_done(Parent, MiniPlt, DocPlt);
     {AnalPid, ext_calls, NewExtCalls} ->
       loop(State, Analysis, NewExtCalls);
     {AnalPid, ext_types, ExtTypes} ->
@@ -161,6 +161,7 @@ analysis_start(Parent, Analysis, LegalWarnings) ->
               begin
                 TmpCServer3 =
                   dialyzer_utils:process_record_remote_types(TmpCServer2),
+                erlang:garbage_collect(),
                 dialyzer_contracts:process_contract_remote_types(TmpCServer3)
               end)
     catch
@@ -171,48 +172,54 @@ analysis_start(Parent, Analysis, LegalWarnings) ->
   NewPlt1 = dialyzer_plt:insert_exported_types(NewPlt0, ExpTypes),
   State0 = State#analysis_state{plt = NewPlt1},
   dump_callgraph(Callgraph, State0, Analysis),
-  State1 = State0#analysis_state{codeserver = NewCServer},
   %% Remove all old versions of the files being analyzed
   AllNodes = dialyzer_callgraph:all_nodes(Callgraph),
-  Plt1 = dialyzer_plt:delete_list(NewPlt1, AllNodes),
+  Plt1_a = dialyzer_plt:delete_list(NewPlt1, AllNodes),
+  Plt1 = dialyzer_plt:insert_callbacks(Plt1_a, NewCServer),
+  State1 = State0#analysis_state{codeserver = NewCServer, plt = Plt1},
   Exports = dialyzer_codeserver:get_exports(NewCServer),
+  NonExports = sets:subtract(sets:from_list(AllNodes), Exports),
+  NonExportsList = sets:to_list(NonExports),
   NewCallgraph =
     case Analysis#analysis.race_detection of
       true -> dialyzer_callgraph:put_race_detection(true, Callgraph);
       false -> Callgraph
     end,
-  State2 = analyze_callgraph(NewCallgraph, State1#analysis_state{plt = Plt1}),
+  %% Calls to erlang:garbage_collect() help to reduce the heap size.
+  %% An alternative is to spawn more processes to do the job(s).
+  erlang:garbage_collect(),
+  State2 = analyze_callgraph(NewCallgraph, State1),
+  #analysis_state{plt = MiniPlt2, doc_plt = DocPlt} = State2,
   dialyzer_callgraph:dispose_race_server(NewCallgraph),
   rcv_and_send_ext_types(Parent),
-  NonExports = sets:subtract(sets:from_list(AllNodes), Exports),
-  NonExportsList = sets:to_list(NonExports),
-  Plt2 = dialyzer_plt:delete_list(State2#analysis_state.plt, NonExportsList),
-  send_codeserver_plt(Parent, CServer, State2#analysis_state.plt),
-  send_analysis_done(Parent, Plt2, State2#analysis_state.doc_plt).
+  %% Since the PLT is never used, a dummy is sent:
+  DummyPlt = dialyzer_plt:new(),
+  send_codeserver_plt(Parent, CServer, DummyPlt),
+  MiniPlt3 = dialyzer_plt:delete_list(MiniPlt2, NonExportsList),
+  send_analysis_done(Parent, MiniPlt3, DocPlt).
 
 analyze_callgraph(Callgraph, #analysis_state{codeserver = Codeserver,
 					     doc_plt = DocPlt,
+                                             plt = Plt,
 					     timing_server = TimingServer,
 					     parent = Parent,
                                              solvers = Solvers} = State) ->
-  Plt = dialyzer_plt:insert_callbacks(State#analysis_state.plt, Codeserver),
-  {NewPlt, NewDocPlt} =
-    case State#analysis_state.analysis_type of
-      plt_build ->
-	NewPlt0 =
-	  dialyzer_succ_typings:analyze_callgraph(Callgraph, Plt, Codeserver,
-						  TimingServer, Solvers, Parent),
-	{NewPlt0, DocPlt};
-      succ_typings ->
-	{Warnings, NewPlt0, NewDocPlt0} =
-	  dialyzer_succ_typings:get_warnings(Callgraph, Plt, DocPlt, Codeserver,
-					     TimingServer, Solvers, Parent),
-        Warnings1 = filter_warnings(Warnings, Codeserver),
-	send_warnings(State#analysis_state.parent, Warnings1),
-	{NewPlt0, NewDocPlt0}
-    end,
-  dialyzer_callgraph:delete(Callgraph),
-  State#analysis_state{plt = NewPlt, doc_plt = NewDocPlt}.
+  case State#analysis_state.analysis_type of
+    plt_build ->
+      NewMiniPlt =
+        dialyzer_succ_typings:analyze_callgraph(Callgraph, Plt, Codeserver,
+                                                TimingServer, Solvers, Parent),
+      dialyzer_callgraph:delete(Callgraph),
+      State#analysis_state{plt = NewMiniPlt, doc_plt = DocPlt};
+    succ_typings ->
+      {Warnings, NewMiniPlt, NewDocPlt} =
+        dialyzer_succ_typings:get_warnings(Callgraph, Plt, DocPlt, Codeserver,
+                                           TimingServer, Solvers, Parent),
+      dialyzer_callgraph:delete(Callgraph),
+      Warnings1 = filter_warnings(Warnings, Codeserver),
+      send_warnings(State#analysis_state.parent, Warnings1),
+      State#analysis_state{plt = NewMiniPlt, doc_plt = NewDocPlt}
+    end.
 
 %%--------------------------------------------------------------------
 %% Build the callgraph and fill the codeserver.
@@ -569,8 +576,9 @@ is_ok_fun({_Filename, _Line, {_M, _F, _A} = MFA}, Codeserver) ->
 is_ok_tag(Tag, {_F, _L, MorMFA}, Codeserver) ->
   not dialyzer_utils:is_suppressed_tag(MorMFA, Tag, Codeserver).
   
-send_analysis_done(Parent, Plt, DocPlt) ->
-  Parent ! {self(), done, Plt, DocPlt},
+send_analysis_done(Parent, MiniPlt, DocPlt) ->
+  ok = dialyzer_plt:give_away(MiniPlt, Parent),
+  Parent ! {self(), done, MiniPlt, DocPlt},
   ok.
 
 send_ext_calls(_Parent, none) ->
@@ -583,7 +591,7 @@ send_ext_types(Parent, ExtTypes) ->
   Parent ! {self(), ext_types, ExtTypes},
   ok.
 
-send_codeserver_plt(Parent, CServer, Plt ) ->
+send_codeserver_plt(Parent, CServer, Plt) ->
   Parent ! {self(), cserver, CServer, Plt},
   ok.
 
diff --git a/lib/dialyzer/src/dialyzer_cl.erl b/lib/dialyzer/src/dialyzer_cl.erl
index fc56693ea3..e8c1613a33 100644
--- a/lib/dialyzer/src/dialyzer_cl.erl
+++ b/lib/dialyzer/src/dialyzer_cl.erl
@@ -637,8 +637,8 @@ cl_loop(State, LogCache) ->
     {BackendPid, warnings, Warnings} ->
       NewState = store_warnings(State, Warnings),
       cl_loop(NewState, LogCache);
-    {BackendPid, done, NewPlt, _NewDocPlt} ->
-      return_value(State, NewPlt);
+    {BackendPid, done, NewMiniPlt, _NewDocPlt} ->
+      return_value(State, NewMiniPlt);
     {BackendPid, ext_calls, ExtCalls} ->
       cl_loop(State#cl_state{external_calls = ExtCalls}, LogCache);
     {BackendPid, ext_types, ExtTypes} ->
@@ -654,6 +654,7 @@ cl_loop(State, LogCache) ->
       cl_error(State, Msg);
     _Other ->
       %% io:format("Received ~p\n", [_Other]),
+      %% Note: {BackendPid, cserver, CodeServer, Plt} is ignored.
       cl_loop(State, LogCache)
   end.
 
@@ -699,10 +700,13 @@ return_value(State = #cl_state{erlang_mode = ErlangMode,
 			       output_plt = OutputPlt,
 			       plt_info = PltInfo,
 			       stored_warnings = StoredWarnings},
-	     Plt) ->
+	     MiniPlt) ->
   case OutputPlt =:= none of
-    true -> ok;
-    false -> dialyzer_plt:to_file(OutputPlt, Plt, ModDeps, PltInfo)
+    true ->
+      dialyzer_plt:delete(MiniPlt);
+    false ->
+      Plt = dialyzer_plt:restore_full_plt(MiniPlt),
+      dialyzer_plt:to_file(OutputPlt, Plt, ModDeps, PltInfo)
   end,
   UnknownWarnings = unknown_warnings(State),
   RetValue =
diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl
index 73b04b305b..331786466f 100644
--- a/lib/dialyzer/src/dialyzer_contracts.erl
+++ b/lib/dialyzer/src/dialyzer_contracts.erl
@@ -170,6 +170,7 @@ process_contract_remote_types(CodeServer) ->
           lists:mapfoldl(ContractFun, C3, dict:to_list(ContractDict)),
         {{ModuleName, dict:from_list(NewContractList)}, C4}
     end,
+  erlang:garbage_collect(),
   Cache = erl_types:cache__new(),
   {NewContractList, C5} =
     lists:mapfoldl(ModuleFun, Cache, dict:to_list(TmpContractDict)),
@@ -177,6 +178,8 @@ process_contract_remote_types(CodeServer) ->
     lists:mapfoldl(ModuleFun, C5, dict:to_list(TmpCallbackDict)),
   NewContractDict = dict:from_list(NewContractList),
   NewCallbackDict = dict:from_list(NewCallbackList),
+  %% Make sure the (huge) cache is garbage collected:
+  erlang:garbage_collect(),
   dialyzer_codeserver:finalize_contracts(NewContractDict, NewCallbackDict,
                                          CodeServer).
 
diff --git a/lib/dialyzer/src/dialyzer_gui_wx.erl b/lib/dialyzer/src/dialyzer_gui_wx.erl
index 4caf64d007..1701aff2f2 100644
--- a/lib/dialyzer/src/dialyzer_gui_wx.erl
+++ b/lib/dialyzer/src/dialyzer_gui_wx.erl
@@ -505,8 +505,9 @@ gui_loop(#gui_state{backend_pid = BackendPid, doc_plt = DocPlt,
 	end,
       ExplanationPid = spawn_link(Fun),
       gui_loop(State#gui_state{expl_pid = ExplanationPid});
-    {BackendPid, done, _NewPlt, NewDocPlt} ->
+    {BackendPid, done, NewMiniPlt, NewDocPlt} ->
       message(State, "Analysis done"),
+      dialyzer_plt:delete(NewMiniPlt),
       config_gui_stop(State),
       gui_loop(State#gui_state{doc_plt = NewDocPlt});
     {'EXIT', BackendPid, {error, Reason}} ->
diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl
index cf2f0e919e..93ec2599b4 100644
--- a/lib/dialyzer/src/dialyzer_plt.erl
+++ b/lib/dialyzer/src/dialyzer_plt.erl
@@ -58,7 +58,9 @@
 	 get_specs/4,
 	 to_file/4,
 	 get_mini_plt/1,
-	 restore_full_plt/2
+	 restore_full_plt/1,
+         delete/1,
+         give_away/2
 	]).
 
 %% Debug utilities
@@ -88,8 +90,10 @@
               exported_types = sets:new()  :: sets:set()}).
 
 -record(mini_plt, {info      :: ets:tid(),
+                   types     :: ets:tid(),
 		   contracts :: ets:tid(),
-		   callbacks :: ets:tid()
+		   callbacks :: ets:tid(),
+                   exported_types :: ets:tid()
 		  }).
 
 -opaque plt() :: #plt{} | #mini_plt{}.
@@ -130,6 +134,10 @@ delete_module(#plt{info = Info, types = Types,
 
 -spec delete_list(plt(), [mfa() | integer()]) -> plt().
 
+delete_list(#mini_plt{info = Info,
+                      contracts = Contracts}=Plt, List) ->
+  Plt#mini_plt{info = ets_table_delete_list(Info, List),
+               contracts = ets_table_delete_list(Contracts, List)};
 delete_list(#plt{info = Info, types = Types,
 		 contracts = Contracts,
 		 callbacks = Callbacks,
@@ -510,32 +518,100 @@ init_md5_list_1(Md5List, [], Acc) ->
 
 -spec get_mini_plt(plt()) -> plt().
 
-get_mini_plt(#plt{info = Info, contracts = Contracts, callbacks = Callbacks}) ->
-  [ETSInfo, ETSContracts, ETSCallbacks] =
-    [ets:new(Name, [public]) || Name <- [plt_info, plt_contracts, plt_callbacks]],
+get_mini_plt(#plt{info = Info,
+                  types = Types,
+                  contracts = Contracts,
+                  callbacks = Callbacks,
+                  exported_types = ExpTypes}) ->
+  [ETSInfo, ETSTypes, ETSContracts, ETSCallbacks, ETSExpTypes] =
+    [ets:new(Name, [public]) ||
+      Name <- [plt_info, plt_types, plt_contracts, plt_callbacks,
+               plt_exported_types]],
   CallbackList = dict:to_list(Callbacks),
   CallbacksByModule =
     [{M, [Cb || {{M1,_,_},_} = Cb <- CallbackList, M1 =:= M]} ||
       M <- lists:usort([M || {{M,_,_},_} <- CallbackList])],
-  [true, true] =
+  [true, true, true] =
     [ets:insert(ETS, dict:to_list(Data)) ||
-      {ETS, Data} <- [{ETSInfo, Info}, {ETSContracts, Contracts}]],
+      {ETS, Data} <- [{ETSInfo, Info},
+                      {ETSTypes, Types},
+                      {ETSContracts, Contracts}]],
   true = ets:insert(ETSCallbacks, CallbacksByModule),
-  #mini_plt{info = ETSInfo, contracts = ETSContracts, callbacks = ETSCallbacks};
+  true = ets:insert(ETSExpTypes, [{ET} || ET <- sets:to_list(ExpTypes)]),
+  #mini_plt{info = ETSInfo,
+            types = ETSTypes,
+            contracts = ETSContracts,
+            callbacks = ETSCallbacks,
+            exported_types = ETSExpTypes};
 get_mini_plt(undefined) ->
   undefined.
 
--spec restore_full_plt(plt(), plt()) -> plt().
-
-restore_full_plt(#mini_plt{info = ETSInfo, contracts = ETSContracts}, Plt) ->
-  Info = dict:from_list(ets:tab2list(ETSInfo)),
-  Contracts = dict:from_list(ets:tab2list(ETSContracts)),
-  ets:delete(ETSContracts),
-  ets:delete(ETSInfo),
-  Plt#plt{info = Info, contracts = Contracts};
-restore_full_plt(undefined, undefined) ->
+-spec restore_full_plt(plt()) -> plt().
+
+restore_full_plt(#mini_plt{info = ETSInfo,
+                           types = ETSTypes,
+                           contracts = ETSContracts,
+                           callbacks = ETSCallbacks,
+                           exported_types = ETSExpTypes} = MiniPlt) ->
+  Info = dict:from_list(tab2list(ETSInfo)),
+  Contracts = dict:from_list(tab2list(ETSContracts)),
+  Types = dict:from_list(tab2list(ETSTypes)),
+  Callbacks =
+    dict:from_list([Cb || {_M, Cbs} <- tab2list(ETSCallbacks), Cb <- Cbs]),
+  ExpTypes = sets:from_list([E || {E} <- tab2list(ETSExpTypes)]),
+  ok = delete(MiniPlt),
+  #plt{info = Info,
+       types = Types,
+       contracts = Contracts,
+       callbacks = Callbacks,
+       exported_types = ExpTypes};
+restore_full_plt(undefined) ->
   undefined.
 
+-spec delete(plt()) -> 'ok'.
+
+delete(#mini_plt{info = ETSInfo,
+                 types = ETSTypes,
+                 contracts = ETSContracts,
+                 callbacks = ETSCallbacks,
+                 exported_types = ETSExpTypes}) ->
+  true = ets:delete(ETSContracts),
+  true = ets:delete(ETSTypes),
+  true = ets:delete(ETSInfo),
+  true = ets:delete(ETSCallbacks),
+  true = ets:delete(ETSExpTypes),
+  ok.
+
+-spec give_away(plt(), pid()) -> 'ok'.
+
+give_away(#mini_plt{info = ETSInfo,
+                    types = ETSTypes,
+                    contracts = ETSContracts,
+                    callbacks = ETSCallbacks,
+                    exported_types = ETSExpTypes},
+          Pid) ->
+  true = ets:give_away(ETSContracts, Pid, any),
+  true = ets:give_away(ETSTypes, Pid, any),
+  true = ets:give_away(ETSInfo, Pid, any),
+  true = ets:give_away(ETSCallbacks, Pid, any),
+  true = ets:give_away(ETSExpTypes, Pid, any),
+  ok.
+
+%% Somewhat slower than ets:tab2list(), but uses less memory.
+tab2list(T) ->
+  tab2list(ets:first(T), T, []).
+
+tab2list('$end_of_table', T, A) ->
+  case ets:first(T) of % no safe_fixtable()...
+    '$end_of_table' -> A;
+    Key -> tab2list(Key, T, A)
+  end;
+tab2list(Key, T, A) ->
+  Vs = ets:lookup(T, Key),
+  Key1 = ets:next(T, Key),
+  ets:delete(T, Key),
+  tab2list(Key1, T, Vs ++ A).
+
 %%---------------------------------------------------------------------------
 %% Edoc
 
@@ -607,6 +683,12 @@ table_delete_module1(Plt, Mod) ->
 table_delete_module2(Plt, Mod) ->
   dict:filter(fun(M, _Val) -> M =/= Mod end, Plt).
 
+ets_table_delete_list(Tab, [H|T]) ->
+  ets:delete(Tab, H),
+  ets_table_delete_list(Tab, T);
+ets_table_delete_list(Tab, []) ->
+  Tab.
+
 table_delete_list(Plt, [H|T]) ->
   table_delete_list(dict:erase(H, Plt), T);
 table_delete_list(Plt, []) ->
diff --git a/lib/dialyzer/src/dialyzer_succ_typings.erl b/lib/dialyzer/src/dialyzer_succ_typings.erl
index 987da3aecf..986def52bf 100644
--- a/lib/dialyzer/src/dialyzer_succ_typings.erl
+++ b/lib/dialyzer/src/dialyzer_succ_typings.erl
@@ -96,7 +96,7 @@ analyze_callgraph(Callgraph, Plt, Codeserver, TimingServer, Solvers, Parent) ->
   NewState =
     init_state_and_get_success_typings(Callgraph, Plt, Codeserver,
 				       TimingServer, Solvers, Parent),
-  dialyzer_plt:restore_full_plt(NewState#st.plt, Plt).
+  NewState#st.plt.
 
 %%--------------------------------------------------------------------
 
@@ -111,6 +111,7 @@ init_state_and_get_success_typings(Callgraph, Plt, Codeserver,
 
 get_refined_success_typings(SCCs, #st{callgraph = Callgraph,
 				      timing_server = TimingServer} = State) ->
+  erlang:garbage_collect(),
   case find_succ_typings(SCCs, State) of
     {fixpoint, State1} -> State1;
     {not_fixpoint, NotFixpoint1, State1} ->
@@ -155,8 +156,8 @@ get_warnings(Callgraph, Plt, DocPlt, Codeserver,
     ?timing(TimingServer, "warning",
 	    get_warnings_from_modules(Mods, InitState, MiniDocPlt)),
   {postprocess_warnings(CWarns ++ ModWarns, Codeserver),
-   dialyzer_plt:restore_full_plt(MiniPlt, Plt),
-   dialyzer_plt:restore_full_plt(MiniDocPlt, DocPlt)}.
+   MiniPlt,
+   dialyzer_plt:restore_full_plt(MiniDocPlt)}.
 
 get_warnings_from_modules(Mods, State, DocPlt) ->
   #st{callgraph = Callgraph, codeserver = Codeserver,
@@ -174,10 +175,10 @@ collect_warnings(M, {Codeserver, Callgraph, Plt, DocPlt}) ->
   %% Check if there are contracts for functions that do not exist
   Warnings1 =
     dialyzer_contracts:contracts_without_fun(Contracts, AllFuns, Callgraph),
+  Attrs = cerl:module_attrs(ModCode),
   {Warnings2, FunTypes} =
     dialyzer_dataflow:get_warnings(ModCode, Plt, Callgraph, Codeserver,
 				   Records),
-  Attrs = cerl:module_attrs(ModCode),
   Warnings3 =
     dialyzer_behaviours:check_callbacks(M, Attrs, Records, Plt, Codeserver),
   DocPlt = insert_into_doc_plt(FunTypes, Callgraph, DocPlt),
diff --git a/lib/typer/src/typer.erl b/lib/typer/src/typer.erl
index 5c82750a21..563b1bbcb5 100644
--- a/lib/typer/src/typer.erl
+++ b/lib/typer/src/typer.erl
@@ -172,8 +172,10 @@ get_type_info(#analysis{callgraph = CallGraph,
   StrippedCallGraph = remove_external(CallGraph, TrustPLT),
   %% io:format("--- Analyzing callgraph... "),
   try 
-    NewPlt = dialyzer_succ_typings:analyze_callgraph(StrippedCallGraph, 
-						     TrustPLT, CodeServer),
+    NewMiniPlt = dialyzer_succ_typings:analyze_callgraph(StrippedCallGraph,
+                                                         TrustPLT,
+                                                         CodeServer),
+    NewPlt = dialyzer_plt:restore_full_plt(NewMiniPlt),
     Analysis#analysis{callgraph = StrippedCallGraph, trust_plt = NewPlt}
   catch
     error:What ->