From 57e21970a394193e929795629fe5c3cd15310660 Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Tue, 6 Dec 2016 14:02:12 +0100
Subject: dialyzer: Do not send full PLTs as messages

The mini PLT is extended to hold all data of the full PLT, and the
full PLT is restored when needed (for storing the PLT on file).
---
 lib/dialyzer/src/dialyzer_analysis_callgraph.erl |  70 ++++++++------
 lib/dialyzer/src/dialyzer_cl.erl                 |  14 ++-
 lib/dialyzer/src/dialyzer_contracts.erl          |   3 +
 lib/dialyzer/src/dialyzer_gui_wx.erl             |   3 +-
 lib/dialyzer/src/dialyzer_plt.erl                | 116 +++++++++++++++++++----
 lib/dialyzer/src/dialyzer_succ_typings.erl       |   9 +-
 6 files changed, 157 insertions(+), 58 deletions(-)

(limited to 'lib/dialyzer/src')

diff --git a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
index 08e55a78bd..8b3f70aa23 100644
--- a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
+++ b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
@@ -101,9 +101,9 @@ loop(#server_state{parent = Parent} = State,
     {AnalPid, cserver, CServer, Plt} ->
       send_codeserver_plt(Parent, CServer, Plt),
       loop(State, Analysis, ExtCalls);
-    {AnalPid, done, Plt, DocPlt} ->
+    {AnalPid, done, MiniPlt, DocPlt} ->
       send_ext_calls(Parent, ExtCalls),
-      send_analysis_done(Parent, Plt, DocPlt);
+      send_analysis_done(Parent, MiniPlt, DocPlt);
     {AnalPid, ext_calls, NewExtCalls} ->
       loop(State, Analysis, NewExtCalls);
     {AnalPid, ext_types, ExtTypes} ->
@@ -161,6 +161,7 @@ analysis_start(Parent, Analysis, LegalWarnings) ->
               begin
                 TmpCServer3 =
                   dialyzer_utils:process_record_remote_types(TmpCServer2),
+                erlang:garbage_collect(),
                 dialyzer_contracts:process_contract_remote_types(TmpCServer3)
               end)
     catch
@@ -171,48 +172,54 @@ analysis_start(Parent, Analysis, LegalWarnings) ->
   NewPlt1 = dialyzer_plt:insert_exported_types(NewPlt0, ExpTypes),
   State0 = State#analysis_state{plt = NewPlt1},
   dump_callgraph(Callgraph, State0, Analysis),
-  State1 = State0#analysis_state{codeserver = NewCServer},
   %% Remove all old versions of the files being analyzed
   AllNodes = dialyzer_callgraph:all_nodes(Callgraph),
-  Plt1 = dialyzer_plt:delete_list(NewPlt1, AllNodes),
+  Plt1_a = dialyzer_plt:delete_list(NewPlt1, AllNodes),
+  Plt1 = dialyzer_plt:insert_callbacks(Plt1_a, NewCServer),
+  State1 = State0#analysis_state{codeserver = NewCServer, plt = Plt1},
   Exports = dialyzer_codeserver:get_exports(NewCServer),
+  NonExports = sets:subtract(sets:from_list(AllNodes), Exports),
+  NonExportsList = sets:to_list(NonExports),
   NewCallgraph =
     case Analysis#analysis.race_detection of
       true -> dialyzer_callgraph:put_race_detection(true, Callgraph);
       false -> Callgraph
     end,
-  State2 = analyze_callgraph(NewCallgraph, State1#analysis_state{plt = Plt1}),
+  %% Calls to erlang:garbage_collect() help to reduce the heap size.
+  %% An alternative is to spawn more processes to do the job(s).
+  erlang:garbage_collect(),
+  State2 = analyze_callgraph(NewCallgraph, State1),
+  #analysis_state{plt = MiniPlt2, doc_plt = DocPlt} = State2,
   dialyzer_callgraph:dispose_race_server(NewCallgraph),
   rcv_and_send_ext_types(Parent),
-  NonExports = sets:subtract(sets:from_list(AllNodes), Exports),
-  NonExportsList = sets:to_list(NonExports),
-  Plt2 = dialyzer_plt:delete_list(State2#analysis_state.plt, NonExportsList),
-  send_codeserver_plt(Parent, CServer, State2#analysis_state.plt),
-  send_analysis_done(Parent, Plt2, State2#analysis_state.doc_plt).
+  %% Since the PLT is never used, a dummy is sent:
+  DummyPlt = dialyzer_plt:new(),
+  send_codeserver_plt(Parent, CServer, DummyPlt),
+  MiniPlt3 = dialyzer_plt:delete_list(MiniPlt2, NonExportsList),
+  send_analysis_done(Parent, MiniPlt3, DocPlt).
 
 analyze_callgraph(Callgraph, #analysis_state{codeserver = Codeserver,
 					     doc_plt = DocPlt,
+                                             plt = Plt,
 					     timing_server = TimingServer,
 					     parent = Parent,
                                              solvers = Solvers} = State) ->
-  Plt = dialyzer_plt:insert_callbacks(State#analysis_state.plt, Codeserver),
-  {NewPlt, NewDocPlt} =
-    case State#analysis_state.analysis_type of
-      plt_build ->
-	NewPlt0 =
-	  dialyzer_succ_typings:analyze_callgraph(Callgraph, Plt, Codeserver,
-						  TimingServer, Solvers, Parent),
-	{NewPlt0, DocPlt};
-      succ_typings ->
-	{Warnings, NewPlt0, NewDocPlt0} =
-	  dialyzer_succ_typings:get_warnings(Callgraph, Plt, DocPlt, Codeserver,
-					     TimingServer, Solvers, Parent),
-        Warnings1 = filter_warnings(Warnings, Codeserver),
-	send_warnings(State#analysis_state.parent, Warnings1),
-	{NewPlt0, NewDocPlt0}
-    end,
-  dialyzer_callgraph:delete(Callgraph),
-  State#analysis_state{plt = NewPlt, doc_plt = NewDocPlt}.
+  case State#analysis_state.analysis_type of
+    plt_build ->
+      NewMiniPlt =
+        dialyzer_succ_typings:analyze_callgraph(Callgraph, Plt, Codeserver,
+                                                TimingServer, Solvers, Parent),
+      dialyzer_callgraph:delete(Callgraph),
+      State#analysis_state{plt = NewMiniPlt, doc_plt = DocPlt};
+    succ_typings ->
+      {Warnings, NewMiniPlt, NewDocPlt} =
+        dialyzer_succ_typings:get_warnings(Callgraph, Plt, DocPlt, Codeserver,
+                                           TimingServer, Solvers, Parent),
+      dialyzer_callgraph:delete(Callgraph),
+      Warnings1 = filter_warnings(Warnings, Codeserver),
+      send_warnings(State#analysis_state.parent, Warnings1),
+      State#analysis_state{plt = NewMiniPlt, doc_plt = NewDocPlt}
+    end.
 
 %%--------------------------------------------------------------------
 %% Build the callgraph and fill the codeserver.
@@ -569,8 +576,9 @@ is_ok_fun({_Filename, _Line, {_M, _F, _A} = MFA}, Codeserver) ->
 is_ok_tag(Tag, {_F, _L, MorMFA}, Codeserver) ->
   not dialyzer_utils:is_suppressed_tag(MorMFA, Tag, Codeserver).
   
-send_analysis_done(Parent, Plt, DocPlt) ->
-  Parent ! {self(), done, Plt, DocPlt},
+send_analysis_done(Parent, MiniPlt, DocPlt) ->
+  ok = dialyzer_plt:give_away(MiniPlt, Parent),
+  Parent ! {self(), done, MiniPlt, DocPlt},
   ok.
 
 send_ext_calls(_Parent, none) ->
@@ -583,7 +591,7 @@ send_ext_types(Parent, ExtTypes) ->
   Parent ! {self(), ext_types, ExtTypes},
   ok.
 
-send_codeserver_plt(Parent, CServer, Plt ) ->
+send_codeserver_plt(Parent, CServer, Plt) ->
   Parent ! {self(), cserver, CServer, Plt},
   ok.
 
diff --git a/lib/dialyzer/src/dialyzer_cl.erl b/lib/dialyzer/src/dialyzer_cl.erl
index fc56693ea3..e8c1613a33 100644
--- a/lib/dialyzer/src/dialyzer_cl.erl
+++ b/lib/dialyzer/src/dialyzer_cl.erl
@@ -637,8 +637,8 @@ cl_loop(State, LogCache) ->
     {BackendPid, warnings, Warnings} ->
       NewState = store_warnings(State, Warnings),
       cl_loop(NewState, LogCache);
-    {BackendPid, done, NewPlt, _NewDocPlt} ->
-      return_value(State, NewPlt);
+    {BackendPid, done, NewMiniPlt, _NewDocPlt} ->
+      return_value(State, NewMiniPlt);
     {BackendPid, ext_calls, ExtCalls} ->
       cl_loop(State#cl_state{external_calls = ExtCalls}, LogCache);
     {BackendPid, ext_types, ExtTypes} ->
@@ -654,6 +654,7 @@ cl_loop(State, LogCache) ->
       cl_error(State, Msg);
     _Other ->
       %% io:format("Received ~p\n", [_Other]),
+      %% Note: {BackendPid, cserver, CodeServer, Plt} is ignored.
       cl_loop(State, LogCache)
   end.
 
@@ -699,10 +700,13 @@ return_value(State = #cl_state{erlang_mode = ErlangMode,
 			       output_plt = OutputPlt,
 			       plt_info = PltInfo,
 			       stored_warnings = StoredWarnings},
-	     Plt) ->
+	     MiniPlt) ->
   case OutputPlt =:= none of
-    true -> ok;
-    false -> dialyzer_plt:to_file(OutputPlt, Plt, ModDeps, PltInfo)
+    true ->
+      dialyzer_plt:delete(MiniPlt);
+    false ->
+      Plt = dialyzer_plt:restore_full_plt(MiniPlt),
+      dialyzer_plt:to_file(OutputPlt, Plt, ModDeps, PltInfo)
   end,
   UnknownWarnings = unknown_warnings(State),
   RetValue =
diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl
index 73b04b305b..331786466f 100644
--- a/lib/dialyzer/src/dialyzer_contracts.erl
+++ b/lib/dialyzer/src/dialyzer_contracts.erl
@@ -170,6 +170,7 @@ process_contract_remote_types(CodeServer) ->
           lists:mapfoldl(ContractFun, C3, dict:to_list(ContractDict)),
         {{ModuleName, dict:from_list(NewContractList)}, C4}
     end,
+  erlang:garbage_collect(),
   Cache = erl_types:cache__new(),
   {NewContractList, C5} =
     lists:mapfoldl(ModuleFun, Cache, dict:to_list(TmpContractDict)),
@@ -177,6 +178,8 @@ process_contract_remote_types(CodeServer) ->
     lists:mapfoldl(ModuleFun, C5, dict:to_list(TmpCallbackDict)),
   NewContractDict = dict:from_list(NewContractList),
   NewCallbackDict = dict:from_list(NewCallbackList),
+  %% Make sure the (huge) cache is garbage collected:
+  erlang:garbage_collect(),
   dialyzer_codeserver:finalize_contracts(NewContractDict, NewCallbackDict,
                                          CodeServer).
 
diff --git a/lib/dialyzer/src/dialyzer_gui_wx.erl b/lib/dialyzer/src/dialyzer_gui_wx.erl
index 4caf64d007..1701aff2f2 100644
--- a/lib/dialyzer/src/dialyzer_gui_wx.erl
+++ b/lib/dialyzer/src/dialyzer_gui_wx.erl
@@ -505,8 +505,9 @@ gui_loop(#gui_state{backend_pid = BackendPid, doc_plt = DocPlt,
 	end,
       ExplanationPid = spawn_link(Fun),
       gui_loop(State#gui_state{expl_pid = ExplanationPid});
-    {BackendPid, done, _NewPlt, NewDocPlt} ->
+    {BackendPid, done, NewMiniPlt, NewDocPlt} ->
       message(State, "Analysis done"),
+      dialyzer_plt:delete(NewMiniPlt),
       config_gui_stop(State),
       gui_loop(State#gui_state{doc_plt = NewDocPlt});
     {'EXIT', BackendPid, {error, Reason}} ->
diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl
index cf2f0e919e..93ec2599b4 100644
--- a/lib/dialyzer/src/dialyzer_plt.erl
+++ b/lib/dialyzer/src/dialyzer_plt.erl
@@ -58,7 +58,9 @@
 	 get_specs/4,
 	 to_file/4,
 	 get_mini_plt/1,
-	 restore_full_plt/2
+	 restore_full_plt/1,
+         delete/1,
+         give_away/2
 	]).
 
 %% Debug utilities
@@ -88,8 +90,10 @@
               exported_types = sets:new()  :: sets:set()}).
 
 -record(mini_plt, {info      :: ets:tid(),
+                   types     :: ets:tid(),
 		   contracts :: ets:tid(),
-		   callbacks :: ets:tid()
+		   callbacks :: ets:tid(),
+                   exported_types :: ets:tid()
 		  }).
 
 -opaque plt() :: #plt{} | #mini_plt{}.
@@ -130,6 +134,10 @@ delete_module(#plt{info = Info, types = Types,
 
 -spec delete_list(plt(), [mfa() | integer()]) -> plt().
 
+delete_list(#mini_plt{info = Info,
+                      contracts = Contracts}=Plt, List) ->
+  Plt#mini_plt{info = ets_table_delete_list(Info, List),
+               contracts = ets_table_delete_list(Contracts, List)};
 delete_list(#plt{info = Info, types = Types,
 		 contracts = Contracts,
 		 callbacks = Callbacks,
@@ -510,32 +518,100 @@ init_md5_list_1(Md5List, [], Acc) ->
 
 -spec get_mini_plt(plt()) -> plt().
 
-get_mini_plt(#plt{info = Info, contracts = Contracts, callbacks = Callbacks}) ->
-  [ETSInfo, ETSContracts, ETSCallbacks] =
-    [ets:new(Name, [public]) || Name <- [plt_info, plt_contracts, plt_callbacks]],
+get_mini_plt(#plt{info = Info,
+                  types = Types,
+                  contracts = Contracts,
+                  callbacks = Callbacks,
+                  exported_types = ExpTypes}) ->
+  [ETSInfo, ETSTypes, ETSContracts, ETSCallbacks, ETSExpTypes] =
+    [ets:new(Name, [public]) ||
+      Name <- [plt_info, plt_types, plt_contracts, plt_callbacks,
+               plt_exported_types]],
   CallbackList = dict:to_list(Callbacks),
   CallbacksByModule =
     [{M, [Cb || {{M1,_,_},_} = Cb <- CallbackList, M1 =:= M]} ||
       M <- lists:usort([M || {{M,_,_},_} <- CallbackList])],
-  [true, true] =
+  [true, true, true] =
     [ets:insert(ETS, dict:to_list(Data)) ||
-      {ETS, Data} <- [{ETSInfo, Info}, {ETSContracts, Contracts}]],
+      {ETS, Data} <- [{ETSInfo, Info},
+                      {ETSTypes, Types},
+                      {ETSContracts, Contracts}]],
   true = ets:insert(ETSCallbacks, CallbacksByModule),
-  #mini_plt{info = ETSInfo, contracts = ETSContracts, callbacks = ETSCallbacks};
+  true = ets:insert(ETSExpTypes, [{ET} || ET <- sets:to_list(ExpTypes)]),
+  #mini_plt{info = ETSInfo,
+            types = ETSTypes,
+            contracts = ETSContracts,
+            callbacks = ETSCallbacks,
+            exported_types = ETSExpTypes};
 get_mini_plt(undefined) ->
   undefined.
 
--spec restore_full_plt(plt(), plt()) -> plt().
-
-restore_full_plt(#mini_plt{info = ETSInfo, contracts = ETSContracts}, Plt) ->
-  Info = dict:from_list(ets:tab2list(ETSInfo)),
-  Contracts = dict:from_list(ets:tab2list(ETSContracts)),
-  ets:delete(ETSContracts),
-  ets:delete(ETSInfo),
-  Plt#plt{info = Info, contracts = Contracts};
-restore_full_plt(undefined, undefined) ->
+-spec restore_full_plt(plt()) -> plt().
+
+restore_full_plt(#mini_plt{info = ETSInfo,
+                           types = ETSTypes,
+                           contracts = ETSContracts,
+                           callbacks = ETSCallbacks,
+                           exported_types = ETSExpTypes} = MiniPlt) ->
+  Info = dict:from_list(tab2list(ETSInfo)),
+  Contracts = dict:from_list(tab2list(ETSContracts)),
+  Types = dict:from_list(tab2list(ETSTypes)),
+  Callbacks =
+    dict:from_list([Cb || {_M, Cbs} <- tab2list(ETSCallbacks), Cb <- Cbs]),
+  ExpTypes = sets:from_list([E || {E} <- tab2list(ETSExpTypes)]),
+  ok = delete(MiniPlt),
+  #plt{info = Info,
+       types = Types,
+       contracts = Contracts,
+       callbacks = Callbacks,
+       exported_types = ExpTypes};
+restore_full_plt(undefined) ->
   undefined.
 
+-spec delete(plt()) -> 'ok'.
+
+delete(#mini_plt{info = ETSInfo,
+                 types = ETSTypes,
+                 contracts = ETSContracts,
+                 callbacks = ETSCallbacks,
+                 exported_types = ETSExpTypes}) ->
+  true = ets:delete(ETSContracts),
+  true = ets:delete(ETSTypes),
+  true = ets:delete(ETSInfo),
+  true = ets:delete(ETSCallbacks),
+  true = ets:delete(ETSExpTypes),
+  ok.
+
+-spec give_away(plt(), pid()) -> 'ok'.
+
+give_away(#mini_plt{info = ETSInfo,
+                    types = ETSTypes,
+                    contracts = ETSContracts,
+                    callbacks = ETSCallbacks,
+                    exported_types = ETSExpTypes},
+          Pid) ->
+  true = ets:give_away(ETSContracts, Pid, any),
+  true = ets:give_away(ETSTypes, Pid, any),
+  true = ets:give_away(ETSInfo, Pid, any),
+  true = ets:give_away(ETSCallbacks, Pid, any),
+  true = ets:give_away(ETSExpTypes, Pid, any),
+  ok.
+
+%% Somewhat slower than ets:tab2list(), but uses less memory.
+tab2list(T) ->
+  tab2list(ets:first(T), T, []).
+
+tab2list('$end_of_table', T, A) ->
+  case ets:first(T) of % no safe_fixtable()...
+    '$end_of_table' -> A;
+    Key -> tab2list(Key, T, A)
+  end;
+tab2list(Key, T, A) ->
+  Vs = ets:lookup(T, Key),
+  Key1 = ets:next(T, Key),
+  ets:delete(T, Key),
+  tab2list(Key1, T, Vs ++ A).
+
 %%---------------------------------------------------------------------------
 %% Edoc
 
@@ -607,6 +683,12 @@ table_delete_module1(Plt, Mod) ->
 table_delete_module2(Plt, Mod) ->
   dict:filter(fun(M, _Val) -> M =/= Mod end, Plt).
 
+ets_table_delete_list(Tab, [H|T]) ->
+  ets:delete(Tab, H),
+  ets_table_delete_list(Tab, T);
+ets_table_delete_list(Tab, []) ->
+  Tab.
+
 table_delete_list(Plt, [H|T]) ->
   table_delete_list(dict:erase(H, Plt), T);
 table_delete_list(Plt, []) ->
diff --git a/lib/dialyzer/src/dialyzer_succ_typings.erl b/lib/dialyzer/src/dialyzer_succ_typings.erl
index 987da3aecf..986def52bf 100644
--- a/lib/dialyzer/src/dialyzer_succ_typings.erl
+++ b/lib/dialyzer/src/dialyzer_succ_typings.erl
@@ -96,7 +96,7 @@ analyze_callgraph(Callgraph, Plt, Codeserver, TimingServer, Solvers, Parent) ->
   NewState =
     init_state_and_get_success_typings(Callgraph, Plt, Codeserver,
 				       TimingServer, Solvers, Parent),
-  dialyzer_plt:restore_full_plt(NewState#st.plt, Plt).
+  NewState#st.plt.
 
 %%--------------------------------------------------------------------
 
@@ -111,6 +111,7 @@ init_state_and_get_success_typings(Callgraph, Plt, Codeserver,
 
 get_refined_success_typings(SCCs, #st{callgraph = Callgraph,
 				      timing_server = TimingServer} = State) ->
+  erlang:garbage_collect(),
   case find_succ_typings(SCCs, State) of
     {fixpoint, State1} -> State1;
     {not_fixpoint, NotFixpoint1, State1} ->
@@ -155,8 +156,8 @@ get_warnings(Callgraph, Plt, DocPlt, Codeserver,
     ?timing(TimingServer, "warning",
 	    get_warnings_from_modules(Mods, InitState, MiniDocPlt)),
   {postprocess_warnings(CWarns ++ ModWarns, Codeserver),
-   dialyzer_plt:restore_full_plt(MiniPlt, Plt),
-   dialyzer_plt:restore_full_plt(MiniDocPlt, DocPlt)}.
+   MiniPlt,
+   dialyzer_plt:restore_full_plt(MiniDocPlt)}.
 
 get_warnings_from_modules(Mods, State, DocPlt) ->
   #st{callgraph = Callgraph, codeserver = Codeserver,
@@ -174,10 +175,10 @@ collect_warnings(M, {Codeserver, Callgraph, Plt, DocPlt}) ->
   %% Check if there are contracts for functions that do not exist
   Warnings1 =
     dialyzer_contracts:contracts_without_fun(Contracts, AllFuns, Callgraph),
+  Attrs = cerl:module_attrs(ModCode),
   {Warnings2, FunTypes} =
     dialyzer_dataflow:get_warnings(ModCode, Plt, Callgraph, Codeserver,
 				   Records),
-  Attrs = cerl:module_attrs(ModCode),
   Warnings3 =
     dialyzer_behaviours:check_callbacks(M, Attrs, Records, Plt, Codeserver),
   DocPlt = insert_into_doc_plt(FunTypes, Callgraph, DocPlt),
-- 
cgit v1.2.3


From c87ddb41318396a0d3acc44be13f9258d57b96f3 Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Tue, 13 Dec 2016 14:01:09 +0100
Subject: dialyzer: Optimize graph condensation

By not using ETS when calculating the condensation of graphs, peak
heap memory consumption is reduced.
---
 lib/dialyzer/src/dialyzer_callgraph.erl | 41 ++++++++++-----------------------
 1 file changed, 12 insertions(+), 29 deletions(-)

(limited to 'lib/dialyzer/src')

diff --git a/lib/dialyzer/src/dialyzer_callgraph.erl b/lib/dialyzer/src/dialyzer_callgraph.erl
index 50abb22009..724628d747 100644
--- a/lib/dialyzer/src/dialyzer_callgraph.erl
+++ b/lib/dialyzer/src/dialyzer_callgraph.erl
@@ -759,36 +759,19 @@ to_ps(#callgraph{} = CG, File, Args) ->
 
 condensation(G) ->
   SCs = digraph_utils:strong_components(G),
-  V2I = ets:new(condensation_v2i, []),
-  I2C = ets:new(condensation_i2c, []),
-  I2I = ets:new(condensation_i2i, [bag]),
-  CFun =
-    fun(SC, N) ->
-	lists:foreach(fun(V) -> true = ets:insert(V2I, {V,N}) end, SC),
-	true = ets:insert(I2C, {N, SC}),
-	N + 1
-    end,
-  lists:foldl(CFun, 1, SCs),
-  Fun1 =
-    fun({V1, V2}) ->
-        I1 = ets:lookup_element(V2I, V1, 2),
-        I2 = ets:lookup_element(V2I, V2, 2),
-	I1 =:= I2 orelse ets:insert(I2I, {I1, I2})
-    end,
-  lists:foreach(Fun1, digraph:edges(G)),
-  Fun3 =
-    fun({I1, I2}, {Out, In}) ->
-        SC1 = ets:lookup_element(I2C, I1, 2),
-        SC2 = ets:lookup_element(I2C, I2, 2),
-        {dict:append(SC1, SC2, Out), dict:append(SC2, SC1, In)}
-    end,
-  {OutDict, InDict} = ets:foldl(Fun3, {dict:new(), dict:new()}, I2I),
+  %% Subsitute strong components for vertices in edges:
+  C2V = sofs:relation([{SC, V} || SC <- SCs, V <- SC], [{scc, v}]),
+  Es = sofs:relation(digraph:edges(G), [{v, v}]),
+  R1 = sofs:relative_product(C2V, Es),
+  R2 = sofs:relative_product(C2V, sofs:converse(R1)),
+  %% Create in- and out-neighbours:
+  In = sofs:relation_to_family(sofs:strict_relation(R2)),
+  R3 = sofs:converse(R2),
+  Out = sofs:relation_to_family(sofs:strict_relation(R3)),
   [OutETS, InETS] =
     [ets:new(Name,[{read_concurrency, true}]) ||
       Name <- [callgraph_deps_out, callgraph_deps_in]],
-  ets:insert(OutETS, dict:to_list(OutDict)),
-  ets:insert(InETS, dict:to_list(InDict)),
-  ets:delete(V2I),
-  ets:delete(I2C),
-  ets:delete(I2I),
+  ets:insert(OutETS, sofs:to_external(Out)),
+  ets:insert(InETS, sofs:to_external(In)),
+  erlang:garbage_collect(),
   {{'e', OutETS, InETS}, SCs}.
-- 
cgit v1.2.3


From 3c6dbbeb252247ef97cad93921deb4c6e313f11b Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Tue, 20 Dec 2016 11:36:52 +0100
Subject: dialyzer: Reduce ETS usage during the typesig phase

The condensed graph of SCCs occupies less ETS memory. A table
translating to and from SCC to a unique integer is introduced.
---
 lib/dialyzer/src/dialyzer_callgraph.erl | 63 +++++++++++++++++++++------------
 1 file changed, 41 insertions(+), 22 deletions(-)

(limited to 'lib/dialyzer/src')

diff --git a/lib/dialyzer/src/dialyzer_callgraph.erl b/lib/dialyzer/src/dialyzer_callgraph.erl
index 724628d747..5e02e7a2cc 100644
--- a/lib/dialyzer/src/dialyzer_callgraph.erl
+++ b/lib/dialyzer/src/dialyzer_callgraph.erl
@@ -119,7 +119,11 @@
 
 -opaque callgraph() :: #callgraph{}.
 
--type active_digraph() :: {'d', digraph:graph()} | {'e', ets:tid(), ets:tid()}.
+-type active_digraph() :: {'d', digraph:graph()}
+                        | {'e',
+                           Out :: ets:tid(),
+                           In :: ets:tid(),
+                           Map :: ets:tid()}.
 
 %%----------------------------------------------------------------------
 
@@ -248,24 +252,30 @@ find_non_local_calls([], Set) ->
 
 -spec get_depends_on(scc() | module(), callgraph()) -> [scc()].
 
-get_depends_on(SCC, #callgraph{active_digraph = {'e', Out, _In}}) ->
-  case ets_lookup_dict(SCC, Out) of
-    {ok, Value} -> Value;
-    error -> []
-  end;
+get_depends_on(SCC, #callgraph{active_digraph = {'e', Out, _In, Maps}}) ->
+  lookup_scc(SCC, Out, Maps);
 get_depends_on(SCC, #callgraph{active_digraph = {'d', DG}}) ->
   digraph:out_neighbours(DG, SCC).
 
 -spec get_required_by(scc() | module(), callgraph()) -> [scc()].
 
-get_required_by(SCC, #callgraph{active_digraph = {'e', _Out, In}}) ->
-  case ets_lookup_dict(SCC, In) of
-    {ok, Value} -> Value;
-    error -> []
-  end;
+get_required_by(SCC, #callgraph{active_digraph = {'e', _Out, In, Maps}}) ->
+  lookup_scc(SCC, In, Maps);
 get_required_by(SCC, #callgraph{active_digraph = {'d', DG}}) ->
   digraph:in_neighbours(DG, SCC).
 
+lookup_scc(SCC, Table, Maps) ->
+  case ets_lookup_dict({'scc', SCC}, Maps) of
+    {ok, SCCInt} ->
+      case ets_lookup_dict(SCCInt, Table) of
+        {ok, Ints} ->
+          [ets:lookup_element(Maps, Int, 2) || Int <- Ints];
+        error ->
+          []
+      end;
+    error -> []
+  end.
+
 %%----------------------------------------------------------------------
 %% Handling of modules & SCCs
 %%----------------------------------------------------------------------
@@ -582,9 +592,10 @@ digraph_delete(DG) ->
 
 active_digraph_delete({'d', DG}) ->
   digraph:delete(DG);
-active_digraph_delete({'e', Out, In}) ->
+active_digraph_delete({'e', Out, In, Maps}) ->
   ets:delete(Out),
-  ets:delete(In).
+  ets:delete(In),
+  ets:delete(Maps).
 
 digraph_edges(DG) ->
   digraph:edges(DG).
@@ -758,20 +769,28 @@ to_ps(#callgraph{} = CG, File, Args) ->
   ok.
 
 condensation(G) ->
-  SCs = digraph_utils:strong_components(G),
-  %% Subsitute strong components for vertices in edges:
-  C2V = sofs:relation([{SC, V} || SC <- SCs, V <- SC], [{scc, v}]),
+  SCCs = digraph_utils:strong_components(G),
+  %% Assign unique numbers to SCCs:
+  Ints = lists:seq(1, length(SCCs)),
+  IntToSCC = lists:zip(Ints, SCCs),
+  IntScc = sofs:relation(IntToSCC, [{int, scc}]),
+  %% Subsitute strong components for vertices in edges using the
+  %% unique numbers:
+  C2V = sofs:relation([{SC, V} || SC <- SCCs, V <- SC], [{scc, v}]),
+  I2V = sofs:relative_product(IntScc, C2V), % [{v, int}]
   Es = sofs:relation(digraph:edges(G), [{v, v}]),
-  R1 = sofs:relative_product(C2V, Es),
-  R2 = sofs:relative_product(C2V, sofs:converse(R1)),
+  R1 = sofs:relative_product(I2V, Es),
+  R2 = sofs:relative_product(I2V, sofs:converse(R1)),
   %% Create in- and out-neighbours:
   In = sofs:relation_to_family(sofs:strict_relation(R2)),
   R3 = sofs:converse(R2),
   Out = sofs:relation_to_family(sofs:strict_relation(R3)),
-  [OutETS, InETS] =
+  [OutETS, InETS, MapsETS] =
     [ets:new(Name,[{read_concurrency, true}]) ||
-      Name <- [callgraph_deps_out, callgraph_deps_in]],
+      Name <- [callgraph_deps_out, callgraph_deps_in, callgraph_scc_map]],
   ets:insert(OutETS, sofs:to_external(Out)),
   ets:insert(InETS, sofs:to_external(In)),
-  erlang:garbage_collect(),
-  {{'e', OutETS, InETS}, SCs}.
+  %% Create mappings from SCCs to unique integers, and the inverse:
+  ets:insert(MapsETS, lists:zip([{'scc', SCC} || SCC<- SCCs], Ints)),
+  ets:insert(MapsETS, IntToSCC),
+  {{'e', OutETS, InETS, MapsETS}, SCCs}.
-- 
cgit v1.2.3


From 2bcf8a6159e3b370739b835b52b95ff73c54776f Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Thu, 15 Dec 2016 09:59:53 +0100
Subject: dialyzer: Update code for finding parallelism

Using erlang:system_info(schedulers_online) for determining
parallelism is recommended by OTP experts. It takes care of "+S 4:1"
and simultaneous loading of HiPE code.
---
 lib/dialyzer/src/dialyzer_utils.erl | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

(limited to 'lib/dialyzer/src')

diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl
index 1f2d3e3aaa..921644422e 100644
--- a/lib/dialyzer/src/dialyzer_utils.erl
+++ b/lib/dialyzer/src/dialyzer_utils.erl
@@ -959,9 +959,7 @@ label(Tree) ->
 -spec parallelism() -> integer().
 
 parallelism() ->
-  CPUs = erlang:system_info(logical_processors_available),
-  Schedulers = erlang:system_info(schedulers),
-  min(CPUs, Schedulers).
+  erlang:system_info(schedulers_online).
 
 -spec family([{K,V}]) -> [{K,[V]}].
 
-- 
cgit v1.2.3


From 7f3b77274958b62523e83ab2b37ad29ec9a6cf3c Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Mon, 19 Dec 2016 10:02:12 +0100
Subject: dialyzer: Reduce memory consumption during 'remote' phase

The cache used for speeding up the translation from forms to types is
no longer global, but reset per module. The peak memory consumption
is reduced, and the added time seems to be small.
---
 lib/dialyzer/src/dialyzer_contracts.erl | 16 +++++++---------
 lib/dialyzer/src/dialyzer_utils.erl     | 33 ++++++++++++++++++---------------
 2 files changed, 25 insertions(+), 24 deletions(-)

(limited to 'lib/dialyzer/src')

diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl
index 331786466f..82275d5df7 100644
--- a/lib/dialyzer/src/dialyzer_contracts.erl
+++ b/lib/dialyzer/src/dialyzer_contracts.erl
@@ -165,17 +165,15 @@ process_contract_remote_types(CodeServer) ->
         {{MFA, {File, Contract, Xtra}}, C2}
     end,
   ModuleFun =
-    fun({ModuleName, ContractDict}, C3) ->
-        {NewContractList, C4} =
-          lists:mapfoldl(ContractFun, C3, dict:to_list(ContractDict)),
-        {{ModuleName, dict:from_list(NewContractList)}, C4}
+    fun({ModuleName, ContractDict}) ->
+        Cache = erl_types:cache__new(),
+        {NewContractList, _NewCache} =
+          lists:mapfoldl(ContractFun, Cache, dict:to_list(ContractDict)),
+        {ModuleName, dict:from_list(NewContractList)}
     end,
   erlang:garbage_collect(),
-  Cache = erl_types:cache__new(),
-  {NewContractList, C5} =
-    lists:mapfoldl(ModuleFun, Cache, dict:to_list(TmpContractDict)),
-  {NewCallbackList, _C6} =
-    lists:mapfoldl(ModuleFun, C5, dict:to_list(TmpCallbackDict)),
+  NewContractList = lists:map(ModuleFun, dict:to_list(TmpContractDict)),
+  NewCallbackList = lists:map(ModuleFun, dict:to_list(TmpCallbackDict)),
   NewContractDict = dict:from_list(NewContractList),
   NewCallbackDict = dict:from_list(NewCallbackList),
   %% Make sure the (huge) cache is garbage collected:
diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl
index 921644422e..5386d5f219 100644
--- a/lib/dialyzer/src/dialyzer_utils.erl
+++ b/lib/dialyzer/src/dialyzer_utils.erl
@@ -302,13 +302,11 @@ get_record_fields([], _RecDict, Acc) ->
 process_record_remote_types(CServer) ->
   TempRecords = dialyzer_codeserver:get_temp_records(CServer),
   ExpTypes = dialyzer_codeserver:get_exported_types(CServer),
-  Cache = erl_types:cache__new(),
-  {TempRecords1, Cache1} =
-    process_opaque_types0(TempRecords, ExpTypes, Cache),
+  TempRecords1 = process_opaque_types0(TempRecords, ExpTypes),
   %% A cache (not the field type cache) is used for speeding things up a bit.
   VarTable = erl_types:var_table__new(),
   ModuleFun =
-    fun({Module, Record}, C0) ->
+    fun({Module, Record}) ->
         RecordFun =
           fun({Key, Value}, C2) ->
               case Key of
@@ -334,24 +332,27 @@ process_record_remote_types(CServer) ->
                 _Other -> {{Key, Value}, C2}
               end
           end,
-        {RecordList, C1} =
-          lists:mapfoldl(RecordFun, C0, dict:to_list(Record)),
-        {{Module, dict:from_list(RecordList)}, C1}
+        Cache = erl_types:cache__new(),
+        {RecordList, _NewCache} =
+          lists:mapfoldl(RecordFun, Cache, dict:to_list(Record)),
+        {Module, dict:from_list(RecordList)}
     end,
-  {NewRecordsList, C1} =
-    lists:mapfoldl(ModuleFun, Cache1, dict:to_list(TempRecords1)),
+  NewRecordsList = lists:map(ModuleFun, dict:to_list(TempRecords1)),
   NewRecords = dict:from_list(NewRecordsList),
-  _C8 = check_record_fields(NewRecords, ExpTypes, C1),
+  check_record_fields(NewRecords, ExpTypes),
   dialyzer_codeserver:finalize_records(NewRecords, CServer).
 
 %% erl_types:t_from_form() substitutes the declaration of opaque types
 %% for the expanded type in some cases. To make sure the initial type,
 %% any(), is not used, the expansion is done twice.
 %% XXX: Recursive opaque types are not handled well.
-process_opaque_types0(TempRecords0, TempExpTypes, Cache) ->
-  {TempRecords1, NewCache} =
+process_opaque_types0(TempRecords0, TempExpTypes) ->
+  Cache = erl_types:cache__new(),
+  {TempRecords1, Cache1} =
     process_opaque_types(TempRecords0, TempExpTypes, Cache),
-  process_opaque_types(TempRecords1, TempExpTypes, NewCache).
+  {TempRecords, _NewCache} =
+    process_opaque_types(TempRecords1, TempExpTypes, Cache1),
+  TempRecords.
 
 process_opaque_types(TempRecords, TempExpTypes, Cache) ->
   VarTable = erl_types:var_table__new(),
@@ -380,7 +381,8 @@ process_opaque_types(TempRecords, TempExpTypes, Cache) ->
   {dict:from_list(TempRecordList), NewCache}.
   %% dict:map(ModuleFun, TempRecords).
 
-check_record_fields(Records, TempExpTypes, Cache) ->
+check_record_fields(Records, TempExpTypes) ->
+  Cache = erl_types:cache__new(),
   VarTable = erl_types:var_table__new(),
   CheckFun =
     fun({Module, Element}, C0) ->
@@ -412,7 +414,8 @@ check_record_fields(Records, TempExpTypes, Cache) ->
           end,
         lists:foldl(ElemFun, C0, dict:to_list(Element))
     end,
-  lists:foldl(CheckFun, Cache, dict:to_list(Records)).
+  _NewCache = lists:foldl(CheckFun, Cache, dict:to_list(Records)),
+  ok.
 
 msg_with_position(Fun, FileLine) ->
   try Fun()
-- 
cgit v1.2.3


From 7a6ea7e9946e72a03167145c476e66ee77098b6e Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Fri, 9 Dec 2016 16:16:11 +0100
Subject: dialyzer: Optimize memory consumption

Some SCC info is no longer cached. It reduces the peak memory
consumption of workers during the typesig phase, and seems to cost
very little time.
---
 lib/dialyzer/src/dialyzer_analysis_callgraph.erl |  4 +-
 lib/dialyzer/src/dialyzer_codeserver.erl         | 10 ++++-
 lib/dialyzer/src/dialyzer_contracts.erl          |  2 +-
 lib/dialyzer/src/dialyzer_succ_typings.erl       | 22 ++++++-----
 lib/dialyzer/src/dialyzer_typesig.erl            | 50 ++++++++++++------------
 5 files changed, 50 insertions(+), 38 deletions(-)

(limited to 'lib/dialyzer/src')

diff --git a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
index 8b3f70aa23..c8129cfbaa 100644
--- a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
+++ b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
@@ -121,6 +121,7 @@ loop(#server_state{parent = Parent} = State,
 %% The Analysis
 %%--------------------------------------------------------------------
 
+%% Calls to erlang:garbage_collect() help to reduce the heap size.
 analysis_start(Parent, Analysis, LegalWarnings) ->
   CServer = dialyzer_codeserver:new(),
   Plt = Analysis#analysis.plt,
@@ -157,6 +158,7 @@ analysis_start(Parent, Analysis, LegalWarnings) ->
       TmpCServer1 = dialyzer_codeserver:set_temp_records(MergedRecords, TmpCServer0),
       TmpCServer2 =
         dialyzer_codeserver:finalize_exported_types(MergedExpTypes, TmpCServer1),
+      erlang:garbage_collect(),
       ?timing(State#analysis_state.timing_server, "remote",
               begin
                 TmpCServer3 =
@@ -185,8 +187,6 @@ analysis_start(Parent, Analysis, LegalWarnings) ->
       true -> dialyzer_callgraph:put_race_detection(true, Callgraph);
       false -> Callgraph
     end,
-  %% Calls to erlang:garbage_collect() help to reduce the heap size.
-  %% An alternative is to spawn more processes to do the job(s).
   erlang:garbage_collect(),
   State2 = analyze_callgraph(NewCallgraph, State1),
   #analysis_state{plt = MiniPlt2, doc_plt = DocPlt} = State2,
diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl
index 03cd9671af..5028c4f3ba 100644
--- a/lib/dialyzer/src/dialyzer_codeserver.erl
+++ b/lib/dialyzer/src/dialyzer_codeserver.erl
@@ -48,6 +48,7 @@
 	 is_exported/2,
 	 lookup_mod_code/2,
 	 lookup_mfa_code/2,
+	 lookup_mfa_var_label/2,
 	 lookup_mod_records/2,
 	 lookup_mod_contracts/2,
 	 lookup_mfa_contract/2,
@@ -173,8 +174,8 @@ insert(Mod, ModCode, CS) ->
   As = cerl:get_ann(ModCode),
   Funs =
     [{{Mod, cerl:fname_id(Var), cerl:fname_arity(Var)},
-      Val} || Val = {Var, _Fun} <- Defs],
-  Keys = [Key || {Key, _Value} <- Funs],
+      Val, {Var, cerl_trees:get_label(Fun)}} || Val = {Var, Fun} <- Defs],
+  Keys = [Key || {Key, _Value, _Label} <- Funs],
   ModEntry = {Mod, {Name, Exports, Attrs, Keys, As}},
   true = ets:insert(CS#codeserver.code, [ModEntry|Funs]),
   CS.
@@ -237,6 +238,11 @@ lookup_mod_code(Mod, CS) when is_atom(Mod) ->
 lookup_mfa_code({_M, _F, _A} = MFA, CS) ->
   table__lookup(CS#codeserver.code, MFA).
 
+-spec lookup_mfa_var_label(mfa(), codeserver()) -> {cerl:c_var(), label()}.
+
+lookup_mfa_var_label({_M, _F, _A} = MFA, CS) ->
+  ets:lookup_element(CS#codeserver.code, MFA, 3).
+
 -spec get_next_core_label(codeserver()) -> label().
 
 get_next_core_label(#codeserver{next_core_label = NCL}) ->
diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl
index 82275d5df7..2cca16d085 100644
--- a/lib/dialyzer/src/dialyzer_contracts.erl
+++ b/lib/dialyzer/src/dialyzer_contracts.erl
@@ -176,7 +176,7 @@ process_contract_remote_types(CodeServer) ->
   NewCallbackList = lists:map(ModuleFun, dict:to_list(TmpCallbackDict)),
   NewContractDict = dict:from_list(NewContractList),
   NewCallbackDict = dict:from_list(NewCallbackList),
-  %% Make sure the (huge) cache is garbage collected:
+  %% Make sure temporary data and the (huge) cache are garbage collected:
   erlang:garbage_collect(),
   dialyzer_codeserver:finalize_contracts(NewContractDict, NewCallbackDict,
                                          CodeServer).
diff --git a/lib/dialyzer/src/dialyzer_succ_typings.erl b/lib/dialyzer/src/dialyzer_succ_typings.erl
index 986def52bf..d92173875c 100644
--- a/lib/dialyzer/src/dialyzer_succ_typings.erl
+++ b/lib/dialyzer/src/dialyzer_succ_typings.erl
@@ -349,21 +349,25 @@ find_succ_typings(SCCs, #st{codeserver = Codeserver, callgraph = Callgraph,
 
 -spec find_succ_types_for_scc(scc(), typesig_init_data()) -> [mfa_or_funlbl()].
 
-find_succ_types_for_scc(SCC, {Codeserver, Callgraph, Plt, Solvers}) ->
-  SCC_Info = [{MFA, 
-	       dialyzer_codeserver:lookup_mfa_code(MFA, Codeserver),
-	       dialyzer_codeserver:lookup_mod_records(M, Codeserver)}
-	      || {M, _, _} = MFA <- SCC],
+find_succ_types_for_scc(SCC0, {Codeserver, Callgraph, Plt, Solvers}) ->
+  SCC = [MFA || {_, _, _} = MFA <- SCC0],
   Contracts1 = [{MFA, dialyzer_codeserver:lookup_mfa_contract(MFA, Codeserver)}
-		|| {_, _, _} = MFA <- SCC],
+		|| MFA <- SCC],
   Contracts2 = [{MFA, Contract} || {MFA, {ok, Contract}} <- Contracts1],
   Contracts3 = orddict:from_list(Contracts2),
   Label = dialyzer_codeserver:get_next_core_label(Codeserver),
-  AllFuns = collect_fun_info([Fun || {_MFA, {_Var, Fun}, _Rec} <- SCC_Info]),
+  AllFuns = lists:append(
+              [begin
+                 {_Var, Fun} =
+                   dialyzer_codeserver:lookup_mfa_code(MFA, Codeserver),
+                 collect_fun_info([Fun])
+               end || MFA <- SCC]),
+  erlang:garbage_collect(),
   PropTypes = get_fun_types_from_plt(AllFuns, Callgraph, Plt),
   %% Assume that the PLT contains the current propagated types
-  FunTypes = dialyzer_typesig:analyze_scc(SCC_Info, Label, Callgraph,
-                                          Plt, PropTypes, Solvers),
+  FunTypes = dialyzer_typesig:analyze_scc(SCC, Label, Callgraph,
+                                          Codeserver, Plt, PropTypes,
+                                          Solvers),
   AllFunSet = sets:from_list([X || {X, _} <- AllFuns]),
   FilteredFunTypes =
     dict:filter(fun(X, _) -> sets:is_element(X, AllFunSet) end, FunTypes),
diff --git a/lib/dialyzer/src/dialyzer_typesig.erl b/lib/dialyzer/src/dialyzer_typesig.erl
index 1787b66192..075b6ee165 100644
--- a/lib/dialyzer/src/dialyzer_typesig.erl
+++ b/lib/dialyzer/src/dialyzer_typesig.erl
@@ -29,7 +29,7 @@
 
 -module(dialyzer_typesig).
 
--export([analyze_scc/6]).
+-export([analyze_scc/7]).
 -export([get_safe_underapprox/2]).
 
 %%-import(helper, %% 'helper' could be any module doing sanity checks...
@@ -101,7 +101,6 @@
 
 -type types() :: erl_types:type_table().
 
--type typesig_scc()    :: [{mfa(), {cerl:c_var(), cerl:c_fun()}, types()}].
 -type typesig_funmap() :: #{type_var() => type_var()}.
 
 -type prop_types() :: dict:dict(label(), types()).
@@ -160,11 +159,10 @@
 %%-----------------------------------------------------------------------------
 %% Analysis of strongly connected components.
 %%
-%% analyze_scc(SCC, NextLabel, CallGraph, PLT, PropTypes, Solvers) -> FunTypes
+%% analyze_scc(SCC, NextLabel, CallGraph, CodeServer,
+%%             PLT, PropTypes, Solvers) -> FunTypes
 %%
-%% SCC       - [{MFA, Def, Records}]
-%%             where Def = {Var, Fun} as in the Core Erlang module definitions.
-%%                   Records = dict(RecName, {Arity, [{FieldName, FieldType}]})
+%% SCC       - [{MFA}]
 %% NextLabel - An integer that is higher than any label in the code.
 %% CallGraph - A callgraph as produced by dialyzer_callgraph.erl
 %%             Note: The callgraph must have been built with all the
@@ -176,28 +174,27 @@
 %% Solvers   - User specified solvers.
 %%-----------------------------------------------------------------------------
 
--spec analyze_scc(typesig_scc(), label(),
+-spec analyze_scc([mfa()], label(),
 		  dialyzer_callgraph:callgraph(),
+                  dialyzer_codeserver:codeserver(),
 		  dialyzer_plt:plt(), prop_types(), [solver()]) -> prop_types().
 
-analyze_scc(SCC, NextLabel, CallGraph, Plt, PropTypes, Solvers0) ->
+analyze_scc(SCC, NextLabel, CallGraph, CServer, Plt, PropTypes, Solvers0) ->
   Solvers = solvers(Solvers0),
-  assert_format_of_scc(SCC),
-  State1 = new_state(SCC, NextLabel, CallGraph, Plt, PropTypes, Solvers),
-  DefSet = add_def_list([Var || {_MFA, {Var, _Fun}, _Rec} <- SCC], sets:new()),
-  State2 = traverse_scc(SCC, DefSet, State1),
+  State1 = new_state(SCC, NextLabel, CallGraph, CServer, Plt, PropTypes,
+                     Solvers),
+  DefSet = add_def_list(maps:values(State1#state.name_map), sets:new()),
+  ModRecs = [{M, dialyzer_codeserver:lookup_mod_records(M, CServer)} ||
+              M <- lists:usort([M || {M, _, _} <- SCC])],
+  State2 = traverse_scc(SCC, CServer, DefSet, ModRecs, State1),
   State3 = state__finalize(State2),
+  erlang:garbage_collect(),
   Funs = state__scc(State3),
   pp_constrs_scc(Funs, State3),
   constraints_to_dot_scc(Funs, State3),
   T = solve(Funs, State3),
   dict:from_list(maps:to_list(T)).
 
-assert_format_of_scc([{_MFA, {_Var, _Fun}, _Records}|Left]) ->
-  assert_format_of_scc(Left);
-assert_format_of_scc([]) ->
-  ok.
-
 solvers([]) -> [v2];
 solvers(Solvers) -> Solvers.
 
@@ -207,12 +204,14 @@ solvers(Solvers) -> Solvers.
 %%
 %% ============================================================================
 
-traverse_scc([{_MFA, Def, Rec}|Left], DefSet, AccState) ->
+traverse_scc([{M,_,_}=MFA|Left], Codeserver, DefSet, ModRecs, AccState) ->
+  Def = dialyzer_codeserver:lookup_mfa_code(MFA, Codeserver),
+  {M, Rec} = lists:keyfind(M, 1, ModRecs),
   TmpState1 = state__set_rec_dict(AccState, Rec),
   DummyLetrec = cerl:c_letrec([Def], cerl:c_atom(foo)),
   {NewAccState, _} = traverse(DummyLetrec, DefSet, TmpState1),
-  traverse_scc(Left, DefSet, NewAccState);
-traverse_scc([], _DefSet, AccState) ->
+  traverse_scc(Left, Codeserver, DefSet, ModRecs, NewAccState);
+traverse_scc([], _Codeserver, _DefSet, _ModRecs, AccState) ->
   AccState.
 
 traverse(Tree, DefinedVars, State) ->
@@ -2702,11 +2701,14 @@ pp_map(_S, _Map) ->
 %%
 %% ============================================================================
 
-new_state(SCC0, NextLabel, CallGraph, Plt, PropTypes, Solvers) ->
-  List = [{MFA, Var} || {MFA, {Var, _Fun}, _Rec} <- SCC0],
+new_state(MFAs, NextLabel, CallGraph, CServer, Plt, PropTypes, Solvers) ->
+  List_SCC =
+    [begin
+       {Var, Label} = dialyzer_codeserver:lookup_mfa_var_label(MFA, CServer),
+       {{MFA, Var}, t_var(Label)}
+   end || MFA <- MFAs],
+  {List, SCC} = lists:unzip(List_SCC),
   NameMap = maps:from_list(List),
-  MFAs = [MFA || {MFA, _Var} <- List],
-  SCC = [mk_var(Fun) || {_MFA, {_Var, Fun}, _Rec} <- SCC0],
   SelfRec =
     case SCC of
       [OneF] ->
-- 
cgit v1.2.3


From e083770cdea7b1dfdc9b74a3ae58b1cc8da8f3a9 Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Thu, 22 Dec 2016 09:10:11 +0100
Subject: dialyzer: Compress a few more ETS tables

---
 lib/dialyzer/src/dialyzer_codeserver.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/dialyzer/src')

diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl
index 5028c4f3ba..72180e89e9 100644
--- a/lib/dialyzer/src/dialyzer_codeserver.erl
+++ b/lib/dialyzer/src/dialyzer_codeserver.erl
@@ -130,7 +130,7 @@ ets_set_to_set(Table) ->
   ets:foldl(Fold, sets:new(), Table).
 
 ets_read_concurrent_table(Name) ->
-  ets:new(Name, [{read_concurrency, true}]).
+  ets:new(Name, [compressed, {read_concurrency, true}]).
 
 %%--------------------------------------------------------------------
 
-- 
cgit v1.2.3


From 8786ac9ef56e34c6c2910320d2b291452853b034 Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Thu, 22 Dec 2016 10:26:19 +0100
Subject: dialyzer: Use maps instead of dict for module contracts map

---
 lib/dialyzer/src/dialyzer_codeserver.erl   | 37 +++++++++++---------
 lib/dialyzer/src/dialyzer_contracts.erl    | 18 +++++-----
 lib/dialyzer/src/dialyzer_succ_typings.erl |  2 +-
 lib/dialyzer/src/dialyzer_utils.erl        | 56 +++++++++++++++---------------
 4 files changed, 59 insertions(+), 54 deletions(-)

(limited to 'lib/dialyzer/src')

diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl
index 72180e89e9..28a9966e42 100644
--- a/lib/dialyzer/src/dialyzer_codeserver.erl
+++ b/lib/dialyzer/src/dialyzer_codeserver.erl
@@ -59,19 +59,20 @@
 	 store_temp_records/3,
 	 store_temp_contracts/4]).
 
--export_type([codeserver/0, fun_meta_info/0]).
+-export_type([codeserver/0, fun_meta_info/0, contracts/0]).
 
 -include("dialyzer.hrl").
 
 %%--------------------------------------------------------------------
 
 -type dict_ets() :: ets:tid().
+-type map_ets()  :: ets:tid().
 -type  set_ets() :: ets:tid().
 
 -type types()         :: erl_types:type_table().
 -type mod_records()   :: dict:dict(module(), types()).
 
--type contracts()     :: dict:dict(mfa(),dialyzer_contracts:file_contract()).
+-type contracts()     :: #{mfa() => dialyzer_contracts:file_contract()}.
 -type mod_contracts() :: dict:dict(module(), contracts()).
 
 %% A property-list of data compiled from -compile and -dialyzer attributes.
@@ -84,8 +85,8 @@
 		     code		 :: dict_ets(),
                      exported_types      :: set_ets() | 'undefined', % set(mfa())
 		     records             :: dict_ets() | 'undefined',
-		     contracts           :: dict_ets() | 'undefined',
-		     callbacks           :: dict_ets() | 'undefined',
+		     contracts           :: map_ets() | 'undefined',
+		     callbacks           :: map_ets() | 'undefined',
                      fun_meta_info       :: dict_ets(), % {mfa(), meta_info()}
 		     exports             :: 'clean' | set_ets(), % set(mfa())
                      temp_exported_types :: 'clean' | set_ets(), % set(mfa())
@@ -109,6 +110,10 @@ ets_dict_store(Key, Element, Table) ->
   true = ets:insert(Table, {Key, Element}),
   Table.
 
+ets_map_store(Key, Element, Table) ->
+  true = ets:insert(Table, {Key, Element}),
+  Table.
+
 ets_dict_store_dict(Dict, Table) ->
   true = ets:insert(Table, dict:to_list(Dict)).
 
@@ -301,9 +306,9 @@ finalize_records(Dict, CS) ->
 lookup_mod_contracts(Mod, #codeserver{contracts = ContDict})
   when is_atom(Mod) ->
   case ets_dict_find(Mod, ContDict) of
-    error -> dict:new();
+    error -> maps:new();
     {ok, Keys} ->
-      dict:from_list([get_file_contract(Key, ContDict)|| Key <- Keys])
+      maps:from_list([get_file_contract(Key, ContDict)|| Key <- Keys])
   end.
 
 get_file_contract(Key, ContDict) ->
@@ -336,20 +341,20 @@ get_callbacks(#codeserver{callbacks = CallbDict}) ->
 -spec store_temp_contracts(module(), contracts(), contracts(), codeserver()) ->
 	 codeserver().
 
-store_temp_contracts(Mod, SpecDict, CallbackDict,
+store_temp_contracts(Mod, SpecMap, CallbackMap,
 		     #codeserver{temp_contracts = Cn,
 				 temp_callbacks = Cb} = CS)
   when is_atom(Mod) ->
   CS1 =
-    case dict:size(SpecDict) =:= 0 of
+    case maps:size(SpecMap) =:= 0 of
       true -> CS;
       false ->
-	CS#codeserver{temp_contracts = ets_dict_store(Mod, SpecDict, Cn)}
+	CS#codeserver{temp_contracts = ets_map_store(Mod, SpecMap, Cn)}
     end,
-  case dict:size(CallbackDict) =:= 0 of
+  case maps:size(CallbackMap) =:= 0 of
     true -> CS1;
     false ->
-      CS1#codeserver{temp_callbacks = ets_dict_store(Mod, CallbackDict, Cb)}
+      CS1#codeserver{temp_callbacks = ets_map_store(Mod, CallbackMap, Cb)}
   end.
 
 -spec get_temp_contracts(codeserver()) -> {mod_contracts(), mod_contracts()}.
@@ -369,14 +374,14 @@ finalize_contracts(SpecDict, CallbackDict, CS)  ->
   CS#codeserver{contracts = Contracts, callbacks = Callbacks,
 		temp_contracts = clean, temp_callbacks = clean}.
 
-decompose_spec_dict(Mod, Dict, Table) ->
-  Keys = dict:fetch_keys(Dict),
-  true = ets:insert(Table, dict:to_list(Dict)),
+decompose_spec_dict(Mod, Map, Table) ->
+  Keys = maps:keys(Map),
+  true = ets:insert(Table, maps:to_list(Map)),
   true = ets:insert(Table, {Mod, Keys}),
   Table.
 
-decompose_cb_dict(_Mod, Dict, Table) ->
-  true = ets:insert(Table, dict:to_list(Dict)),
+decompose_cb_dict(_Mod, Map, Table) ->
+  true = ets:insert(Table, maps:to_list(Map)),
   Table.
 
 table__lookup(TablePid, M) when is_atom(M) ->
diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl
index 2cca16d085..d3ed4435ad 100644
--- a/lib/dialyzer/src/dialyzer_contracts.erl
+++ b/lib/dialyzer/src/dialyzer_contracts.erl
@@ -165,11 +165,11 @@ process_contract_remote_types(CodeServer) ->
         {{MFA, {File, Contract, Xtra}}, C2}
     end,
   ModuleFun =
-    fun({ModuleName, ContractDict}) ->
+    fun({ModuleName, ContractMap}) ->
         Cache = erl_types:cache__new(),
         {NewContractList, _NewCache} =
-          lists:mapfoldl(ContractFun, Cache, dict:to_list(ContractDict)),
-        {ModuleName, dict:from_list(NewContractList)}
+          lists:mapfoldl(ContractFun, Cache, maps:to_list(ContractMap)),
+        {ModuleName, maps:from_list(NewContractList)}
     end,
   erlang:garbage_collect(),
   NewContractList = lists:map(ModuleFun, dict:to_list(TmpContractDict)),
@@ -398,7 +398,7 @@ solve_constraints(Contract, Call, Constraints) ->
   %%  ?debug("Inf: ~s\n", [erl_types:t_to_string(Inf)]),
   %%  erl_types:t_assign_variables_to_subtype(Contract, Inf).
 
--type contracts() :: dict:dict(mfa(),dialyzer_contracts:file_contract()).
+-type contracts() :: dialyzer_codeserver:contracts().
 
 %% Checks the contracts for functions that are not implemented
 -spec contracts_without_fun(contracts(), [_], dialyzer_callgraph:callgraph()) ->
@@ -408,12 +408,12 @@ contracts_without_fun(Contracts, AllFuns0, Callgraph) ->
   AllFuns1 = [{dialyzer_callgraph:lookup_name(Label, Callgraph), Arity}
 	      || {Label, Arity} <- AllFuns0],
   AllFuns2 = [{M, F, A} || {{ok, {M, F, _}}, A} <- AllFuns1],
-  AllContractMFAs = dict:fetch_keys(Contracts),
+  AllContractMFAs = maps:keys(Contracts),
   ErrorContractMFAs = AllContractMFAs -- AllFuns2,
   [warn_spec_missing_fun(MFA, Contracts) || MFA <- ErrorContractMFAs].
 
 warn_spec_missing_fun({M, F, A} = MFA, Contracts) ->
-  {{File, Line}, _Contract, _Xtra} = dict:fetch(MFA, Contracts),
+  {{File, Line}, _Contract, _Xtra} = maps:get(MFA, Contracts),
   WarningInfo = {File, Line, MFA},
   {?WARN_CONTRACT_SYNTAX, WarningInfo, {spec_missing_fun, [M, F, A]}}.
 
@@ -446,11 +446,11 @@ insert_constraints([], Map) -> Map.
 -spec store_tmp_contract(mfa(), file_line(), spec_data(), contracts(), types()) ->
         contracts().
 
-store_tmp_contract(MFA, FileLine, {TypeSpec, Xtra}, SpecDict, RecordsDict) ->
+store_tmp_contract(MFA, FileLine, {TypeSpec, Xtra}, SpecMap, RecordsDict) ->
   %% io:format("contract from form: ~p\n", [TypeSpec]),
   TmpContract = contract_from_form(TypeSpec, MFA, RecordsDict, FileLine),
   %% io:format("contract: ~p\n", [TmpContract]),
-  dict:store(MFA, {FileLine, TmpContract, Xtra}, SpecDict).
+  maps:put(MFA, {FileLine, TmpContract, Xtra}, SpecMap).
 
 contract_from_form(Forms, MFA, RecDict, FileLine) ->
   {CFuns, Forms1} = contract_from_form(Forms, MFA, RecDict, FileLine, [], []),
@@ -678,7 +678,7 @@ get_invalid_contract_warnings(Modules, CodeServer, Plt, FindOpaques) ->
 
 get_invalid_contract_warnings_modules([Mod|Mods], CodeServer, Plt, FindOpaques, Acc) ->
   Contracts1 = dialyzer_codeserver:lookup_mod_contracts(Mod, CodeServer),
-  Contracts2 = dict:to_list(Contracts1),
+  Contracts2 = maps:to_list(Contracts1),
   Records = dialyzer_codeserver:lookup_mod_records(Mod, CodeServer),
   NewAcc = get_invalid_contract_warnings_funs(Contracts2, Plt, Records, FindOpaques, Acc),
   get_invalid_contract_warnings_modules(Mods, CodeServer, Plt, FindOpaques, NewAcc);
diff --git a/lib/dialyzer/src/dialyzer_succ_typings.erl b/lib/dialyzer/src/dialyzer_succ_typings.erl
index d92173875c..0e44a5223f 100644
--- a/lib/dialyzer/src/dialyzer_succ_typings.erl
+++ b/lib/dialyzer/src/dialyzer_succ_typings.erl
@@ -263,7 +263,7 @@ refine_one_module(M, {CodeServer, Callgraph, Plt, _Solvers}) ->
   NewFunTypes =
     dialyzer_dataflow:get_fun_types(ModCode, Plt, Callgraph, CodeServer, Records),
   Contracts1 = dialyzer_codeserver:lookup_mod_contracts(M, CodeServer),
-  Contracts = orddict:from_list(dict:to_list(Contracts1)),
+  Contracts = orddict:from_list(maps:to_list(Contracts1)),
   FindOpaques = find_opaques_fun(Records),
   DecoratedFunTypes =
     decorate_succ_typings(Contracts, Callgraph, NewFunTypes, FindOpaques),
diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl
index 5386d5f219..275b56cecd 100644
--- a/lib/dialyzer/src/dialyzer_utils.erl
+++ b/lib/dialyzer/src/dialyzer_utils.erl
@@ -438,17 +438,17 @@ merge_records(NewRecords, OldRecords) ->
 %%
 %% ============================================================================
 
--type spec_dict()     :: dict:dict().
--type callback_dict() :: dict:dict().
+-type spec_map()     :: dialyzer_codeserver:contracts().
+-type callback_map() :: dialyzer_codeserver:contracts().
 
 -spec get_spec_info(module(), abstract_code(), type_table()) ->
-        {'ok', spec_dict(), callback_dict()} | {'error', string()}.
+        {'ok', spec_map(), callback_map()} | {'error', string()}.
 
-get_spec_info(ModName, AbstractCode, RecordsDict) ->
+get_spec_info(ModName, AbstractCode, RecordsMap) ->
   OptionalCallbacks0 = get_optional_callbacks(AbstractCode, ModName),
   OptionalCallbacks = gb_sets:from_list(OptionalCallbacks0),
-  get_spec_info(AbstractCode, dict:new(), dict:new(),
-		RecordsDict, ModName, OptionalCallbacks, "nofile").
+  get_spec_info(AbstractCode, maps:new(), maps:new(),
+		RecordsMap, ModName, OptionalCallbacks, "nofile").
 
 get_optional_callbacks(Abs, ModName) ->
   [{ModName, F, A} || {F, A} <- get_optional_callbacks(Abs)].
@@ -466,7 +466,7 @@ get_optional_callbacks(Abs) ->
 %%    are erl_types:erl_type()
 
 get_spec_info([{attribute, Anno, Contract, {Id, TypeSpec}}|Left],
-	      SpecDict, CallbackDict, RecordsDict, ModName, OptCb, File)
+	      SpecMap, CallbackMap, RecordsMap, ModName, OptCb, File)
   when ((Contract =:= 'spec') or (Contract =:= 'callback')),
        is_list(TypeSpec) ->
   Ln = erl_anno:line(Anno),
@@ -475,24 +475,24 @@ get_spec_info([{attribute, Anno, Contract, {Id, TypeSpec}}|Left],
 	  {F, A} -> {ModName, F, A}
 	end,
   Xtra = [optional_callback || gb_sets:is_member(MFA, OptCb)],
-  ActiveDict =
+  ActiveMap =
     case Contract of
-      spec     -> SpecDict;
-      callback -> CallbackDict
+      spec     -> SpecMap;
+      callback -> CallbackMap
     end,
-  try dict:find(MFA, ActiveDict) of
+  try maps:find(MFA, ActiveMap) of
     error ->
       SpecData = {TypeSpec, Xtra},
-      NewActiveDict =
+      NewActiveMap =
 	dialyzer_contracts:store_tmp_contract(MFA, {File, Ln}, SpecData,
-					      ActiveDict, RecordsDict),
-      {NewSpecDict, NewCallbackDict} =
+					      ActiveMap, RecordsMap),
+      {NewSpecMap, NewCallbackMap} =
 	case Contract of
-	  spec     -> {NewActiveDict, CallbackDict};
-	  callback -> {SpecDict, NewActiveDict}
+	  spec     -> {NewActiveMap, CallbackMap};
+	  callback -> {SpecMap, NewActiveMap}
 	end,
-      get_spec_info(Left, NewSpecDict, NewCallbackDict,
-		    RecordsDict, ModName, OptCb, File);
+      get_spec_info(Left, NewSpecMap, NewCallbackMap,
+		    RecordsMap, ModName, OptCb, File);
     {ok, {{OtherFile, L}, _D}} ->
       {Mod, Fun, Arity} = MFA,
       Msg = flat_format("  Contract/callback for function ~w:~w/~w "
@@ -505,16 +505,16 @@ get_spec_info([{attribute, Anno, Contract, {Id, TypeSpec}}|Left],
 			  [Ln, Error])}
   end;
 get_spec_info([{attribute, _, file, {IncludeFile, _}}|Left],
-	      SpecDict, CallbackDict, RecordsDict, ModName, OptCb, _File) ->
-  get_spec_info(Left, SpecDict, CallbackDict,
-		RecordsDict, ModName, OptCb, IncludeFile);
-get_spec_info([_Other|Left], SpecDict, CallbackDict,
-	      RecordsDict, ModName, OptCb, File) ->
-  get_spec_info(Left, SpecDict, CallbackDict,
-                RecordsDict, ModName, OptCb, File);
-get_spec_info([], SpecDict, CallbackDict,
-              _RecordsDict, _ModName, _OptCb, _File) ->
-  {ok, SpecDict, CallbackDict}.
+	      SpecMap, CallbackMap, RecordsMap, ModName, OptCb, _File) ->
+  get_spec_info(Left, SpecMap, CallbackMap,
+		RecordsMap, ModName, OptCb, IncludeFile);
+get_spec_info([_Other|Left], SpecMap, CallbackMap,
+	      RecordsMap, ModName, OptCb, File) ->
+  get_spec_info(Left, SpecMap, CallbackMap,
+                RecordsMap, ModName, OptCb, File);
+get_spec_info([], SpecMap, CallbackMap,
+              _RecordsMap, _ModName, _OptCb, _File) ->
+  {ok, SpecMap, CallbackMap}.
 
 -spec get_fun_meta_info(module(), abstract_code(), [dial_warn_tag()]) ->
                 dialyzer_codeserver:fun_meta_info() | {'error', string()}.
-- 
cgit v1.2.3


From 66d387c039156989a6ea834033888ac50f1efc1e Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Sat, 24 Dec 2016 17:24:05 +0100
Subject: dialyzer: Use maps instaed of dict

---
 lib/dialyzer/src/dialyzer_codeserver.erl | 24 ++++++++++--------------
 lib/dialyzer/src/dialyzer_plt.erl        | 14 +++++++++-----
 lib/dialyzer/src/dialyzer_typesig.erl    |  4 ++--
 lib/dialyzer/src/dialyzer_utils.erl      | 24 ++++++++++++------------
 4 files changed, 33 insertions(+), 33 deletions(-)

(limited to 'lib/dialyzer/src')

diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl
index 28a9966e42..c05a7a075b 100644
--- a/lib/dialyzer/src/dialyzer_codeserver.erl
+++ b/lib/dialyzer/src/dialyzer_codeserver.erl
@@ -70,7 +70,7 @@
 -type  set_ets() :: ets:tid().
 
 -type types()         :: erl_types:type_table().
--type mod_records()   :: dict:dict(module(), types()).
+-type mod_records()   :: erl_types:mod_records().
 
 -type contracts()     :: #{mfa() => dialyzer_contracts:file_contract()}.
 -type mod_contracts() :: dict:dict(module(), contracts()).
@@ -84,15 +84,15 @@
 -record(codeserver, {next_core_label = 0 :: label(),
 		     code		 :: dict_ets(),
                      exported_types      :: set_ets() | 'undefined', % set(mfa())
-		     records             :: dict_ets() | 'undefined',
+		     records             :: map_ets() | 'undefined',
 		     contracts           :: map_ets() | 'undefined',
 		     callbacks           :: map_ets() | 'undefined',
                      fun_meta_info       :: dict_ets(), % {mfa(), meta_info()}
 		     exports             :: 'clean' | set_ets(), % set(mfa())
                      temp_exported_types :: 'clean' | set_ets(), % set(mfa())
-		     temp_records        :: 'clean' | dict_ets(),
-		     temp_contracts      :: 'clean' | dict_ets(),
-		     temp_callbacks      :: 'clean' | dict_ets()
+		     temp_records        :: 'clean' | map_ets(),
+		     temp_contracts      :: 'clean' | map_ets(),
+		     temp_callbacks      :: 'clean' | map_ets()
 		    }).
 
 -opaque codeserver() :: #codeserver{}.
@@ -106,10 +106,6 @@ ets_dict_find(Key, Table) ->
     _:_ -> error
   end.
 
-ets_dict_store(Key, Element, Table) ->
-  true = ets:insert(Table, {Key, Element}),
-  Table.
-
 ets_map_store(Key, Element, Table) ->
   true = ets:insert(Table, {Key, Element}),
   Table.
@@ -262,8 +258,8 @@ set_next_core_label(NCL, CS) ->
 
 lookup_mod_records(Mod, #codeserver{records = RecDict}) when is_atom(Mod) ->
   case ets_dict_find(Mod, RecDict) of
-    error -> dict:new();
-    {ok, Dict} -> Dict
+    error -> maps:new();
+    {ok, Map} -> Map
   end.
 
 -spec get_records(codeserver()) -> mod_records().
@@ -273,11 +269,11 @@ get_records(#codeserver{records = RecDict}) ->
 
 -spec store_temp_records(module(), types(), codeserver()) -> codeserver().
 
-store_temp_records(Mod, Dict, #codeserver{temp_records = TempRecDict} = CS)
+store_temp_records(Mod, Map, #codeserver{temp_records = TempRecDict} = CS)
   when is_atom(Mod) ->
-  case dict:size(Dict) =:= 0 of
+  case maps:size(Map) =:= 0 of
     true -> CS;
-    false -> CS#codeserver{temp_records = ets_dict_store(Mod, Dict, TempRecDict)}
+    false -> CS#codeserver{temp_records = ets_map_store(Mod, Map, TempRecDict)}
   end.
 
 -spec get_temp_records(codeserver()) -> mod_records().
diff --git a/lib/dialyzer/src/dialyzer_plt.erl b/lib/dialyzer/src/dialyzer_plt.erl
index 93ec2599b4..0eda73a208 100644
--- a/lib/dialyzer/src/dialyzer_plt.erl
+++ b/lib/dialyzer/src/dialyzer_plt.erl
@@ -84,7 +84,7 @@
 %%----------------------------------------------------------------------
 
 -record(plt, {info           = table_new() :: dict:dict(),
-	      types          = table_new() :: dict:dict(),
+	      types          = table_new() :: erl_types:mod_records(),
 	      contracts      = table_new() :: dict:dict(),
 	      callbacks      = table_new() :: dict:dict(),
               exported_types = sets:new()  :: sets:set()}).
@@ -191,7 +191,7 @@ lookup(Plt, Label) when is_integer(Label) ->
 lookup_1(#mini_plt{info = Info}, MFAorLabel) ->
   ets_table_lookup(Info, MFAorLabel).
 
--spec insert_types(plt(), dict:dict()) -> plt().
+-spec insert_types(plt(), erl_types:mod_records()) -> plt().
 
 insert_types(PLT, Rec) ->
   PLT#plt{types = Rec}.
@@ -201,7 +201,7 @@ insert_types(PLT, Rec) ->
 insert_exported_types(PLT, Set) ->
   PLT#plt{exported_types = Set}.
 
--spec get_types(plt()) -> dict:dict().
+-spec get_types(plt()) -> erl_types:mod_records().
 
 get_types(#plt{types = Types}) ->
   Types.
@@ -261,8 +261,10 @@ from_file(FileName, ReturnInfo) ->
 	  Msg = io_lib:format("Old PLT file ~s\n", [FileName]),
 	  plt_error(Msg);
 	ok ->
+          Types = [{Mod, maps:from_list(dict:to_list(Types))} ||
+                    {Mod, Types} <- dict:to_list(Rec#file_plt.types)],
 	  Plt = #plt{info = Rec#file_plt.info,
-		     types = Rec#file_plt.types,
+		     types = dict:from_list(Types),
 		     contracts = Rec#file_plt.contracts,
 		     callbacks = Rec#file_plt.callbacks,
                      exported_types = Rec#file_plt.exported_types},
@@ -379,12 +381,14 @@ to_file(FileName,
 			  end,
 			  OldModDeps, ModDeps),
   ImplMd5 = compute_implementation_md5(),
+  FileTypes = dict:from_list([{Mod, dict:from_list(maps:to_list(MTypes))} ||
+                               {Mod, MTypes} <- dict:to_list(Types)]),
   Record = #file_plt{version = ?VSN,
 		     file_md5_list = MD5,
 		     info = Info,
 		     contracts = Contracts,
 		     callbacks = Callbacks,
-		     types = Types,
+		     types = FileTypes,
                      exported_types = ExpTypes,
 		     mod_deps = NewModDeps,
 		     implementation_md5 = ImplMd5},
diff --git a/lib/dialyzer/src/dialyzer_typesig.erl b/lib/dialyzer/src/dialyzer_typesig.erl
index 075b6ee165..a9ebac6c8b 100644
--- a/lib/dialyzer/src/dialyzer_typesig.erl
+++ b/lib/dialyzer/src/dialyzer_typesig.erl
@@ -103,7 +103,7 @@
 
 -type typesig_funmap() :: #{type_var() => type_var()}.
 
--type prop_types() :: dict:dict(label(), types()).
+-type prop_types() :: dict:dict(label(), erl_types:erl_type()).
 
 -record(state, {callgraph                :: dialyzer_callgraph:callgraph()
                                           | 'undefined',
@@ -120,7 +120,7 @@
 		plt                      :: dialyzer_plt:plt()
                                           | 'undefined',
 		prop_types  = dict:new() :: prop_types(),
-		records     = dict:new() :: types(),
+		records     = maps:new() :: types(),
 		scc         = []         :: ordsets:ordset(type_var()),
 		mfas                     :: [mfa()],
                 solvers     = []         :: [solver()]
diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl
index 275b56cecd..e0a9628fb3 100644
--- a/lib/dialyzer/src/dialyzer_utils.erl
+++ b/lib/dialyzer/src/dialyzer_utils.erl
@@ -202,7 +202,7 @@ get_core_from_abstract_code(AbstrCode, Opts) ->
 
 get_record_and_type_info(AbstractCode) ->
   Module = get_module(AbstractCode),
-  get_record_and_type_info(AbstractCode, Module, dict:new()).
+  get_record_and_type_info(AbstractCode, Module, maps:new()).
 
 -spec get_record_and_type_info(abstract_code(), module(), type_table()) ->
 	{'ok', type_table()} | {'error', string()}.
@@ -215,7 +215,7 @@ get_record_and_type_info([{attribute, A, record, {Name, Fields0}}|Left],
   {ok, Fields} = get_record_fields(Fields0, RecDict),
   Arity = length(Fields),
   FN = {File, erl_anno:line(A)},
-  NewRecDict = dict:store({record, Name}, {FN, [{Arity,Fields}]}, RecDict),
+  NewRecDict = maps:put({record, Name}, {FN, [{Arity,Fields}]}, RecDict),
   get_record_and_type_info(Left, Module, NewRecDict, File);
 get_record_and_type_info([{attribute, A, type, {{record, Name}, Fields0, []}}
 			  |Left], Module, RecDict, File) ->
@@ -223,7 +223,7 @@ get_record_and_type_info([{attribute, A, type, {{record, Name}, Fields0, []}}
   {ok, Fields} = get_record_fields(Fields0, RecDict),
   Arity = length(Fields),
   FN = {File, erl_anno:line(A)},
-  NewRecDict = dict:store({record, Name}, {FN, [{Arity, Fields}]}, RecDict),
+  NewRecDict = maps:put({record, Name}, {FN, [{Arity, Fields}]}, RecDict),
   get_record_and_type_info(Left, Module, NewRecDict, File);
 get_record_and_type_info([{attribute, A, Attr, {Name, TypeForm}}|Left],
 			 Module, RecDict, File)
@@ -263,9 +263,9 @@ add_new_type(TypeOrOpaque, Name, TypeForm, ArgForms, Module, FN,
     false ->
       try erl_types:t_var_names(ArgForms) of
         ArgNames ->
-	  dict:store({TypeOrOpaque, Name, Arity},
-                     {{Module, FN, TypeForm, ArgNames},
-                      erl_types:t_any()}, RecDict)
+	  maps:put({TypeOrOpaque, Name, Arity},
+                   {{Module, FN, TypeForm, ArgNames},
+                    erl_types:t_any()}, RecDict)
       catch
         _:_ ->
 	  throw({error, flat_format("Type declaration for ~w does not "
@@ -334,8 +334,8 @@ process_record_remote_types(CServer) ->
           end,
         Cache = erl_types:cache__new(),
         {RecordList, _NewCache} =
-          lists:mapfoldl(RecordFun, Cache, dict:to_list(Record)),
-        {Module, dict:from_list(RecordList)}
+          lists:mapfoldl(RecordFun, Cache, maps:to_list(Record)),
+        {Module, maps:from_list(RecordList)}
     end,
   NewRecordsList = lists:map(ModuleFun, dict:to_list(TempRecords1)),
   NewRecords = dict:from_list(NewRecordsList),
@@ -372,8 +372,8 @@ process_opaque_types(TempRecords, TempExpTypes, Cache) ->
               end
           end,
         {RecordList, C1} =
-          lists:mapfoldl(RecordFun, C0, dict:to_list(Record)),
-        {{Module, dict:from_list(RecordList)}, C1}
+          lists:mapfoldl(RecordFun, C0, maps:to_list(Record)),
+        {{Module, maps:from_list(RecordList)}, C1}
         %% dict:map(RecordFun, Record)
     end,
   {TempRecordList, NewCache} =
@@ -412,7 +412,7 @@ check_record_fields(Records, TempExpTypes) ->
                   msg_with_position(Fun, FileLine)
               end
           end,
-        lists:foldl(ElemFun, C0, dict:to_list(Element))
+        lists:foldl(ElemFun, C0, maps:to_list(Element))
     end,
   _NewCache = lists:foldl(CheckFun, Cache, dict:to_list(Records)),
   ok.
@@ -710,7 +710,7 @@ format_errors([]) ->
 -spec format_sig(erl_types:erl_type()) -> string().
 
 format_sig(Type) ->
-  format_sig(Type, dict:new()).
+  format_sig(Type, maps:new()).
 
 -spec format_sig(erl_types:erl_type(), type_table()) -> string().
 
-- 
cgit v1.2.3


From 5d9e51a4271833855519df37df8f964216a0e594 Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Wed, 28 Dec 2016 19:22:42 +0100
Subject: dialyzer: Use less memory when translating contracts

---
 lib/dialyzer/src/dialyzer_codeserver.erl | 83 +++++++++++++++++---------------
 lib/dialyzer/src/dialyzer_contracts.erl  | 28 +++++------
 2 files changed, 58 insertions(+), 53 deletions(-)

(limited to 'lib/dialyzer/src')

diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl
index c05a7a075b..ba278b627a 100644
--- a/lib/dialyzer/src/dialyzer_codeserver.erl
+++ b/lib/dialyzer/src/dialyzer_codeserver.erl
@@ -29,7 +29,8 @@
 -module(dialyzer_codeserver).
 
 -export([delete/1,
-	 finalize_contracts/3,
+	 store_temp_contracts/4,
+	 finalize_contracts/1,
          finalize_exported_types/2,
 	 finalize_records/2,
 	 get_contracts/1,
@@ -38,7 +39,9 @@
 	 get_exports/1,
 	 get_records/1,
 	 get_next_core_label/1,
-	 get_temp_contracts/1,
+         get_temp_contracts/2,
+         contracts_modules/1,
+         store_contracts/4,
          get_temp_exported_types/1,
 	 get_temp_records/1,
 	 insert/3,
@@ -56,8 +59,7 @@
 	 new/0,
 	 set_next_core_label/2,
 	 set_temp_records/2,
-	 store_temp_records/3,
-	 store_temp_contracts/4]).
+	 store_temp_records/3]).
 
 -export_type([codeserver/0, fun_meta_info/0, contracts/0]).
 
@@ -85,8 +87,8 @@
 		     code		 :: dict_ets(),
                      exported_types      :: set_ets() | 'undefined', % set(mfa())
 		     records             :: map_ets() | 'undefined',
-		     contracts           :: map_ets() | 'undefined',
-		     callbacks           :: map_ets() | 'undefined',
+		     contracts           :: map_ets(),
+		     callbacks           :: map_ets(),
                      fun_meta_info       :: dict_ets(), % {mfa(), meta_info()}
 		     exports             :: 'clean' | set_ets(), % set(mfa())
                      temp_exported_types :: 'clean' | set_ets(), % set(mfa())
@@ -141,6 +143,8 @@ new() ->
   CodeOptions = [compressed, public, {read_concurrency, true}],
   Code = ets:new(dialyzer_codeserver_code, CodeOptions),
   TempOptions = [public, {write_concurrency, true}],
+  Contracts = ets_read_concurrent_table(dialyzer_codeserver_contracts),
+  Callbacks = ets_read_concurrent_table(dialyzer_codeserver_callbacks),
   [Exports, FunMetaInfo, TempExportedTypes, TempRecords, TempContracts,
    TempCallbacks] =
     [ets:new(Name, TempOptions) ||
@@ -152,6 +156,8 @@ new() ->
   #codeserver{code                = Code,
 	      exports             = Exports,
               fun_meta_info       = FunMetaInfo,
+              contracts           = Contracts,
+              callbacks           = Callbacks,
 	      temp_exported_types = TempExportedTypes,
 	      temp_records        = TempRecords,
 	      temp_contracts      = TempContracts,
@@ -341,44 +347,43 @@ store_temp_contracts(Mod, SpecMap, CallbackMap,
 		     #codeserver{temp_contracts = Cn,
 				 temp_callbacks = Cb} = CS)
   when is_atom(Mod) ->
-  CS1 =
-    case maps:size(SpecMap) =:= 0 of
-      true -> CS;
-      false ->
-	CS#codeserver{temp_contracts = ets_map_store(Mod, SpecMap, Cn)}
-    end,
-  case maps:size(CallbackMap) =:= 0 of
-    true -> CS1;
-    false ->
-      CS1#codeserver{temp_callbacks = ets_map_store(Mod, CallbackMap, Cb)}
-  end.
+  CS1 = CS#codeserver{temp_contracts = ets_map_store(Mod, SpecMap, Cn)},
+  CS1#codeserver{temp_callbacks = ets_map_store(Mod, CallbackMap, Cb)}.
 
--spec get_temp_contracts(codeserver()) -> {mod_contracts(), mod_contracts()}.
+-spec contracts_modules(codeserver()) -> [module()].
 
-get_temp_contracts(#codeserver{temp_contracts = TempContDict,
-			       temp_callbacks = TempCallDict}) ->
-  {ets_dict_to_dict(TempContDict), ets_dict_to_dict(TempCallDict)}.
+contracts_modules(#codeserver{temp_contracts = TempContTable}) ->
+  ets:select(TempContTable, [{{'$1', '$2'}, [], ['$1']}]).
 
--spec finalize_contracts(mod_contracts(), mod_contracts(), codeserver()) ->
-                           codeserver().
+-spec store_contracts(module(), contracts(), contracts(), codeserver()) ->
+                         codeserver().
 
-finalize_contracts(SpecDict, CallbackDict, CS)  ->
-  Contracts = ets_read_concurrent_table(dialyzer_codeserver_contracts),
-  Callbacks = ets_read_concurrent_table(dialyzer_codeserver_callbacks),
-  Contracts = dict:fold(fun decompose_spec_dict/3, Contracts, SpecDict),
-  Callbacks = dict:fold(fun decompose_cb_dict/3, Callbacks, CallbackDict),
-  CS#codeserver{contracts = Contracts, callbacks = Callbacks,
-		temp_contracts = clean, temp_callbacks = clean}.
-
-decompose_spec_dict(Mod, Map, Table) ->
-  Keys = maps:keys(Map),
-  true = ets:insert(Table, maps:to_list(Map)),
-  true = ets:insert(Table, {Mod, Keys}),
-  Table.
+store_contracts(Mod, SpecMap, CallbackMap, CS) ->
+  #codeserver{contracts = SpecDict, callbacks = CallbackDict} = CS,
+  Keys = maps:keys(SpecMap),
+  true = ets:insert(SpecDict, maps:to_list(SpecMap)),
+  true = ets:insert(SpecDict, {Mod, Keys}),
+  true = ets:insert(CallbackDict, maps:to_list(CallbackMap)),
+  CS.
 
-decompose_cb_dict(_Mod, Map, Table) ->
-  true = ets:insert(Table, maps:to_list(Map)),
-  Table.
+-spec get_temp_contracts(module(), codeserver()) ->
+                            {contracts(), contracts()}.
+
+get_temp_contracts(Mod, #codeserver{temp_contracts = TempContDict,
+                                    temp_callbacks = TempCallDict}) ->
+  [{Mod, Contracts}] = ets:lookup(TempContDict, Mod),
+  true = ets:delete(TempContDict, Mod),
+  [{Mod, Callbacks}] = ets:lookup(TempCallDict, Mod),
+  true = ets:delete(TempCallDict, Mod),
+  {Contracts, Callbacks}.
+
+-spec finalize_contracts(codeserver()) -> codeserver().
+
+finalize_contracts(#codeserver{temp_contracts = TempContDict,
+                               temp_callbacks = TempCallDict} = CS)  ->
+  true = ets:delete(TempContDict),
+  true = ets:delete(TempCallDict),
+  CS#codeserver{temp_contracts = clean, temp_callbacks = clean}.
 
 table__lookup(TablePid, M) when is_atom(M) ->
   {Name, Exports, Attrs, Keys, As} = ets:lookup_element(TablePid, M, 2),
diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl
index d3ed4435ad..c0a4c6892c 100644
--- a/lib/dialyzer/src/dialyzer_contracts.erl
+++ b/lib/dialyzer/src/dialyzer_contracts.erl
@@ -150,8 +150,7 @@ sequence([H|T], Delimiter) -> H ++ Delimiter ++ sequence(T, Delimiter).
 	  dialyzer_codeserver:codeserver().
 
 process_contract_remote_types(CodeServer) ->
-  {TmpContractDict, TmpCallbackDict} =
-    dialyzer_codeserver:get_temp_contracts(CodeServer),
+  Mods = dialyzer_codeserver:contracts_modules(CodeServer),
   ExpTypes = dialyzer_codeserver:get_exported_types(CodeServer),
   RecordDict = dialyzer_codeserver:get_records(CodeServer),
   ContractFun =
@@ -165,21 +164,22 @@ process_contract_remote_types(CodeServer) ->
         {{MFA, {File, Contract, Xtra}}, C2}
     end,
   ModuleFun =
-    fun({ModuleName, ContractMap}) ->
+    fun(ModuleName) ->
         Cache = erl_types:cache__new(),
-        {NewContractList, _NewCache} =
+        {ContractMap, CallbackMap} =
+          dialyzer_codeserver:get_temp_contracts(ModuleName, CodeServer),
+        {NewContractList, Cache1} =
           lists:mapfoldl(ContractFun, Cache, maps:to_list(ContractMap)),
-        {ModuleName, maps:from_list(NewContractList)}
+        {NewCallbackList, _NewCache} =
+          lists:mapfoldl(ContractFun, Cache1, maps:to_list(CallbackMap)),
+        dialyzer_codeserver:store_contracts(ModuleName,
+                                            maps:from_list(NewContractList),
+                                            maps:from_list(NewCallbackList),
+                                            CodeServer)
     end,
-  erlang:garbage_collect(),
-  NewContractList = lists:map(ModuleFun, dict:to_list(TmpContractDict)),
-  NewCallbackList = lists:map(ModuleFun, dict:to_list(TmpCallbackDict)),
-  NewContractDict = dict:from_list(NewContractList),
-  NewCallbackDict = dict:from_list(NewCallbackList),
-  %% Make sure temporary data and the (huge) cache are garbage collected:
-  erlang:garbage_collect(),
-  dialyzer_codeserver:finalize_contracts(NewContractDict, NewCallbackDict,
-                                         CodeServer).
+  lists:foreach(ModuleFun, Mods),
+  %% erlang:garbage_collect(),
+  dialyzer_codeserver:finalize_contracts(CodeServer).
 
 -type opaques_fun() :: fun((module()) -> [erl_types:erl_type()]).
 
-- 
cgit v1.2.3


From 12b3790003ca2c060b6ab143dffd0c23580b5476 Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Sun, 1 Jan 2017 19:53:36 +0100
Subject: dialyzer: Try to reduce memory usage

The translation from forms to types is done in a separate process
in an attempt to reduce peak memory usage.

Expect further optimizations as it is probably not feasible
in the long run to keep all type information on the heap.
---
 lib/dialyzer/src/dialyzer_analysis_callgraph.erl | 32 +++++++++++----
 lib/dialyzer/src/dialyzer_codeserver.erl         | 51 ++++++++++++++++--------
 lib/dialyzer/src/dialyzer_contracts.erl          | 11 +++--
 lib/dialyzer/src/dialyzer_utils.erl              |  5 ++-
 4 files changed, 68 insertions(+), 31 deletions(-)

(limited to 'lib/dialyzer/src')

diff --git a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
index c8129cfbaa..fed9a7d1f2 100644
--- a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
+++ b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
@@ -160,12 +160,7 @@ analysis_start(Parent, Analysis, LegalWarnings) ->
         dialyzer_codeserver:finalize_exported_types(MergedExpTypes, TmpCServer1),
       erlang:garbage_collect(),
       ?timing(State#analysis_state.timing_server, "remote",
-              begin
-                TmpCServer3 =
-                  dialyzer_utils:process_record_remote_types(TmpCServer2),
-                erlang:garbage_collect(),
-                dialyzer_contracts:process_contract_remote_types(TmpCServer3)
-              end)
+              contracts_and_records(TmpCServer2))
     catch
       throw:{error, _ErrorMsg} = Error -> exit(Error)
     end,
@@ -187,7 +182,6 @@ analysis_start(Parent, Analysis, LegalWarnings) ->
       true -> dialyzer_callgraph:put_race_detection(true, Callgraph);
       false -> Callgraph
     end,
-  erlang:garbage_collect(),
   State2 = analyze_callgraph(NewCallgraph, State1),
   #analysis_state{plt = MiniPlt2, doc_plt = DocPlt} = State2,
   dialyzer_callgraph:dispose_race_server(NewCallgraph),
@@ -198,6 +192,30 @@ analysis_start(Parent, Analysis, LegalWarnings) ->
   MiniPlt3 = dialyzer_plt:delete_list(MiniPlt2, NonExportsList),
   send_analysis_done(Parent, MiniPlt3, DocPlt).
 
+contracts_and_records(CodeServer) ->
+  Fun = contrs_and_recs(CodeServer),
+  {Pid, Ref} = erlang:spawn_monitor(Fun),
+  dialyzer_codeserver:give_away(CodeServer, Pid),
+  Pid ! {self(), go},
+  receive {'DOWN', Ref, process, Pid, Return} ->
+      Return
+  end.
+
+-spec contrs_and_recs(dialyzer_codeserver:codeserver()) ->
+                         fun(() -> no_return()).
+
+contrs_and_recs(TmpCServer2) ->
+  fun() ->
+      Parent = receive {Pid, go} -> Pid end,
+      {TmpCServer3, RecordDict} =
+        dialyzer_utils:process_record_remote_types(TmpCServer2),
+      TmpServer4 =
+        dialyzer_contracts:process_contract_remote_types(TmpCServer3,
+                                                         RecordDict),
+      dialyzer_codeserver:give_away(TmpServer4, Parent),
+      exit(TmpServer4)
+  end.
+
 analyze_callgraph(Callgraph, #analysis_state{codeserver = Codeserver,
 					     doc_plt = DocPlt,
                                              plt = Plt,
diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl
index ba278b627a..786ed229df 100644
--- a/lib/dialyzer/src/dialyzer_codeserver.erl
+++ b/lib/dialyzer/src/dialyzer_codeserver.erl
@@ -30,6 +30,7 @@
 
 -export([delete/1,
 	 store_temp_contracts/4,
+         give_away/2,
 	 finalize_contracts/1,
          finalize_exported_types/2,
 	 finalize_records/2,
@@ -85,8 +86,8 @@
 
 -record(codeserver, {next_core_label = 0 :: label(),
 		     code		 :: dict_ets(),
-                     exported_types      :: set_ets() | 'undefined', % set(mfa())
-		     records             :: map_ets() | 'undefined',
+                     exported_types      :: set_ets(), % set(mfa())
+		     records             :: map_ets(),
 		     contracts           :: map_ets(),
 		     callbacks           :: map_ets(),
                      fun_meta_info       :: dict_ets(), % {mfa(), meta_info()}
@@ -132,9 +133,6 @@ ets_set_to_set(Table) ->
   Fold = fun({E}, Set) -> sets:add_element(E, Set) end,
   ets:foldl(Fold, sets:new(), Table).
 
-ets_read_concurrent_table(Name) ->
-  ets:new(Name, [compressed, {read_concurrency, true}]).
-
 %%--------------------------------------------------------------------
 
 -spec new() -> codeserver().
@@ -142,9 +140,14 @@ ets_read_concurrent_table(Name) ->
 new() ->
   CodeOptions = [compressed, public, {read_concurrency, true}],
   Code = ets:new(dialyzer_codeserver_code, CodeOptions),
+  ReadOptions = [compressed, {read_concurrency, true}],
+  [Contracts, Callbacks, Records, ExportedTypes] =
+    [ets:new(Name, ReadOptions) ||
+      Name <- [dialyzer_codeserver_contracts,
+               dialyzer_codeserver_callbacks,
+               dialyzer_codeserver_records,
+               dialyzer_codeserver_exported_types]],
   TempOptions = [public, {write_concurrency, true}],
-  Contracts = ets_read_concurrent_table(dialyzer_codeserver_contracts),
-  Callbacks = ets_read_concurrent_table(dialyzer_codeserver_callbacks),
   [Exports, FunMetaInfo, TempExportedTypes, TempRecords, TempContracts,
    TempCallbacks] =
     [ets:new(Name, TempOptions) ||
@@ -156,6 +159,8 @@ new() ->
   #codeserver{code                = Code,
 	      exports             = Exports,
               fun_meta_info       = FunMetaInfo,
+              exported_types      = ExportedTypes,
+              records             = Records,
               contracts           = Contracts,
               callbacks           = Callbacks,
 	      temp_exported_types = TempExportedTypes,
@@ -228,12 +233,12 @@ get_exports(#codeserver{exports = Exports}) ->
 
 -spec finalize_exported_types(sets:set(mfa()), codeserver()) -> codeserver().
 
-finalize_exported_types(Set, CS) ->
-  ExportedTypes = ets_read_concurrent_table(dialyzer_codeserver_exported_types),
+finalize_exported_types(Set,
+                        #codeserver{exported_types = ExportedTypes,
+                                    temp_exported_types = TempETypes} = CS) ->
   true = ets_set_insert_set(Set, ExportedTypes),
-  TempExpTypes = CS#codeserver.temp_exported_types,
-  true = ets:delete(TempExpTypes),
-  CS#codeserver{exported_types = ExportedTypes, temp_exported_types = clean}.
+  true = ets:delete(TempETypes),
+  CS#codeserver{temp_exported_types = clean}.
 
 -spec lookup_mod_code(atom(), codeserver()) -> cerl:c_module().
 
@@ -297,11 +302,11 @@ set_temp_records(Dict, CS) ->
 
 -spec finalize_records(mod_records(), codeserver()) -> codeserver().
 
-finalize_records(Dict, CS) ->
-  true = ets:delete(CS#codeserver.temp_records),
-  Records = ets_read_concurrent_table(dialyzer_codeserver_records),
+finalize_records(Dict, #codeserver{temp_records = TmpRecords,
+                                   records = Records} = CS) ->
+  true = ets:delete(TmpRecords),
   true = ets_dict_store_dict(Dict, Records),
-  CS#codeserver{records = Records, temp_records = clean}.
+  CS#codeserver{temp_records = clean}.
 
 -spec lookup_mod_contracts(atom(), codeserver()) -> contracts().
 
@@ -377,6 +382,20 @@ get_temp_contracts(Mod, #codeserver{temp_contracts = TempContDict,
   true = ets:delete(TempCallDict, Mod),
   {Contracts, Callbacks}.
 
+-spec give_away(codeserver(), pid()) -> 'ok'.
+
+give_away(#codeserver{temp_records = TempRecords,
+                      temp_contracts = TempContracts,
+                      temp_callbacks = TempCallbacks,
+                      records = Records,
+                      contracts = Contracts,
+                      callbacks = Callbacks}, Pid) ->
+  _ = [true = ets:give_away(Table, Pid, any) ||
+        Table <- [TempRecords, TempContracts, TempCallbacks,
+                  Records, Contracts, Callbacks],
+        Table =/= clean],
+  ok.
+
 -spec finalize_contracts(codeserver()) -> codeserver().
 
 finalize_contracts(#codeserver{temp_contracts = TempContDict,
diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl
index c0a4c6892c..f3fba68e84 100644
--- a/lib/dialyzer/src/dialyzer_contracts.erl
+++ b/lib/dialyzer/src/dialyzer_contracts.erl
@@ -31,7 +31,7 @@
 	 get_contract_return/2,
 	 %% get_contract_signature/1,
 	 is_overloaded/1,
-	 process_contract_remote_types/1,
+	 process_contract_remote_types/2,
 	 store_tmp_contract/5]).
 
 -export_type([file_contract/0, plt_contracts/0]).
@@ -146,13 +146,13 @@ sequence([], _Delimiter) -> "";
 sequence([H], _Delimiter) -> H;
 sequence([H|T], Delimiter) -> H ++ Delimiter ++ sequence(T, Delimiter).
 
--spec process_contract_remote_types(dialyzer_codeserver:codeserver()) ->
-	  dialyzer_codeserver:codeserver().
+-spec process_contract_remote_types(dialyzer_codeserver:codeserver(),
+                                    erl_types:mod_records()) ->
+                                       dialyzer_codeserver:codeserver().
 
-process_contract_remote_types(CodeServer) ->
+process_contract_remote_types(CodeServer, RecordDict) ->
   Mods = dialyzer_codeserver:contracts_modules(CodeServer),
   ExpTypes = dialyzer_codeserver:get_exported_types(CodeServer),
-  RecordDict = dialyzer_codeserver:get_records(CodeServer),
   ContractFun =
     fun({{_M, _F, _A}=MFA, {File, TmpContract, Xtra}}, C0) ->
         #tmp_contract{contract_funs = CFuns, forms = Forms} = TmpContract,
@@ -178,7 +178,6 @@ process_contract_remote_types(CodeServer) ->
                                             CodeServer)
     end,
   lists:foreach(ModuleFun, Mods),
-  %% erlang:garbage_collect(),
   dialyzer_codeserver:finalize_contracts(CodeServer).
 
 -type opaques_fun() :: fun((module()) -> [erl_types:erl_type()]).
diff --git a/lib/dialyzer/src/dialyzer_utils.erl b/lib/dialyzer/src/dialyzer_utils.erl
index e0a9628fb3..e71a953279 100644
--- a/lib/dialyzer/src/dialyzer_utils.erl
+++ b/lib/dialyzer/src/dialyzer_utils.erl
@@ -296,7 +296,8 @@ get_record_fields([{record_field, _Line, Name, _Init}|Left], RecDict, Acc) ->
 get_record_fields([], _RecDict, Acc) ->
   lists:reverse(Acc).
 
--spec process_record_remote_types(codeserver()) -> codeserver().
+-spec process_record_remote_types(codeserver()) ->
+                                     {codeserver(), mod_records()}.
 
 %% The field types are cached. Used during analysis when handling records.
 process_record_remote_types(CServer) ->
@@ -340,7 +341,7 @@ process_record_remote_types(CServer) ->
   NewRecordsList = lists:map(ModuleFun, dict:to_list(TempRecords1)),
   NewRecords = dict:from_list(NewRecordsList),
   check_record_fields(NewRecords, ExpTypes),
-  dialyzer_codeserver:finalize_records(NewRecords, CServer).
+  {dialyzer_codeserver:finalize_records(NewRecords, CServer), NewRecords}.
 
 %% erl_types:t_from_form() substitutes the declaration of opaque types
 %% for the expanded type in some cases. To make sure the initial type,
-- 
cgit v1.2.3


From 7cbac17f33c6f3dc2d8a0224eec8d378f00c560f Mon Sep 17 00:00:00 2001
From: Hans Bolinder <hasse@erlang.org>
Date: Thu, 5 Jan 2017 08:09:04 +0100
Subject: dialyzer: Compact 'file' annotations in Core code

File annotations {file, File} in Core Erlang code is replaced by
{file, FakeFile}, where FakeFile is "0", "1", ..., in order to save
memory. When a warning message is created, FakeFile is translated to
the original File (a bit awkward but easy to do).

The Core code stored in ETS tables needs less space, but more
important is that a less heap memory is used in the dataflow,
warnings, and typesig phases, allowing more parallelism.
---
 lib/dialyzer/src/dialyzer_analysis_callgraph.erl | 15 +++++----
 lib/dialyzer/src/dialyzer_behaviours.erl         | 13 +++++---
 lib/dialyzer/src/dialyzer_codeserver.erl         | 39 ++++++++++++++++++++++--
 lib/dialyzer/src/dialyzer_dataflow.erl           | 22 +++++++++----
 4 files changed, 69 insertions(+), 20 deletions(-)

(limited to 'lib/dialyzer/src')

diff --git a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
index fed9a7d1f2..4e18058993 100644
--- a/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
+++ b/lib/dialyzer/src/dialyzer_analysis_callgraph.erl
@@ -628,14 +628,14 @@ format_bad_calls([{{_, _, _}, {_, module_info, A}}|Left], CodeServer, Acc)
 format_bad_calls([{FromMFA, {M, F, A} = To}|Left], CodeServer, Acc) ->
   {_Var, FunCode} = dialyzer_codeserver:lookup_mfa_code(FromMFA, CodeServer),
   Msg = {call_to_missing, [M, F, A]},
-  {File, Line} = find_call_file_and_line(FunCode, To),
+  {File, Line} = find_call_file_and_line(FromMFA, FunCode, To, CodeServer),
   WarningInfo = {File, Line, FromMFA},
   NewAcc = [{?WARN_CALLGRAPH, WarningInfo, Msg}|Acc],
   format_bad_calls(Left, CodeServer, NewAcc);
 format_bad_calls([], _CodeServer, Acc) ->
   Acc.
 
-find_call_file_and_line(Tree, MFA) ->
+find_call_file_and_line({Module, _, _}, Tree, MFA, CodeServer) ->
   Fun =
     fun(SubTree, Acc) ->
 	case cerl:is_c_call(SubTree) of
@@ -648,7 +648,7 @@ find_call_file_and_line(Tree, MFA) ->
 		case {cerl:concrete(M), cerl:concrete(F), A} of
 		  MFA ->
 		    Ann = cerl:get_ann(SubTree),
-		    [{get_file(Ann), get_line(Ann)}|Acc];
+		    [{get_file(CodeServer, Module, Ann), get_line(Ann)}|Acc];
 		  {erlang, make_fun, 3} ->
 		    [CA1, CA2, CA3] = cerl:call_args(SubTree),
 		    case
@@ -664,7 +664,8 @@ find_call_file_and_line(Tree, MFA) ->
 			of
 			  MFA ->
 			    Ann = cerl:get_ann(SubTree),
-			    [{get_file(Ann), get_line(Ann)}|Acc];
+			    [{get_file(CodeServer, Module, Ann),
+                              get_line(Ann)}|Acc];
 			  _ ->
 			    Acc
 			end;
@@ -684,8 +685,10 @@ get_line([Line|_]) when is_integer(Line) -> Line;
 get_line([_|Tail]) -> get_line(Tail);
 get_line([]) -> -1.
 
-get_file([{file, File}|_]) -> File;
-get_file([_|Tail]) -> get_file(Tail).
+get_file(Codeserver, Module, [{file, FakeFile}|_]) ->
+  dialyzer_codeserver:translate_fake_file(Codeserver, Module, FakeFile);
+get_file(Codeserver, Module, [_|Tail]) ->
+  get_file(Codeserver, Module, Tail).
 
 -spec dump_callgraph(dialyzer_callgraph:callgraph(), #analysis_state{}, #analysis{}) ->
   'ok'.
diff --git a/lib/dialyzer/src/dialyzer_behaviours.erl b/lib/dialyzer/src/dialyzer_behaviours.erl
index 5623929a43..524ae047e2 100644
--- a/lib/dialyzer/src/dialyzer_behaviours.erl
+++ b/lib/dialyzer/src/dialyzer_behaviours.erl
@@ -62,9 +62,9 @@ check_callbacks(Module, Attrs, Records, Plt, Codeserver) ->
     _ ->
       MFA = {Module,module_info,0},
       {_Var,Code} = dialyzer_codeserver:lookup_mfa_code(MFA, Codeserver),
-      File = get_file(cerl:get_ann(Code)),
+      File = get_file(Codeserver, Module, cerl:get_ann(Code)),
       State = #state{plt = Plt, filename = File, behlines = BehLines,
-		     codeserver = Codeserver, records = Records},
+                     codeserver = Codeserver, records = Records},
       Warnings = get_warnings(Module, Behaviours, State),
       [add_tag_warning_info(Module, W, State) || W <- Warnings]
   end.
@@ -213,12 +213,15 @@ add_tag_warning_info(Module, {_Tag, [_B, Fun, Arity|_R]} = Warn, State) ->
     dialyzer_codeserver:lookup_mfa_code({Module, Fun, Arity},
 					State#state.codeserver),
   Anns = cerl:get_ann(FunCode),
-  WarningInfo = {get_file(Anns), get_line(Anns), {Module, Fun, Arity}},
+  File = get_file(State#state.codeserver, Module, Anns),
+  WarningInfo = {File, get_line(Anns), {Module, Fun, Arity}},
   {?WARN_BEHAVIOUR, WarningInfo, Warn}.
 
 get_line([Line|_]) when is_integer(Line) -> Line;
 get_line([_|Tail]) -> get_line(Tail);
 get_line([]) -> -1.
 
-get_file([{file, File}|_]) -> File;
-get_file([_|Tail]) -> get_file(Tail).
+get_file(Codeserver, Module, [{file, FakeFile}|_]) ->
+  dialyzer_codeserver:translate_fake_file(Codeserver, Module, FakeFile);
+get_file(Codeserver, Module, [_|Tail]) ->
+  get_file(Codeserver, Module, Tail).
diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl
index 786ed229df..a5bb4e209c 100644
--- a/lib/dialyzer/src/dialyzer_codeserver.erl
+++ b/lib/dialyzer/src/dialyzer_codeserver.erl
@@ -60,7 +60,8 @@
 	 new/0,
 	 set_next_core_label/2,
 	 set_temp_records/2,
-	 store_temp_records/3]).
+	 store_temp_records/3,
+         translate_fake_file/3]).
 
 -export_type([codeserver/0, fun_meta_info/0, contracts/0]).
 
@@ -183,13 +184,15 @@ insert(Mod, ModCode, CS) ->
   Exports = cerl:module_exports(ModCode),
   Attrs = cerl:module_attrs(ModCode),
   Defs = cerl:module_defs(ModCode),
+  {Files, SmallDefs} = compress_file_anno(Defs),
   As = cerl:get_ann(ModCode),
   Funs =
     [{{Mod, cerl:fname_id(Var), cerl:fname_arity(Var)},
-      Val, {Var, cerl_trees:get_label(Fun)}} || Val = {Var, Fun} <- Defs],
+      Val, {Var, cerl_trees:get_label(Fun)}} || Val = {Var, Fun} <- SmallDefs],
   Keys = [Key || {Key, _Value, _Label} <- Funs],
   ModEntry = {Mod, {Name, Exports, Attrs, Keys, As}},
-  true = ets:insert(CS#codeserver.code, [ModEntry|Funs]),
+  ModFileEntry = {{mod, Mod}, Files},
+  true = ets:insert(CS#codeserver.code, [ModEntry, ModFileEntry|Funs]),
   CS.
 
 -spec get_temp_exported_types(codeserver()) -> sets:set(mfa()).
@@ -404,9 +407,39 @@ finalize_contracts(#codeserver{temp_contracts = TempContDict,
   true = ets:delete(TempCallDict),
   CS#codeserver{temp_contracts = clean, temp_callbacks = clean}.
 
+-spec translate_fake_file(codeserver(), module(), file:filename()) ->
+                             file:filename().
+
+translate_fake_file(#codeserver{code = Code}, Module, FakeFile) ->
+  Files = ets:lookup_element(Code, {mod, Module}, 2),
+  {FakeFile, File} = lists:keyfind(FakeFile, 1, Files),
+  File.
+
 table__lookup(TablePid, M) when is_atom(M) ->
   {Name, Exports, Attrs, Keys, As} = ets:lookup_element(TablePid, M, 2),
   Defs = [table__lookup(TablePid, Key) || Key <- Keys],
   cerl:ann_c_module(As, Name, Exports, Attrs, Defs);
 table__lookup(TablePid, MFA) ->
   ets:lookup_element(TablePid, MFA, 2).
+
+compress_file_anno(Term) ->
+  {Files, SmallTerm} = compress_file_anno(Term, []),
+  {[{FakeFile, File} || {File, {file, FakeFile}} <- Files], SmallTerm}.
+
+compress_file_anno({file, F}, Fs) when is_list(F) ->
+  case lists:keyfind(F, 1, Fs) of
+    false ->
+      I = integer_to_list(length(Fs)),
+      FileI = {file, I},
+      NFs = [{F, FileI}|Fs],
+      {NFs, FileI};
+    {F, FileI} -> {Fs, FileI}
+  end;
+compress_file_anno(T, Fs) when is_tuple(T) ->
+  {NFs, NL} = compress_file_anno(tuple_to_list(T), Fs),
+  {NFs, list_to_tuple(NL)};
+compress_file_anno([E|L], Fs) ->
+  {Fs1, NE} = compress_file_anno(E, Fs),
+  {NFs, NL} = compress_file_anno(L, Fs1),
+  {NFs, [NE|NL]};
+compress_file_anno(T, Fs) -> {Fs, T}.
diff --git a/lib/dialyzer/src/dialyzer_dataflow.erl b/lib/dialyzer/src/dialyzer_dataflow.erl
index 639ed426df..ce292e1140 100644
--- a/lib/dialyzer/src/dialyzer_dataflow.erl
+++ b/lib/dialyzer/src/dialyzer_dataflow.erl
@@ -529,7 +529,7 @@ handle_apply_or_call([{TypeOfApply, {Fun, Sig, Contr, LocalRet}}|Left],
     case is_race_analysis_enabled(State) of
       true ->
         Ann = cerl:get_ann(Tree),
-        File = get_file(Ann),
+        File = get_file(Ann, State),
         Line = abs(get_line(Ann)),
         dialyzer_races:store_race_call(Fun, ArgTypes, Args,
                                        {File, Line}, State);
@@ -3090,7 +3090,7 @@ state__add_warning(#state{warnings = Warnings, warning_mode = true} = State,
   Ann = cerl:get_ann(Tree),
   case Force of
     true ->
-      WarningInfo = {get_file(Ann),
+      WarningInfo = {get_file(Ann, State),
                      abs(get_line(Ann)),
                      State#state.curr_fun},
       Warn = {Tag, WarningInfo, Msg},
@@ -3100,7 +3100,9 @@ state__add_warning(#state{warnings = Warnings, warning_mode = true} = State,
       case is_compiler_generated(Ann) of
         true -> State;
         false ->
-          WarningInfo = {get_file(Ann), get_line(Ann), State#state.curr_fun},
+          WarningInfo = {get_file(Ann, State),
+                         get_line(Ann),
+                         State#state.curr_fun},
           Warn = {Tag, WarningInfo, Msg},
           case Tag of
             ?WARN_CONTRACT_RANGE -> ok;
@@ -3499,6 +3501,12 @@ state__put_races(Races, State) ->
 state__records_only(#state{records = Records}) ->
   #state{records = Records}.
 
+-spec state__translate_file(file:filename(), state()) -> file:filename().
+
+state__translate_file(FakeFile, State) ->
+  #state{codeserver = CodeServer, module = Module} = State,
+  dialyzer_codeserver:translate_fake_file(CodeServer, Module, FakeFile).
+
 %%% ===========================================================================
 %%%
 %%%  Races
@@ -3570,9 +3578,11 @@ get_line([Line|_]) when is_integer(Line) -> Line;
 get_line([_|Tail]) -> get_line(Tail);
 get_line([]) -> -1.
 
-get_file([]) -> [];
-get_file([{file, File}|_]) -> File;
-get_file([_|Tail]) -> get_file(Tail).
+get_file([], _State) -> [];
+get_file([{file, FakeFile}|_], State) ->
+  state__translate_file(FakeFile, State);
+get_file([_|Tail], State) ->
+  get_file(Tail, State).
 
 is_compiler_generated(Ann) ->
   lists:member(compiler_generated, Ann) orelse (get_line(Ann) < 1).
-- 
cgit v1.2.3