Merge tag 'OTP-20.0' into sverker/20/binary_to_atom-utf8-crash/ERL-474/OTP-14590

1 files changed, 201 insertions, 107 deletions

diff --git a/lib/dialyzer/src/dialyzer_codeserver.erl b/lib/dialyzer/src/dialyzer_codeserver.erl
index 03cd9671af..5587cf2bdf 100644
--- a/lib/dialyzer/src/dialyzer_codeserver.erl
+++ b/lib/dialyzer/src/dialyzer_codeserver.erl
@@ -1,8 +1,4 @@
 %% -*- erlang-indent-level: 2 -*-
-%%-----------------------------------------------------------------------
-%% %CopyrightBegin%
-%%
-%% Copyright Ericsson AB 2006-2015. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -15,9 +11,6 @@
 %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
-%%
-%% %CopyrightEnd%
-%%
 
 %%%-------------------------------------------------------------------
 %%% File    : dialyzer_codeserver.erl
@@ -29,18 +22,25 @@
 -module(dialyzer_codeserver).
 
 -export([delete/1,
-	 finalize_contracts/3,
+	 store_temp_contracts/4,
+         give_away/2,
+	 finalize_contracts/1,
          finalize_exported_types/2,
-	 finalize_records/2,
+	 finalize_records/1,
 	 get_contracts/1,
 	 get_callbacks/1,
          get_exported_types/1,
+         extract_exported_types/1,
 	 get_exports/1,
-	 get_records/1,
+	 get_records_table/1,
+         extract_records/1,
 	 get_next_core_label/1,
-	 get_temp_contracts/1,
+         get_temp_contracts/2,
+         all_temp_modules/1,
+         store_contracts/4,
          get_temp_exported_types/1,
-	 get_temp_records/1,
+         get_temp_records_table/1,
+	 lookup_temp_mod_records/2,
 	 insert/3,
 	 insert_exports/2,
          insert_temp_exported_types/2,
@@ -48,30 +48,29 @@
 	 is_exported/2,
 	 lookup_mod_code/2,
 	 lookup_mfa_code/2,
+	 lookup_mfa_var_label/2,
 	 lookup_mod_records/2,
 	 lookup_mod_contracts/2,
 	 lookup_mfa_contract/2,
          lookup_meta_info/2,
 	 new/0,
 	 set_next_core_label/2,
-	 set_temp_records/2,
 	 store_temp_records/3,
-	 store_temp_contracts/4]).
+         translate_fake_file/3]).
 
--export_type([codeserver/0, fun_meta_info/0]).
+-export_type([codeserver/0, fun_meta_info/0, contracts/0]).
 
 -include("dialyzer.hrl").
 
 %%--------------------------------------------------------------------
 
 -type dict_ets() :: ets:tid().
+-type map_ets()  :: ets:tid().
 -type  set_ets() :: ets:tid().
 
 -type types()         :: erl_types:type_table().
--type mod_records()   :: dict:dict(module(), types()).
 
--type contracts()     :: dict:dict(mfa(),dialyzer_contracts:file_contract()).
--type mod_contracts() :: dict:dict(module(), contracts()).
+-type contracts()     :: #{mfa() => dialyzer_contracts:file_contract()}.
 
 %% A property-list of data compiled from -compile and -dialyzer attributes.
 -type meta_info()     :: [{{'nowarn_function' | dial_warn_tag()},
@@ -81,16 +80,16 @@
 
 -record(codeserver, {next_core_label = 0 :: label(),
 		     code		 :: dict_ets(),
-                     exported_types      :: set_ets() | 'undefined', % set(mfa())
-		     records             :: dict_ets() | 'undefined',
-		     contracts           :: dict_ets() | 'undefined',
-		     callbacks           :: dict_ets() | 'undefined',
+                     exported_types      :: 'clean' | set_ets(), % set(mfa())
+		     records             :: 'clean' | map_ets(),
+		     contracts           :: map_ets(),
+		     callbacks           :: map_ets(),
                      fun_meta_info       :: dict_ets(), % {mfa(), meta_info()}
 		     exports             :: 'clean' | set_ets(), % set(mfa())
                      temp_exported_types :: 'clean' | set_ets(), % set(mfa())
-		     temp_records        :: 'clean' | dict_ets(),
-		     temp_contracts      :: 'clean' | dict_ets(),
-		     temp_callbacks      :: 'clean' | dict_ets()
+		     temp_records        :: 'clean' | map_ets(),
+		     temp_contracts      :: 'clean' | map_ets(),
+		     temp_callbacks      :: 'clean' | map_ets()
 		    }).
 
 -opaque codeserver() :: #codeserver{}.
@@ -104,13 +103,10 @@ ets_dict_find(Key, Table) ->
     _:_ -> error
   end.
 
-ets_dict_store(Key, Element, Table) ->
+ets_map_store(Key, Element, Table) ->
   true = ets:insert(Table, {Key, Element}),
   Table.
 
-ets_dict_store_dict(Dict, Table) ->
-  true = ets:insert(Table, dict:to_list(Dict)).
-
 ets_dict_to_dict(Table) ->
   Fold = fun({Key,Value}, Dict) -> dict:store(Key, Value, Dict) end,
   ets:foldl(Fold, dict:new(), Table).
@@ -128,9 +124,6 @@ ets_set_to_set(Table) ->
   Fold = fun({E}, Set) -> sets:add_element(E, Set) end,
   ets:foldl(Fold, sets:new(), Table).
 
-ets_read_concurrent_table(Name) ->
-  ets:new(Name, [{read_concurrency, true}]).
-
 %%--------------------------------------------------------------------
 
 -spec new() -> codeserver().
@@ -138,6 +131,13 @@ ets_read_concurrent_table(Name) ->
 new() ->
   CodeOptions = [compressed, public, {read_concurrency, true}],
   Code = ets:new(dialyzer_codeserver_code, CodeOptions),
+  ReadOptions = [compressed, {read_concurrency, true}],
+  [Contracts, Callbacks, Records, ExportedTypes] =
+    [ets:new(Name, ReadOptions) ||
+      Name <- [dialyzer_codeserver_contracts,
+               dialyzer_codeserver_callbacks,
+               dialyzer_codeserver_records,
+               dialyzer_codeserver_exported_types]],
   TempOptions = [public, {write_concurrency, true}],
   [Exports, FunMetaInfo, TempExportedTypes, TempRecords, TempContracts,
    TempCallbacks] =
@@ -150,6 +150,10 @@ new() ->
   #codeserver{code                = Code,
 	      exports             = Exports,
               fun_meta_info       = FunMetaInfo,
+              exported_types      = ExportedTypes,
+              records             = Records,
+              contracts           = Contracts,
+              callbacks           = Callbacks,
 	      temp_exported_types = TempExportedTypes,
 	      temp_records        = TempRecords,
 	      temp_contracts      = TempContracts,
@@ -157,11 +161,8 @@ new() ->
 
 -spec delete(codeserver()) -> 'ok'.
 
-delete(#codeserver{code = Code, exported_types = ExportedTypes,
-		   records = Records, contracts = Contracts,
-		   callbacks = Callbacks}) ->
-  lists:foreach(fun ets:delete/1,
-		[Code, ExportedTypes, Records, Contracts, Callbacks]).
+delete(CServer) ->
+  lists:foreach(fun(Table) -> true = ets:delete(Table) end, tables(CServer)).
 
 -spec insert(atom(), cerl:c_module(), codeserver()) -> codeserver().
 
@@ -170,13 +171,15 @@ insert(Mod, ModCode, CS) ->
   Exports = cerl:module_exports(ModCode),
   Attrs = cerl:module_attrs(ModCode),
   Defs = cerl:module_defs(ModCode),
+  {Files, SmallDefs} = compress_file_anno(Defs),
   As = cerl:get_ann(ModCode),
   Funs =
     [{{Mod, cerl:fname_id(Var), cerl:fname_arity(Var)},
-      Val} || Val = {Var, _Fun} <- Defs],
-  Keys = [Key || {Key, _Value} <- Funs],
+      Val, {Var, cerl_trees:get_label(Fun)}} || Val = {Var, Fun} <- SmallDefs],
+  Keys = [Key || {Key, _Value, _Label} <- Funs],
   ModEntry = {Mod, {Name, Exports, Attrs, Keys, As}},
-  true = ets:insert(CS#codeserver.code, [ModEntry|Funs]),
+  ModFileEntry = {{mod, Mod}, Files},
+  true = ets:insert(CS#codeserver.code, [ModEntry, ModFileEntry|Funs]),
   CS.
 
 -spec get_temp_exported_types(codeserver()) -> sets:set(mfa()).
@@ -213,6 +216,11 @@ is_exported(MFA, #codeserver{exports = Exports}) ->
 get_exported_types(#codeserver{exported_types = ExpTypes}) ->
   ets_set_to_set(ExpTypes).
 
+-spec extract_exported_types(codeserver()) -> {codeserver(), set_ets()}.
+
+extract_exported_types(#codeserver{exported_types = ExpTypes} = CS) ->
+  {CS#codeserver{exported_types = 'clean'}, ExpTypes}.
+
 -spec get_exports(codeserver()) -> sets:set(mfa()).
 
 get_exports(#codeserver{exports = Exports}) ->
@@ -220,12 +228,12 @@ get_exports(#codeserver{exports = Exports}) ->
 
 -spec finalize_exported_types(sets:set(mfa()), codeserver()) -> codeserver().
 
-finalize_exported_types(Set, CS) ->
-  ExportedTypes = ets_read_concurrent_table(dialyzer_codeserver_exported_types),
+finalize_exported_types(Set,
+                        #codeserver{exported_types = ExportedTypes,
+                                    temp_exported_types = TempETypes} = CS) ->
   true = ets_set_insert_set(Set, ExportedTypes),
-  TempExpTypes = CS#codeserver.temp_exported_types,
-  true = ets:delete(TempExpTypes),
-  CS#codeserver{exported_types = ExportedTypes, temp_exported_types = clean}.
+  true = ets:delete(TempETypes),
+  CS#codeserver{temp_exported_types = clean}.
 
 -spec lookup_mod_code(atom(), codeserver()) -> cerl:c_module().
 
@@ -237,6 +245,11 @@ lookup_mod_code(Mod, CS) when is_atom(Mod) ->
 lookup_mfa_code({_M, _F, _A} = MFA, CS) ->
   table__lookup(CS#codeserver.code, MFA).
 
+-spec lookup_mfa_var_label(mfa(), codeserver()) -> {cerl:c_var(), label()}.
+
+lookup_mfa_var_label({_M, _F, _A} = MFA, CS) ->
+  ets:lookup_element(CS#codeserver.code, MFA, 3).
+
 -spec get_next_core_label(codeserver()) -> label().
 
 get_next_core_label(#codeserver{next_core_label = NCL}) ->
@@ -251,53 +264,78 @@ set_next_core_label(NCL, CS) ->
 
 lookup_mod_records(Mod, #codeserver{records = RecDict}) when is_atom(Mod) ->
   case ets_dict_find(Mod, RecDict) of
-    error -> dict:new();
-    {ok, Dict} -> Dict
+    error -> maps:new();
+    {ok, Map} -> Map
   end.
 
--spec get_records(codeserver()) -> mod_records().
+-spec get_records_table(codeserver()) -> map_ets().
+
+get_records_table(#codeserver{records = RecDict}) ->
+  RecDict.
+
+-spec extract_records(codeserver()) -> {codeserver(), map_ets()}.
 
-get_records(#codeserver{records = RecDict}) ->
-  ets_dict_to_dict(RecDict).
+extract_records(#codeserver{records = RecDict} = CS) ->
+  {CS#codeserver{records = clean}, RecDict}.
 
 -spec store_temp_records(module(), types(), codeserver()) -> codeserver().
 
-store_temp_records(Mod, Dict, #codeserver{temp_records = TempRecDict} = CS)
+store_temp_records(Mod, Map, #codeserver{temp_records = TempRecDict} = CS)
   when is_atom(Mod) ->
-  case dict:size(Dict) =:= 0 of
+  case maps:size(Map) =:= 0 of
     true -> CS;
-    false -> CS#codeserver{temp_records = ets_dict_store(Mod, Dict, TempRecDict)}
+    false -> CS#codeserver{temp_records = ets_map_store(Mod, Map, TempRecDict)}
   end.
 
--spec get_temp_records(codeserver()) -> mod_records().
+-spec get_temp_records_table(codeserver()) -> map_ets().
 
-get_temp_records(#codeserver{temp_records = TempRecDict}) ->
-  ets_dict_to_dict(TempRecDict).
+get_temp_records_table(#codeserver{temp_records = TempRecDict}) ->
+  TempRecDict.
 
--spec set_temp_records(mod_records(), codeserver()) -> codeserver().
+-spec lookup_temp_mod_records(module(), codeserver()) -> types().
 
-set_temp_records(Dict, CS) ->
-  true = ets:delete(CS#codeserver.temp_records),
-  TempRecords = ets:new(dialyzer_codeserver_temp_records,[]),
-  true = ets_dict_store_dict(Dict, TempRecords),
-  CS#codeserver{temp_records = TempRecords}.
-
--spec finalize_records(mod_records(), codeserver()) -> codeserver().
+lookup_temp_mod_records(Mod, #codeserver{temp_records = TempRecDict}) ->
+  case ets_dict_find(Mod, TempRecDict) of
+    error -> maps:new();
+    {ok, Map} -> Map
+  end.
 
-finalize_records(Dict, CS) ->
-  true = ets:delete(CS#codeserver.temp_records),
-  Records = ets_read_concurrent_table(dialyzer_codeserver_records),
-  true = ets_dict_store_dict(Dict, Records),
-  CS#codeserver{records = Records, temp_records = clean}.
+-spec finalize_records(codeserver()) -> codeserver().
+
+finalize_records(#codeserver{temp_records = TmpRecords,
+                             records = Records} = CS) ->
+  %% The annotations of the abstract code are reset as they are no
+  %% longer needed, which makes the ETS table compression better.
+  A0 = erl_anno:new(0),
+  AFun = fun(_) -> A0 end,
+  FFun = fun({F, Abs, Type}) ->
+               NewAbs = erl_parse:map_anno(AFun, Abs),
+               {F, NewAbs, Type}
+         end,
+  ArFun = fun({Arity, Fields}) -> {Arity, lists:map(FFun, Fields)} end,
+  List = dialyzer_utils:ets_tab2list(TmpRecords),
+  true = ets:delete(TmpRecords),
+  Fun = fun({Mod, Map}) ->
+            MFun =
+              fun({record, _}, {FileLine, ArityFields}) ->
+                    {FileLine, lists:map(ArFun, ArityFields)};
+                 (_, {{M, FileLine, Abs, Args}, Type}) ->
+                    {{M, FileLine, erl_parse:map_anno(AFun, Abs), Args}, Type}
+              end,
+            {Mod, maps:map(MFun, Map)}
+        end,
+  NewList = lists:map(Fun, List),
+  true = ets:insert(Records, NewList),
+  CS#codeserver{temp_records = clean}.
 
 -spec lookup_mod_contracts(atom(), codeserver()) -> contracts().
 
 lookup_mod_contracts(Mod, #codeserver{contracts = ContDict})
   when is_atom(Mod) ->
   case ets_dict_find(Mod, ContDict) of
-    error -> dict:new();
+    error -> maps:new();
     {ok, Keys} ->
-      dict:from_list([get_file_contract(Key, ContDict)|| Key <- Keys])
+      maps:from_list([get_file_contract(Key, ContDict)|| Key <- Keys])
   end.
 
 get_file_contract(Key, ContDict) ->
@@ -317,10 +355,13 @@ lookup_meta_info(MorMFA, #codeserver{fun_meta_info = FunMetaInfo}) ->
     {ok, PropList} -> PropList
   end.
 
--spec get_contracts(codeserver()) -> mod_contracts().
+-spec get_contracts(codeserver()) ->
+                       dict:dict(mfa(), dialyzer_contracts:file_contract()).
 
 get_contracts(#codeserver{contracts = ContDict}) ->
-  ets_dict_to_dict(ContDict).
+  dict:filter(fun({_M, _F, _A}, _) -> true;
+                 (_, _) -> false
+              end, ets_dict_to_dict(ContDict)).
 
 -spec get_callbacks(codeserver()) -> list().
 
@@ -330,48 +371,79 @@ get_callbacks(#codeserver{callbacks = CallbDict}) ->
 -spec store_temp_contracts(module(), contracts(), contracts(), codeserver()) ->
 	 codeserver().
 
-store_temp_contracts(Mod, SpecDict, CallbackDict,
+store_temp_contracts(Mod, SpecMap, CallbackMap,
 		     #codeserver{temp_contracts = Cn,
 				 temp_callbacks = Cb} = CS)
   when is_atom(Mod) ->
-  CS1 =
-    case dict:size(SpecDict) =:= 0 of
-      true -> CS;
-      false ->
-	CS#codeserver{temp_contracts = ets_dict_store(Mod, SpecDict, Cn)}
-    end,
-  case dict:size(CallbackDict) =:= 0 of
-    true -> CS1;
-    false ->
-      CS1#codeserver{temp_callbacks = ets_dict_store(Mod, CallbackDict, Cb)}
-  end.
-
--spec get_temp_contracts(codeserver()) -> {mod_contracts(), mod_contracts()}.
+  %% Make sure Mod is stored even if there are no callbacks or
+  %% contracts.
+  CS1 = CS#codeserver{temp_contracts = ets_map_store(Mod, SpecMap, Cn)},
+  CS1#codeserver{temp_callbacks = ets_map_store(Mod, CallbackMap, Cb)}.
 
-get_temp_contracts(#codeserver{temp_contracts = TempContDict,
-			       temp_callbacks = TempCallDict}) ->
-  {ets_dict_to_dict(TempContDict), ets_dict_to_dict(TempCallDict)}.
+-spec all_temp_modules(codeserver()) -> [module()].
 
--spec finalize_contracts(mod_contracts(), mod_contracts(), codeserver()) ->
-                           codeserver().
+all_temp_modules(#codeserver{temp_contracts = TempContTable}) ->
+  ets:select(TempContTable, [{{'$1', '$2'}, [], ['$1']}]).
 
-finalize_contracts(SpecDict, CallbackDict, CS)  ->
-  Contracts = ets_read_concurrent_table(dialyzer_codeserver_contracts),
-  Callbacks = ets_read_concurrent_table(dialyzer_codeserver_callbacks),
-  Contracts = dict:fold(fun decompose_spec_dict/3, Contracts, SpecDict),
-  Callbacks = dict:fold(fun decompose_cb_dict/3, Callbacks, CallbackDict),
-  CS#codeserver{contracts = Contracts, callbacks = Callbacks,
-		temp_contracts = clean, temp_callbacks = clean}.
+-spec store_contracts(module(), contracts(), contracts(), codeserver()) ->
+                         codeserver().
 
-decompose_spec_dict(Mod, Dict, Table) ->
-  Keys = dict:fetch_keys(Dict),
-  true = ets:insert(Table, dict:to_list(Dict)),
-  true = ets:insert(Table, {Mod, Keys}),
-  Table.
+store_contracts(Mod, SpecMap, CallbackMap, CS) ->
+  #codeserver{contracts = SpecDict, callbacks = CallbackDict} = CS,
+  Keys = maps:keys(SpecMap),
+  true = ets:insert(SpecDict, maps:to_list(SpecMap)),
+  true = ets:insert(SpecDict, {Mod, Keys}),
+  true = ets:insert(CallbackDict, maps:to_list(CallbackMap)),
+  CS.
 
-decompose_cb_dict(_Mod, Dict, Table) ->
-  true = ets:insert(Table, dict:to_list(Dict)),
-  Table.
+-spec get_temp_contracts(module(), codeserver()) ->
+                            {contracts(), contracts()}.
+
+get_temp_contracts(Mod, #codeserver{temp_contracts = TempContDict,
+                                    temp_callbacks = TempCallDict}) ->
+  [{Mod, Contracts}] = ets:lookup(TempContDict, Mod),
+  true = ets:delete(TempContDict, Mod),
+  [{Mod, Callbacks}] = ets:lookup(TempCallDict, Mod),
+  true = ets:delete(TempCallDict, Mod),
+  {Contracts, Callbacks}.
+
+-spec give_away(codeserver(), pid()) -> 'ok'.
+
+give_away(CServer, Pid) ->
+  lists:foreach(fun(Table) -> true = ets:give_away(Table, Pid, any)
+                end, tables(CServer)).
+
+tables(#codeserver{code = Code,
+                   fun_meta_info = FunMetaInfo,
+                   exports = Exports,
+                   temp_exported_types = TempExpTypes,
+                   temp_records = TempRecords,
+                   temp_contracts = TempContracts,
+                   temp_callbacks = TempCallbacks,
+                   exported_types = ExportedTypes,
+                   records = Records,
+                   contracts = Contracts,
+                   callbacks = Callbacks}) ->
+  [Table ||  Table <- [Code, FunMetaInfo, Exports, TempExpTypes,
+                       TempRecords, TempContracts, TempCallbacks,
+                       ExportedTypes, Records, Contracts, Callbacks],
+             Table =/= clean].
+
+-spec finalize_contracts(codeserver()) -> codeserver().
+
+finalize_contracts(#codeserver{temp_contracts = TempContDict,
+                               temp_callbacks = TempCallDict} = CS)  ->
+  true = ets:delete(TempContDict),
+  true = ets:delete(TempCallDict),
+  CS#codeserver{temp_contracts = clean, temp_callbacks = clean}.
+
+-spec translate_fake_file(codeserver(), module(), file:filename()) ->
+                             file:filename().
+
+translate_fake_file(#codeserver{code = Code}, Module, FakeFile) ->
+  Files = ets:lookup_element(Code, {mod, Module}, 2),
+  {FakeFile, File} = lists:keyfind(FakeFile, 1, Files),
+  File.
 
 table__lookup(TablePid, M) when is_atom(M) ->
   {Name, Exports, Attrs, Keys, As} = ets:lookup_element(TablePid, M, 2),
@@ -379,3 +451,25 @@ table__lookup(TablePid, M) when is_atom(M) ->
   cerl:ann_c_module(As, Name, Exports, Attrs, Defs);
 table__lookup(TablePid, MFA) ->
   ets:lookup_element(TablePid, MFA, 2).
+
+compress_file_anno(Term) ->
+  {Files, SmallTerm} = compress_file_anno(Term, []),
+  {[{FakeFile, File} || {File, {file, FakeFile}} <- Files], SmallTerm}.
+
+compress_file_anno({file, F}, Fs) when is_list(F) ->
+  case lists:keyfind(F, 1, Fs) of
+    false ->
+      I = integer_to_list(length(Fs)),
+      FileI = {file, I},
+      NFs = [{F, FileI}|Fs],
+      {NFs, FileI};
+    {F, FileI} -> {Fs, FileI}
+  end;
+compress_file_anno(T, Fs) when is_tuple(T) ->
+  {NFs, NL} = compress_file_anno(tuple_to_list(T), Fs),
+  {NFs, list_to_tuple(NL)};
+compress_file_anno([E|L], Fs) ->
+  {Fs1, NE} = compress_file_anno(E, Fs),
+  {NFs, NL} = compress_file_anno(L, Fs1),
+  {NFs, [NE|NL]};
+compress_file_anno(T, Fs) -> {Fs, T}.