Merge branch 'hasse/dialyzer/improve_from_form/OTP-13547'

* hasse/dialyzer/improve_from_form/OTP-13547:
  Update primary bootstrap
  stdlib: Correct types and specs
  dialyzer: Minor adjustments
  dialyzer: Suppress unmatched_return for send/2
  dialyzer: Improve the translation of forms to types
  dialyzer: Use a cache when translating forms to types
  dialyzer: Prepare erl_types:t_from_form() for a cache
  dialyzer: Optimize erl_types:t_form_form()
  dialyzer: Correct types
  syntax_tools: Correct types
  erts: Correct character repr in doc of the abstract format
  stdlib: Correct types and specs

1 files changed, 97 insertions, 64 deletions

diff --git a/lib/dialyzer/src/dialyzer_contracts.erl b/lib/dialyzer/src/dialyzer_contracts.erl
index 272ad10e90..976a2b8955 100644
--- a/lib/dialyzer/src/dialyzer_contracts.erl
+++ b/lib/dialyzer/src/dialyzer_contracts.erl
@@ -53,7 +53,9 @@
 %% to expand records and/or remote types that they might contain.
 %%-----------------------------------------------------------------------
 
--type tmp_contract_fun() :: fun((sets:set(mfa()), types()) -> contract_pair()).
+-type cache() :: ets:tid().
+-type tmp_contract_fun() ::
+        fun((sets:set(mfa()), types(), cache()) -> contract_pair()).
 
 -record(tmp_contract, {contract_funs = [] :: [tmp_contract_fun()],
 		       forms	     = [] :: [{_, _}]}).
@@ -153,19 +155,30 @@ process_contract_remote_types(CodeServer) ->
   ExpTypes = dialyzer_codeserver:get_exported_types(CodeServer),
   RecordDict = dialyzer_codeserver:get_records(CodeServer),
   ContractFun =
-    fun({_M, _F, _A}, {File, #tmp_contract{contract_funs = CFuns, forms = Forms}, Xtra}) ->
-	NewCs = [CFun(ExpTypes, RecordDict) || CFun <- CFuns],
-	Args = general_domain(NewCs),
-	{File, #contract{contracts = NewCs, args = Args, forms = Forms}, Xtra}
+    fun({{_M, _F, _A}=MFA, {File, TmpContract, Xtra}}, C0) ->
+        #tmp_contract{contract_funs = CFuns, forms = Forms} = TmpContract,
+        {NewCs, C2} = lists:mapfoldl(fun(CFun, C1) ->
+                                         CFun(ExpTypes, RecordDict, C1)
+                                     end, C0, CFuns),
+        Args = general_domain(NewCs),
+        Contract = #contract{contracts = NewCs, args = Args, forms = Forms},
+        {{MFA, {File, Contract, Xtra}}, C2}
     end,
   ModuleFun =
-    fun(_ModuleName, ContractDict) ->
-	dict:map(ContractFun, ContractDict)
+    fun({ModuleName, ContractDict}, C3) ->
+        {NewContractList, C4} =
+          lists:mapfoldl(ContractFun, C3, dict:to_list(ContractDict)),
+        {{ModuleName, dict:from_list(NewContractList)}, C4}
     end,
-  NewContractDict = dict:map(ModuleFun, TmpContractDict),
-  NewCallbackDict = dict:map(ModuleFun, TmpCallbackDict),
+  Cache = erl_types:cache__new(),
+  {NewContractList, C5} =
+    lists:mapfoldl(ModuleFun, Cache, dict:to_list(TmpContractDict)),
+  {NewCallbackList, _C6} =
+    lists:mapfoldl(ModuleFun, C5, dict:to_list(TmpCallbackDict)),
+  NewContractDict = dict:from_list(NewContractList),
+  NewCallbackDict = dict:from_list(NewCallbackList),
   dialyzer_codeserver:finalize_contracts(NewContractDict, NewCallbackDict,
-					 CodeServer).
+                                         CodeServer).
 
 -type opaques_fun() :: fun((module()) -> [erl_types:erl_type()]).
 
@@ -431,19 +444,19 @@ contract_from_form(Forms, MFA, RecDict, FileLine) ->
 contract_from_form([{type, _, 'fun', [_, _]} = Form | Left], MFA, RecDict,
 		   FileLine, TypeAcc, FormAcc) ->
   TypeFun =
-    fun(ExpTypes, AllRecords) ->
-	NewType =
+    fun(ExpTypes, AllRecords, Cache) ->
+	{NewType, NewCache} =
 	  try
-            from_form_with_check(Form, ExpTypes, MFA, AllRecords)
+            from_form_with_check(Form, ExpTypes, MFA, AllRecords, Cache)
 	  catch
 	    throw:{error, Msg} ->
 	      {File, Line} = FileLine,
 	      NewMsg = io_lib:format("~s:~p: ~s", [filename:basename(File),
-						     Line, Msg]),
+                                                   Line, Msg]),
 	      throw({error, NewMsg})
 	  end,
         NewTypeNoVars = erl_types:subst_all_vars_to_any(NewType),
-	{NewTypeNoVars, []}
+	{{NewTypeNoVars, []}, NewCache}
     end,
   NewTypeAcc = [TypeFun | TypeAcc],
   NewFormAcc = [{Form, []} | FormAcc],
@@ -452,13 +465,15 @@ contract_from_form([{type, _L1, bounded_fun,
 		     [{type, _L2, 'fun', [_, _]} = Form, Constr]}| Left],
 		   MFA, RecDict, FileLine, TypeAcc, FormAcc) ->
   TypeFun =
-    fun(ExpTypes, AllRecords) ->
-	{Constr1, VarDict} =
-	  process_constraints(Constr, MFA, RecDict, ExpTypes, AllRecords),
-        NewType = from_form_with_check(Form, ExpTypes, MFA, AllRecords,
-                                       VarDict),
+    fun(ExpTypes, AllRecords, Cache) ->
+	{Constr1, VarTable, Cache1} =
+	  process_constraints(Constr, MFA, RecDict, ExpTypes, AllRecords,
+                              Cache),
+        {NewType, NewCache} =
+          from_form_with_check(Form, ExpTypes, MFA, AllRecords,
+                               VarTable, Cache1),
         NewTypeNoVars = erl_types:subst_all_vars_to_any(NewType),
-	{NewTypeNoVars, Constr1}
+	{{NewTypeNoVars, Constr1}, NewCache}
     end,
   NewTypeAcc = [TypeFun | TypeAcc],
   NewFormAcc = [{Form, Constr} | FormAcc],
@@ -466,74 +481,91 @@ contract_from_form([{type, _L1, bounded_fun,
 contract_from_form([], _MFA, _RecDict, _FileLine, TypeAcc, FormAcc) ->
   {lists:reverse(TypeAcc), lists:reverse(FormAcc)}.
 
-process_constraints(Constrs, MFA, RecDict, ExpTypes, AllRecords) ->
-  Init0 = initialize_constraints(Constrs, MFA, RecDict, ExpTypes, AllRecords),
+process_constraints(Constrs, MFA, RecDict, ExpTypes, AllRecords, Cache) ->
+  {Init0, NewCache} = initialize_constraints(Constrs, MFA, RecDict, ExpTypes,
+                                             AllRecords, Cache),
   Init = remove_cycles(Init0),
-  constraints_fixpoint(Init, MFA, RecDict, ExpTypes, AllRecords).
+  constraints_fixpoint(Init, MFA, RecDict, ExpTypes, AllRecords, NewCache).
 
-initialize_constraints(Constrs, MFA, RecDict, ExpTypes, AllRecords) ->
-  initialize_constraints(Constrs, MFA, RecDict, ExpTypes, AllRecords, []).
+initialize_constraints(Constrs, MFA, RecDict, ExpTypes, AllRecords, Cache) ->
+  initialize_constraints(Constrs, MFA, RecDict, ExpTypes, AllRecords,
+                         Cache, []).
 
-initialize_constraints([], _MFA, _RecDict, _ExpTypes, _AllRecords, Acc) ->
-  Acc;
-initialize_constraints([Constr|Rest], MFA, RecDict, ExpTypes, AllRecords, Acc) ->
+initialize_constraints([], _MFA, _RecDict, _ExpTypes, _AllRecords,
+                       Cache, Acc) ->
+  {Acc, Cache};
+initialize_constraints([Constr|Rest], MFA, RecDict, ExpTypes, AllRecords,
+                       Cache, Acc) ->
   case Constr of
     {type, _, constraint, [{atom, _, is_subtype}, [Type1, Type2]]} ->
       VarTable = erl_types:var_table__new(),
-      T1 = final_form(Type1, ExpTypes, MFA, AllRecords, VarTable),
+      {T1, NewCache} =
+        final_form(Type1, ExpTypes, MFA, AllRecords, VarTable, Cache),
       Entry = {T1, Type2},
-      initialize_constraints(Rest, MFA, RecDict, ExpTypes, AllRecords, [Entry|Acc]);
+      initialize_constraints(Rest, MFA, RecDict, ExpTypes, AllRecords,
+                             NewCache, [Entry|Acc]);
     {type, _, constraint, [{atom,_,Name}, List]} ->
       N = length(List),
       throw({error,
 	     io_lib:format("Unsupported type guard ~w/~w\n", [Name, N])})
   end.
 
-constraints_fixpoint(Constrs, MFA, RecDict, ExpTypes, AllRecords) ->
+constraints_fixpoint(Constrs, MFA, RecDict, ExpTypes, AllRecords, Cache) ->
   VarTable = erl_types:var_table__new(),
-  VarDict =
-    constraints_to_dict(Constrs, MFA, RecDict, ExpTypes, AllRecords, VarTable),
-  constraints_fixpoint(VarDict, MFA, Constrs, RecDict, ExpTypes, AllRecords).
-
-constraints_fixpoint(OldVarDict, MFA, Constrs, RecDict, ExpTypes, AllRecords) ->
-  NewVarDict =
-    constraints_to_dict(Constrs, MFA, RecDict, ExpTypes, AllRecords, OldVarDict),
-  case NewVarDict of
-    OldVarDict ->
+  {VarTab, NewCache} =
+    constraints_to_dict(Constrs, MFA, RecDict, ExpTypes, AllRecords,
+                        VarTable, Cache),
+  constraints_fixpoint(VarTab, MFA, Constrs, RecDict, ExpTypes,
+                       AllRecords, NewCache).
+
+constraints_fixpoint(OldVarTab, MFA, Constrs, RecDict, ExpTypes,
+                     AllRecords, Cache) ->
+  {NewVarTab, NewCache} =
+    constraints_to_dict(Constrs, MFA, RecDict, ExpTypes, AllRecords,
+                        OldVarTab, Cache),
+  case NewVarTab of
+    OldVarTab ->
       Fun =
 	fun(Key, Value, Acc) ->
 	    [{subtype, erl_types:t_var(Key), Value}|Acc]
 	end,
-      FinalConstrs = maps:fold(Fun, [], NewVarDict),
-      {FinalConstrs, NewVarDict};
+      FinalConstrs = maps:fold(Fun, [], NewVarTab),
+      {FinalConstrs, NewVarTab, NewCache};
     _Other ->
-      constraints_fixpoint(NewVarDict, MFA, Constrs, RecDict, ExpTypes, AllRecords)
+      constraints_fixpoint(NewVarTab, MFA, Constrs, RecDict, ExpTypes,
+                           AllRecords, NewCache)
   end.
 
-final_form(Form, ExpTypes, MFA, AllRecords, VarDict) ->
-  from_form_with_check(Form, ExpTypes, MFA, AllRecords, VarDict).
+final_form(Form, ExpTypes, MFA, AllRecords, VarTable, Cache) ->
+  from_form_with_check(Form, ExpTypes, MFA, AllRecords, VarTable, Cache).
 
-from_form_with_check(Form, ExpTypes, MFA, AllRecords) ->
+from_form_with_check(Form, ExpTypes, MFA, AllRecords, Cache) ->
   VarTable = erl_types:var_table__new(),
-  from_form_with_check(Form, ExpTypes, MFA, AllRecords, VarTable).
+  from_form_with_check(Form, ExpTypes, MFA, AllRecords, VarTable, Cache).
 
-from_form_with_check(Form, ExpTypes, MFA, AllRecords, VarDict) ->
+from_form_with_check(Form, ExpTypes, MFA, AllRecords, VarTable, Cache) ->
   Site = {spec, MFA},
-  erl_types:t_check_record_fields(Form, ExpTypes, Site, AllRecords, VarDict),
-  erl_types:t_from_form(Form, ExpTypes, Site, AllRecords, VarDict).
-
-constraints_to_dict(Constrs, MFA, RecDict, ExpTypes, AllRecords, VarDict) ->
-  Subtypes =
-    constraints_to_subs(Constrs, MFA, RecDict, ExpTypes, AllRecords, VarDict, []),
-  insert_constraints(Subtypes).
-
-constraints_to_subs([], _MFA, _RecDict, _ExpTypes, _AllRecords, _VarDict, Acc) ->
-  Acc;
-constraints_to_subs([C|Rest], MFA, RecDict, ExpTypes, AllRecords, VarDict, Acc) ->
-  {T1, Form2} = C,
-  T2 = final_form(Form2, ExpTypes, MFA, AllRecords, VarDict),
+  C1 = erl_types:t_check_record_fields(Form, ExpTypes, Site, AllRecords,
+                                       VarTable, Cache),
+  erl_types:t_from_form(Form, ExpTypes, Site, AllRecords, VarTable, C1).
+
+constraints_to_dict(Constrs, MFA, RecDict, ExpTypes, AllRecords,
+                    VarTab, Cache) ->
+  {Subtypes, NewCache} =
+    constraints_to_subs(Constrs, MFA, RecDict, ExpTypes, AllRecords,
+                        VarTab, Cache, []),
+  {insert_constraints(Subtypes), NewCache}.
+
+constraints_to_subs([], _MFA, _RecDict, _ExpTypes, _AllRecords,
+                    _VarTab, Cache, Acc) ->
+  {Acc, Cache};
+constraints_to_subs([{T1, Form2}|Rest], MFA, RecDict, ExpTypes, AllRecords,
+                    VarTab, Cache, Acc) ->
+  {T2, NewCache} =
+    final_form(Form2, ExpTypes, MFA, AllRecords, VarTab, Cache),
   NewAcc = [{subtype, T1, T2}|Acc],
-  constraints_to_subs(Rest, MFA, RecDict, ExpTypes, AllRecords, VarDict, NewAcc).
+  constraints_to_subs(Rest, MFA, RecDict, ExpTypes, AllRecords,
+                      VarTab, NewCache, NewAcc).
 
 %% Replaces variables with '_' when necessary to break up cycles among
 %% the constraints.
@@ -647,6 +679,7 @@ get_invalid_contract_warnings_funs([{MFA, {FileLine, Contract, _Xtra}}|Left],
     {value, {Ret, Args}} ->
       Sig = erl_types:t_fun(Args, Ret),
       {M, _F, _A} = MFA,
+      %% io:format("MFA ~p~n", [MFA]),
       Opaques = FindOpaques(M),
       {File, Line} = FileLine,
       WarningInfo = {File, Line, MFA},
@@ -795,7 +828,7 @@ is_remote_types_related(Contract, CSig, Sig, MFA, RecDict) ->
 
 t_from_forms_without_remote([{FType, []}], MFA, RecDict) ->
   Site = {spec, MFA},
-  Type1 = erl_types:t_from_form_without_remote(FType, Site, RecDict),
+  {Type1, _} = erl_types:t_from_form_without_remote(FType, Site, RecDict),
   {ok, erl_types:subst_all_vars_to_any(Type1)};
 t_from_forms_without_remote([{_FType, _Constrs}], _MFA, _RecDict) ->
   %% 'When' constraints