erts: Improve hipe load/upgrade/purge machinery

A step toward better integration of hipe load and purge

Highlights:

* code_server no longer needs to call hipe_unified_loader:post_beam_load/1
  Instead new internal function hipe_redirect_to_module()
  is called by loading BIFs to patch native call sites if needed.

* hipe_purge_module() is called by erts_internal:purge_module/2
  to purge any native code.

* struct hipe_mfa_info redesigned and only used for exported
  functions that are called from or implemented by native code.
  A list of native call sites (struct hipe_ref) are kept for each hipe_mfa_info.

* struct hipe_sdesc used by hipe_find_mfa_from_ra()
  to build native stack traces.

2 files changed, 73 insertions, 184 deletions

diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl
index 835b7eb588..90ecb22074 100644
--- a/lib/kernel/src/code_server.erl
+++ b/lib/kernel/src/code_server.erl
@@ -1130,7 +1130,7 @@ try_load_module_2(File, Mod, Bin, From, Architecture,
                   #state{moddb=Db}=St) ->
     case catch hipe_unified_loader:load_native_code(Mod, Bin, Architecture) of
         {module,Mod} = Module ->
-	    ets:insert(Db, [{{native,Mod},true},{Mod,File}]),
+	    ets:insert(Db, {Mod,File}),
             {reply,Module,St};
         no_native ->
             try_load_module_3(File, Mod, Bin, From, Architecture, St);
@@ -1139,10 +1139,9 @@ try_load_module_2(File, Mod, Bin, From, Architecture,
             {reply,ok,St}
     end.
 
-try_load_module_3(File, Mod, Bin, From, Architecture, St0) ->
+try_load_module_3(File, Mod, Bin, From, _Architecture, St0) ->
     Action = fun({module,_}=Module, #state{moddb=Db}=S) ->
 		     ets:insert(Db, {Mod,File}),
-		     post_beam_load([Mod], Architecture, S),
 		     {reply,Module,S};
 		({error,on_load_failure}=Error, S) ->
 		     {reply,Error,S};
@@ -1153,24 +1152,14 @@ try_load_module_3(File, Mod, Bin, From, Architecture, St0) ->
     Res = erlang:load_module(Mod, Bin),
     handle_on_load(Res, Action, Mod, From, St0).
 
-hipe_result_to_status(Result, #state{moddb=Db}) ->
+hipe_result_to_status(Result, #state{}) ->
     case Result of
-	{module,Mod} ->
-	    ets:insert(Db, [{{native,Mod},true}]),
+	{module,_} ->
 	    Result;
 	_ ->
 	    {error,Result}
     end.
 
-post_beam_load(_, undefined, _) ->
-    %% HiPE is disabled.
-    ok;
-post_beam_load(Mods0, _Architecture, #state{moddb=Db}) ->
-    %% post_beam_load/2 can potentially be very expensive because it
-    %% blocks multi-scheduling. Therefore, we only want to call
-    %% it with modules that are known to have native code loaded.
-    Mods = [M || M <- Mods0, ets:member(Db, {native,M})],
-    hipe_unified_loader:post_beam_load(Mods).
 
 int_list([H|T]) when is_integer(H) -> int_list(T);
 int_list([_|_])                    -> false;
@@ -1313,15 +1302,12 @@ abort_if_sticky(L, Db) ->
 	[_|_] -> {error,Sticky}
     end.
 
-do_finish_loading(Prepared, #state{moddb=Db}=St) ->
+do_finish_loading(Prepared, #state{moddb=Db}) ->
     MagicBins = [B || {_,{B,_}} <- Prepared],
     case erlang:finish_loading(MagicBins) of
 	ok ->
 	    MFs = [{M,F} || {M,{_,F}} <- Prepared],
 	    true = ets:insert(Db, MFs),
-	    Ms = [M || {M,_} <- MFs],
-	    Architecture = erlang:system_info(hipe_architecture),
-	    post_beam_load(Ms, Architecture, St),
 	    ok;
 	{Reason,Ms} ->
 	    {error,[{M,Reason} || M <- Ms]}
diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl
index 7b1e6c2591..75a5995695 100644
--- a/lib/kernel/src/hipe_unified_loader.erl
+++ b/lib/kernel/src/hipe_unified_loader.erl
@@ -41,10 +41,11 @@
 % I think the real solution would be to let BIF erlang:load_module/2 redirect all
 % hipe calls to the module and thereby remove post_beam_load.
 
+% SVERK: Can we remove -compile(no_native) now when post_beam_load is gone?
+
 -export([chunk_name/1,
 	 %% Only the code and code_server modules may call the entries below!
 	 load_native_code/3,
-	 post_beam_load/1,
 	 load_module/4,
 	 load/3]).
 
@@ -101,15 +102,13 @@ word_size(Architecture) ->
 load_native_code(_Mod, _Bin, undefined) ->
   no_native;
 load_native_code(Mod, Bin, Architecture) when is_atom(Mod), is_binary(Bin) ->
-  %% patch_to_emu(Mod),
   case code:get_chunk(Bin, chunk_name(Architecture)) of
     undefined -> no_native;
     NativeCode when is_binary(NativeCode) ->
       erlang:system_flag(multi_scheduling, block_normal),
       try
-        OldReferencesToPatch = patch_to_emu_step1(Mod),
-        case load_module(Mod, NativeCode, Bin, OldReferencesToPatch,
-                         Architecture) of
+        put(hipe_patch_closures, false),
+        case load_common(Mod, NativeCode, Bin, Architecture) of
           bad_crc -> no_native;
           Result -> Result
         end
@@ -120,22 +119,6 @@ load_native_code(Mod, Bin, Architecture) when is_atom(Mod), is_binary(Bin) ->
 
 %%========================================================================
 
--spec post_beam_load([module()]) -> 'ok'.
-
-post_beam_load([])->
-  ok;
-post_beam_load([_|_]=Mods) ->
-  erlang:system_flag(multi_scheduling, block_normal),
-  try
-    _ = [patch_to_emu(Mod) || Mod <- Mods],
-    ok
-  after
-    erlang:system_flag(multi_scheduling, unblock_normal)
-  end,
-  ok.
-
-%%========================================================================
-
 version_check(Version, Mod) when is_atom(Mod) ->
   Ver = ?VERSION_STRING(),
   case Version < Ver of
@@ -153,19 +136,12 @@ version_check(Version, Mod) when is_atom(Mod) ->
 load_module(Mod, Bin, Beam, Architecture) ->
   erlang:system_flag(multi_scheduling, block_normal),
   try
-    load_module_nosmp(Mod, Bin, Beam, Architecture)
+    put(hipe_patch_closures, false),
+    load_common(Mod, Bin, Beam, Architecture)
   after
     erlang:system_flag(multi_scheduling, unblock_normal)
   end.
 
-load_module_nosmp(Mod, Bin, Beam, Architecture) ->
-  load_module(Mod, Bin, Beam, [], Architecture).
-
-load_module(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
-  ?debug_msg("************ Loading Module ~w ************\n",[Mod]),
-  %% Loading a whole module, let the BEAM loader patch closures.
-  put(hipe_patch_closures, false),
-  load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture).
 
 %%========================================================================
 
@@ -175,20 +151,17 @@ load_module(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
 load(Mod, Bin, Architecture) ->
   erlang:system_flag(multi_scheduling, block_normal),
   try
-    load_nosmp(Mod, Bin, Architecture)
+    ?debug_msg("********* Loading funs in module ~w *********\n",[Mod]),
+    %% Loading just some functions in a module; patch closures separately.
+    put(hipe_patch_closures, true),
+    load_common(Mod, Bin, [], Architecture)
   after
     erlang:system_flag(multi_scheduling, unblock_normal)
   end.
 
-load_nosmp(Mod, Bin, Architecture) ->
-  ?debug_msg("********* Loading funs in module ~w *********\n",[Mod]),
-  %% Loading just some functions in a module; patch closures separately.
-  put(hipe_patch_closures, true),
-  load_common(Mod, Bin, [], [], Architecture).
-
 %%------------------------------------------------------------------------
 
-load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
+load_common(Mod, Bin, Beam, Architecture) ->
   %% Unpack the binary.
   [{Version, CheckSum},
    ConstAlign, ConstSize, ConstMap, LabelMap, ExportMap,
@@ -228,15 +201,14 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
                                         Architecture),
       %% Patch references to code labels in data seg.
       ok = patch_consts(LabelMap, ConstAddr, CodeAddress, WriteWord),
+
       %% Find out which functions are being loaded (and where).
-      %% Note: FunDefs are sorted descending.
-      {MFAs,FunDefs} = exports(ExportMap, CodeAddress),
-      %% Remove references to old versions of the module.
-      ReferencesToPatch = get_refs_from(MFAs, []),
-      %% io:format("References to patch: ~w~n", [ReferencesToPatch]),
-      ok = remove_refs_from(MFAs),
+      %% Note: FunDefs are sorted descending address order.
+      FunDefs = exports(ExportMap, CodeAddress),
+
       %% Patch all dynamic references in the code.
       %%  Function calls, Atoms, Constants, System calls
+
       ok = patch(Refs, CodeAddress, ConstMap2, FunDefs, TrampolineMap),
 
       %% Tell the system where the loaded funs are. 
@@ -250,7 +222,7 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
 	  lists:foreach(fun({FE, DestAddress}) ->
 			    hipe_bifs:set_native_address_in_fe(FE, DestAddress)
 			end, erase(closures_to_patch)),
-	  export_funs(FunDefs),
+	  set_beam_call_traps(FunDefs),
 	  ok;
 	BeamBinary when is_binary(BeamBinary) ->
 	  %% Find all closures in the code.
@@ -259,15 +231,10 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
 	  AddressesOfClosuresToPatch =
 	    calculate_addresses(ClosurePatches, CodeAddress, FunDefs),
 	  export_funs(FunDefs),
-	  export_funs(Mod, MD5, BeamBinary,
-                      FunDefs, AddressesOfClosuresToPatch)
+	  make_beam_stub(Mod, MD5, BeamBinary, FunDefs, AddressesOfClosuresToPatch,
+			 CodeAddress, byte_size(CodeBinary))
       end,
-      %% Redirect references to the old module to the new module's BEAM stub.
-      patch_to_emu_step2(OldReferencesToPatch),
-      %% Patch referring functions to call the new function
-      %% The call to export_funs/1 above updated the native addresses
-      %% for the targets, so passing 'FunDefs' is not needed.
-      redirect(ReferencesToPatch),
+
       %% Final clean up.
       _ = erase(hipe_patch_closures),
       _ = erase(hipe_assert_code_area),
@@ -371,21 +338,21 @@ trampoline_map_lookup(Primop, Map) ->
 		 is_exported :: boolean()}).
 
 exports(ExportMap, BaseAddress) ->
-  exports(ExportMap, BaseAddress, [], []).
+  exports(ExportMap, BaseAddress, []).
 
-exports([Offset,M,F,A,IsClosure,IsExported|Rest], BaseAddress, MFAs, FunDefs) ->
+exports([Offset,M,F,A,IsClosure,IsExported|Rest], BaseAddress, FunDefs) ->
   case IsExported andalso erlang:is_builtin(M, F, A) of
     true ->
-      exports(Rest, BaseAddress, MFAs, FunDefs);
+      exports(Rest, BaseAddress, FunDefs);
     _false ->
       MFA = {M,F,A},
       Address = BaseAddress + Offset,
       FunDef = #fundef{address=Address, mfa=MFA, is_closure=IsClosure,
 		       is_exported=IsExported},
-      exports(Rest, BaseAddress, [MFA|MFAs], [FunDef|FunDefs])
+      exports(Rest, BaseAddress, [FunDef|FunDefs])
   end;
-exports([], _, MFAs, FunDefs) ->
-  {MFAs, FunDefs}.
+exports([], _, FunDefs) ->
+  FunDefs.
 
 mod({M,_F,_A}) -> M.
 
@@ -424,9 +391,9 @@ find_closure_refs([], Refs) ->
 
 %%------------------------------------------------------------------------
 
-export_funs([FunDef | FunDefs]) ->
+set_beam_call_traps([FunDef | FunDefs]) ->
   #fundef{address=Address, mfa=MFA, is_closure=IsClosure,
-	  is_exported=IsExported} = FunDef,
+	  is_exported=_IsExported} = FunDef,
   ?IF_DEBUG({M,F,A} = MFA, no_debug),
   ?IF_DEBUG(
      case IsClosure of
@@ -437,15 +404,32 @@ export_funs([FunDef | FunDefs]) ->
 	 ?debug_msg("LINKING: ~w:~w/~w to closure (0x~.16b)\n",
 		    [M,F,A, Address])
      end, no_debug),
-  hipe_bifs:set_funinfo_native_address(MFA, Address, IsExported),
   hipe_bifs:set_native_address(MFA, Address, IsClosure),
+  set_beam_call_traps(FunDefs);
+set_beam_call_traps([]) ->
+  ok.
+
+export_funs([FunDef | FunDefs]) ->
+  #fundef{address=Address, mfa=MFA, is_closure=_IsClosure,
+	  is_exported=IsExported} = FunDef,
+  ?IF_DEBUG({M,F,A} = MFA, no_debug),
+  ?IF_DEBUG(
+     case _IsClosure of
+       false ->
+	 ?debug_msg("LINKING: ~w:~w/~w to (0x~.16b)\n",
+		    [M,F,A, Address]);
+       true ->
+	 ?debug_msg("LINKING: ~w:~w/~w to closure (0x~.16b)\n",
+		    [M,F,A, Address])
+     end, no_debug),
+  hipe_bifs:set_funinfo_native_address(MFA, Address, IsExported),
   export_funs(FunDefs);
 export_funs([]) ->
   ok.
 
-export_funs(Mod, MD5, Beam, FunDefs, ClosuresToPatch) ->
+make_beam_stub(Mod, MD5, Beam, FunDefs, ClosuresToPatch, CodeAddress, CodeSize) ->
   Fs = [{F,A,Address} || #fundef{address=Address, mfa={_M,F,A}} <- FunDefs],
-  Mod = code:make_stub_module(Mod, Beam, {Fs,ClosuresToPatch,MD5}),
+  Mod = code:make_stub_module(Mod, Beam, {Fs,ClosuresToPatch,MD5,CodeAddress,CodeSize}),
   ok.
 
 %%========================================================================
@@ -515,8 +499,7 @@ patch_mfa_call_list([Offset|Offsets], BaseAddress, DestMFA, DestAddress, FunDefs
 patch_mfa_call_list([], _, _, _, _, _, _) -> ok.
 
 patch_call_insn(CallAddress, DestAddress, Trampoline) ->
-  %% This assertion is false when we're called from redirect/2.
-  %% ?ASSERT(assert_local_patch(CallAddress)),
+  ?ASSERT(assert_local_patch(CallAddress)),
   hipe_bifs:patch_call(CallAddress, DestAddress, Trampoline).
 
 %% ____________________________________________________________________
@@ -561,15 +544,15 @@ patch_atom(Address, Atom) ->
   patch_instr(Address, hipe_bifs:atom_to_word(Atom), atom).
 
 patch_sdesc(?STACK_DESC(SymExnRA, FSize, Arity, Live),
-	    Address, {_ConstMap2,CodeAddress}, _FunDefs) ->
+	    Address, {_ConstMap2,CodeAddress}, FunDefs) ->
   ExnRA =
     case SymExnRA of
       [] -> 0; % No catch
       LabelOffset -> CodeAddress + LabelOffset
     end,
   ?ASSERT(assert_local_patch(Address)),
-  DBG_MFA = ?IF_DEBUG(address_to_mfa_lth(Address, _FunDefs), {undefined,undefined,0}),
-  hipe_bifs:enter_sdesc({Address, ExnRA, FSize, Arity, Live, DBG_MFA}).
+  MFA = address_to_mfa_lth(Address, FunDefs),
+  hipe_bifs:enter_sdesc({Address, ExnRA, FSize, Arity, Live, MFA, get(hipe_patch_closures)}).
 
 
 %%----------------------------------------------------------------
@@ -617,6 +600,7 @@ patch_closure(DestMFA, Uniq, Index, Address, FunDefs) ->
 %% RemoteOrLocal ::= 'remote' | 'local'
 %%
 patch_load_mfa(CodeAddress, DestMFA, FunDefs, RemoteOrLocal) ->
+  ?ASSERT(assert_local_patch(CodeAddress)),
   DestAddress =
     case bif_address(DestMFA) of
       false ->
@@ -626,7 +610,6 @@ patch_load_mfa(CodeAddress, DestMFA, FunDefs, RemoteOrLocal) ->
       BifAddress when is_integer(BifAddress) ->
 	BifAddress
     end,
-  ?ASSERT(assert_local_patch(CodeAddress)),
   patch_instr(CodeAddress, DestAddress, 'load_mfa').
 
 %%----------------------------------------------------------------
@@ -781,23 +764,17 @@ find_const(ConstNo, []) ->
 %% RemoteOrLocal ::= 'remote' | 'local'.
 %%
 
-%%
-%% -record(ref, {caller_mfa, address, ref_type, trampoline, remote_or_local}).
-%%
-
 add_ref(CalleeMFA, Address, FunDefs, RefType, Trampoline, RemoteOrLocal) ->
   CallerMFA = address_to_mfa_lth(Address, FunDefs),
-  %% just a sanity assertion below
-  true = case RemoteOrLocal of
-	   local ->
-	     {M1,_,_} = CalleeMFA,
-	     {M2,_,_} = CallerMFA,
-	     M1 =:= M2;
-	   remote ->
-	     true
-	 end,
-  %% io:format("Adding ref ~w\n",[{CallerMFA, CalleeMFA, Address, RefType}]),
-  hipe_bifs:add_ref(CalleeMFA, {CallerMFA,Address,RefType,Trampoline,RemoteOrLocal}).
+  case RemoteOrLocal of
+    local ->
+      %% just a sanity assertion
+      {M,_,_} = CalleeMFA,
+      {M,_,_} = CallerMFA;
+    remote ->
+      hipe_bifs:add_ref(CalleeMFA, {CallerMFA,Address,RefType,Trampoline,get(hipe_patch_closures)})
+  end,
+  ok.
 
 % For FunDefs sorted from low to high addresses
 address_to_mfa_lth(Address, FunDefs) ->
@@ -825,81 +802,6 @@ address_to_mfa_lth(_Address, [], Prev) ->
 %%   ?error_msg("Local adddress not found ~w\n",[Address]),
 %%   exit({?MODULE, local_address_not_found}).
 
-%%----------------------------------------------------------------
-%% Change callers of the given module to instead trap to BEAM.
-%% load_native_code/3 calls this just before loading native code.
-%%
-patch_to_emu(Mod) ->
-  patch_to_emu_step2(patch_to_emu_step1(Mod)).
-
-%% Step 1 must occur before the loading of native code updates
-%% references information or creates a new BEAM stub module.
-patch_to_emu_step1(Mod) ->
-  case is_loaded(Mod) of
-    true ->
-      %% Get exported functions
-      MFAs = [{Mod,Fun,Arity} || {Fun,Arity} <- Mod:module_info(exports)],
-      %% get_refs_from/2 only finds references from compiled static
-      %% call sites to the module, but some native address entries
-      %% were added as the result of dynamic apply calls. We must
-      %% purge them too, but we have no explicit record of them.
-      %% Therefore invalidate all native addresses for the module.
-      hipe_bifs:invalidate_funinfo_native_addresses(MFAs),
-      %% Find all call sites that call these MFAs. As a side-effect,
-      %% create native stubs for any MFAs that are referred.
-      ReferencesToPatch = get_refs_from(MFAs, []),
-      ok = remove_refs_from(MFAs),
-      ReferencesToPatch;
-    false ->
-      %% The first time we load the module, no redirection needs to be done.
-      []
-  end.
-
-%% Step 2 must occur after the new BEAM stub module is created.
-patch_to_emu_step2(ReferencesToPatch) ->
-  redirect(ReferencesToPatch).
-
--spec is_loaded(Module::atom()) -> boolean().
-%% @doc Checks whether a module is loaded or not.
-is_loaded(M) when is_atom(M) ->
-  try hipe_bifs:fun_to_address({M,module_info,0}) of
-    I when is_integer(I) -> true
-  catch _:_ -> false
-  end.
-
-%%--------------------------------------------------------------------
-%% Given a list of MFAs, tag them with their referred_from references.
-%% The resulting {MFA,Refs} list is later passed to redirect/1, once
-%% the MFAs have been bound to (possibly new) native-code addresses.
-%%
-get_refs_from(MFAs, []) ->
-  mark_referred_from(MFAs),
-  MFAs.
-
-mark_referred_from(MFAs) ->
-  lists:foreach(fun(MFA) -> hipe_bifs:mark_referred_from(MFA) end, MFAs).
-
-%%--------------------------------------------------------------------
-%% Given a list of MFAs with referred_from references, update their
-%% callers to refer to their new native-code addresses.
-%%
-%% The {MFA,Refs} list must come from get_refs_from/2.
-%%
-redirect(MFAs) ->
-  lists:foreach(fun(MFA) -> hipe_bifs:redirect_referred_from(MFA) end, MFAs).
-
-%%--------------------------------------------------------------------
-%% Given a list of MFAs, remove all referred_from references having
-%% any of them as CallerMFA.
-%%
-%% This is the only place using refers_to. Whenever a reference is
-%% added from CallerMFA to CalleeMFA, CallerMFA is added to CalleeMFA's
-%% referred_from list, and CalleeMFA is added to CallerMFA's refers_to
-%% list. The refers_to list is used here to find the CalleeMFAs whose
-%% referred_from lists should be updated.
-%%
-remove_refs_from(MFAs) ->
-  lists:foreach(fun(MFA) -> hipe_bifs:remove_refs_from(MFA) end, MFAs).
 
 %%--------------------------------------------------------------------
 
@@ -913,12 +815,13 @@ get_native_address(MFA, FunDefs, RemoteOrLocal) ->
   case mfa_to_address(MFA, FunDefs, RemoteOrLocal) of
     Adr when is_integer(Adr) -> Adr;
     false ->
-      IsRemote =
 	case RemoteOrLocal of
-	  remote -> true;
-	  local -> false
-	end,
-      hipe_bifs:find_na_or_make_stub(MFA, IsRemote)
+	  remote ->
+	    hipe_bifs:find_na_or_make_stub(MFA);
+	  local ->
+	    ?error_msg("Local function ~p not found\n",[MFA]),
+	    exit({function_not_found,MFA})
+	end
   end.
 
 mfa_to_address(MFA, [#fundef{address=Adr, mfa=MFA,