erts: Improve hipe load/upgrade/purge machinery

A step toward better integration of hipe load and purge

Highlights:

* code_server no longer needs to call hipe_unified_loader:post_beam_load/1
  Instead new internal function hipe_redirect_to_module()
  is called by loading BIFs to patch native call sites if needed.

* hipe_purge_module() is called by erts_internal:purge_module/2
  to purge any native code.

* struct hipe_mfa_info redesigned and only used for exported
  functions that are called from or implemented by native code.
  A list of native call sites (struct hipe_ref) are kept for each hipe_mfa_info.

* struct hipe_sdesc used by hipe_find_mfa_from_ra()
  to build native stack traces.

3 files changed, 114 insertions, 16 deletions

diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl
index c5167efa56..1be4c364d5 100644
--- a/lib/kernel/test/code_SUITE.erl
+++ b/lib/kernel/test/code_SUITE.erl
@@ -483,23 +483,25 @@ load_binary(Config) when is_list(Config) ->
     code:delete(code_b_test),
     ok.
 
+
 upgrade(Config) ->
     DataDir = proplists:get_value(data_dir, Config),
 
-    %%T = [beam, hipe],
-    T = [beam],
-
-    [upgrade_do(DataDir, Client, U1, U2, O1, O2)
-     || Client<-T, U1<-T, U2<-T, O1<-T, O2<-T],
+    T = [beam, hipe],
+    %%T = [beam],
+    %%T = [hipe],
 
+    [upgrade_do(DataDir, Client, T) || Client <- T],
     ok.
 
-upgrade_do(DataDir, Client, U1, U2, O1, O2) ->
+upgrade_do(DataDir, Client, T) ->
     compile_load(upgrade_client, DataDir, undefined, Client),
-    upgrade_client:run(DataDir, U1, U2, O1, O2),
+    [upgrade_client:run(DataDir, U1, U2, O1, O2)
+     || U1<-T, U2<-T, O1<-T, O2<-T],
     ok.
 
 compile_load(Mod, Dir, Ver, CodeType) ->
+    erlang:display({"{{{{{{{{{{{{{{{{Loading",Mod,Ver,CodeType}),
     Version = case Ver of
 	undefined ->
 	    io:format("Compiling '~p' as ~p\n", [Mod, CodeType]),
@@ -516,9 +518,15 @@ compile_load(Mod, Dir, Ver, CodeType) ->
     CompOpts = [binary, report] ++ Target ++ Version,
 
     Src = filename:join(Dir, atom_to_list(Mod) ++ ".erl"),
+    T1 = erlang:now(),
     {ok,Mod,Code} = compile:file(Src, CompOpts),
+    T2 = erlang:now(),
     ObjFile = filename:basename(Src,".erl") ++ ".beam",
     {module,Mod} = code:load_binary(Mod, ObjFile, Code),
+    T3 = erlang:now(),
+    io:format("Compile time ~p ms, Load time ~p ms\n",
+	      [timer:now_diff(T2,T1) div 1000, timer:now_diff(T3,T2) div 1000]),
+    erlang:display({"}}}}}}}}}}}}}}}Loaded",Mod,Ver,CodeType}),
     ok.
 
 dir_req(Config) when is_list(Config) ->
@@ -810,8 +818,6 @@ check_funs({'$M_EXPR','$F_EXPR',_},
 	    {code_server,start_link,1}]) -> 0;
 check_funs({'$M_EXPR','$F_EXPR',_},
 	   [{erlang,spawn_link,1},{code_server,start_link,1}]) -> 0;
-check_funs({'$M_EXPR',module_info,1},
-	   [{hipe_unified_loader,patch_to_emu_step1,1} | _]) -> 0;
 check_funs({'$M_EXPR','$F_EXPR',2},
 	   [{hipe_unified_loader,write_words,3} | _]) -> 0;
 check_funs({'$M_EXPR','$F_EXPR',2},
@@ -823,11 +829,7 @@ check_funs({'$M_EXPR','$F_EXPR',2},
 	    {hipe_unified_loader,sort_and_write,5} | _]) -> 0;
 check_funs({'$M_EXPR','$F_EXPR',1},
 	   [{lists,foreach,2},
-	    {hipe_unified_loader,patch_consts,3} | _]) -> 0;
-check_funs({'$M_EXPR','$F_EXPR',1},
-	   [{lists,foreach,2},
-	    {hipe_unified_loader,mark_referred_from,1},
-	    {hipe_unified_loader,get_refs_from,2}| _]) -> 0;
+	    {hipe_unified_loader,patch_consts,4} | _]) -> 0;
 check_funs({'$M_EXPR',warning_msg,2},
 	   [{code_server,finish_on_load_report,2} | _]) -> 0;
 check_funs({'$M_EXPR','$F_EXPR',1},
diff --git a/lib/kernel/test/code_SUITE_data/upgrade_client.erl b/lib/kernel/test/code_SUITE_data/upgrade_client.erl
index bb655e01d3..7ca0df7f5c 100644
--- a/lib/kernel/test/code_SUITE_data/upgrade_client.erl
+++ b/lib/kernel/test/code_SUITE_data/upgrade_client.erl
@@ -9,6 +9,8 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
     %% Load version 1 of upgradee
     code_SUITE:compile_load(upgradee, Dir, 1, Upgradee1),
 
+    Tracer = start_tracing(),
+
     ?line 1 = upgradee:exp1(),
     ?line 1 = upgradee:exp1exp2(),
     ?line 1 = upgradee:exp1loc2(),
@@ -56,6 +58,16 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc1),
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
 
+    Env1 = "Env1",
+    put(loc1_fun, upgradee:get_local_fun(Env1)),
+    erlang:display(sverk_break),
+    ?line {1,Env1} = (get(loc1_fun))(),
+
+    put(exp1exp2_fun, upgradee:get_exp1exp2_fun()),
+    ?line 1 = (get(exp1exp2_fun))(),
+
+    ?line 13 = check_tracing(Tracer),
+
     %%
     %% Load version 1 of other
     %%
@@ -78,6 +90,8 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, exp2),
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
 
+    ?line 5 = check_tracing(Tracer),
+
     %%
     %% Load version 2 of upgradee
     %%
@@ -130,6 +144,15 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, exp2),
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
 
+    ?line {1,Env1} = (get(loc1_fun))(),
+    Env2 = "Env2",
+    put(loc2_fun, upgradee:get_local_fun(Env2)),
+    ?line {2,Env2} = (get(loc2_fun))(),
+
+    ?line 2 = (get(exp1exp2_fun))(),
+
+    ?line 10 = check_tracing(Tracer),
+
     %%
     %% Load version 2 of other
     %%
@@ -182,17 +205,26 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc1loc2),
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
 
+    ?line {1,Env1} = (get(loc1_fun))(),
+    ?line {2,Env2} = (get(loc2_fun))(),
+    ?line 2 = (get(exp1exp2_fun))(),
+
+    ?line 10 = check_tracing(Tracer),
 
     %%
     %% Upgrade proxy to version 2
     %%
     P ! upgrade_order,
 
-    
     %%
-    io:format("Delete version 2 of 'upgradee'\n",[]),
+    io:format("Purge version 1 of 'upgradee'\n",[]),
     %%
+    put(loc1_fun,undefined),
     code:purge(upgradee),
+
+    %%
+    io:format("Delete version 2 of 'upgradee'\n",[]),
+    %%
     code:delete(upgradee),
     
     ?line {'EXIT',{undef,_}} = (catch upgradee:exp2()),
@@ -239,17 +271,24 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, exp1loc2),
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc1loc2),
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
+
+    ?line {'EXIT',{undef,_}} = (catch (get(exp1exp2_fun))()),
     
+    ?line 14 = check_tracing(Tracer),
+
     unlink(P),
     exit(P, die_please),
 
     io:format("Purge 'upgradee'\n",[]),
+    put(loc2_fun,undefined),
     code:purge(upgradee),
 
     io:format("Delete and purge 'other'\n",[]),
     code:purge(other),
     code:delete(other),
     code:purge(other),
+
+    stop_tracing(Tracer),
     ok.
 
 proxy_call(Pid, CallType, Func) ->
@@ -257,3 +296,48 @@ proxy_call(Pid, CallType, Func) ->
     receive
 	{Pid, call_result, Func, Ret} -> Ret
     end.
+
+
+start_tracing() ->
+    Self = self(),
+    {Tracer,_} = spawn_opt(fun() -> tracer_loop(Self) end, [link,monitor]),
+    ?line 1 = erlang:trace_pattern({error_handler,undefined_function,3},
+				   true, [global]),
+    ?line 1 = erlang:trace(Self, true, [call,{tracer,Tracer}]),
+    Tracer.
+
+
+tracer_loop(Receiver) ->
+    receive
+	die_please ->
+	    ok;
+	{do_trace_delivered, Tracee} ->
+	    _ = erlang:trace_delivered(Tracee),
+	    tracer_loop(Receiver);
+
+	Msg ->
+	    Receiver ! Msg,
+	    tracer_loop(Receiver)
+    end.
+
+check_tracing(Tracer) ->
+    Tracer ! {do_trace_delivered, self()},
+    check_tracing_loop(0).
+
+check_tracing_loop(N) ->
+    Self = self(),
+    receive
+	{trace, _Pid, call, {_M, _F, _Args}} = Msg ->
+	    io:format("Trace: ~p\n",[Msg]),
+	    check_tracing_loop(N+1);
+	{trace_delivered, Self, _} ->
+	    N
+    end.
+
+
+stop_tracing(Tracer) ->
+    erlang:trace(self(), false, [call]),
+    Tracer ! die_please,
+    receive
+	{'DOWN', _, process, Tracer, _} -> ok
+    end.
diff --git a/lib/kernel/test/code_SUITE_data/upgradee.erl b/lib/kernel/test/code_SUITE_data/upgradee.erl
index 62b1d95e30..8ca660c19c 100644
--- a/lib/kernel/test/code_SUITE_data/upgradee.erl
+++ b/lib/kernel/test/code_SUITE_data/upgradee.erl
@@ -8,6 +8,9 @@
 -export([exp1/0]).     % only exported in v1
 -export([exp1loc2/0]). % exported in v1, local in v2
 -export([exp1exp2/0]). % exported in v1 and v2
+-export([get_local_fun/1]).
+-export([get_exp1exp2_fun/0]).
+-export([exp1exp2_fun/0]).
 
 exp1() -> ?VERSION.
 loc1() -> ?VERSION.
@@ -20,6 +23,9 @@ loc1() -> ?VERSION.
 -export([exp2/0]).
 -export([loc1exp2/0]).
 -export([exp1exp2/0]).
+-export([get_local_fun/1]).
+-export([get_exp1exp2_fun/0]).
+-export([exp1exp2_fun/0]).
 
 exp2() -> ?VERSION.
 loc2() -> ?VERSION.
@@ -31,6 +37,12 @@ exp1loc2() -> ?VERSION.
 loc1exp2() -> ?VERSION.
 loc1loc2() -> ?VERSION.
 
+get_local_fun(Env) -> fun() -> {?VERSION,Env} end.
+get_exp1exp2_fun() -> fun ?MODULE:exp1exp2_fun/0.
+
+exp1exp2_fun() ->
+    ?VERSION.
+
 dispatch_loop() ->
     receive 
 	upgrade_order ->