erts: Fix early hipe patch loading

by introducing hipe_bifs:commit_patch_load/1
that creates the HipeModule.

10 files changed, 76 insertions, 50 deletions

diff --git a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam
index 876407a6d6..73e95b03c7 100644
--- a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam
+++ b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 0eb390bf4c..4a833689e3 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -6540,6 +6540,45 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
     BIF_ERROR(p, BADARG);
 }
 
+int erts_commit_hipe_patch_load(Eterm hipe_magic_bin)
+{
+    Binary* hipe_magic;
+    HipeLoaderState* hipe_stp;
+    HipeModule *hipe_code;
+    Module* modp;
+
+    if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) ||
+	!(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val,
+	  hipe_stp = hipe_get_loader_state(hipe_magic)) ||
+	hipe_stp->module == NIL || hipe_stp->text_segment == 0) {
+	return 0;
+    }
+
+    modp = erts_get_module(hipe_stp->module, erts_active_code_ix());
+    if (!modp)
+	return 0;
+
+    /*
+     * Initialise HiPE module
+     */
+    hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code));
+    hipe_code->text_segment = hipe_stp->text_segment;
+    hipe_code->text_segment_size = hipe_stp->text_segment_size;
+    hipe_code->data_segment = hipe_stp->data_segment;
+    hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs;
+    hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc;
+
+    modp->curr.hipe_code = hipe_code;
+
+    /* Prevent code from being freed */
+    hipe_stp->text_segment = 0;
+    hipe_stp->data_segment = 0;
+    hipe_stp->new_hipe_refs = NULL;
+    hipe_stp->new_hipe_sdesc = NULL;
+
+    return 1;
+}
+
 #undef WORDS_PER_FUNCTION
 
 static int safe_mul(UWord a, UWord b, UWord* resp)
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 93996e8b41..29a0bc8d0b 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1048,6 +1048,7 @@ void erts_set_current_function(FunctionInfo* fi, BeamInstr* current);
 Eterm erts_module_info_0(Process* p, Eterm module);
 Eterm erts_module_info_1(Process* p, Eterm module, Eterm what);
 Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info);
+int erts_commit_hipe_patch_load(Eterm hipe_magic_bin);
 
 /* beam_ranges.c */
 void erts_init_ranges(void);
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index da895e4e8b..67508e6f5d 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -649,6 +649,14 @@ BIF_RETTYPE hipe_bifs_fun_to_address_1(BIF_ALIST_1)
     BIF_RET(address_to_term(pc, BIF_P));
 }
 
+BIF_RETTYPE hipe_bifs_commit_patch_load_1(BIF_ALIST_1)
+{
+    if (!erts_commit_hipe_patch_load(BIF_ARG_1))
+        BIF_ERROR(BIF_P, BADARG);
+
+    BIF_RET(am_ok);
+}
+
 BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
 {
     Eterm *pc;
@@ -689,14 +697,12 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2)
 {
     struct hipe_sdesc *sdesc;
     HipeLoaderState* stp;
-    Module* modp;
-    int do_commit;
 
     stp = get_loader_state(BIF_ARG_2);
     if (!stp)
 	BIF_ERROR(BIF_P, BADARG);
 
-    sdesc = hipe_decode_sdesc(BIF_ARG_1, &do_commit);
+    sdesc = hipe_decode_sdesc(BIF_ARG_1);
     if (!sdesc) {
 	fprintf(stderr, "%s: bad sdesc!\r\n", __FUNCTION__);
 	BIF_ERROR(BIF_P, BADARG);
@@ -709,17 +715,8 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2)
     /*
      * Link into list of sdesc's in same module instance
      */
-    modp = erts_put_active_module(make_atom(sdesc->m_aix));
-    ASSERT(modp);
-    if (do_commit) { /* Direct "hipe-patching" of early loaded module */
-        ASSERT(modp->curr.hipe_code);
-        sdesc->next_in_modi = modp->curr.hipe_code->first_hipe_sdesc;
-        modp->curr.hipe_code->first_hipe_sdesc = sdesc;
-    }
-    else {           /* Normal module loading/upgrade */
-        sdesc->next_in_modi = stp->new_hipe_sdesc;
-        stp->new_hipe_sdesc = sdesc;
-    }
+    sdesc->next_in_modi = stp->new_hipe_sdesc;
+    stp->new_hipe_sdesc = sdesc;
 
     BIF_RET(NIL);
 }
@@ -1483,7 +1480,7 @@ int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a)
 }
 
 
-/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,DoCommit})
+/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,LoaderState})
  */
 BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 {
@@ -1495,8 +1492,6 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     unsigned int flags;
     struct hipe_mfa_info *callee_mfa;
     struct hipe_ref *ref;
-    Module* modp;
-    int do_commit;
     HipeLoaderState* stp;
 
     if (!term_to_mfa(BIF_ARG_1, &callee))
@@ -1504,7 +1499,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     if (is_not_tuple(BIF_ARG_2))
 	goto badarg;
     tuple = tuple_val(BIF_ARG_2);
-    if (tuple[0] != make_arityval(6))
+    if (tuple[0] != make_arityval(5))
 	goto badarg;
     if (!term_to_mfa(tuple[1], &caller))
 	goto badarg;
@@ -1528,12 +1523,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 	if (!trampoline)
 	    goto badarg;
     }
-    switch (tuple[5]) {
-    case am_true: do_commit = 1; break;
-    case am_false: do_commit = 0; break;
-    default: goto badarg;
-    }
-    stp = get_loader_state(tuple[6]);
+    stp = get_loader_state(tuple[5]);
     if (!stp)
         goto badarg;
 
@@ -1558,17 +1548,8 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     /*
      * Link into list of refs from same module instance
      */
-    modp = erts_put_active_module(caller.mod);
-    ASSERT(modp);
-    if (do_commit) { /* Direct "hipe-patching" of early loaded module */
-        ASSERT(modp->curr.hipe_code);
-        ref->next_from_modi = modp->curr.hipe_code->first_hipe_ref;
-        modp->curr.hipe_code->first_hipe_ref = ref;
-    }
-    else {           /* Normal module loading/upgrade */
-        ref->next_from_modi = stp->new_hipe_refs;
-        stp->new_hipe_refs = ref;
-    }
+    ref->next_from_modi = stp->new_hipe_refs;
+    stp->new_hipe_refs = ref;
 
 #if defined(DEBUG)
     ref->callee = callee_mfa;
@@ -1578,10 +1559,10 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 #endif
     hipe_mfa_info_table_rwunlock();
 
-    DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p) do_commit=%d",
-		  ref, callee.mod, callee.fun, callee.ari, ref->address, do_commit);
-    DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p) do_commit=%d",
-		  ref, caller.mod, caller.fun, caller.ari, ref->address, do_commit);
+    DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p)",
+		  ref, callee.mod, callee.fun, callee.ari, ref->address);
+    DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p)",
+		  ref, caller.mod, caller.fun, caller.ari, ref->address);
     BIF_RET(NIL);
 
  badarg:
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 8475b8ce76..264ea2c34a 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -50,6 +50,7 @@ bif hipe_bifs:constants_size/0
 bif hipe_bifs:merge_term/1
 
 bif hipe_bifs:fun_to_address/1
+bif hipe_bifs:commit_patch_load/1
 bif hipe_bifs:set_native_address/3
 #bif hipe_bifs:address_to_fun/1
 
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
index cebbbafdd3..48f0cab293 100644
--- a/erts/emulator/hipe/hipe_load.c
+++ b/erts/emulator/hipe/hipe_load.c
@@ -57,6 +57,11 @@ hipe_loader_state_dtor(Binary* magic)
     stp->data_segment_size = 0;
 
     stp->module = NIL;
+
+    ASSERT(!stp->new_hipe_refs && !stp->new_hipe_sdesc);
+    /* ToDO: Purge lists 'new_hipe_refs' and 'new_hipe_sdesc'
+     *       if this is a failed load.
+     */
 }
 
 Binary *hipe_alloc_loader_state(Eterm module)
diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c
index 3d082668a6..b80e44bc37 100644
--- a/erts/emulator/hipe/hipe_stack.c
+++ b/erts/emulator/hipe/hipe_stack.c
@@ -143,7 +143,7 @@ void hipe_init_sdesc_table(struct hipe_sdesc *sdesc)
  * representation. If different representations are needed in
  * the future, this code has to be made target dependent.
  */
-struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp)
+struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
 {
     Uint ra, exnra;
     Eterm *live;
@@ -158,7 +158,7 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp)
         return 0;
 
     tp = tuple_val(arg);
-    if (tp[0] != make_arityval(7) ||
+    if (tp[0] != make_arityval(6) ||
         term_to_Uint(tp[1], &ra) == 0 ||
 	term_to_Uint(tp[2], &exnra) == 0 ||
 	is_not_small(tp[3]) ||
@@ -191,12 +191,6 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp)
 	    return 0;
     }
 
-    switch(tp[7]) {
-    case am_true: *do_commitp = 1; break;
-    case am_false: *do_commitp = 0; break;
-    default: return 0;
-    }
-
     /* Calculate number of words for the live bitmap. */
     livebitswords = (fsize + stk_nargs + 1 + 31) / 32;
     /* Calculate number of bytes needed for the stack descriptor. */
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index 537755fefa..7e30358767 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -82,7 +82,7 @@ extern struct hipe_sdesc_table hipe_sdesc_table;
 extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*);
 extern void hipe_destruct_sdesc(struct hipe_sdesc*);
 extern void hipe_init_sdesc_table(struct hipe_sdesc*);
-extern struct hipe_sdesc *hipe_decode_sdesc(Eterm, int* do_commitp);
+extern struct hipe_sdesc *hipe_decode_sdesc(Eterm);
 
 #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
 #define __builtin_expect(x, expected_value) (x)
diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl
index f53f6d5816..42a038f2d1 100644
--- a/lib/hipe/cerl/erl_bif_types.erl
+++ b/lib/hipe/cerl/erl_bif_types.erl
@@ -1094,6 +1094,9 @@ type(hipe_bifs, remove_refs_from, 1, Xs, Opaques) ->
 type(hipe_bifs, set_funinfo_native_address, 3, Xs, Opaques) ->
   strict(hipe_bifs, set_funinfo_native_address, 3, Xs,
 	 fun (_) -> t_nil() end, Opaques);
+type(hipe_bifs, commit_patch_load, 1, Xs, Opaques) ->
+  strict(hipe_bifs, commit_patch_load, 1, Xs,
+	 fun (_) -> t_atom() end, Opaques);
 type(hipe_bifs, set_native_address, 3, Xs, Opaques) ->
   strict(hipe_bifs, set_native_address, 3, Xs,
 	 fun (_) -> t_nil() end, Opaques);
@@ -2531,6 +2534,8 @@ arg_types(hipe_bifs, remove_refs_from, 1) ->
   [t_sup([t_mfa(), t_atom('all')])];
 arg_types(hipe_bifs, set_funinfo_native_address, 3) ->
   arg_types(hipe_bifs, set_native_address, 3);
+arg_types(hipe_bifs, commit_patch_load, 1) ->
+  [t_binary()];
 arg_types(hipe_bifs, set_native_address, 3) ->
   [t_mfa(), t_integer(), t_boolean()];
 arg_types(hipe_bifs, set_native_address_in_fe, 2) ->
diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl
index cde2eb059a..03116758de 100644
--- a/lib/kernel/src/hipe_unified_loader.erl
+++ b/lib/kernel/src/hipe_unified_loader.erl
@@ -225,6 +225,7 @@ load_common(Mod, Bin, Beam, Architecture) ->
 	  lists:foreach(fun({FE, DestAddress}) ->
 			    hipe_bifs:set_native_address_in_fe(FE, DestAddress)
 			end, erase(closures_to_patch)),
+          ok = hipe_bifs:commit_patch_load(LoaderState),
 	  set_beam_call_traps(FunDefs),
 	  ok;
 	BeamBinary when is_binary(BeamBinary) ->
@@ -556,7 +557,7 @@ patch_sdesc(?STACK_DESC(SymExnRA, FSize, Arity, Live),
     end,
   ?ASSERT(assert_local_patch(Address)),
   MFA = address_to_mfa_lth(Address, FunDefs),
-  hipe_bifs:enter_sdesc({Address, ExnRA, FSize, Arity, Live, MFA, get(hipe_patch_closures)},
+  hipe_bifs:enter_sdesc({Address, ExnRA, FSize, Arity, Live, MFA},
 		       get(hipe_loader_state)).
 
 
@@ -778,7 +779,6 @@ add_ref(CalleeMFA, Address, FunDefs, RefType, Trampoline, RemoteOrLocal) ->
       {M,_,_} = CallerMFA;
     remote ->
       hipe_bifs:add_ref(CalleeMFA, {CallerMFA,Address,RefType,Trampoline,
-				    get(hipe_patch_closures),
 				    get(hipe_loader_state)})
   end,
   ok.