erts: Move new hipe ref and sdesc lists to loader state

12 files changed, 115 insertions, 90 deletions

diff --git a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam
index 9877283cd2..876407a6d6 100644
--- a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam
+++ b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index ec1db8d9c2..f5d1ca7e54 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -1798,7 +1798,6 @@ delete_code(Module* modp)
 
     ASSERT(modp->curr.num_breakpoints == 0);
     ASSERT(modp->curr.num_traced_exports == 0);
-    DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "delete_code old.first_hipe_ref=%p", modp->curr.first_hipe_ref);
     modp->old = modp->curr;
     erts_module_instance_init(&modp->curr);
 }
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 7d29f393e5..0eb390bf4c 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -1126,11 +1126,8 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
     modp->curr.code_length = size;
     modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */
 #if defined(HIPE)
-    DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code new_hipe_refs = %p", modp->new_hipe_refs);
-    modp->curr.first_hipe_ref = modp->new_hipe_refs;
-    modp->curr.first_hipe_sdesc = modp->new_hipe_sdesc;
-    modp->new_hipe_refs = NULL;
-    modp->new_hipe_sdesc = NULL;
+    DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code "
+                  "first_hipe_ref = %p", hipe_code->first_hipe_ref);
     modp->curr.hipe_code = hipe_code;
 #endif
 
@@ -6498,6 +6495,8 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
     hipe_code->text_segment = hipe_stp->text_segment;
     hipe_code->text_segment_size = hipe_stp->text_segment_size;
     hipe_code->data_segment = hipe_stp->data_segment;
+    hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs;
+    hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc;
 
     /*
      * Insert the module in the module table.
@@ -6524,6 +6523,8 @@ erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
 	/* Prevent code from being freed */
 	hipe_stp->text_segment = 0;
 	hipe_stp->data_segment = 0;
+        hipe_stp->new_hipe_refs = NULL;
+        hipe_stp->new_hipe_sdesc = NULL;
 
 	erts_free_aligned_binary_bytes(temp_alloc);
 	free_loader_state(magic);
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index 5c56ad20df..e9b18ba525 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -77,8 +77,6 @@ void erts_module_instance_init(struct erl_module_instance* modi)
     modi->num_breakpoints = 0;
     modi->num_traced_exports = 0;
 #ifdef HIPE
-    modi->first_hipe_ref = NULL;
-    modi->first_hipe_sdesc = NULL;
     modi->hipe_code = NULL;
 #endif
 }
@@ -95,8 +93,6 @@ static Module* module_alloc(Module* tmpl)
     obj->on_load = 0;
 #ifdef HIPE
     obj->first_hipe_mfa = NULL;
-    obj->new_hipe_refs = NULL;
-    obj->new_hipe_sdesc = NULL;
 #endif
     DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc");
     return obj;
@@ -214,8 +210,6 @@ static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod)
     dst_mod->on_load = src_mod->on_load;
 #ifdef HIPE
     dst_mod->first_hipe_mfa = src_mod->first_hipe_mfa;
-    dst_mod->new_hipe_refs  = src_mod->new_hipe_refs;
-    dst_mod->new_hipe_sdesc = src_mod->new_hipe_sdesc;
 #endif
 }
 
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index c3adaf12fc..9de610ccf7 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -35,8 +35,6 @@ struct erl_module_instance {
     int num_breakpoints;
     int num_traced_exports;
 #ifdef HIPE
-    struct hipe_ref* first_hipe_ref;  /* all external hipe calls from this module */
-    struct hipe_sdesc* first_hipe_sdesc;  /* all stack descriptors for this module */
     HipeModule *hipe_code;
 #endif
 };
@@ -51,8 +49,6 @@ typedef struct erl_module {
     struct erl_module_instance* on_load;
 #ifdef HIPE
     struct hipe_mfa_info* first_hipe_mfa;
-    struct hipe_ref* new_hipe_refs;
-    struct hipe_sdesc* new_hipe_sdesc;
 #endif
 } Module; 
 
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 46d4378f21..da895e4e8b 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -685,12 +685,17 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
     BIF_RET(am_false);
 }
 
-BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
+BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2)
 {
     struct hipe_sdesc *sdesc;
+    HipeLoaderState* stp;
     Module* modp;
     int do_commit;
 
+    stp = get_loader_state(BIF_ARG_2);
+    if (!stp)
+	BIF_ERROR(BIF_P, BADARG);
+
     sdesc = hipe_decode_sdesc(BIF_ARG_1, &do_commit);
     if (!sdesc) {
 	fprintf(stderr, "%s: bad sdesc!\r\n", __FUNCTION__);
@@ -707,12 +712,13 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
     modp = erts_put_active_module(make_atom(sdesc->m_aix));
     ASSERT(modp);
     if (do_commit) { /* Direct "hipe-patching" of early loaded module */
-        sdesc->next_in_modi = modp->curr.first_hipe_sdesc;
-        modp->curr.first_hipe_sdesc = sdesc;
+        ASSERT(modp->curr.hipe_code);
+        sdesc->next_in_modi = modp->curr.hipe_code->first_hipe_sdesc;
+        modp->curr.hipe_code->first_hipe_sdesc = sdesc;
     }
     else {           /* Normal module loading/upgrade */
-        sdesc->next_in_modi = modp->new_hipe_sdesc;
-        modp->new_hipe_sdesc = sdesc;
+        sdesc->next_in_modi = stp->new_hipe_sdesc;
+        stp->new_hipe_sdesc = sdesc;
     }
 
     BIF_RET(NIL);
@@ -1491,13 +1497,14 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     struct hipe_ref *ref;
     Module* modp;
     int do_commit;
+    HipeLoaderState* stp;
 
     if (!term_to_mfa(BIF_ARG_1, &callee))
 	goto badarg;
     if (is_not_tuple(BIF_ARG_2))
 	goto badarg;
     tuple = tuple_val(BIF_ARG_2);
-    if (tuple[0] != make_arityval(5))
+    if (tuple[0] != make_arityval(6))
 	goto badarg;
     if (!term_to_mfa(tuple[1], &caller))
 	goto badarg;
@@ -1526,6 +1533,10 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     case am_false: do_commit = 0; break;
     default: goto badarg;
     }
+    stp = get_loader_state(tuple[6]);
+    if (!stp)
+        goto badarg;
+
     hipe_mfa_info_table_rwlock();
     callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari);
 
@@ -1550,12 +1561,13 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     modp = erts_put_active_module(caller.mod);
     ASSERT(modp);
     if (do_commit) { /* Direct "hipe-patching" of early loaded module */
-        ref->next_from_modi = modp->curr.first_hipe_ref;
-        modp->curr.first_hipe_ref = ref;
+        ASSERT(modp->curr.hipe_code);
+        ref->next_from_modi = modp->curr.hipe_code->first_hipe_ref;
+        modp->curr.hipe_code->first_hipe_ref = ref;
     }
     else {           /* Normal module loading/upgrade */
-        ref->next_from_modi = modp->new_hipe_refs;
-        modp->new_hipe_refs = ref;
+        ref->next_from_modi = stp->new_hipe_refs;
+        stp->new_hipe_refs = ref;
     }
 
 #if defined(DEBUG)
@@ -1653,9 +1665,12 @@ BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1)
 int hipe_purge_need_blocking(Module* modp)
 {
     /* SVERK: Verify if this is really necessary */
-    return (modp->old.first_hipe_ref ||
-            modp->old.first_hipe_sdesc ||
-            (!modp->curr.code_hdr && modp->first_hipe_mfa));
+    if (modp->old.hipe_code) {
+        if (modp->old.hipe_code->first_hipe_ref ||
+            modp->old.hipe_code->first_hipe_sdesc)
+            return 1;
+    }
+    return !modp->curr.code_hdr && modp->first_hipe_mfa;
 }
 
 void hipe_purge_module(Module* modp)
@@ -1667,68 +1682,75 @@ void hipe_purge_module(Module* modp)
 
     DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module");
 
-    /*
-     * Remove all hipe_ref's (external calls) from the old module instance
-     */
-    ref = modp->old.first_hipe_ref;
-
-    while (ref) {
-	struct hipe_ref* free_ref = ref;
-
-        ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
-
-	DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref,
-		      ref->callee->m, ref->callee->f, ref->callee->a);
-	DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref,
-		      ref->caller_m, ref->caller_f, ref->caller_a);
-	ASSERT(ref->caller_m == make_atom(modp->module));
-
+    if (modp->old.hipe_code) {
         /*
-         * Unlink from other refs to same callee
+         * Remove all hipe_ref's (external calls) from the old module instance
          */
-        ASSERT(ref->head.next->prev == &ref->head);
-        ASSERT(ref->head.prev->next == &ref->head);
-        ASSERT(ref->head.next != &ref->head);
-        ASSERT(ref->head.prev != &ref->head);
-        ref->head.next->prev = ref->head.prev;
-        ref->head.prev->next = ref->head.next;
+        ref = modp->old.hipe_code->first_hipe_ref;
+
+        while (ref) {
+            struct hipe_ref* free_ref = ref;
+
+            ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+            DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref,
+                          ref->callee->m, ref->callee->f, ref->callee->a);
+            DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref,
+                          ref->caller_m, ref->caller_f, ref->caller_a);
+            ASSERT(ref->caller_m == make_atom(modp->module));
+
+            /*
+             * Unlink from other refs to same callee
+             */
+            ASSERT(ref->head.next->prev == &ref->head);
+            ASSERT(ref->head.prev->next == &ref->head);
+            ASSERT(ref->head.next != &ref->head);
+            ASSERT(ref->head.prev != &ref->head);
+            ref->head.next->prev = ref->head.prev;
+            ref->head.prev->next = ref->head.next;
+
+            /*
+             * Was this the last ref to that callee?
+             */
+            if (ref->head.next == ref->head.prev) {
+                struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers);
+                if (p->is_stub) {
+                    unlink_mfa_from_mod(p);
+                    purge_mfa(p);
+               }
+            }
 
-        /*
-         * Was this the last ref to that callee?
-         */
-        if (ref->head.next == ref->head.prev) {
-            struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers);
-            if (p->is_stub) {
-                unlink_mfa_from_mod(p);
-                purge_mfa(p);
-           }
+            ref = ref->next_from_modi;
+            erts_free(ERTS_ALC_T_HIPE, free_ref);
         }
+        modp->old.hipe_code->first_hipe_ref = NULL;
 
-	ref = ref->next_from_modi;
-	erts_free(ERTS_ALC_T_HIPE, free_ref);
-    }
-    modp->old.first_hipe_ref = NULL;
+        /*
+         * Remove all hipe_sdesc's for the old module instance
+         */
+        sdesc = modp->old.hipe_code->first_hipe_sdesc;
 
-    /*
-     * Remove all hipe_sdesc's for the old module instance
-     */
-    sdesc = modp->old.first_hipe_sdesc;
+        while (sdesc) {
+            struct hipe_sdesc* free_sdesc = sdesc;
 
-    while (sdesc) {
-	struct hipe_sdesc* free_sdesc = sdesc;
+            ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
 
-        ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+            DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue);
+            ASSERT(sdesc->m_aix == modp->module);
 
-	DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue);
-	ASSERT(sdesc->m_aix == modp->module);
+            sdesc = sdesc->next_in_modi;
+            hipe_destruct_sdesc(free_sdesc);
+        }
+        modp->old.hipe_code->first_hipe_sdesc = NULL;
 
-	sdesc = sdesc->next_in_modi;
-        hipe_destruct_sdesc(free_sdesc);
+        hipe_free_module(modp->old.hipe_code);
+        modp->old.hipe_code = NULL;
     }
-    modp->old.first_hipe_sdesc = NULL;
+
 
     /*
-     * Remove unreferred hipe_mfa_info's
+     * Remove unreferred hipe_mfa_info's 
+     * when all module instances are removed (like in init:restart) 
      */
     if (modp->curr.code_hdr == NULL) {
         struct hipe_mfa_info** prevp = &modp->first_hipe_mfa;
@@ -1743,10 +1765,6 @@ void hipe_purge_module(Module* modp)
                 prevp = &p->next_in_mod;
         }
     }
-    if (modp->old.hipe_code) {
-        hipe_free_module(modp->old.hipe_code);
-        modp->old.hipe_code = NULL;
-    }
 }
 
 
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 1e4bde5ef2..8475b8ce76 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -56,7 +56,7 @@ bif hipe_bifs:set_native_address/3
 bif hipe_bifs:set_funinfo_native_address/3
 #bif hipe_bifs:invalidate_funinfo_native_addresses/1
 
-bif hipe_bifs:enter_sdesc/1
+bif hipe_bifs:enter_sdesc/2
 
 bif hipe_bifs:bif_address/3
 bif hipe_bifs:primop_address/1
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
index 5bce3f1aee..cebbbafdd3 100644
--- a/erts/emulator/hipe/hipe_load.c
+++ b/erts/emulator/hipe/hipe_load.c
@@ -79,6 +79,9 @@ Binary *hipe_alloc_loader_state(Eterm module)
     stp->data_segment = NULL;
     stp->data_segment_size = 0;
 
+    stp->new_hipe_refs = NULL;
+    stp->new_hipe_sdesc = NULL;
+
     return magic;
 }
 
diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h
index 9e24e915ed..17dd3c3c05 100644
--- a/erts/emulator/hipe/hipe_load.h
+++ b/erts/emulator/hipe/hipe_load.h
@@ -36,6 +36,9 @@ typedef struct hipe_loader_state {
     void *data_segment;
     Uint data_segment_size;
 
+    struct hipe_ref* new_hipe_refs;
+    struct hipe_sdesc* new_hipe_sdesc;
+
 } HipeLoaderState;
 
 extern Binary *hipe_alloc_loader_state(Eterm module);
diff --git a/erts/emulator/hipe/hipe_module.h b/erts/emulator/hipe/hipe_module.h
index 1e1302fc0a..b489f567cb 100644
--- a/erts/emulator/hipe/hipe_module.h
+++ b/erts/emulator/hipe/hipe_module.h
@@ -35,6 +35,9 @@ struct hipe_module {
     void *text_segment;
     Uint text_segment_size;
     void *data_segment;
+
+    struct hipe_ref* first_hipe_ref;  /* all external hipe calls from this module */
+    struct hipe_sdesc* first_hipe_sdesc;  /* all stack descriptors for this module */
 };
 
 extern void hipe_free_module(HipeModule *mod);
diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl
index 9a08305686..f53f6d5816 100644
--- a/lib/hipe/cerl/erl_bif_types.erl
+++ b/lib/hipe/cerl/erl_bif_types.erl
@@ -1058,8 +1058,8 @@ type(hipe_bifs, enter_code, 3, Xs, Opaques) ->
 			     %% XXX: The tuple below contains integers and
 			     %% is of size same as the length of the MFA list
 			     t_sup(t_nil(), t_binary())]) end, Opaques);
-type(hipe_bifs, enter_sdesc, 1, Xs, Opaques) ->
-  strict(hipe_bifs, enter_sdesc, 1, Xs, fun (_) -> t_nil() end, Opaques);
+type(hipe_bifs, enter_sdesc, 2, Xs, Opaques) ->
+  strict(hipe_bifs, enter_sdesc, 2, Xs, fun (_) -> t_nil() end, Opaques);
 type(hipe_bifs, find_na_or_make_stub, 1, Xs, Opaques) ->
   strict(hipe_bifs, find_na_or_make_stub, 1, Xs,
 	 fun (_) -> t_integer() end, Opaques); % address
@@ -2459,7 +2459,9 @@ arg_types(hipe_bifs, add_ref, 2) ->
   [t_mfa(), t_tuple([t_mfa(),
 		     t_integer(),
 		     t_sup(t_atom('call'), t_atom('load_mfa')),
-		     t_trampoline()])];
+		     t_trampoline(),
+		     t_atom(),
+		     t_binary()])];
 arg_types(hipe_bifs, alloc_data, 3) ->
   [t_integer(), t_integer(), t_binary()];
 arg_types(hipe_bifs, array, 2) ->
@@ -2498,8 +2500,9 @@ arg_types(hipe_bifs, check_crc, 1) ->
   [t_crc32()];
 arg_types(hipe_bifs, enter_code, 3) ->
   [t_binary(), t_sup(t_nil(), t_tuple()), t_binary()];
-arg_types(hipe_bifs, enter_sdesc, 1) ->
-  [t_tuple([t_integer(), t_integer(), t_integer(), t_integer(), t_integer(), t_mfa()])];
+arg_types(hipe_bifs, enter_sdesc, 2) ->
+  [t_tuple([t_integer(), t_integer(), t_integer(), t_integer(), t_integer(), t_mfa(), t_atom()]),
+   t_binary()];
 arg_types(hipe_bifs, find_na_or_make_stub, 1) ->
   [t_mfa()];
 arg_types(hipe_bifs, fun_to_address, 1) ->
diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl
index 6245646263..cde2eb059a 100644
--- a/lib/kernel/src/hipe_unified_loader.erl
+++ b/lib/kernel/src/hipe_unified_loader.erl
@@ -189,6 +189,7 @@ load_common(Mod, Bin, Beam, Architecture) ->
       WordSize = word_size(Architecture),
       WriteWord = write_word_fun(WordSize),
       LoaderState = hipe_bifs:alloc_loader_state(Mod),
+      put(hipe_loader_state, LoaderState),
       %% Create data segment
       {ConstAddr,ConstMap2} =
 	create_data_segment(ConstAlign, ConstSize, ConstMap, WriteWord,
@@ -238,6 +239,7 @@ load_common(Mod, Bin, Beam, Architecture) ->
       end,
 
       %% Final clean up.
+      _ = erase(hipe_loader_state),
       _ = erase(hipe_patch_closures),
       _ = erase(hipe_assert_code_area),
       ?debug_msg("****************Loader Finished****************\n", []),
@@ -554,7 +556,8 @@ patch_sdesc(?STACK_DESC(SymExnRA, FSize, Arity, Live),
     end,
   ?ASSERT(assert_local_patch(Address)),
   MFA = address_to_mfa_lth(Address, FunDefs),
-  hipe_bifs:enter_sdesc({Address, ExnRA, FSize, Arity, Live, MFA, get(hipe_patch_closures)}).
+  hipe_bifs:enter_sdesc({Address, ExnRA, FSize, Arity, Live, MFA, get(hipe_patch_closures)},
+		       get(hipe_loader_state)).
 
 
 %%----------------------------------------------------------------
@@ -774,7 +777,9 @@ add_ref(CalleeMFA, Address, FunDefs, RefType, Trampoline, RemoteOrLocal) ->
       {M,_,_} = CalleeMFA,
       {M,_,_} = CallerMFA;
     remote ->
-      hipe_bifs:add_ref(CalleeMFA, {CallerMFA,Address,RefType,Trampoline,get(hipe_patch_closures)})
+      hipe_bifs:add_ref(CalleeMFA, {CallerMFA,Address,RefType,Trampoline,
+				    get(hipe_patch_closures),
+				    get(hipe_loader_state)})
   end,
   ok.