erts: Improve hipe load/upgrade/purge machinery

A step toward better integration of hipe load and purge

Highlights:

* code_server no longer needs to call hipe_unified_loader:post_beam_load/1
  Instead new internal function hipe_redirect_to_module()
  is called by loading BIFs to patch native call sites if needed.

* hipe_purge_module() is called by erts_internal:purge_module/2
  to purge any native code.

* struct hipe_mfa_info redesigned and only used for exported
  functions that are called from or implemented by native code.
  A list of native call sites (struct hipe_ref) are kept for each hipe_mfa_info.

* struct hipe_sdesc used by hipe_find_mfa_from_ra()
  to build native stack traces.

30 files changed, 1005 insertions, 734 deletions

diff --git a/bootstrap/lib/kernel/ebin/code_server.beam b/bootstrap/lib/kernel/ebin/code_server.beam
index 0763c4c93b..e369cc2f36 100644
--- a/bootstrap/lib/kernel/ebin/code_server.beam
+++ b/bootstrap/lib/kernel/ebin/code_server.beam
diff --git a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam
index 2e639db331..f560389756 100644
--- a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam
+++ b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 153fa205ed..237513095a 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -36,6 +36,12 @@
 #include "erl_nif.h"
 #include "erl_bits.h"
 #include "erl_thr_progress.h"
+#ifdef HIPE
+#  include "hipe_bif0.h"
+#  define IF_HIPE(X) (X)
+#else
+#  define IF_HIPE(X) (0)
+#endif
 
 #ifdef HIPE
 #  include "hipe_stack.h"
@@ -169,6 +175,11 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
     if (res == BIF_ARG_1) {
 	erts_end_staging_code_ix();
 	erts_commit_staging_code_ix();
+#ifdef HIPE
+        if (!modp)
+	    modp = erts_get_module(BIF_ARG_1, erts_active_code_ix());
+        hipe_redirect_to_module(modp);
+#endif
     }
     else {
 	erts_abort_staging_code_ix();
@@ -241,7 +252,7 @@ struct m {
     Uint exception;
 };
 
-static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int);
+static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int, int);
 #ifdef ERTS_SMP
 static void smp_code_ix_commiter(void*);
 
@@ -377,8 +388,9 @@ finish_loading_1(BIF_ALIST_1)
     for (i = 0; i < n; i++) {
 	if (p[i].modp->curr.num_breakpoints > 0 ||
 	    p[i].modp->curr.num_traced_exports > 0 ||
-	    erts_is_default_trace_enabled()) {
-	    /* tracing involved, fallback with thread blocking */
+	    erts_is_default_trace_enabled() ||
+	    IF_HIPE(hipe_need_blocking(p[i].modp))) {
+	    /* tracing or hipe need thread blocking */
 	    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 	    erts_smp_thr_progress_block();
 	    is_blocking = 1;
@@ -436,32 +448,36 @@ finish_loading_1(BIF_ALIST_1)
     }
 
 done:
-    return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n);
+    return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n, 1);
 }
 
 static Eterm
 staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
-		 struct m* loaded, int nloaded)
+		 struct m* mods, int nmods, int free_mods)
 {    
 #ifdef ERTS_SMP
     if (is_blocking || !commit)
 #endif
     {
 	if (commit) {
+	    int i;
 	    erts_end_staging_code_ix();
 	    erts_commit_staging_code_ix();
-	    if (loaded) {
-		int i;
-		for (i=0; i < nloaded; i++) {		
-		    set_default_trace_pattern(loaded[i].module);
+
+	    for (i=0; i < nmods; i++) {
+		if (mods[i].modp->curr.code_hdr) {
+		    set_default_trace_pattern(mods[i].module);
 		}
+	      #ifdef HIPE
+		hipe_redirect_to_module(mods[i].modp);
+	      #endif
 	    }
 	}
 	else {
 	    erts_abort_staging_code_ix();
 	}
-	if (loaded) {
-	    erts_free(ERTS_ALC_T_LOADER_TMP, loaded);
+	if (free_mods) {
+	    erts_free(ERTS_ALC_T_LOADER_TMP, mods);
 	}
 	if (is_blocking) {
 	    erts_smp_thr_progress_unblock();
@@ -474,8 +490,8 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
     else {
 	ASSERT(is_value(res));
 
-	if (loaded) {
-	    erts_free(ERTS_ALC_T_LOADER_TMP, loaded);
+	if (free_mods) {
+	    erts_free(ERTS_ALC_T_LOADER_TMP, mods);
 	}
 	erts_end_staging_code_ix();
 	/*
@@ -653,8 +669,9 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
 	}
 	else {
 	    if (modp->curr.num_breakpoints > 0 ||
-		modp->curr.num_traced_exports > 0) {
-		/* we have tracing, retry single threaded */
+		modp->curr.num_traced_exports > 0 ||
+		IF_HIPE(hipe_need_blocking(modp))) {
+		/* tracing or hipe need to go single threaded */
 		erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 		erts_smp_thr_progress_block();
 		is_blocking = 1;
@@ -668,7 +685,14 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
 	    success = 1;
 	}
     }
-    return staging_epilogue(BIF_P, success, res, is_blocking, NULL, 0);  
+    {
+	struct m mod;
+	Eterm retval;
+	mod.module = BIF_ARG_1;
+	mod.modp = modp;
+	retval = staging_epilogue(BIF_P, success, res, is_blocking, &mod, 1, 0);
+	return retval;
+    }
 }
 
 BIF_RETTYPE module_loaded_1(BIF_ALIST_1)
@@ -809,6 +833,9 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	}
 	modp->curr.code_hdr->on_load_function_ptr = NULL;
 	set_default_trace_pattern(BIF_ARG_1);
+      #ifdef HIPE
+        hipe_redirect_to_module(modp);
+      #endif
     } else if (BIF_ARG_2 == am_false) {
 	int i;
 
@@ -1619,15 +1646,18 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 		/*
 		 * Unload any NIF library
 		 */
-		if (modp->old.nif != NULL) {
+		if (modp->old.nif != NULL
+		    || IF_HIPE(hipe_purge_need_blocking(modp))) {
 		    /* ToDo: Do unload nif without blocking */
 		    erts_rwunlock_old_code(code_ix);
 		    erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
 		    erts_smp_thr_progress_block();
 		    is_blocking = 1;
 		    erts_rwlock_old_code(code_ix);
-		    erts_unload_nif(modp->old.nif);
-		    modp->old.nif = NULL;
+		    if (modp->old.nif) {
+		      erts_unload_nif(modp->old.nif);
+		      modp->old.nif = NULL;
+		    }
 		}
 
 		/*
@@ -1646,7 +1676,9 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
 		modp->old.code_length = 0;
 		modp->old.catches = BEAM_CATCHES_NIL;
 		erts_remove_from_ranges(code);
-
+#ifdef HIPE
+		hipe_purge_module(modp);
+#endif
 		ERTS_BIF_PREP_RET(ret, am_true);
 	    }
 
@@ -1719,6 +1751,8 @@ delete_code(Module* modp)
 			 (BeamInstr) BeamOp(op_i_generic_breakpoint)) {
 		    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
 		    ASSERT(modp->curr.num_traced_exports > 0);
+		    DBG_TRACE_MFA(ep->code[0],ep->code[1],ep->code[2],
+				  "export trace cleared, code_ix=%d", code_ix);
 		    erts_clear_export_break(modp, ep->code+3);
 		}
 		else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler
@@ -1727,17 +1761,16 @@ delete_code(Module* modp)
 	    ep->addressv[code_ix] = ep->code+3;
 	    ep->code[3] = (BeamInstr) em_call_error_handler;
 	    ep->code[4] = 0;
+	    DBG_TRACE_MFA(ep->code[0],ep->code[1],ep->code[2],
+			  "export invalidation, code_ix=%d", code_ix);
 	}
     }
 
     ASSERT(modp->curr.num_breakpoints == 0);
     ASSERT(modp->curr.num_traced_exports == 0);
+    DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "delete_code old.first_hipe_ref=%p", modp->curr.first_hipe_ref);
     modp->old = modp->curr;
-    modp->curr.code_hdr = NULL;
-    modp->curr.code_length = 0;
-    modp->curr.catches = BEAM_CATCHES_NIL;
-    modp->curr.nif = NULL;
-
+    erts_module_instance_init(&modp->curr);
 }
 
 
@@ -1751,9 +1784,11 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module)
      * if not, delete old code; error if old code already exists.
      */
 
-    if (modp->curr.code_hdr && modp->old.code_hdr)  {
-	return am_not_purged;
-    } else if (!modp->old.code_hdr) { /* Make the current version old. */
+    if (modp->curr.code_hdr) {
+	if (modp->old.code_hdr)  {
+	    return am_not_purged;
+	}
+	/* Make the current version old. */
 	delete_code(modp);
     }
     return NIL;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 0ba06058a5..ccb0f786ec 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6105,6 +6105,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
     Uint sz;
     int i;
 
+    DBG_TRACE_MFA(fi[0], fi[1], fi[2], "call_error_handler");
     /*
      * Search for the error_handler module.
      */
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index d69b18e22f..9b206f9a23 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -482,7 +482,8 @@ static void free_literal_fragment(ErlHeapFragment*);
 static void loader_state_dtor(Binary* magic);
 static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 				  Eterm group_leader, Eterm module,
-				  BeamCodeHeader* code, Uint size);
+				  BeamCodeHeader* code, Uint size,
+				  void* hipe_code_start, UWord hipe_code_size);
 static int init_iff_file(LoaderState* stp, byte* code, Uint size);
 static int scan_iff_file(LoaderState* stp, Uint* chunk_types,
 			 Uint num_types, Uint num_mandatory);
@@ -842,9 +843,7 @@ erts_finish_loading(Binary* magic, Process* c_p,
 	    erts_alloc(ERTS_ALC_T_PREPARED_CODE,
 		       sizeof(struct erl_module_instance));
 	inst_p = mod_tab_p->on_load;
-	inst_p->nif = 0;
-        inst_p->num_breakpoints = 0;
-        inst_p->num_traced_exports = 0;
+        erts_module_instance_init(inst_p);
     }
 
     inst_p->code_hdr = stp->hdr;
@@ -1094,7 +1093,8 @@ loader_state_dtor(Binary* magic)
 static Eterm
 stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
 		     Eterm group_leader, Eterm module,
-		     BeamCodeHeader* code_hdr, Uint size)
+		     BeamCodeHeader* code_hdr, Uint size,
+		     void* hipe_code_start, UWord hipe_code_size)
 {
     Module* modp;
     Eterm retval;
@@ -1117,6 +1117,17 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
     modp->curr.code_hdr = code_hdr;
     modp->curr.code_length = size;
     modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */
+#if defined(HIPE)
+    DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code new_hipe_refs = %p", modp->new_hipe_refs);
+    modp->curr.first_hipe_ref = modp->new_hipe_refs;
+    modp->curr.first_hipe_sdesc = modp->new_hipe_sdesc;
+    modp->curr.hipe_code_start = hipe_code_start;
+    modp->new_hipe_refs = NULL;
+    modp->new_hipe_sdesc = NULL;
+# ifdef DEBUG
+    modp->curr.hipe_code_size  = hipe_code_size;
+# endif
+#endif
 
     /*
      * Update ranges (used for finding a function from a PC value).
@@ -4773,9 +4784,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
 	}
 	ep = erts_export_put(stp->module, stp->export[i].function,
 			     stp->export[i].arity);
-	if (!on_load) {
-	    ep->addressv[erts_staging_code_ix()] = address;
-	} else {
+	if (on_load) {
 	    /*
 	     * on_load: Don't make any of the exported functions
 	     * callable yet. Keep any function in the current
@@ -4783,6 +4792,8 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
 	     */
 	    ep->code[4] = (BeamInstr) address;
 	}
+        else
+            ep->addressv[erts_staging_code_ix()] = address;
     }
 
     /*
@@ -5989,17 +6000,12 @@ code_module_md5_1(BIF_ALIST_1)
 static BeamInstr*
 make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode)
 {
+    DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", &fp[5]);
     fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
     fp[1] = native;
     fp[2] = mod;
     fp[3] = func;
     fp[4] = arity;
-#ifdef HIPE
-    if (native) {
-	fp[5] = BeamOpCode(op_move_return_n);
-	hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5);
-    }
-#endif
     fp[5] = OpCode;
     return fp + WORDS_PER_FUNCTION;
 }
@@ -6087,6 +6093,8 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp)
 	if (stp->export[i].function == function && stp->export[i].arity == arity) {
 	    Export* ep = erts_export_put(mod, function, arity);
 	    ep->addressv[erts_staging_code_ix()] = fp+5;
+	    DBG_TRACE_MFA(mod,function,arity,"set beam stub at %p in export at %p (code_ix=%d)",
+			  fp+5, ep, erts_staging_code_ix());
 	    return;
 	}
     }
@@ -6282,6 +6290,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     byte* temp_alloc = NULL;
     byte* bytes;
     Uint size;
+    UWord hipe_code_start = NULL, hipe_code_size = 0;
 
     /*
      * Must initialize stp->lambdas here because the error handling code
@@ -6297,7 +6306,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
 	goto error;
     }
     tp = tuple_val(Info);
-    if (tp[0] != make_arityval(3)) {
+    if (tp[0] != make_arityval(5)) {
       goto error;
     }
     Funcs = tp[1];
@@ -6314,6 +6323,15 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
     }
     size = binary_size(Beam);
 
+#ifdef HIPE
+    if (!term_to_Uint(tp[4], &hipe_code_start))
+        goto error;
+# ifdef DEBUG
+    if (!term_to_Uint(tp[5], &hipe_code_size))
+        goto error;
+# endif
+#endif
+
     /*
      * Scan the Beam binary and read the interesting sections.
      */
@@ -6476,7 +6494,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
      */
 
     rval = stub_insert_new_code(p, 0, p->group_leader, Mod,
-				code_hdr, code_size);
+				code_hdr, code_size,
+				(void*)hipe_code_start, hipe_code_size);
     if (rval != NIL) {
 	goto error;
     }
@@ -6517,3 +6536,46 @@ static int safe_mul(UWord a, UWord b, UWord* resp)
     }
 }
 
+#ifdef ENABLE_DBG_TRACE_MFA
+
+#define MFA_MAX 10
+Eterm dbg_trace_m[MFA_MAX];
+Eterm dbg_trace_f[MFA_MAX];
+Uint  dbg_trace_a[MFA_MAX];
+unsigned int dbg_trace_ix = 0;
+
+void dbg_set_traced_mfa(const char* m, const char* f, Uint a)
+{
+    unsigned i = dbg_trace_ix++;
+    ASSERT(i < MFA_MAX);
+    dbg_trace_m[i] = am_atom_put(m, strlen(m));
+    dbg_trace_f[i] = am_atom_put(f, strlen(f));
+    dbg_trace_a[i] = a;
+}
+
+int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a)
+{
+    unsigned int i;
+    for (i = 0; i < dbg_trace_ix; ++i) {
+        if (m == dbg_trace_m[i] &&
+            (!f || (f == dbg_trace_f[i] && a == dbg_trace_a[i]))) {
+
+            return i+1;
+        }
+    }
+    return 0;
+}
+
+void dbg_vtrace_mfa(unsigned ix, const char* format, ...)
+{
+    va_list arglist;
+    va_start(arglist, format);
+    ASSERT(--ix < MFA_MAX);
+    erts_fprintf(stderr, "MFA TRACE %T:%T/%u: ",
+                 dbg_trace_m[ix], dbg_trace_f[ix], (int)dbg_trace_a[ix]);
+
+    erts_vfprintf(stderr, format, arglist);
+    va_end(arglist);
+}
+
+#endif /* ENABLE_DBG_TRACE_MFA */
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 6be4031822..5f0f34fcdd 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -151,4 +151,24 @@ struct BeamCodeLineTab_ {
 #define LOC_FILE(Loc) ((Loc) >> 24)
 #define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1))
 
+#ifdef DEBUG
+# define ENABLE_DBG_TRACE_MFA
+#endif
+
+#ifdef  ENABLE_DBG_TRACE_MFA
+
+void dbg_set_traced_mfa(const char* m, const char* f, Uint a);
+int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a);
+void dbg_vtrace_mfa(unsigned ix, const char* format, ...);
+#define DBG_TRACE_MFA(M,F,A,FMT, ...) do {\
+    unsigned ix;\
+    if ((ix=dbg_is_traced_mfa(M,F,A))) \
+        dbg_vtrace_mfa(ix, FMT"\n", ##__VA_ARGS__);\
+  }while(0)
+
+#else
+#  define dbg_set_traced_mfa(M,F,A)
+#  define DBG_TRACE_MFA(M,F,A,FMT, ...)
+#endif /* ENABLE_DBG_TRACE_MFA */
+
 #endif /* _BEAM_LOAD_H */
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 2a19211987..9da7fae1dc 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -145,6 +145,9 @@ export_alloc(struct export_entry* tmpl_e)
 	    blob->entryv[ix].ep = &blob->exp;
 	}
 	ix = 0;
+
+	DBG_TRACE_MFA(obj->code[0], obj->code[1], obj->code[2],
+		      "export allocation at %p", obj);
     }
     else { /* Existing entry in another table, use free entry in blob */
 	blob = entry_to_blob(tmpl_e);
@@ -163,9 +166,14 @@ export_free(struct export_entry* obj)
     obj->slot.index = -1;
     for (i=0; i < ERTS_NUM_CODE_IX; i++) {
 	if (blob->entryv[i].slot.index >= 0) {
+	    DBG_TRACE_MFA(blob->exp.code[0], blob->exp.code[1], blob->exp.code[2],
+			  "export entry slot %u freed for %p",
+			  (obj - blob->entryv), &blob->exp);
 	    return;
 	}
     }
+    DBG_TRACE_MFA(blob->exp.code[0], blob->exp.code[1], blob->exp.code[2],
+		      "export blob deallocation at %p", &blob->exp);
     erts_free(ERTS_ALC_T_EXPORT, blob);
     erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob));
 }
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index d4f6e17c56..93ce8fcf9f 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -26,6 +26,7 @@
 #include "erl_vm.h"
 #include "global.h"
 #include "module.h"
+#include "beam_catches.h"
 
 #ifdef DEBUG
 #  define IF_DEBUG(x) x
@@ -67,6 +68,23 @@ static int module_cmp(Module* tmpl, Module* obj)
     return tmpl->module != obj->module;
 }
 
+void erts_module_instance_init(struct erl_module_instance* modi)
+{
+    modi->code_hdr = 0;
+    modi->code_length = 0;
+    modi->catches = BEAM_CATCHES_NIL;
+    modi->nif = NULL;
+    modi->num_breakpoints = 0;
+    modi->num_traced_exports = 0;
+#ifdef HIPE
+    modi->first_hipe_ref = NULL;
+    modi->first_hipe_sdesc = NULL;
+    modi->hipe_code_start = NULL;
+# ifdef DEBUG
+    modi->hipe_code_size = 0;
+# endif
+#endif
+}
 
 static Module* module_alloc(Module* tmpl)
 {
@@ -74,18 +92,16 @@ static Module* module_alloc(Module* tmpl)
     erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module));
 
     obj->module = tmpl->module;
-    obj->curr.code_hdr = 0;
-    obj->old.code_hdr = 0;
-    obj->curr.code_length = 0;
-    obj->old.code_length = 0;
     obj->slot.index = -1;
-    obj->curr.nif = NULL;
-    obj->old.nif = NULL;
-    obj->curr.num_breakpoints = 0;
-    obj->old.num_breakpoints  = 0;
-    obj->curr.num_traced_exports = 0;
-    obj->old.num_traced_exports = 0;
+    erts_module_instance_init(&obj->curr);
+    erts_module_instance_init(&obj->old);
     obj->on_load = 0;
+#ifdef HIPE
+    obj->first_hipe_mfa = NULL;
+    obj->new_hipe_refs = NULL;
+    obj->new_hipe_sdesc = NULL;
+#endif
+    DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc");
     return obj;
 }
 
@@ -139,19 +155,14 @@ erts_get_module(Eterm mod, ErtsCodeIndex code_ix)
     }
 }
 
-Module*
-erts_put_module(Eterm mod)
+
+static Module* put_module(Eterm mod, IndexTable* mod_tab)
 {
     Module e;
-    IndexTable* mod_tab;
     int oldsz, newsz;
     Module* res;
 
     ASSERT(is_atom(mod));
-    ERTS_SMP_LC_ASSERT(erts_initialized == 0
-		       || erts_has_code_write_permission());
-
-    mod_tab = &module_tables[erts_staging_code_ix()];
     e.module = atom_val(mod);
     oldsz = index_table_sz(mod_tab);
     res = (Module*) index_put_entry(mod_tab, (void*) &e);
@@ -160,6 +171,24 @@ erts_put_module(Eterm mod)
     return res;
 }
 
+Module*
+erts_put_module(Eterm mod)
+{
+    ERTS_SMP_LC_ASSERT(erts_initialized == 0
+		       || erts_has_code_write_permission());
+
+    return put_module(mod, &module_tables[erts_staging_code_ix()]);
+}
+
+Module*
+erts_put_active_module(Eterm mod)
+{
+    ASSERT(is_atom(mod));
+    //SVERK Why not? ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+    return put_module(mod, &module_tables[erts_active_code_ix()]);
+}
+
 Module *module_code(int i, ErtsCodeIndex code_ix)
 {
     return (Module*) erts_index_lookup(&module_tables[code_ix], i);
@@ -186,6 +215,11 @@ static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod)
     dst_mod->curr = src_mod->curr;
     dst_mod->old = src_mod->old;
     dst_mod->on_load = src_mod->on_load;
+#ifdef HIPE
+    dst_mod->first_hipe_mfa = src_mod->first_hipe_mfa;
+    dst_mod->new_hipe_refs  = src_mod->new_hipe_refs;
+    dst_mod->new_hipe_sdesc = src_mod->new_hipe_sdesc;
+#endif
 }
 
 void module_start_staging(void)
@@ -217,6 +251,7 @@ void module_start_staging(void)
 	src_mod = (Module*) erts_index_lookup(src, i);
 	dst_mod = (Module*) index_put_entry(dst, src_mod);
 	ASSERT(dst_mod != src_mod);
+
         copy_module(dst_mod, src_mod);
     }
     newsz = index_table_sz(dst);
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 1c1afc8461..694583597b 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -30,6 +30,14 @@ struct erl_module_instance {
     struct erl_module_nif* nif;
     int num_breakpoints;
     int num_traced_exports;
+#ifdef HIPE
+    struct hipe_ref* first_hipe_ref;  /* all external hipe calls from this module */
+    struct hipe_sdesc* first_hipe_sdesc;  /* all stack descriptors for this module */
+    void* hipe_code_start;
+# ifdef DEBUG
+    UWord hipe_code_size;
+# endif
+#endif
 };
 
 typedef struct erl_module {
@@ -40,10 +48,17 @@ typedef struct erl_module {
     struct erl_module_instance curr;
     struct erl_module_instance old; /* protected by "old_code" rwlock */
     struct erl_module_instance* on_load;
+#ifdef HIPE
+    struct hipe_mfa_info* first_hipe_mfa;
+    struct hipe_ref* new_hipe_refs;
+    struct hipe_sdesc* new_hipe_sdesc;
+#endif
 } Module; 
 
+void erts_module_instance_init(struct erl_module_instance* modi);
 Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix);
 Module* erts_put_module(Eterm mod);
+Module* erts_put_active_module(Eterm mod); /* only while blocked */
 
 void init_module_table(void);
 void module_start_staging(void);
diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index 62739d2a78..df53f4db30 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -130,6 +130,13 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
     return alloc_code(nrbytes);
 }
 
+void hipe_free_code(void* code)
+{
+    ALLOC_CODE_STATS(--nr_allocs);
+    /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/
+    erts_free(ERTS_ALC_T_HIPE_EXEC, code);
+}
+
 /* Make stub for native code calling exported beam function.
 */
 void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
@@ -234,6 +241,14 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
     return code;
 }
 
+void hipe_free_native_stub(void* stub)
+{
+    ALLOC_CODE_STATS(++nr_allocs);
+    /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/
+
+    erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
+}
+
 void hipe_arch_print_pcb(struct hipe_process_state *p)
 {
 #define U(n,x) \
diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h
index 6f959815bb..df38a80069 100644
--- a/erts/emulator/hipe/hipe_arch.h
+++ b/erts/emulator/hipe/hipe_arch.h
@@ -31,22 +31,30 @@ extern int hipe_patch_insn(void *address, Uint value, Eterm type);
 extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline);
 
 extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p);
-extern void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity);
+extern void  hipe_free_code(void*);
+extern void *hipe_make_native_stub(void *exp, unsigned int beamArity);
+extern void  hipe_free_native_stub(void*);
+
 
 #if defined(__sparc__)
 #include "hipe_sparc.h"
+#include "hipe_sparc_asm.h"
 #endif
 #if defined(__i386__)
 #include "hipe_x86.h"
+#include "hipe_x86_asm.h"
 #endif
 #if defined(__x86_64__)
 #include "hipe_amd64.h"
+#include "hipe_amd64_asm.h"
 #endif
 #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
 #include "hipe_ppc.h"
+#include "hipe_ppc_asm.h"
 #endif
 #if defined(__arm__)
 #include "hipe_arm.h"
+#include "hipe_arm_asm.h"
 #endif
 
 #if !defined(AEXTERN)
diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c
index f8ef468341..64e35b62d8 100644
--- a/erts/emulator/hipe/hipe_arm.c
+++ b/erts/emulator/hipe/hipe_arm.c
@@ -198,6 +198,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
     return address;
 }
 
+void  hipe_free_code(void* code, unsigned int bytes)
+{
+    /*SVERK: Leaking code memory */
+}
+
 static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu)
 {
     unsigned int *address;
@@ -229,6 +234,11 @@ static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu)
     return address;
 }
 
+void hipe_free_native_stub(void* stub)
+{
+    /*SVERK: Leaking code stub */
+}
+
 /*
  * ARMv5's support for 32-bit immediates is effectively non-existent.
  * Hence, every 32-bit immediate is stored in memory and loaded via
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 95c2d73971..8be6dc9aa9 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -54,6 +54,7 @@
 
 #define BeamOpCode(Op)	((Uint)BeamOp(Op))
 
+
 int term_to_Sint32(Eterm term, Sint *sp)
 {
     Sint val;
@@ -644,30 +645,21 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
     pc = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari);
 
     if (pc) {
-	hipe_mfa_save_orig_beam_op(mfa.mod, mfa.fun, mfa.ari, pc);
-#if HIPE
-#ifdef DEBUG_LINKER
-	printf("%s: ", __FUNCTION__);
-	print_mfa(mfa.mod, mfa.fun, mfa.ari);
-	printf(": planting call trap to %p at BEAM pc %p\r\n", address, pc);
-#endif
+	DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", pc, address);
 	hipe_set_call_trap(pc, address, is_closure);
 	BIF_RET(am_true);
-#endif
     }
-#ifdef DEBUG_LINKER
-    printf("%s: ", __FUNCTION__);
-    print_mfa(mfa.mod, mfa.fun, mfa.ari);
-    printf(": no BEAM pc found\r\n");
-#endif
+    DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "failed set call trap to %p, no beam code found", address);
     BIF_RET(am_false);
 }
 
 BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
 {
     struct hipe_sdesc *sdesc;
+    Module* modp;
+    int do_commit;
 
-    sdesc = hipe_decode_sdesc(BIF_ARG_1);
+    sdesc = hipe_decode_sdesc(BIF_ARG_1, &do_commit);
     if (!sdesc) {
 	fprintf(stderr, "%s: bad sdesc!\r\n", __FUNCTION__);
 	BIF_ERROR(BIF_P, BADARG);
@@ -676,6 +668,21 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
 	fprintf(stderr, "%s: duplicate entry!\r\n", __FUNCTION__);
 	BIF_ERROR(BIF_P, BADARG);
     }
+
+    /*
+     * Link into list of sdesc's in same module instance
+     */
+    modp = erts_put_active_module(make_atom(sdesc->m_aix));
+    ASSERT(modp);
+    if (do_commit) { /* Direct "hipe-patching" of early loaded module */
+        sdesc->next_in_modi = modp->curr.first_hipe_sdesc;
+        modp->curr.first_hipe_sdesc = sdesc;
+    }
+    else {           /* Normal module loading/upgrade */
+        sdesc->next_in_modi = modp->new_hipe_sdesc;
+        modp->new_hipe_sdesc = sdesc;
+    }
+
     BIF_RET(NIL);
 }
 
@@ -970,7 +977,7 @@ BIF_RETTYPE hipe_bifs_get_fe_2(BIF_ALIST_2)
 
 	atom_buf[0] = '\0';
 	strncat(atom_buf, (char*)atom_tab(i)->name, atom_tab(i)->len);
-	printf("no fun entry for %s %ld:%ld\n", atom_buf, uniq, index);
+	printf("no fun entry for %s %ld:%ld\n", atom_buf, (unsigned long)uniq, (unsigned long)index);
 	BIF_ERROR(BIF_P, BADARG);
     }
     BIF_RET(address_to_term((void *)fe, BIF_P));
@@ -997,11 +1004,15 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2)
     BIF_RET(am_true);
 }
 
+struct hipe_ref_head {
+    struct hipe_ref_head* next;
+    struct hipe_ref_head* prev;
+};
+
 /*
- * MFA info hash table:
+ * An exported function called from or implemented by native code
  * - maps MFA to native code entry point
- * - the MFAs it calls (refers_to)
- * - the references to it (referred_from)
+ * - all references to it (callers)
  * - maps MFA to most recent trampoline [if powerpc or arm]
  */
 struct hipe_mfa_info {
@@ -1011,16 +1022,19 @@ struct hipe_mfa_info {
     } bucket;
     Eterm m;	/* atom */
     Eterm f;	/* atom */
-    unsigned int a;
+    unsigned int a : sizeof(int)*8 - 1;
+    unsigned int is_stub : 1;       /* if beam or not (yet) loaded */
     void *remote_address;
-    void *local_address;
-    Eterm *beam_code;
-    Uint orig_beam_op;
-    struct hipe_mfa_info_list *refers_to;
-    struct hipe_ref *referred_from;
+    void *new_address;
+    struct hipe_ref_head callers;   /* sentinel in list of hipe_ref's */
+    struct hipe_mfa_info* next_in_mod;
 #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
     void *trampoline;
 #endif
+#ifdef DEBUG
+    Export* dbg_export;
+#endif
+
 };
 
 static struct {
@@ -1038,6 +1052,25 @@ static struct {
     erts_smp_rwmtx_t lock;
 } hipe_mfa_info_table;
 
+
+/*
+ * An external native call site M:F(...)
+ * to be patched when the callee changes.
+ */
+struct hipe_ref {
+    struct hipe_ref_head head;    /* list of refs to same calleee */
+    void *address;
+    void *trampoline;
+    unsigned int flags;
+    struct hipe_ref* next_from_modi; /* list of refs from same module instance */
+#if defined(DEBUG)
+    struct hipe_mfa_info* callee;
+    Eterm caller_m, caller_f, caller_a;
+#endif
+};
+#define REF_FLAG_IS_LOAD_MFA		1	/* bit 0: 0 == call, 1 == load_mfa */
+
+
 static inline void hipe_mfa_info_table_init_lock(void)
 {
     erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock");
@@ -1108,15 +1141,18 @@ static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigne
     res->m = m;
     res->f = f;
     res->a = arity;
+    res->is_stub = 0;
     res->remote_address = NULL;
-    res->local_address = NULL;
-    res->beam_code = NULL;
-    res->orig_beam_op = 0;
-    res->refers_to = NULL;
-    res->referred_from = NULL;
+    res->new_address = NULL;
+    res->callers.next = &res->callers;
+    res->callers.prev = &res->callers;
+    res->next_in_mod = NULL;
 #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
     res->trampoline = NULL;
 #endif
+#ifdef DEBUG
+    res->dbg_export = NULL;
+#endif
 
     return res;
 }
@@ -1154,22 +1190,13 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter
     return NULL;
 }
 
-#if 0 /* XXX: unused */
-void *hipe_mfa_find_na(Eterm m, Eterm f, unsigned int arity)
-{
-    const struct hipe_mfa_info *p;
-
-    p = hipe_mfa_info_table_get(m, f, arity);
-    return p ? p->address : NULL;
-}
-#endif
-
 static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned int arity)
 {
     unsigned long h;
     unsigned int i;
     struct hipe_mfa_info *p;
     unsigned int size;
+    Module* modp;
 
     h = HIPE_MFA_HASH(m, f, arity);
     i = h & hipe_mfa_info_table.mask;
@@ -1189,23 +1216,47 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f,
     size = 1 << hipe_mfa_info_table.log2size;
     if (hipe_mfa_info_table.used > (4*size/5))		/* rehash at 80% */
 	hipe_mfa_info_table_grow();
+
+    modp = erts_put_active_module(m);
+    ASSERT(modp);
+    p->next_in_mod = modp->first_hipe_mfa;
+    modp->first_hipe_mfa = p;
+
+    DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p);
+
     return p;
 }
 
-static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address, int is_exported)
+static void remove_mfa_info(struct hipe_mfa_info* rm)
+{
+    unsigned int i;
+    struct hipe_mfa_info *p;
+    struct hipe_mfa_info **prevp;
+
+    i = rm->bucket.hvalue & hipe_mfa_info_table.mask;
+    prevp = &hipe_mfa_info_table.bucket[i];
+    for (;;) {
+        p = *prevp;
+        ASSERT(p);
+        if (p == rm) {
+            *prevp = p->bucket.next;
+            ASSERT(hipe_mfa_info_table.used > 0);
+            hipe_mfa_info_table.used--;
+            return;
+        }
+        prevp = &p->bucket.next;
+    }
+}
+
+static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address)
 {
     struct hipe_mfa_info *p;
 
     hipe_mfa_info_table_rwlock();
     p = hipe_mfa_info_table_put_rwlocked(m, f, arity);
-#ifdef DEBUG_LINKER
-    printf("%s: ", __FUNCTION__);
-    print_mfa(m, f, arity);
-    printf(": changing address from %p to %p\r\n", p->local_address, address);
-#endif
-    p->local_address = address;
-    if (is_exported)
-	p->remote_address = address;
+    DBG_TRACE_MFA(m,f,arity,"set native address in hipe_mfa_info at %p", p);
+    p->new_address = address;
+
     hipe_mfa_info_table_rwunlock();
 }
 
@@ -1237,168 +1288,85 @@ BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3)
 {
     struct hipe_mfa mfa;
     void *address;
-    int is_exported;
 
-    if (!term_to_mfa(BIF_ARG_1, &mfa))
-	BIF_ERROR(BIF_P, BADARG);
-    address = term_to_address(BIF_ARG_2);
-    if (!address)
-	BIF_ERROR(BIF_P, BADARG);
-    if (BIF_ARG_3 == am_true)
-	is_exported = 1;
-    else if (BIF_ARG_3 == am_false)
-	is_exported = 0;
-    else
+    switch (BIF_ARG_3) {
+    case am_true: /* is_exported */
+        if (!term_to_mfa(BIF_ARG_1, &mfa))
+            BIF_ERROR(BIF_P, BADARG);
+        address = term_to_address(BIF_ARG_2);
+        if (!address)
+            BIF_ERROR(BIF_P, BADARG);
+        hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address);
+        break;
+    case am_false:
+        break; /* ignore local functions */
+    default:
 	BIF_ERROR(BIF_P, BADARG);
-    hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address, is_exported);
-    BIF_RET(NIL);
-}
-
-BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1)
-{
-    Eterm lst;
-    struct hipe_mfa mfa;
-    struct hipe_mfa_info *p;
-
-    hipe_mfa_info_table_rwlock();
-    lst = BIF_ARG_1;
-    while (is_list(lst)) {
-	if (!term_to_mfa(CAR(list_val(lst)), &mfa))
-	    break;
-	lst = CDR(list_val(lst));
-	p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
-	if (p) {
-	    p->remote_address = NULL;
-	    p->local_address = NULL;
-	    if (p->beam_code) {
-#ifdef DEBUG_LINKER
-		printf("%s: ", __FUNCTION__);
-		print_mfa(mfa.mod, mfa.fun, mfa.ari);
-		printf(": removing call trap from BEAM pc %p (new op %#lx)\r\n",
-		       p->beam_code, p->orig_beam_op);
-#endif
-		p->beam_code[0] = p->orig_beam_op;
-		p->beam_code = NULL;
-		p->orig_beam_op = 0;
-	    } else {
-#ifdef DEBUG_LINKER
-		printf("%s: ", __FUNCTION__);
-		print_mfa(mfa.mod, mfa.fun, mfa.ari);
-		printf(": no call trap to remove\r\n");
-#endif
-	    }
-	}
     }
-    hipe_mfa_info_table_rwunlock();
-    if (is_not_nil(lst))
-	BIF_ERROR(BIF_P, BADARG);
     BIF_RET(NIL);
 }
 
-void hipe_mfa_save_orig_beam_op(Eterm mod, Eterm fun, unsigned int ari, Eterm *pc)
+
+/* Ask if we need to block all threads
+ * while loading/deleting (beam) code for this module?
+ */
+int hipe_need_blocking(Module* modp)
 {
-    Uint orig_beam_op;
     struct hipe_mfa_info *p;
 
-    orig_beam_op = pc[0];
-    if (orig_beam_op != BeamOpCode(op_hipe_trap_call_closure) &&
-	orig_beam_op != BeamOpCode(op_hipe_trap_call)) {
-	hipe_mfa_info_table_rwlock();
-	p = hipe_mfa_info_table_put_rwlocked(mod, fun, ari);
-#ifdef DEBUG_LINKER
-	printf("%s: ", __FUNCTION__);
-	print_mfa(mod, fun, ari);
-	printf(": saving orig op %#lx from BEAM pc %p\r\n", orig_beam_op, pc);
-#endif
-	p->beam_code = pc;
-	p->orig_beam_op = orig_beam_op;
-	hipe_mfa_info_table_rwunlock();
-    } else {
-#ifdef DEBUG_LINKER
-	printf("%s: ", __FUNCTION__);
-	print_mfa(mod, fun, ari);
-	printf(": orig op %#lx already saved\r\n", orig_beam_op);
-#endif
+    /* Need to block if we have at least one native caller to this module
+     * or native code to make unaccessible.
+     */
+    hipe_mfa_info_table_rlock();
+    for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) {
+        ASSERT(!p->new_address);
+        if (p->callers.next != &p->callers || !p->is_stub) {
+            break;
+	}
     }
+    hipe_mfa_info_table_runlock();
+    return (p != NULL);
 }
 
-static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote)
-{
-    Export *export_entry;
-    void *StubAddress;
-
-    ASSERT(is_remote);
-
-    export_entry = erts_export_get_or_make_stub(m, f, arity);
-    StubAddress = hipe_make_native_stub(export_entry, arity);
-    if (!StubAddress)
-	erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
-    return StubAddress;
-}
-
-static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info **pp)
+static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a)
 {
     struct hipe_mfa_info *p;
-    void *address;
 
     p = hipe_mfa_info_table_get_locked(m, f, a);
-    if (p) {
-	/* find address, predicting for a runtime apply call */
-	address = p->remote_address;
-	if (!is_remote)
-	    address = p->local_address;
-	if (address)
-	    return address;
-
-	/* bummer, install stub, checking if one already existed */
-	address = p->remote_address;
-	if (address)
-	    return address;
-    }
-    /* Caller must take the slow path with the write lock held, but allow
-       it to avoid some work if it already holds the write lock.  */
-    if (pp)
-	*pp = p;
-    return NULL;
-}
-
-static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info *p)
-{
-    void *address;
-
-    if (!p)
-	p = hipe_mfa_info_table_put_rwlocked(m, f, a);
-    address = hipe_make_stub(m, f, a, is_remote);
-    /* XXX: how to tell if a BEAM MFA is exported or not? */
-    p->remote_address = address;
-    return address;
+    return p ? p->remote_address : NULL;
 }
 
-static void *hipe_get_na_nofail_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote)
+static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a)
 {
-    struct hipe_mfa_info *p;
-    void *address;
+    struct hipe_mfa_info *p = hipe_mfa_info_table_put_rwlocked(m, f, a);
 
-    address = hipe_get_na_try_locked(m, f, a, is_remote, &p);
-    if (address)
-	return address;
+    if (!p->remote_address) {
+        Export* export_entry = erts_export_get_or_make_stub(m, f, a);
+        void* stubAddress = hipe_make_native_stub(export_entry, a);
+        if (!stubAddress)
+            erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
 
-    address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, p);
-    return address;
+        p->remote_address = stubAddress;
+        p->is_stub = 1;
+#ifdef DEBUG
+        p->dbg_export = export_entry;
+#endif
+    }
+    return p->remote_address;
 }
 
-static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a, int is_remote)
+static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a)
 {
     void *address;
 
     hipe_mfa_info_table_rlock();
-    address = hipe_get_na_try_locked(m, f, a, is_remote, NULL);
+    address = hipe_get_na_try_locked(m, f, a);
     hipe_mfa_info_table_runlock();
     if (address)
 	return address;
 
     hipe_mfa_info_table_rwlock();
-    address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, NULL);
+    address = hipe_get_na_slow_rwlocked(m, f, a);
     hipe_mfa_info_table_rwunlock();
     return address;
 }
@@ -1408,7 +1376,7 @@ void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a)
 {
     if (is_not_atom(m) || is_not_atom(f) || a > 255)
 	return NULL;
-    return hipe_get_na_nofail(m, f, a, 1);
+    return hipe_get_na_nofail(m, f, a);
 }
 
 /* primop, but called like a BIF for error handling purposes */
@@ -1420,25 +1388,19 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3)
     if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2))
 	BIF_ERROR(BIF_P, BADARG);
     arity = unsigned_val(BIF_ARG_3); /* no error check */
-    address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity, 1);
+    address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity);
     BIF_RET((Eterm)address);	/* semi-Ok */
 }
 
-BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2)
+BIF_RETTYPE hipe_bifs_find_na_or_make_stub_1(BIF_ALIST_1)
 {
     struct hipe_mfa mfa;
     void *address;
-    int is_remote;
 
     if (!term_to_mfa(BIF_ARG_1, &mfa))
 	BIF_ERROR(BIF_P, BADARG);
-    if (BIF_ARG_2 == am_true)
-	is_remote = 1;
-    else if (BIF_ARG_2 == am_false)
-	is_remote = 0;
-    else
-	BIF_ERROR(BIF_P, BADARG);
-    address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari, is_remote);
+
+    address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari);
     BIF_RET(address_to_term(address, BIF_P));
 }
 
@@ -1460,7 +1422,7 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2)
 	f = ep->code[1];
     } else
 	goto badfun;
-    address = hipe_get_na_nofail(m, f, BIF_ARG_2, 1);
+    address = hipe_get_na_nofail(m, f, BIF_ARG_2);
     BIF_RET((Eterm)address);
 
  badfun:
@@ -1471,60 +1433,19 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2)
 
 int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a)
 {
-    struct hipe_mfa_info *mfa;
-    long mfa_offset, ra_offset;
-    struct hipe_mfa_info **bucket;
-    unsigned int i, nrbuckets;
+    const struct hipe_sdesc* sdesc = hipe_find_sdesc((unsigned long)ra);
 
-    /* Note about locking: the table is only updated from the
-       loader, which runs with the rest of the system suspended. */
-    /* XXX: alas not true; see comment at hipe_mfa_info_table.lock */
-    hipe_mfa_info_table_rlock();
-    bucket = hipe_mfa_info_table.bucket;
-    nrbuckets = 1 << hipe_mfa_info_table.log2size;
-    mfa = NULL;
-    mfa_offset = LONG_MAX;
-    for (i = 0; i < nrbuckets; ++i) {
-	struct hipe_mfa_info *b = bucket[i];
-	while (b != NULL) {
-	    ra_offset = (char*)ra - (char*)b->local_address;
-	    if (ra_offset > 0 && ra_offset < mfa_offset) {
-		mfa_offset = ra_offset;
-		mfa = b;
-	    }
-	    b = b->bucket.next;
-	}
-    }
-    if (mfa) {
-	*m = mfa->m;
-	*f = mfa->f;
-	*a = mfa->a;
-    }
-    hipe_mfa_info_table_runlock();
-    return mfa ? 1 : 0;
-}
+    if (!sdesc || sdesc->m_aix == atom_val(am_Empty))
+        return 0;
 
-/*
- * Patch Reference Handling.
- */
-struct hipe_mfa_info_list {
-    struct hipe_mfa_info *mfa;
-    struct hipe_mfa_info_list *next;
-};
+    *m = make_atom(sdesc->m_aix);
+    *f = make_atom(sdesc->f_aix);
+    *a = sdesc->a;
+    return 1;
+}
 
-struct hipe_ref {
-    struct hipe_mfa_info *caller_mfa;
-    void *address;
-    void *trampoline;
-    unsigned int flags;
-    struct hipe_ref *next;
-};
-#define REF_FLAG_IS_LOAD_MFA		1	/* bit 0: 0 == call, 1 == load_mfa */
-#define REF_FLAG_IS_REMOTE		2	/* bit 1: 0 == local, 1 == remote */
-#define REF_FLAG_PENDING_REDIRECT	4	/* bit 2: 1 == pending redirect */
-#define REF_FLAG_PENDING_REMOVE		8	/* bit 3: 1 == pending remove */
 
-/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,'remote'|'local'})
+/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,DoCommit})
  */
 BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 {
@@ -1535,9 +1456,9 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
     void *trampoline;
     unsigned int flags;
     struct hipe_mfa_info *callee_mfa;
-    struct hipe_mfa_info *caller_mfa;
-    struct hipe_mfa_info_list *refers_to;
     struct hipe_ref *ref;
+    Module* modp;
+    int do_commit;
 
     if (!term_to_mfa(BIF_ARG_1, &callee))
 	goto badarg;
@@ -1569,62 +1490,90 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
 	    goto badarg;
     }
     switch (tuple[5]) {
-      case am_local:
-	break;
-      case am_remote:
-	flags |= REF_FLAG_IS_REMOTE;
-	break;
-      default:
-	goto badarg;
+    case am_true: do_commit = 1; break;
+    case am_false: do_commit = 0; break;
+    default: goto badarg;
     }
     hipe_mfa_info_table_rwlock();
     callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari);
-    caller_mfa = hipe_mfa_info_table_put_rwlocked(caller.mod, caller.fun, caller.ari);
-
-    refers_to = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*refers_to));
-    refers_to->mfa = callee_mfa;
-    refers_to->next = caller_mfa->refers_to;
-    caller_mfa->refers_to = refers_to;
 
-    ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*ref));
-    ref->caller_mfa = caller_mfa;
+    ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref));
     ref->address = address;
     ref->trampoline = trampoline;
     ref->flags = flags;
-    ref->next = callee_mfa->referred_from;
-    callee_mfa->referred_from = ref;
+
+    /*
+     * Link into list of refs to same callee
+     */
+    ASSERT(callee_mfa->callers.next->prev == &callee_mfa->callers);
+    ASSERT(callee_mfa->callers.prev->next == &callee_mfa->callers);
+    ref->head.next = callee_mfa->callers.next;
+    ref->head.prev = &callee_mfa->callers;
+    ref->head.next->prev = &ref->head;
+    ref->head.prev->next = &ref->head;
+
+    /*
+     * Link into list of refs from same module instance
+     */
+    modp = erts_put_active_module(caller.mod);
+    ASSERT(modp);
+    if (do_commit) { /* Direct "hipe-patching" of early loaded module */
+        ref->next_from_modi = modp->curr.first_hipe_ref;
+        modp->curr.first_hipe_ref = ref;
+    }
+    else {           /* Normal module loading/upgrade */
+        ref->next_from_modi = modp->new_hipe_refs;
+        modp->new_hipe_refs = ref;
+    }
+
+#if defined(DEBUG)
+    ref->callee = callee_mfa;
+    ref->caller_m = caller.mod;
+    ref->caller_f = caller.fun;
+    ref->caller_a = caller.ari;
+#endif
     hipe_mfa_info_table_rwunlock();
 
+    DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p) do_commit=%d",
+		  ref, callee.mod, callee.fun, callee.ari, ref->address, do_commit);
+    DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p) do_commit=%d",
+		  ref, caller.mod, caller.fun, caller.ari, ref->address, do_commit);
     BIF_RET(NIL);
 
  badarg:
     BIF_ERROR(BIF_P, BADARG);
 }
 
-/* Given a CalleeMFA, mark each ref to it as pending-redirect.
- * This ensures that remove_refs_from() won't remove them: any
- * removal is instead done at the end of redirect_referred_from().
- */
-BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */
+
+static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me)
 {
-    struct hipe_mfa mfa;
-    const struct hipe_mfa_info *p;
-    struct hipe_ref *ref;
+    Module* modp = erts_get_module(unlink_me->m, erts_active_code_ix());
+    struct hipe_mfa_info** prevp = &modp->first_hipe_mfa;
+    struct hipe_mfa_info* p;
 
-    if (!term_to_mfa(BIF_ARG_1, &mfa))
-	BIF_ERROR(BIF_P, BADARG);
-    hipe_mfa_info_table_rwlock();
-    p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
-    if (p)
-	for (ref = p->referred_from; ref != NULL; ref = ref->next)
-	    ref->flags |= REF_FLAG_PENDING_REDIRECT;
-    hipe_mfa_info_table_rwunlock();
-    BIF_RET(NIL);
+    ASSERT(modp);
+    for (;;) {
+        p = *prevp;
+        ASSERT(p && p->m == unlink_me->m);
+        if (p == unlink_me) {
+            *prevp = p->next_in_mod;
+            break;
+        }
+         prevp = &p->next_in_mod;
+    }
+}
+
+static void purge_mfa(struct hipe_mfa_info* p)
+{
+    ASSERT(p->is_stub);
+    remove_mfa_info(p);
+    hipe_free_native_stub(p->remote_address);
+    erts_free(ERTS_ALC_T_HIPE, p);
 }
 
 /* Called by init:restart after unloading all hipe compiled modules
- * to work around bug causing execution of deallocated beam code.
- * Can be removed when delete/purge of native modules works better.
+ * to work around old bug that caused execution of deallocated beam code.
+ * Can be removed now when delete/purge of native modules works better.
  * Test: Do init:restart in debug compiled vm with hipe compiled kernel. 
  */
 static void hipe_purge_all_refs(void)
@@ -1634,126 +1583,205 @@ static void hipe_purge_all_refs(void)
 
     hipe_mfa_info_table_rwlock();
 
+    ASSERT(hipe_mfa_info_table.used == 0);
     bucket = hipe_mfa_info_table.bucket;
     nrbuckets = 1 << hipe_mfa_info_table.log2size;
     for (i = 0; i < nrbuckets; ++i) {	
+        ASSERT(bucket[i] == NULL);
 	while (bucket[i] != NULL) {
+            Module* modp;
 	    struct hipe_mfa_info* mfa = bucket[i];
 	    bucket[i] = mfa->bucket.next;
 
-	    while (mfa->refers_to) {
-		struct hipe_mfa_info_list *to = mfa->refers_to;
-		mfa->refers_to = to->next;
-		erts_free(ERTS_ALC_T_HIPE, to);
-	    }
-	    while (mfa->referred_from) {
-		struct hipe_ref* from = mfa->referred_from;
-		mfa->referred_from = from->next;
-		erts_free(ERTS_ALC_T_HIPE, from);
-	    }
+	    ASSERT(mfa->callers.next == &mfa->callers);
+
+            modp = erts_get_module(mfa->m, erts_active_code_ix());
+            if (modp) {
+                /* unsafe write to active module */
+                modp->first_hipe_mfa = NULL;
+            }
 	    erts_free(ERTS_ALC_T_HIPE, mfa);
 	}
     }
+    hipe_mfa_info_table.used = 0;
     hipe_mfa_info_table_rwunlock();
 }
 
 BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1)
 {
-    struct hipe_mfa mfa;
-    struct hipe_mfa_info *caller_mfa, *callee_mfa;
-    struct hipe_mfa_info_list *refers_to, *tmp_refers_to;
-    struct hipe_ref **prev, *ref;
-
     if (BIF_ARG_1 == am_all) {
 	hipe_purge_all_refs();
 	BIF_RET(am_ok);
     }
 
-    if (!term_to_mfa(BIF_ARG_1, &mfa))
-	BIF_ERROR(BIF_P, BADARG);
-    hipe_mfa_info_table_rwlock();
-    caller_mfa = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
-    if (caller_mfa) {
-	refers_to = caller_mfa->refers_to;
-	while (refers_to) {
-	    callee_mfa = refers_to->mfa;
-	    prev = &callee_mfa->referred_from;
-	    ref = *prev;
-	    while (ref) {
-		if (ref->caller_mfa == caller_mfa) {
-		    if (ref->flags & REF_FLAG_PENDING_REDIRECT) {
-			ref->flags |= REF_FLAG_PENDING_REMOVE;
-			prev = &ref->next;
-			ref = ref->next;
-		    } else {
-			struct hipe_ref *tmp = ref;
-			ref = ref->next;
-			*prev = ref;
-			erts_free(ERTS_ALC_T_HIPE, tmp);
-		    }
-		} else {
-		    prev = &ref->next;
-		    ref = ref->next;
-		}
-	    }
-	    tmp_refers_to = refers_to;
-	    refers_to = refers_to->next;
-	    erts_free(ERTS_ALC_T_HIPE, tmp_refers_to);
-	}
-	caller_mfa->refers_to = NULL;
+    ASSERT(!"hipe_bifs_remove_refs_from_1() called");
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+int hipe_purge_need_blocking(Module* modp)
+{
+    /* SVERK: Verify if this is really necessary */
+    return (modp->old.first_hipe_ref ||
+            modp->old.first_hipe_sdesc ||
+            (!modp->curr.code_hdr && modp->first_hipe_mfa));
+}
+
+void hipe_purge_module(Module* modp)
+{
+    struct hipe_ref* ref;
+    struct hipe_sdesc* sdesc;
+
+    ASSERT(modp);
+
+    DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module");
+
+    /*
+     * Remove all hipe_ref's (external calls) from the old module instance
+     */
+    ref = modp->old.first_hipe_ref;
+
+    while (ref) {
+	struct hipe_ref* free_ref = ref;
+
+        ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+	DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref,
+		      ref->callee->m, ref->callee->f, ref->callee->a);
+	DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref,
+		      ref->caller_m, ref->caller_f, ref->caller_a);
+	ASSERT(ref->caller_m == make_atom(modp->module));
+
+        /*
+         * Unlink from other refs to same callee
+         */
+        ASSERT(ref->head.next->prev == &ref->head);
+        ASSERT(ref->head.prev->next == &ref->head);
+        ASSERT(ref->head.next != &ref->head);
+        ASSERT(ref->head.prev != &ref->head);
+        ref->head.next->prev = ref->head.prev;
+        ref->head.prev->next = ref->head.next;
+
+        /*
+         * Was this the last ref to that callee?
+         */
+        if (ref->head.next == ref->head.prev) {
+            struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers);
+            if (p->is_stub) {
+                unlink_mfa_from_mod(p);
+                purge_mfa(p);
+           }
+        }
+
+	ref = ref->next_from_modi;
+	erts_free(ERTS_ALC_T_HIPE, free_ref);
+    }
+    modp->old.first_hipe_ref = NULL;
+
+    /*
+     * Remove all hipe_sdesc's for the old module instance
+     */
+    sdesc = modp->old.first_hipe_sdesc;
+
+    while (sdesc) {
+	struct hipe_sdesc* free_sdesc = sdesc;
+
+        ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+	DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue);
+	ASSERT(sdesc->m_aix == modp->module);
+
+	sdesc = sdesc->next_in_modi;
+        hipe_destruct_sdesc(free_sdesc);
+    }
+    modp->old.first_hipe_sdesc = NULL;
+
+    /*
+     * Remove unreferred hipe_mfa_info's
+     */
+    if (modp->curr.code_hdr == NULL) {
+        struct hipe_mfa_info** prevp = &modp->first_hipe_mfa;
+        struct hipe_mfa_info* p = *prevp;
+        ERTS_SMP_LC_ASSERT(!p || erts_smp_thr_progress_is_blocking());
+        for (; p; p = *prevp) {
+            if (p->callers.next == &p->callers) {
+                *prevp = p->next_in_mod;
+                purge_mfa(p);
+            }
+            else
+                prevp = &p->next_in_mod;
+        }
+    }
+    if (modp->old.hipe_code_start) {
+#ifdef DEBUG
+        sys_memset(modp->old.hipe_code_start, 0xfe, modp->old.hipe_code_size);
+#endif
+        hipe_free_code(modp->old.hipe_code_start);
+        modp->old.hipe_code_start = NULL;
     }
-    hipe_mfa_info_table_rwunlock();
-    BIF_RET(am_ok);
 }
 
 
-/* redirect_referred_from(CalleeMFA)
- * Redirect all pending-redirect refs in CalleeMFA's referred_from.
- * Then remove any pending-redirect && pending-remove refs from CalleeMFA's referred_from.
+/*
+ * Redirect all existing native calls to this module
  */
-BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1)
+void hipe_redirect_to_module(Module* modp)
 {
-    struct hipe_mfa mfa;
     struct hipe_mfa_info *p;
-    struct hipe_ref **prev, *ref;
-    int is_remote, res;
-    void *new_address;
+    struct hipe_ref_head* refh;
+
+    ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+    for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) {
+        if (p->new_address) {
+            if (p->is_stub) {
+                hipe_free_native_stub(p->remote_address);
+                p->is_stub = 0;
+            }
+            DBG_TRACE_MFA(p->m, p->f, p->a, "Commit new_address %p", p->new_address);
+            p->remote_address = p->new_address;
+            p->new_address = NULL;
+#ifdef DEBUG
+            p->dbg_export = NULL;
+#endif
+        }
+        else if (!p->is_stub) {
+            Export* exp = erts_export_get_or_make_stub(p->m, p->f, p->a);
+            p->remote_address = hipe_make_native_stub(exp, p->a);
+            DBG_TRACE_MFA(p->m, p->f, p->a, "Commit stub %p", p->remote_address);
+            if (!p->remote_address)
+                erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
+            p->is_stub = 1;
+#ifdef DEBUG
+            p->dbg_export = exp;
+#endif
+        }
+        else {
+            DBG_TRACE_MFA(p->m, p->f, p->a, "Commit no-op, already stub");
+            ASSERT(p->remote_address && p->dbg_export);
+        }
 
-    if (!term_to_mfa(BIF_ARG_1, &mfa))
-	BIF_ERROR(BIF_P, BADARG);
-    hipe_mfa_info_table_rwlock();
-    p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
-    if (p) {
-	prev = &p->referred_from;
-	ref = *prev;
-	while (ref) {
-	    if (ref->flags & REF_FLAG_PENDING_REDIRECT) {
-		is_remote = ref->flags & REF_FLAG_IS_REMOTE;
-		new_address = hipe_get_na_nofail_rwlocked(p->m, p->f, p->a, is_remote);
-		if (ref->flags & REF_FLAG_IS_LOAD_MFA)
-		    res = hipe_patch_insn(ref->address, (Uint)new_address, am_load_mfa);
-		else
-		    res = hipe_patch_call(ref->address, new_address, ref->trampoline);
-		if (res)
-		    fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__);
-		ref->flags &= ~REF_FLAG_PENDING_REDIRECT;
-		if (ref->flags & REF_FLAG_PENDING_REMOVE) {
-		    struct hipe_ref *tmp = ref;
-		    ref = ref->next;
-		    *prev = ref;
-		    erts_free(ERTS_ALC_T_HIPE, tmp);
-		} else {
-		    prev = &ref->next;
-		    ref = ref->next;
-		}
-	    } else {
-		prev = &ref->next;
-		ref = ref->next;
-	    }
+	DBG_TRACE_MFA(p->m,p->f,p->a,"START REDIRECT towards hipe_mfa_info at %p", p);
+	for (refh = p->callers.next; refh != &p->callers; refh = refh->next) {
+            struct hipe_ref* ref = (struct hipe_ref*) refh;
+	    int res;
+
+	    DBG_TRACE_MFA(p->m,p->f,p->a, "  REDIRECT ref at %p FROM %T:%T/%u (%p -> %p)",
+			  ref, ref->caller_m, ref->caller_f, ref->caller_a,
+			  ref->address, p->remote_address);
+
+	    DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a,
+			  "  REDIRECT ref at %p TO %T:%T/%u (%p -> %p)",
+			  ref, p->m,p->f,p->a, ref->address, p->remote_address);
+
+	    if (ref->flags & REF_FLAG_IS_LOAD_MFA)
+		res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa);
+	    else
+		res = hipe_patch_call(ref->address, p->remote_address, ref->trampoline);
+	    if (res)
+		fprintf(stderr, "%s: patch failed", __FUNCTION__);
 	}
+	DBG_TRACE_MFA(p->m,p->f,p->a,"DONE REDIRECT towards hipe_mfa_info at %p", p);
     }
-    hipe_mfa_info_table_rwunlock();
-    BIF_RET(NIL);
 }
 
 BIF_RETTYPE hipe_bifs_check_crc_1(BIF_ALIST_1)
diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h
index c9a8216368..1914405e2d 100644
--- a/erts/emulator/hipe/hipe_bif0.h
+++ b/erts/emulator/hipe/hipe_bif0.h
@@ -42,7 +42,10 @@ extern void hipe_primop_set_trampoline(Eterm name, void *trampoline);
 #endif
 
 /* needed in beam_load.c */
-void hipe_mfa_save_orig_beam_op(Eterm m, Eterm f, unsigned int a, Eterm *pc);
+int hipe_need_blocking(Module*);
+int hipe_purge_need_blocking(Module*);
+void hipe_purge_module(Module*);
+void hipe_redirect_to_module(Module* modp);
 
 /* these are also needed in hipe_amd64.c */
 extern void *term_to_address(Eterm);
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 99237aae05..eae84383df 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -54,7 +54,7 @@ bif hipe_bifs:set_native_address/3
 #bif hipe_bifs:address_to_fun/1
 
 bif hipe_bifs:set_funinfo_native_address/3
-bif hipe_bifs:invalidate_funinfo_native_addresses/1
+#bif hipe_bifs:invalidate_funinfo_native_addresses/1
 
 bif hipe_bifs:update_code_size/3
 bif hipe_bifs:code_size/1
@@ -72,7 +72,7 @@ bif hipe_bifs:term_to_word/1
 bif hipe_bifs:get_fe/2
 bif hipe_bifs:set_native_address_in_fe/2
 
-bif hipe_bifs:find_na_or_make_stub/2
+bif hipe_bifs:find_na_or_make_stub/1
 
 bif hipe_bifs:check_crc/1
 bif hipe_bifs:system_crc/0
@@ -84,9 +84,7 @@ bif hipe_bifs:patch_insn/3
 bif hipe_bifs:patch_call/3
 
 bif hipe_bifs:add_ref/2
-bif hipe_bifs:mark_referred_from/1
 bif hipe_bifs:remove_refs_from/1
-bif hipe_bifs:redirect_referred_from/1
 
 # atoms used by add_ref/2
 atom call
diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c
index 9b2048c457..a1a6e2ad02 100644
--- a/erts/emulator/hipe/hipe_ppc.c
+++ b/erts/emulator/hipe/hipe_ppc.c
@@ -214,6 +214,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
     return address;
 }
 
+void hipe_free_code(void* code, unsigned int bytes)
+{
+    /*SVERK: Leaking code memory */
+}
+
 static unsigned int *alloc_stub(Uint nrwords)
 {
     unsigned int *address;
@@ -241,6 +246,11 @@ static unsigned int *alloc_stub(Uint nrwords)
     return address;
 }
 
+void hipe_free_native_stub(void* stub)
+{
+    /*SVERK: Leaking code stubs */
+}
+
 static void patch_imm16(Uint32 *address, unsigned int imm16)
 {
     unsigned int insn = *address;
diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h
index 09804e3016..1369b392fe 100644
--- a/erts/emulator/hipe/hipe_risc_glue.h
+++ b/erts/emulator/hipe/hipe_risc_glue.h
@@ -66,15 +66,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y)
 
 static __inline__ void hipe_arch_glue_init(void)
 {
-    static struct hipe_sdesc_with_exnra nbif_return_sdesc = {
-	.exnra = (unsigned long)&nbif_fail,
-	.sdesc = {
-	    .bucket = { .hvalue = (unsigned long)&nbif_return },
-            .fsize = 0,
-            .has_exnra = 1,
-            .arity = 0
-	},
-    };
+    static struct hipe_sdesc_with_exnra nbif_return_sdesc;
+
+    nbif_return_sdesc.exnra = (unsigned long)nbif_fail;
+    nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return;
+    nbif_return_sdesc.sdesc.fsize = 0;
+    nbif_return_sdesc.sdesc.has_exnra = 1;
+    nbif_return_sdesc.sdesc.stk_nargs = 0;
+    nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty);
+    nbif_return_sdesc.sdesc.f_aix = atom_val(am_return);
+    nbif_return_sdesc.sdesc.a     = 0;
+
     hipe_init_sdesc_table(&nbif_return_sdesc.sdesc);
 }
 
diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c
index 17bef0718c..3d082668a6 100644
--- a/erts/emulator/hipe/hipe_stack.c
+++ b/erts/emulator/hipe/hipe_stack.c
@@ -102,6 +102,28 @@ struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc)
     return sdesc;
 }
 
+void hipe_destruct_sdesc(struct hipe_sdesc *sdesc)
+{
+    unsigned int i;
+    struct hipe_sdesc** prevp;
+    void* free_me;
+
+    i = (sdesc->bucket.hvalue >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask;
+    prevp = &hipe_sdesc_table.bucket[i];
+
+    for (; *prevp != sdesc; prevp = &(*prevp)->bucket.next)
+        ASSERT(*prevp);
+
+    *prevp = sdesc->bucket.next;
+    hipe_sdesc_table.used -= 1;
+
+    if (sdesc->has_exnra)
+        free_me = ErtsContainerStruct(sdesc, struct hipe_sdesc_with_exnra, sdesc);
+    else
+        free_me = sdesc;
+    erts_free(ERTS_ALC_T_HIPE, free_me);
+}
+
 void hipe_init_sdesc_table(struct hipe_sdesc *sdesc)
 {
     unsigned int log2size, size;
@@ -121,31 +143,46 @@ void hipe_init_sdesc_table(struct hipe_sdesc *sdesc)
  * representation. If different representations are needed in
  * the future, this code has to be made target dependent.
  */
-struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
+struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp)
 {
     Uint ra, exnra;
     Eterm *live;
-    Uint fsize, arity, nlive, i, nslots, off;
+    Uint fsize, nargs, stk_nargs, nlive, i, nslots, off;
     Uint livebitswords, sdescbytes;
     void *p;
     struct hipe_sdesc *sdesc;
-
-    if (is_not_tuple(arg) ||
-	(tuple_val(arg))[0] != make_arityval(6) ||
-	term_to_Uint((tuple_val(arg))[1], &ra) == 0 ||
-	term_to_Uint((tuple_val(arg))[2], &exnra) == 0 ||
-	is_not_small((tuple_val(arg))[3]) ||
-	(fsize = unsigned_val((tuple_val(arg))[3])) > 65535 ||
-	is_not_small((tuple_val(arg))[4]) ||
-	(arity = unsigned_val((tuple_val(arg))[4])) > 255 ||
-	is_not_tuple((tuple_val(arg))[5]))
+    Eterm* mfa_tpl;
+    Eterm* tp;
+
+    if (is_not_tuple(arg))
+        return 0;
+
+    tp = tuple_val(arg);
+    if (tp[0] != make_arityval(7) ||
+        term_to_Uint(tp[1], &ra) == 0 ||
+	term_to_Uint(tp[2], &exnra) == 0 ||
+	is_not_small(tp[3]) ||
+	(fsize = unsigned_val(tp[3])) > 65535 ||
+	is_not_small(tp[4]) ||
+	(stk_nargs = unsigned_val(tp[4])) > 255 ||
+	is_not_tuple(tp[5]) ||
+        is_not_tuple(tp[6]) ||
+        (mfa_tpl = tuple_val(tp[6]))[0] != make_arityval(3) ||
+        is_not_atom(mfa_tpl[1]) ||
+        is_not_atom(mfa_tpl[2]) ||
+        is_not_small(mfa_tpl[3]) ||
+        (nargs = unsigned_val(mfa_tpl[3])) > 255)
 	return 0;
+
+    if (stk_nargs > nargs)
+        return 0;
+
     /* Get tuple with live slots */
-    live = tuple_val((tuple_val(arg))[5]) + 1;
+    live = tuple_val(tp[5]) + 1;
     /* Get number of live slots */
     nlive = arityval(live[-1]);
-    /* Calculate size of frame = locals + ra + arguments */
-    nslots = fsize + 1 + arity;
+    /* Calculate size of frame = locals + ra + stack arguments */
+    nslots = fsize + 1 + stk_nargs;
     /* Check that only valid slots are given. */
     for (i = 0; i < nlive; ++i) {
 	if (is_not_small(live[i]) ||
@@ -154,8 +191,14 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
 	    return 0;
     }
 
+    switch(tp[7]) {
+    case am_true: *do_commitp = 1; break;
+    case am_false: *do_commitp = 0; break;
+    default: return 0;
+    }
+
     /* Calculate number of words for the live bitmap. */
-    livebitswords = (fsize + arity + 1 + 31) / 32;
+    livebitswords = (fsize + stk_nargs + 1 + 31) / 32;
     /* Calculate number of bytes needed for the stack descriptor. */
     sdescbytes =
 	(exnra
@@ -172,12 +215,17 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
     } else
 	sdesc = p;
 
+    sdesc->m_aix = atom_val(mfa_tpl[1]);
+    sdesc->f_aix = atom_val(mfa_tpl[2]);
+    sdesc->a = nargs;
+
+
     /* Initialise head of sdesc. */
     sdesc->bucket.next = 0;
     sdesc->bucket.hvalue = ra;
     sdesc->fsize = fsize;
     sdesc->has_exnra = (exnra ? 1 : 0);
-    sdesc->arity = arity;
+    sdesc->stk_nargs = stk_nargs;
     /* Clear all live-bits */
     for (i = 0; i < livebitswords; ++i)
 	sdesc->livebits[i] = 0;
@@ -186,13 +234,5 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
 	off = unsigned_val(live[i]);
 	sdesc->livebits[off / 32] |= (1 << (off & 31));
     }
-#ifdef DEBUG
-    {
-	Eterm mfa_tpl = tuple_val(arg)[6];	
-	sdesc->dbg_M = tuple_val(mfa_tpl)[1];
-	sdesc->dbg_F = tuple_val(mfa_tpl)[2];
-	sdesc->dbg_A = tuple_val(mfa_tpl)[3];
-    }
-#endif
     return sdesc;
 }
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index 81f2e53dbc..1863b0db8c 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -37,12 +37,12 @@ struct hipe_sdesc {
     } bucket;
     unsigned int fsize : 23;    /* frame size */
     unsigned int has_exnra : 1; /* exn handler presence flag */
-    unsigned int arity : 8;
-#ifdef DEBUG
-    Eterm dbg_M, dbg_F;
-    unsigned dbg_A;
-#endif
-    unsigned int livebits[1]; /* size depends on arch & data in summary field */
+    unsigned int stk_nargs : 8; /* arguments on stack */
+    Uint32 m_aix;
+    Uint32 f_aix;
+    Uint32 a;
+    struct hipe_sdesc* next_in_modi;
+    Uint32 livebits[1]; /* size depends on arch & data in summary field */
 };
 
 struct hipe_sdesc_with_exnra {
@@ -55,9 +55,10 @@ static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc)
     return sdesc->fsize;
 }
 
+/* Nr of arguments pushed on stack */
 static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc)
 {
-    return sdesc->arity;
+    return sdesc->stk_nargs;
 }
 
 static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc)
@@ -79,8 +80,9 @@ struct hipe_sdesc_table {
 extern struct hipe_sdesc_table hipe_sdesc_table;
 
 extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*);
+extern void hipe_destruct_sdesc(struct hipe_sdesc*);
 extern void hipe_init_sdesc_table(struct hipe_sdesc*);
-extern struct hipe_sdesc *hipe_decode_sdesc(Eterm);
+extern struct hipe_sdesc *hipe_decode_sdesc(Eterm, int* do_commitp);
 
 #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
 #define __builtin_expect(x, expected_value) (x)
diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c
index 5f6c8c200e..8a3ba8bc7f 100644
--- a/erts/emulator/hipe/hipe_x86.c
+++ b/erts/emulator/hipe/hipe_x86.c
@@ -184,6 +184,15 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
     return alloc_code(nrbytes);
 }
 
+void hipe_free_code(void* code, unsigned int bytes)
+{
+    /* SVERK: Leak !!!
+    ALLOC_CODE_STATS(--nr_allocs);
+    ALLOC_CODE_STATS(total_alloc -= bytes);
+
+    erts_free(ERTS_ALC_T_HIPE_EXEC, code);*/
+}
+
 void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
 {
     /*
@@ -264,6 +273,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
     return code;
 }
 
+void hipe_free_native_stub(void* stub)
+{
+    /* SVERK: leak leak drip drop */
+}
+
 void hipe_arch_print_pcb(struct hipe_process_state *p)
 {
 #define U(n,x) \
diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h
index 7802076f70..e92e6ea718 100644
--- a/erts/emulator/hipe/hipe_x86_gc.h
+++ b/erts/emulator/hipe/hipe_x86_gc.h
@@ -65,18 +65,14 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state)
     state->sdesc0 = sdesc;
     return sdesc;
 #else
-    unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS;
-    if ((int)nstkarity < 0)
-	nstkarity = 0;
     state->sdesc0[0].fsize = 0;
     state->sdesc0[0].has_exnra = 0;
-    state->sdesc0[0].arity = nstkarity;
+    state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 :
+                                  p->hipe.narity - NR_ARG_REGS);
     state->sdesc0[0].livebits[0] = 0;
-# ifdef DEBUG
-    state->sdesc0[0].dbg_M = 0;
-    state->sdesc0[0].dbg_F = am_undefined;
-    state->sdesc0[0].dbg_A = 0;
-# endif
+    state->sdesc0[0].m_aix = 0;
+    state->sdesc0[0].f_aix = atom_val(am_undefined);
+    state->sdesc0[0].a     = 0;
     /* XXX: this appears to prevent a gcc-4.1.1 bug on x86 */
     __asm__ __volatile__("" : : "m"(*state) : "memory");
     return &state->sdesc0[0];
diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h
index 0e9fcd61f1..de2b061706 100644
--- a/erts/emulator/hipe/hipe_x86_glue.h
+++ b/erts/emulator/hipe/hipe_x86_glue.h
@@ -58,18 +58,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y)
 
 static __inline__ void hipe_arch_glue_init(void)
 {
-    static struct hipe_sdesc_with_exnra nbif_return_sdesc = {
-	.exnra = (unsigned long)nbif_fail,
-	.sdesc = {
-	    .bucket = { .hvalue = (unsigned long)nbif_return },
-            .fsize = 0,
-            .has_exnra = 1,
-            .arity = 0,
-	#ifdef DEBUG
-	    .dbg_F = am_return,
-	#endif
-	},
-    };
+    static struct hipe_sdesc_with_exnra nbif_return_sdesc;
+
+    nbif_return_sdesc.exnra = (unsigned long)nbif_fail;
+    nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return;
+    nbif_return_sdesc.sdesc.fsize = 0;
+    nbif_return_sdesc.sdesc.has_exnra = 1;
+    nbif_return_sdesc.sdesc.stk_nargs = 0;
+    nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty);
+    nbif_return_sdesc.sdesc.f_aix = atom_val(am_return);
+    nbif_return_sdesc.sdesc.a     = 0;
+
     hipe_init_sdesc_table(&nbif_return_sdesc.sdesc);
 }
 
diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c
index 9426b166e2..31582b3a2e 100644
--- a/erts/emulator/hipe/hipe_x86_stack.c
+++ b/erts/emulator/hipe/hipe_x86_stack.c
@@ -60,7 +60,6 @@ void hipe_print_nstack(Process *p)
     unsigned int mask;
     unsigned int sdesc_size;
     unsigned int i;
-    unsigned int nstkarity;
     static const char dashes[2*sizeof(long)+5] = {
 	[0 ... 2*sizeof(long)+3] = '-'
     };
@@ -68,12 +67,10 @@ void hipe_print_nstack(Process *p)
     nsp = p->hipe.nsp;
     nsp_end = p->hipe.nstend;
 
-    nstkarity = p->hipe.narity - NR_ARG_REGS;
-    if ((int)nstkarity < 0)
-	nstkarity = 0;
     sdesc0.fsize = 0;
     sdesc0.has_exnra = 0;
-    sdesc0.arity = nstkarity;
+    sdesc0.stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 :
+                        p->hipe.narity - NR_ARG_REGS);
     sdesc0.livebits[0] = ~1;
     sdesc = &sdesc0;
 
diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 8427bb134d..b76ba6427f 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -445,62 +445,64 @@ module_md5_ok(Code) ->
 make_stub(Config) when is_list(Config) ->
     catch erlang:purge_module(my_code_test),
     MD5 = erlang:md5(<<>>),
+    Arg3 = {[], [], MD5, 0, 0},
 
     Data = proplists:get_value(data_dir, Config),
     File = filename:join(Data, "my_code_test"),
     {ok,my_code_test,Code} = compile:file(File, [binary]),
 
-    my_code_test = code:make_stub_module(my_code_test, Code, {[],[],MD5}),
+    my_code_test = code:make_stub_module(my_code_test, Code, Arg3),
     true = erlang:delete_module(my_code_test),
     true = erlang:purge_module(my_code_test),
 
     my_code_test = code:make_stub_module(my_code_test, 
                                          make_unaligned_sub_binary(Code),
-                                         {[],[],MD5}),
+                                         Arg3),
     true = erlang:delete_module(my_code_test),
     true = erlang:purge_module(my_code_test),
 
     my_code_test = code:make_stub_module(my_code_test, zlib:gzip(Code),
-                                         {[],[],MD5}),
+                                         Arg3),
     true = erlang:delete_module(my_code_test),
     true = erlang:purge_module(my_code_test),
 
     %% Should fail.
     {'EXIT',{badarg,_}} =
-    (catch code:make_stub_module(my_code_test, <<"bad">>, {[],[],MD5})),
+    (catch code:make_stub_module(my_code_test, <<"bad">>, Arg3)),
     {'EXIT',{badarg,_}} =
     (catch code:make_stub_module(my_code_test,
                                  bit_sized_binary(Code),
-                                 {[],[],MD5})),
+                                 Arg3)),
     {'EXIT',{badarg,_}} =
     (catch code:make_stub_module(my_code_test_with_wrong_name,
-                                 Code, {[],[],MD5})),
+                                 Code, Arg3)),
     ok.
 
 make_stub_many_funs(Config) when is_list(Config) ->
     catch erlang:purge_module(many_funs),
     MD5 = erlang:md5(<<>>),
+    Arg3 = {[], [], MD5, 0, 0},
 
     Data = proplists:get_value(data_dir, Config),
     File = filename:join(Data, "many_funs"),
     {ok,many_funs,Code} = compile:file(File, [binary]),
 
-    many_funs = code:make_stub_module(many_funs, Code, {[],[],MD5}),
+    many_funs = code:make_stub_module(many_funs, Code, Arg3),
     true = erlang:delete_module(many_funs),
     true = erlang:purge_module(many_funs),
     many_funs = code:make_stub_module(many_funs, 
                                       make_unaligned_sub_binary(Code),
-                                      {[],[],MD5}),
+                                      Arg3),
     true = erlang:delete_module(many_funs),
     true = erlang:purge_module(many_funs),
 
     %% Should fail.
     {'EXIT',{badarg,_}} =
-    (catch code:make_stub_module(many_funs, <<"bad">>, {[],[],MD5})),
+    (catch code:make_stub_module(many_funs, <<"bad">>, Arg3)),
     {'EXIT',{badarg,_}} =
     (catch code:make_stub_module(many_funs,
                                  bit_sized_binary(Code),
-                                 {[],[],MD5})),
+                                 Arg3)),
     ok.
 
 constant_pools(Config) when is_list(Config) ->
diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl
index c9cc1cfe25..0e43af4f60 100644
--- a/lib/hipe/cerl/erl_bif_types.erl
+++ b/lib/hipe/cerl/erl_bif_types.erl
@@ -1060,8 +1060,8 @@ type(hipe_bifs, enter_code, 2, Xs, Opaques) ->
 			     t_sup(t_nil(), t_binary())]) end, Opaques);
 type(hipe_bifs, enter_sdesc, 1, Xs, Opaques) ->
   strict(hipe_bifs, enter_sdesc, 1, Xs, fun (_) -> t_nil() end, Opaques);
-type(hipe_bifs, find_na_or_make_stub, 2, Xs, Opaques) ->
-  strict(hipe_bifs, find_na_or_make_stub, 2, Xs,
+type(hipe_bifs, find_na_or_make_stub, 1, Xs, Opaques) ->
+  strict(hipe_bifs, find_na_or_make_stub, 1, Xs,
 	 fun (_) -> t_integer() end, Opaques); % address
 type(hipe_bifs, fun_to_address, 1, Xs, Opaques) ->
   strict(hipe_bifs, fun_to_address, 1, Xs,
@@ -1071,12 +1071,6 @@ type(hipe_bifs, get_fe, 2, Xs, Opaques) ->
 type(hipe_bifs, get_rts_param, 1, Xs, Opaques) ->
   strict(hipe_bifs, get_rts_param, 1, Xs,
 	 fun (_) -> t_sup(t_integer(), t_nil()) end, Opaques);
-type(hipe_bifs, invalidate_funinfo_native_addresses, 1, Xs, Opaques) ->
-  strict(hipe_bifs, invalidate_funinfo_native_addresses, 1, Xs,
-	 fun (_) -> t_nil() end, Opaques);
-type(hipe_bifs, mark_referred_from, 1, Xs, Opaques) ->
-  strict(hipe_bifs, mark_referred_from, 1, Xs,
-	 fun (_) -> t_nil() end, Opaques);
 type(hipe_bifs, merge_term, 1, Xs, Opaques) ->
   strict(hipe_bifs, merge_term, 1, Xs, fun ([X]) -> X end, Opaques);
 type(hipe_bifs, nstack_used_size, 0, _, _Opaques) ->
@@ -1088,9 +1082,6 @@ type(hipe_bifs, patch_insn, 3, Xs, Opaques) ->
 type(hipe_bifs, primop_address, 1, Xs, Opaques) ->
   strict(hipe_bifs, primop_address, 1, Xs,
 	 fun (_) -> t_sup(t_integer(), t_atom('false')) end, Opaques);
-type(hipe_bifs, redirect_referred_from, 1, Xs, Opaques) ->
-  strict(hipe_bifs, redirect_referred_from, 1, Xs,
-	 fun (_) -> t_nil() end, Opaques);
 type(hipe_bifs, ref, 1, Xs, Opaques) ->
   strict(hipe_bifs, ref, 1, Xs, fun (_) -> t_immarray() end, Opaques);
 type(hipe_bifs, ref_get, 1, Xs, Opaques) ->
@@ -2469,8 +2460,7 @@ arg_types(hipe_bifs, add_ref, 2) ->
   [t_mfa(), t_tuple([t_mfa(),
 		     t_integer(),
 		     t_sup(t_atom('call'), t_atom('load_mfa')),
-		     t_trampoline(),
-		     t_sup(t_atom('remote'), t_atom('local'))])];
+		     t_trampoline()])];
 arg_types(hipe_bifs, alloc_data, 2) ->
   [t_integer(), t_integer()];
 arg_types(hipe_bifs, array, 2) ->
@@ -2511,18 +2501,14 @@ arg_types(hipe_bifs, enter_code, 2) ->
   [t_binary(), t_sup(t_nil(), t_tuple())];
 arg_types(hipe_bifs, enter_sdesc, 1) ->
   [t_tuple([t_integer(), t_integer(), t_integer(), t_integer(), t_integer(), t_mfa()])];
-arg_types(hipe_bifs, find_na_or_make_stub, 2) ->
-  [t_mfa(), t_boolean()];
+arg_types(hipe_bifs, find_na_or_make_stub, 1) ->
+  [t_mfa()];
 arg_types(hipe_bifs, fun_to_address, 1) ->
   [t_mfa()];
 arg_types(hipe_bifs, get_fe, 2) ->
   [t_atom(), t_tuple([t_integer(), t_integer(), t_integer()])];
 arg_types(hipe_bifs, get_rts_param, 1) ->
   [t_fixnum()];
-arg_types(hipe_bifs, invalidate_funinfo_native_addresses, 1) ->
-  [t_list(t_mfa())];
-arg_types(hipe_bifs, mark_referred_from, 1) ->
-  [t_mfa()];
 arg_types(hipe_bifs, merge_term, 1) ->
   [t_any()];
 arg_types(hipe_bifs, nstack_used_size, 0) ->
@@ -2533,8 +2519,6 @@ arg_types(hipe_bifs, patch_insn, 3) ->
   [t_integer(), t_integer(), t_insn_type()];
 arg_types(hipe_bifs, primop_address, 1) ->
   [t_atom()];
-arg_types(hipe_bifs, redirect_referred_from, 1) ->
-  [t_mfa()];
 arg_types(hipe_bifs, ref, 1) ->
   [t_immediate()];
 arg_types(hipe_bifs, ref_get, 1) ->
diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl
index 835b7eb588..90ecb22074 100644
--- a/lib/kernel/src/code_server.erl
+++ b/lib/kernel/src/code_server.erl
@@ -1130,7 +1130,7 @@ try_load_module_2(File, Mod, Bin, From, Architecture,
                   #state{moddb=Db}=St) ->
     case catch hipe_unified_loader:load_native_code(Mod, Bin, Architecture) of
         {module,Mod} = Module ->
-	    ets:insert(Db, [{{native,Mod},true},{Mod,File}]),
+	    ets:insert(Db, {Mod,File}),
             {reply,Module,St};
         no_native ->
             try_load_module_3(File, Mod, Bin, From, Architecture, St);
@@ -1139,10 +1139,9 @@ try_load_module_2(File, Mod, Bin, From, Architecture,
             {reply,ok,St}
     end.
 
-try_load_module_3(File, Mod, Bin, From, Architecture, St0) ->
+try_load_module_3(File, Mod, Bin, From, _Architecture, St0) ->
     Action = fun({module,_}=Module, #state{moddb=Db}=S) ->
 		     ets:insert(Db, {Mod,File}),
-		     post_beam_load([Mod], Architecture, S),
 		     {reply,Module,S};
 		({error,on_load_failure}=Error, S) ->
 		     {reply,Error,S};
@@ -1153,24 +1152,14 @@ try_load_module_3(File, Mod, Bin, From, Architecture, St0) ->
     Res = erlang:load_module(Mod, Bin),
     handle_on_load(Res, Action, Mod, From, St0).
 
-hipe_result_to_status(Result, #state{moddb=Db}) ->
+hipe_result_to_status(Result, #state{}) ->
     case Result of
-	{module,Mod} ->
-	    ets:insert(Db, [{{native,Mod},true}]),
+	{module,_} ->
 	    Result;
 	_ ->
 	    {error,Result}
     end.
 
-post_beam_load(_, undefined, _) ->
-    %% HiPE is disabled.
-    ok;
-post_beam_load(Mods0, _Architecture, #state{moddb=Db}) ->
-    %% post_beam_load/2 can potentially be very expensive because it
-    %% blocks multi-scheduling. Therefore, we only want to call
-    %% it with modules that are known to have native code loaded.
-    Mods = [M || M <- Mods0, ets:member(Db, {native,M})],
-    hipe_unified_loader:post_beam_load(Mods).
 
 int_list([H|T]) when is_integer(H) -> int_list(T);
 int_list([_|_])                    -> false;
@@ -1313,15 +1302,12 @@ abort_if_sticky(L, Db) ->
 	[_|_] -> {error,Sticky}
     end.
 
-do_finish_loading(Prepared, #state{moddb=Db}=St) ->
+do_finish_loading(Prepared, #state{moddb=Db}) ->
     MagicBins = [B || {_,{B,_}} <- Prepared],
     case erlang:finish_loading(MagicBins) of
 	ok ->
 	    MFs = [{M,F} || {M,{_,F}} <- Prepared],
 	    true = ets:insert(Db, MFs),
-	    Ms = [M || {M,_} <- MFs],
-	    Architecture = erlang:system_info(hipe_architecture),
-	    post_beam_load(Ms, Architecture, St),
 	    ok;
 	{Reason,Ms} ->
 	    {error,[{M,Reason} || M <- Ms]}
diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl
index 7b1e6c2591..75a5995695 100644
--- a/lib/kernel/src/hipe_unified_loader.erl
+++ b/lib/kernel/src/hipe_unified_loader.erl
@@ -41,10 +41,11 @@
 % I think the real solution would be to let BIF erlang:load_module/2 redirect all
 % hipe calls to the module and thereby remove post_beam_load.
 
+% SVERK: Can we remove -compile(no_native) now when post_beam_load is gone?
+
 -export([chunk_name/1,
 	 %% Only the code and code_server modules may call the entries below!
 	 load_native_code/3,
-	 post_beam_load/1,
 	 load_module/4,
 	 load/3]).
 
@@ -101,15 +102,13 @@ word_size(Architecture) ->
 load_native_code(_Mod, _Bin, undefined) ->
   no_native;
 load_native_code(Mod, Bin, Architecture) when is_atom(Mod), is_binary(Bin) ->
-  %% patch_to_emu(Mod),
   case code:get_chunk(Bin, chunk_name(Architecture)) of
     undefined -> no_native;
     NativeCode when is_binary(NativeCode) ->
       erlang:system_flag(multi_scheduling, block_normal),
       try
-        OldReferencesToPatch = patch_to_emu_step1(Mod),
-        case load_module(Mod, NativeCode, Bin, OldReferencesToPatch,
-                         Architecture) of
+        put(hipe_patch_closures, false),
+        case load_common(Mod, NativeCode, Bin, Architecture) of
           bad_crc -> no_native;
           Result -> Result
         end
@@ -120,22 +119,6 @@ load_native_code(Mod, Bin, Architecture) when is_atom(Mod), is_binary(Bin) ->
 
 %%========================================================================
 
--spec post_beam_load([module()]) -> 'ok'.
-
-post_beam_load([])->
-  ok;
-post_beam_load([_|_]=Mods) ->
-  erlang:system_flag(multi_scheduling, block_normal),
-  try
-    _ = [patch_to_emu(Mod) || Mod <- Mods],
-    ok
-  after
-    erlang:system_flag(multi_scheduling, unblock_normal)
-  end,
-  ok.
-
-%%========================================================================
-
 version_check(Version, Mod) when is_atom(Mod) ->
   Ver = ?VERSION_STRING(),
   case Version < Ver of
@@ -153,19 +136,12 @@ version_check(Version, Mod) when is_atom(Mod) ->
 load_module(Mod, Bin, Beam, Architecture) ->
   erlang:system_flag(multi_scheduling, block_normal),
   try
-    load_module_nosmp(Mod, Bin, Beam, Architecture)
+    put(hipe_patch_closures, false),
+    load_common(Mod, Bin, Beam, Architecture)
   after
     erlang:system_flag(multi_scheduling, unblock_normal)
   end.
 
-load_module_nosmp(Mod, Bin, Beam, Architecture) ->
-  load_module(Mod, Bin, Beam, [], Architecture).
-
-load_module(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
-  ?debug_msg("************ Loading Module ~w ************\n",[Mod]),
-  %% Loading a whole module, let the BEAM loader patch closures.
-  put(hipe_patch_closures, false),
-  load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture).
 
 %%========================================================================
 
@@ -175,20 +151,17 @@ load_module(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
 load(Mod, Bin, Architecture) ->
   erlang:system_flag(multi_scheduling, block_normal),
   try
-    load_nosmp(Mod, Bin, Architecture)
+    ?debug_msg("********* Loading funs in module ~w *********\n",[Mod]),
+    %% Loading just some functions in a module; patch closures separately.
+    put(hipe_patch_closures, true),
+    load_common(Mod, Bin, [], Architecture)
   after
     erlang:system_flag(multi_scheduling, unblock_normal)
   end.
 
-load_nosmp(Mod, Bin, Architecture) ->
-  ?debug_msg("********* Loading funs in module ~w *********\n",[Mod]),
-  %% Loading just some functions in a module; patch closures separately.
-  put(hipe_patch_closures, true),
-  load_common(Mod, Bin, [], [], Architecture).
-
 %%------------------------------------------------------------------------
 
-load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
+load_common(Mod, Bin, Beam, Architecture) ->
   %% Unpack the binary.
   [{Version, CheckSum},
    ConstAlign, ConstSize, ConstMap, LabelMap, ExportMap,
@@ -228,15 +201,14 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
                                         Architecture),
       %% Patch references to code labels in data seg.
       ok = patch_consts(LabelMap, ConstAddr, CodeAddress, WriteWord),
+
       %% Find out which functions are being loaded (and where).
-      %% Note: FunDefs are sorted descending.
-      {MFAs,FunDefs} = exports(ExportMap, CodeAddress),
-      %% Remove references to old versions of the module.
-      ReferencesToPatch = get_refs_from(MFAs, []),
-      %% io:format("References to patch: ~w~n", [ReferencesToPatch]),
-      ok = remove_refs_from(MFAs),
+      %% Note: FunDefs are sorted descending address order.
+      FunDefs = exports(ExportMap, CodeAddress),
+
       %% Patch all dynamic references in the code.
       %%  Function calls, Atoms, Constants, System calls
+
       ok = patch(Refs, CodeAddress, ConstMap2, FunDefs, TrampolineMap),
 
       %% Tell the system where the loaded funs are. 
@@ -250,7 +222,7 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
 	  lists:foreach(fun({FE, DestAddress}) ->
 			    hipe_bifs:set_native_address_in_fe(FE, DestAddress)
 			end, erase(closures_to_patch)),
-	  export_funs(FunDefs),
+	  set_beam_call_traps(FunDefs),
 	  ok;
 	BeamBinary when is_binary(BeamBinary) ->
 	  %% Find all closures in the code.
@@ -259,15 +231,10 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
 	  AddressesOfClosuresToPatch =
 	    calculate_addresses(ClosurePatches, CodeAddress, FunDefs),
 	  export_funs(FunDefs),
-	  export_funs(Mod, MD5, BeamBinary,
-                      FunDefs, AddressesOfClosuresToPatch)
+	  make_beam_stub(Mod, MD5, BeamBinary, FunDefs, AddressesOfClosuresToPatch,
+			 CodeAddress, byte_size(CodeBinary))
       end,
-      %% Redirect references to the old module to the new module's BEAM stub.
-      patch_to_emu_step2(OldReferencesToPatch),
-      %% Patch referring functions to call the new function
-      %% The call to export_funs/1 above updated the native addresses
-      %% for the targets, so passing 'FunDefs' is not needed.
-      redirect(ReferencesToPatch),
+
       %% Final clean up.
       _ = erase(hipe_patch_closures),
       _ = erase(hipe_assert_code_area),
@@ -371,21 +338,21 @@ trampoline_map_lookup(Primop, Map) ->
 		 is_exported :: boolean()}).
 
 exports(ExportMap, BaseAddress) ->
-  exports(ExportMap, BaseAddress, [], []).
+  exports(ExportMap, BaseAddress, []).
 
-exports([Offset,M,F,A,IsClosure,IsExported|Rest], BaseAddress, MFAs, FunDefs) ->
+exports([Offset,M,F,A,IsClosure,IsExported|Rest], BaseAddress, FunDefs) ->
   case IsExported andalso erlang:is_builtin(M, F, A) of
     true ->
-      exports(Rest, BaseAddress, MFAs, FunDefs);
+      exports(Rest, BaseAddress, FunDefs);
     _false ->
       MFA = {M,F,A},
       Address = BaseAddress + Offset,
       FunDef = #fundef{address=Address, mfa=MFA, is_closure=IsClosure,
 		       is_exported=IsExported},
-      exports(Rest, BaseAddress, [MFA|MFAs], [FunDef|FunDefs])
+      exports(Rest, BaseAddress, [FunDef|FunDefs])
   end;
-exports([], _, MFAs, FunDefs) ->
-  {MFAs, FunDefs}.
+exports([], _, FunDefs) ->
+  FunDefs.
 
 mod({M,_F,_A}) -> M.
 
@@ -424,9 +391,9 @@ find_closure_refs([], Refs) ->
 
 %%------------------------------------------------------------------------
 
-export_funs([FunDef | FunDefs]) ->
+set_beam_call_traps([FunDef | FunDefs]) ->
   #fundef{address=Address, mfa=MFA, is_closure=IsClosure,
-	  is_exported=IsExported} = FunDef,
+	  is_exported=_IsExported} = FunDef,
   ?IF_DEBUG({M,F,A} = MFA, no_debug),
   ?IF_DEBUG(
      case IsClosure of
@@ -437,15 +404,32 @@ export_funs([FunDef | FunDefs]) ->
 	 ?debug_msg("LINKING: ~w:~w/~w to closure (0x~.16b)\n",
 		    [M,F,A, Address])
      end, no_debug),
-  hipe_bifs:set_funinfo_native_address(MFA, Address, IsExported),
   hipe_bifs:set_native_address(MFA, Address, IsClosure),
+  set_beam_call_traps(FunDefs);
+set_beam_call_traps([]) ->
+  ok.
+
+export_funs([FunDef | FunDefs]) ->
+  #fundef{address=Address, mfa=MFA, is_closure=_IsClosure,
+	  is_exported=IsExported} = FunDef,
+  ?IF_DEBUG({M,F,A} = MFA, no_debug),
+  ?IF_DEBUG(
+     case _IsClosure of
+       false ->
+	 ?debug_msg("LINKING: ~w:~w/~w to (0x~.16b)\n",
+		    [M,F,A, Address]);
+       true ->
+	 ?debug_msg("LINKING: ~w:~w/~w to closure (0x~.16b)\n",
+		    [M,F,A, Address])
+     end, no_debug),
+  hipe_bifs:set_funinfo_native_address(MFA, Address, IsExported),
   export_funs(FunDefs);
 export_funs([]) ->
   ok.
 
-export_funs(Mod, MD5, Beam, FunDefs, ClosuresToPatch) ->
+make_beam_stub(Mod, MD5, Beam, FunDefs, ClosuresToPatch, CodeAddress, CodeSize) ->
   Fs = [{F,A,Address} || #fundef{address=Address, mfa={_M,F,A}} <- FunDefs],
-  Mod = code:make_stub_module(Mod, Beam, {Fs,ClosuresToPatch,MD5}),
+  Mod = code:make_stub_module(Mod, Beam, {Fs,ClosuresToPatch,MD5,CodeAddress,CodeSize}),
   ok.
 
 %%========================================================================
@@ -515,8 +499,7 @@ patch_mfa_call_list([Offset|Offsets], BaseAddress, DestMFA, DestAddress, FunDefs
 patch_mfa_call_list([], _, _, _, _, _, _) -> ok.
 
 patch_call_insn(CallAddress, DestAddress, Trampoline) ->
-  %% This assertion is false when we're called from redirect/2.
-  %% ?ASSERT(assert_local_patch(CallAddress)),
+  ?ASSERT(assert_local_patch(CallAddress)),
   hipe_bifs:patch_call(CallAddress, DestAddress, Trampoline).
 
 %% ____________________________________________________________________
@@ -561,15 +544,15 @@ patch_atom(Address, Atom) ->
   patch_instr(Address, hipe_bifs:atom_to_word(Atom), atom).
 
 patch_sdesc(?STACK_DESC(SymExnRA, FSize, Arity, Live),
-	    Address, {_ConstMap2,CodeAddress}, _FunDefs) ->
+	    Address, {_ConstMap2,CodeAddress}, FunDefs) ->
   ExnRA =
     case SymExnRA of
       [] -> 0; % No catch
       LabelOffset -> CodeAddress + LabelOffset
     end,
   ?ASSERT(assert_local_patch(Address)),
-  DBG_MFA = ?IF_DEBUG(address_to_mfa_lth(Address, _FunDefs), {undefined,undefined,0}),
-  hipe_bifs:enter_sdesc({Address, ExnRA, FSize, Arity, Live, DBG_MFA}).
+  MFA = address_to_mfa_lth(Address, FunDefs),
+  hipe_bifs:enter_sdesc({Address, ExnRA, FSize, Arity, Live, MFA, get(hipe_patch_closures)}).
 
 
 %%----------------------------------------------------------------
@@ -617,6 +600,7 @@ patch_closure(DestMFA, Uniq, Index, Address, FunDefs) ->
 %% RemoteOrLocal ::= 'remote' | 'local'
 %%
 patch_load_mfa(CodeAddress, DestMFA, FunDefs, RemoteOrLocal) ->
+  ?ASSERT(assert_local_patch(CodeAddress)),
   DestAddress =
     case bif_address(DestMFA) of
       false ->
@@ -626,7 +610,6 @@ patch_load_mfa(CodeAddress, DestMFA, FunDefs, RemoteOrLocal) ->
       BifAddress when is_integer(BifAddress) ->
 	BifAddress
     end,
-  ?ASSERT(assert_local_patch(CodeAddress)),
   patch_instr(CodeAddress, DestAddress, 'load_mfa').
 
 %%----------------------------------------------------------------
@@ -781,23 +764,17 @@ find_const(ConstNo, []) ->
 %% RemoteOrLocal ::= 'remote' | 'local'.
 %%
 
-%%
-%% -record(ref, {caller_mfa, address, ref_type, trampoline, remote_or_local}).
-%%
-
 add_ref(CalleeMFA, Address, FunDefs, RefType, Trampoline, RemoteOrLocal) ->
   CallerMFA = address_to_mfa_lth(Address, FunDefs),
-  %% just a sanity assertion below
-  true = case RemoteOrLocal of
-	   local ->
-	     {M1,_,_} = CalleeMFA,
-	     {M2,_,_} = CallerMFA,
-	     M1 =:= M2;
-	   remote ->
-	     true
-	 end,
-  %% io:format("Adding ref ~w\n",[{CallerMFA, CalleeMFA, Address, RefType}]),
-  hipe_bifs:add_ref(CalleeMFA, {CallerMFA,Address,RefType,Trampoline,RemoteOrLocal}).
+  case RemoteOrLocal of
+    local ->
+      %% just a sanity assertion
+      {M,_,_} = CalleeMFA,
+      {M,_,_} = CallerMFA;
+    remote ->
+      hipe_bifs:add_ref(CalleeMFA, {CallerMFA,Address,RefType,Trampoline,get(hipe_patch_closures)})
+  end,
+  ok.
 
 % For FunDefs sorted from low to high addresses
 address_to_mfa_lth(Address, FunDefs) ->
@@ -825,81 +802,6 @@ address_to_mfa_lth(_Address, [], Prev) ->
 %%   ?error_msg("Local adddress not found ~w\n",[Address]),
 %%   exit({?MODULE, local_address_not_found}).
 
-%%----------------------------------------------------------------
-%% Change callers of the given module to instead trap to BEAM.
-%% load_native_code/3 calls this just before loading native code.
-%%
-patch_to_emu(Mod) ->
-  patch_to_emu_step2(patch_to_emu_step1(Mod)).
-
-%% Step 1 must occur before the loading of native code updates
-%% references information or creates a new BEAM stub module.
-patch_to_emu_step1(Mod) ->
-  case is_loaded(Mod) of
-    true ->
-      %% Get exported functions
-      MFAs = [{Mod,Fun,Arity} || {Fun,Arity} <- Mod:module_info(exports)],
-      %% get_refs_from/2 only finds references from compiled static
-      %% call sites to the module, but some native address entries
-      %% were added as the result of dynamic apply calls. We must
-      %% purge them too, but we have no explicit record of them.
-      %% Therefore invalidate all native addresses for the module.
-      hipe_bifs:invalidate_funinfo_native_addresses(MFAs),
-      %% Find all call sites that call these MFAs. As a side-effect,
-      %% create native stubs for any MFAs that are referred.
-      ReferencesToPatch = get_refs_from(MFAs, []),
-      ok = remove_refs_from(MFAs),
-      ReferencesToPatch;
-    false ->
-      %% The first time we load the module, no redirection needs to be done.
-      []
-  end.
-
-%% Step 2 must occur after the new BEAM stub module is created.
-patch_to_emu_step2(ReferencesToPatch) ->
-  redirect(ReferencesToPatch).
-
--spec is_loaded(Module::atom()) -> boolean().
-%% @doc Checks whether a module is loaded or not.
-is_loaded(M) when is_atom(M) ->
-  try hipe_bifs:fun_to_address({M,module_info,0}) of
-    I when is_integer(I) -> true
-  catch _:_ -> false
-  end.
-
-%%--------------------------------------------------------------------
-%% Given a list of MFAs, tag them with their referred_from references.
-%% The resulting {MFA,Refs} list is later passed to redirect/1, once
-%% the MFAs have been bound to (possibly new) native-code addresses.
-%%
-get_refs_from(MFAs, []) ->
-  mark_referred_from(MFAs),
-  MFAs.
-
-mark_referred_from(MFAs) ->
-  lists:foreach(fun(MFA) -> hipe_bifs:mark_referred_from(MFA) end, MFAs).
-
-%%--------------------------------------------------------------------
-%% Given a list of MFAs with referred_from references, update their
-%% callers to refer to their new native-code addresses.
-%%
-%% The {MFA,Refs} list must come from get_refs_from/2.
-%%
-redirect(MFAs) ->
-  lists:foreach(fun(MFA) -> hipe_bifs:redirect_referred_from(MFA) end, MFAs).
-
-%%--------------------------------------------------------------------
-%% Given a list of MFAs, remove all referred_from references having
-%% any of them as CallerMFA.
-%%
-%% This is the only place using refers_to. Whenever a reference is
-%% added from CallerMFA to CalleeMFA, CallerMFA is added to CalleeMFA's
-%% referred_from list, and CalleeMFA is added to CallerMFA's refers_to
-%% list. The refers_to list is used here to find the CalleeMFAs whose
-%% referred_from lists should be updated.
-%%
-remove_refs_from(MFAs) ->
-  lists:foreach(fun(MFA) -> hipe_bifs:remove_refs_from(MFA) end, MFAs).
 
 %%--------------------------------------------------------------------
 
@@ -913,12 +815,13 @@ get_native_address(MFA, FunDefs, RemoteOrLocal) ->
   case mfa_to_address(MFA, FunDefs, RemoteOrLocal) of
     Adr when is_integer(Adr) -> Adr;
     false ->
-      IsRemote =
 	case RemoteOrLocal of
-	  remote -> true;
-	  local -> false
-	end,
-      hipe_bifs:find_na_or_make_stub(MFA, IsRemote)
+	  remote ->
+	    hipe_bifs:find_na_or_make_stub(MFA);
+	  local ->
+	    ?error_msg("Local function ~p not found\n",[MFA]),
+	    exit({function_not_found,MFA})
+	end
   end.
 
 mfa_to_address(MFA, [#fundef{address=Adr, mfa=MFA,
diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl
index c5167efa56..1be4c364d5 100644
--- a/lib/kernel/test/code_SUITE.erl
+++ b/lib/kernel/test/code_SUITE.erl
@@ -483,23 +483,25 @@ load_binary(Config) when is_list(Config) ->
     code:delete(code_b_test),
     ok.
 
+
 upgrade(Config) ->
     DataDir = proplists:get_value(data_dir, Config),
 
-    %%T = [beam, hipe],
-    T = [beam],
-
-    [upgrade_do(DataDir, Client, U1, U2, O1, O2)
-     || Client<-T, U1<-T, U2<-T, O1<-T, O2<-T],
+    T = [beam, hipe],
+    %%T = [beam],
+    %%T = [hipe],
 
+    [upgrade_do(DataDir, Client, T) || Client <- T],
     ok.
 
-upgrade_do(DataDir, Client, U1, U2, O1, O2) ->
+upgrade_do(DataDir, Client, T) ->
     compile_load(upgrade_client, DataDir, undefined, Client),
-    upgrade_client:run(DataDir, U1, U2, O1, O2),
+    [upgrade_client:run(DataDir, U1, U2, O1, O2)
+     || U1<-T, U2<-T, O1<-T, O2<-T],
     ok.
 
 compile_load(Mod, Dir, Ver, CodeType) ->
+    erlang:display({"{{{{{{{{{{{{{{{{Loading",Mod,Ver,CodeType}),
     Version = case Ver of
 	undefined ->
 	    io:format("Compiling '~p' as ~p\n", [Mod, CodeType]),
@@ -516,9 +518,15 @@ compile_load(Mod, Dir, Ver, CodeType) ->
     CompOpts = [binary, report] ++ Target ++ Version,
 
     Src = filename:join(Dir, atom_to_list(Mod) ++ ".erl"),
+    T1 = erlang:now(),
     {ok,Mod,Code} = compile:file(Src, CompOpts),
+    T2 = erlang:now(),
     ObjFile = filename:basename(Src,".erl") ++ ".beam",
     {module,Mod} = code:load_binary(Mod, ObjFile, Code),
+    T3 = erlang:now(),
+    io:format("Compile time ~p ms, Load time ~p ms\n",
+	      [timer:now_diff(T2,T1) div 1000, timer:now_diff(T3,T2) div 1000]),
+    erlang:display({"}}}}}}}}}}}}}}}Loaded",Mod,Ver,CodeType}),
     ok.
 
 dir_req(Config) when is_list(Config) ->
@@ -810,8 +818,6 @@ check_funs({'$M_EXPR','$F_EXPR',_},
 	    {code_server,start_link,1}]) -> 0;
 check_funs({'$M_EXPR','$F_EXPR',_},
 	   [{erlang,spawn_link,1},{code_server,start_link,1}]) -> 0;
-check_funs({'$M_EXPR',module_info,1},
-	   [{hipe_unified_loader,patch_to_emu_step1,1} | _]) -> 0;
 check_funs({'$M_EXPR','$F_EXPR',2},
 	   [{hipe_unified_loader,write_words,3} | _]) -> 0;
 check_funs({'$M_EXPR','$F_EXPR',2},
@@ -823,11 +829,7 @@ check_funs({'$M_EXPR','$F_EXPR',2},
 	    {hipe_unified_loader,sort_and_write,5} | _]) -> 0;
 check_funs({'$M_EXPR','$F_EXPR',1},
 	   [{lists,foreach,2},
-	    {hipe_unified_loader,patch_consts,3} | _]) -> 0;
-check_funs({'$M_EXPR','$F_EXPR',1},
-	   [{lists,foreach,2},
-	    {hipe_unified_loader,mark_referred_from,1},
-	    {hipe_unified_loader,get_refs_from,2}| _]) -> 0;
+	    {hipe_unified_loader,patch_consts,4} | _]) -> 0;
 check_funs({'$M_EXPR',warning_msg,2},
 	   [{code_server,finish_on_load_report,2} | _]) -> 0;
 check_funs({'$M_EXPR','$F_EXPR',1},
diff --git a/lib/kernel/test/code_SUITE_data/upgrade_client.erl b/lib/kernel/test/code_SUITE_data/upgrade_client.erl
index bb655e01d3..7ca0df7f5c 100644
--- a/lib/kernel/test/code_SUITE_data/upgrade_client.erl
+++ b/lib/kernel/test/code_SUITE_data/upgrade_client.erl
@@ -9,6 +9,8 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
     %% Load version 1 of upgradee
     code_SUITE:compile_load(upgradee, Dir, 1, Upgradee1),
 
+    Tracer = start_tracing(),
+
     ?line 1 = upgradee:exp1(),
     ?line 1 = upgradee:exp1exp2(),
     ?line 1 = upgradee:exp1loc2(),
@@ -56,6 +58,16 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc1),
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
 
+    Env1 = "Env1",
+    put(loc1_fun, upgradee:get_local_fun(Env1)),
+    erlang:display(sverk_break),
+    ?line {1,Env1} = (get(loc1_fun))(),
+
+    put(exp1exp2_fun, upgradee:get_exp1exp2_fun()),
+    ?line 1 = (get(exp1exp2_fun))(),
+
+    ?line 13 = check_tracing(Tracer),
+
     %%
     %% Load version 1 of other
     %%
@@ -78,6 +90,8 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, exp2),
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
 
+    ?line 5 = check_tracing(Tracer),
+
     %%
     %% Load version 2 of upgradee
     %%
@@ -130,6 +144,15 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, exp2),
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
 
+    ?line {1,Env1} = (get(loc1_fun))(),
+    Env2 = "Env2",
+    put(loc2_fun, upgradee:get_local_fun(Env2)),
+    ?line {2,Env2} = (get(loc2_fun))(),
+
+    ?line 2 = (get(exp1exp2_fun))(),
+
+    ?line 10 = check_tracing(Tracer),
+
     %%
     %% Load version 2 of other
     %%
@@ -182,17 +205,26 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc1loc2),
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
 
+    ?line {1,Env1} = (get(loc1_fun))(),
+    ?line {2,Env2} = (get(loc2_fun))(),
+    ?line 2 = (get(exp1exp2_fun))(),
+
+    ?line 10 = check_tracing(Tracer),
 
     %%
     %% Upgrade proxy to version 2
     %%
     P ! upgrade_order,
 
-    
     %%
-    io:format("Delete version 2 of 'upgradee'\n",[]),
+    io:format("Purge version 1 of 'upgradee'\n",[]),
     %%
+    put(loc1_fun,undefined),
     code:purge(upgradee),
+
+    %%
+    io:format("Delete version 2 of 'upgradee'\n",[]),
+    %%
     code:delete(upgradee),
     
     ?line {'EXIT',{undef,_}} = (catch upgradee:exp2()),
@@ -239,17 +271,24 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, exp1loc2),
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc1loc2),
     ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
+
+    ?line {'EXIT',{undef,_}} = (catch (get(exp1exp2_fun))()),
     
+    ?line 14 = check_tracing(Tracer),
+
     unlink(P),
     exit(P, die_please),
 
     io:format("Purge 'upgradee'\n",[]),
+    put(loc2_fun,undefined),
     code:purge(upgradee),
 
     io:format("Delete and purge 'other'\n",[]),
     code:purge(other),
     code:delete(other),
     code:purge(other),
+
+    stop_tracing(Tracer),
     ok.
 
 proxy_call(Pid, CallType, Func) ->
@@ -257,3 +296,48 @@ proxy_call(Pid, CallType, Func) ->
     receive
 	{Pid, call_result, Func, Ret} -> Ret
     end.
+
+
+start_tracing() ->
+    Self = self(),
+    {Tracer,_} = spawn_opt(fun() -> tracer_loop(Self) end, [link,monitor]),
+    ?line 1 = erlang:trace_pattern({error_handler,undefined_function,3},
+				   true, [global]),
+    ?line 1 = erlang:trace(Self, true, [call,{tracer,Tracer}]),
+    Tracer.
+
+
+tracer_loop(Receiver) ->
+    receive
+	die_please ->
+	    ok;
+	{do_trace_delivered, Tracee} ->
+	    _ = erlang:trace_delivered(Tracee),
+	    tracer_loop(Receiver);
+
+	Msg ->
+	    Receiver ! Msg,
+	    tracer_loop(Receiver)
+    end.
+
+check_tracing(Tracer) ->
+    Tracer ! {do_trace_delivered, self()},
+    check_tracing_loop(0).
+
+check_tracing_loop(N) ->
+    Self = self(),
+    receive
+	{trace, _Pid, call, {_M, _F, _Args}} = Msg ->
+	    io:format("Trace: ~p\n",[Msg]),
+	    check_tracing_loop(N+1);
+	{trace_delivered, Self, _} ->
+	    N
+    end.
+
+
+stop_tracing(Tracer) ->
+    erlang:trace(self(), false, [call]),
+    Tracer ! die_please,
+    receive
+	{'DOWN', _, process, Tracer, _} -> ok
+    end.
diff --git a/lib/kernel/test/code_SUITE_data/upgradee.erl b/lib/kernel/test/code_SUITE_data/upgradee.erl
index 62b1d95e30..8ca660c19c 100644
--- a/lib/kernel/test/code_SUITE_data/upgradee.erl
+++ b/lib/kernel/test/code_SUITE_data/upgradee.erl
@@ -8,6 +8,9 @@
 -export([exp1/0]).     % only exported in v1
 -export([exp1loc2/0]). % exported in v1, local in v2
 -export([exp1exp2/0]). % exported in v1 and v2
+-export([get_local_fun/1]).
+-export([get_exp1exp2_fun/0]).
+-export([exp1exp2_fun/0]).
 
 exp1() -> ?VERSION.
 loc1() -> ?VERSION.
@@ -20,6 +23,9 @@ loc1() -> ?VERSION.
 -export([exp2/0]).
 -export([loc1exp2/0]).
 -export([exp1exp2/0]).
+-export([get_local_fun/1]).
+-export([get_exp1exp2_fun/0]).
+-export([exp1exp2_fun/0]).
 
 exp2() -> ?VERSION.
 loc2() -> ?VERSION.
@@ -31,6 +37,12 @@ exp1loc2() -> ?VERSION.
 loc1exp2() -> ?VERSION.
 loc1loc2() -> ?VERSION.
 
+get_local_fun(Env) -> fun() -> {?VERSION,Env} end.
+get_exp1exp2_fun() -> fun ?MODULE:exp1exp2_fun/0.
+
+exp1exp2_fun() ->
+    ?VERSION.
+
 dispatch_loop() ->
     receive 
 	upgrade_order ->