aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--bootstrap/lib/kernel/ebin/code_server.beambin25132 -> 24588 bytes
-rw-r--r--bootstrap/lib/kernel/ebin/hipe_unified_loader.beambin13800 -> 12624 bytes
-rw-r--r--erts/configure.in23
-rw-r--r--erts/emulator/Makefile.in2
-rw-r--r--erts/emulator/beam/beam_bif_load.c132
-rw-r--r--erts/emulator/beam/beam_emu.c4
-rw-r--r--erts/emulator/beam/beam_load.c185
-rw-r--r--erts/emulator/beam/beam_load.h30
-rw-r--r--erts/emulator/beam/erl_alloc.c26
-rw-r--r--erts/emulator/beam/erl_alloc_util.c48
-rw-r--r--erts/emulator/beam/erl_alloc_util.h6
-rw-r--r--erts/emulator/beam/erl_fun.h3
-rw-r--r--erts/emulator/beam/export.c5
-rw-r--r--erts/emulator/beam/external.c4
-rw-r--r--erts/emulator/beam/global.h1
-rw-r--r--erts/emulator/beam/module.c70
-rw-r--r--erts/emulator/beam/module.h9
-rw-r--r--erts/emulator/hipe/hipe_amd64.c40
-rw-r--r--erts/emulator/hipe/hipe_arch.h12
-rw-r--r--erts/emulator/hipe/hipe_arm.c193
-rw-r--r--erts/emulator/hipe/hipe_bif0.c1111
-rw-r--r--erts/emulator/hipe/hipe_bif0.h15
-rw-r--r--erts/emulator/hipe/hipe_bif0.tab18
-rw-r--r--erts/emulator/hipe/hipe_gc.c41
-rw-r--r--erts/emulator/hipe/hipe_load.c105
-rw-r--r--erts/emulator/hipe/hipe_load.h48
-rw-r--r--erts/emulator/hipe/hipe_mkliterals.c3
-rw-r--r--erts/emulator/hipe/hipe_module.c35
-rw-r--r--erts/emulator/hipe/hipe_module.h45
-rw-r--r--erts/emulator/hipe/hipe_ppc.c178
-rw-r--r--erts/emulator/hipe/hipe_risc_gc.h16
-rw-r--r--erts/emulator/hipe/hipe_risc_glue.h18
-rw-r--r--erts/emulator/hipe/hipe_risc_stack.c16
-rw-r--r--erts/emulator/hipe/hipe_sparc.c113
-rw-r--r--erts/emulator/hipe/hipe_stack.c112
-rw-r--r--erts/emulator/hipe/hipe_stack.h56
-rw-r--r--erts/emulator/hipe/hipe_x86.c118
-rw-r--r--erts/emulator/hipe/hipe_x86_gc.h35
-rw-r--r--erts/emulator/hipe/hipe_x86_glue.h21
-rw-r--r--erts/emulator/hipe/hipe_x86_stack.c23
-rw-r--r--erts/emulator/sys/common/erl_mmap.c4
-rw-r--r--erts/emulator/sys/common/erl_mmap.h5
-rw-r--r--erts/emulator/sys/common/erl_mseg.c2
-rw-r--r--erts/emulator/test/code_SUITE.erl67
-rw-r--r--erts/emulator/test/hipe_SUITE.erl57
-rw-r--r--lib/hipe/cerl/erl_bif_types.erl66
-rw-r--r--lib/hipe/llvm/hipe_llvm_main.erl17
-rw-r--r--lib/hipe/llvm/hipe_rtl_to_llvm.erl29
-rw-r--r--lib/hipe/rtl/hipe_rtl_primops.erl3
-rw-r--r--lib/hipe/rtl/hipe_tagscheme.erl14
-rw-r--r--lib/kernel/src/code_server.erl26
-rw-r--r--lib/kernel/src/hipe_unified_loader.erl346
-rw-r--r--lib/kernel/test/code_SUITE.erl31
-rw-r--r--lib/kernel/test/code_SUITE_data/upgrade_client.erl87
-rw-r--r--lib/kernel/test/code_SUITE_data/upgradee.erl12
55 files changed, 1990 insertions, 1696 deletions
diff --git a/bootstrap/lib/kernel/ebin/code_server.beam b/bootstrap/lib/kernel/ebin/code_server.beam
index 0763c4c93b..e369cc2f36 100644
--- a/bootstrap/lib/kernel/ebin/code_server.beam
+++ b/bootstrap/lib/kernel/ebin/code_server.beam
Binary files differ
diff --git a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam
index 2e639db331..73e95b03c7 100644
--- a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam
+++ b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam
Binary files differ
diff --git a/erts/configure.in b/erts/configure.in
index b276f8bdfb..0e599a65ab 100644
--- a/erts/configure.in
+++ b/erts/configure.in
@@ -2031,7 +2031,7 @@ AC_CHECK_FUNCS([getipnodebyname getipnodebyaddr gethostbyname2])
AC_CHECK_FUNCS([ieee_handler fpsetmask finite isnan isinf res_gethostbyname dlopen \
pread pwrite memmove strerror strerror_r strncasecmp \
- gethrtime localtime_r gmtime_r inet_pton \
+ gethrtime localtime_r gmtime_r inet_pton mprotect \
mmap mremap memcpy mallopt sbrk _sbrk __sbrk brk _brk __brk \
flockfile fstat strlcpy strlcat setsid posix2time time2posix \
setlocale nl_langinfo poll mlockall ppoll])
@@ -2750,6 +2750,18 @@ if test X${enable_hipe} != Xno && test X$ac_cv_sizeof_void_p != X4; then
fi
fi
+dnl HiPE cannot run on 32-bit without mprotect()
+if test X${enable_hipe} != Xno && test X$ac_cv_sizeof_void_p == X4; then
+ if test X$ac_cv_func_mprotect != Xyes; then
+ if test X${enable_hipe} = Xyes; then
+ AC_MSG_ERROR([HiPE on 32-bit needs mprotect()])
+ else
+ enable_hipe=no
+ AC_MSG_WARN([Disable HiPE due to lack of mprotect()])
+ fi
+ fi
+fi
+
dnl check to auto-enable hipe here...
if test "$cross_compiling" != "yes" && test X${enable_hipe} != Xno; then
if test -z "$M4"; then
@@ -3379,13 +3391,8 @@ if test X${enable_hipe} = Xyes; then
AC_DEFINE(HIPE,[1],[Define to enable HiPE])
HIPE_HELPERS="xmerl syntax_tools edoc"
ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS hipe"
- case "$ARCH" in
- amd64)
- # For now exec_alloc is only used for hipe on amd64
- AC_MSG_NOTICE([Enable exec_alloc for hipe code allocation])
- ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS exec_alloc"
- ;;
- esac
+ AC_MSG_NOTICE([Enable exec_alloc for hipe code allocation])
+ ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS exec_alloc"
fi
fi
AC_SUBST(HIPE_HELPERS)
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 21bcbbab27..fa4de5805e 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -884,7 +884,9 @@ HIPE_OBJS= \
$(OBJDIR)/hipe_bif2.o \
$(OBJDIR)/hipe_debug.o \
$(OBJDIR)/hipe_gc.o \
+ $(OBJDIR)/hipe_load.o \
$(OBJDIR)/hipe_mode_switch.o \
+ $(OBJDIR)/hipe_module.o \
$(OBJDIR)/hipe_native_bif.o \
$(OBJDIR)/hipe_stack.o $(HIPE_ARCH_OBJS)
ifdef HIPE_ENABLED
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index bce57fb7c1..099abad4e1 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -36,6 +36,12 @@
#include "erl_nif.h"
#include "erl_bits.h"
#include "erl_thr_progress.h"
+#ifdef HIPE
+# include "hipe_bif0.h"
+# define IF_HIPE(X) (X)
+#else
+# define IF_HIPE(X) (0)
+#endif
#ifdef HIPE
# include "hipe_stack.h"
@@ -143,8 +149,17 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1)
BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
{
+#if !defined(HIPE)
+ BIF_ERROR(BIF_P, EXC_NOTSUP);
+#else
Module* modp;
- Eterm res;
+ Eterm res, mod;
+
+ if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1) ||
+ is_not_atom(mod = erts_module_for_prepared_code
+ (((ProcBin*)binary_val(BIF_ARG_1))->val))) {
+ BIF_ERROR(BIF_P, BADARG);
+ }
if (!erts_try_seize_code_write_permission(BIF_P)) {
ERTS_BIF_YIELD3(bif_export[BIF_code_make_stub_module_3],
@@ -154,7 +169,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
erts_smp_thr_progress_block();
- modp = erts_get_module(BIF_ARG_1, erts_active_code_ix());
+ modp = erts_get_module(mod, erts_active_code_ix());
if (modp && modp->curr.num_breakpoints > 0) {
ASSERT(modp->curr.code_hdr != NULL);
@@ -166,9 +181,12 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3);
- if (res == BIF_ARG_1) {
+ if (res == mod) {
erts_end_staging_code_ix();
erts_commit_staging_code_ix();
+ if (!modp)
+ modp = erts_get_module(mod, erts_active_code_ix());
+ hipe_redirect_to_module(modp);
}
else {
erts_abort_staging_code_ix();
@@ -177,6 +195,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
erts_release_code_write_permission();
return res;
+#endif
}
BIF_RETTYPE
@@ -241,7 +260,7 @@ struct m {
Uint exception;
};
-static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int);
+static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int, int);
#ifdef ERTS_SMP
static void smp_code_ix_commiter(void*);
@@ -377,8 +396,9 @@ finish_loading_1(BIF_ALIST_1)
for (i = 0; i < n; i++) {
if (p[i].modp->curr.num_breakpoints > 0 ||
p[i].modp->curr.num_traced_exports > 0 ||
- erts_is_default_trace_enabled()) {
- /* tracing involved, fallback with thread blocking */
+ erts_is_default_trace_enabled() ||
+ IF_HIPE(hipe_need_blocking(p[i].modp))) {
+ /* tracing or hipe need thread blocking */
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
erts_smp_thr_progress_block();
is_blocking = 1;
@@ -436,32 +456,36 @@ finish_loading_1(BIF_ALIST_1)
}
done:
- return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n);
+ return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n, 1);
}
static Eterm
staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
- struct m* loaded, int nloaded)
+ struct m* mods, int nmods, int free_mods)
{
#ifdef ERTS_SMP
if (is_blocking || !commit)
#endif
{
if (commit) {
+ int i;
erts_end_staging_code_ix();
erts_commit_staging_code_ix();
- if (loaded) {
- int i;
- for (i=0; i < nloaded; i++) {
- set_default_trace_pattern(loaded[i].module);
+
+ for (i=0; i < nmods; i++) {
+ if (mods[i].modp->curr.code_hdr) {
+ set_default_trace_pattern(mods[i].module);
}
+ #ifdef HIPE
+ hipe_redirect_to_module(mods[i].modp);
+ #endif
}
}
else {
erts_abort_staging_code_ix();
}
- if (loaded) {
- erts_free(ERTS_ALC_T_LOADER_TMP, loaded);
+ if (free_mods) {
+ erts_free(ERTS_ALC_T_LOADER_TMP, mods);
}
if (is_blocking) {
erts_smp_thr_progress_unblock();
@@ -474,8 +498,8 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
else {
ASSERT(is_value(res));
- if (loaded) {
- erts_free(ERTS_ALC_T_LOADER_TMP, loaded);
+ if (free_mods) {
+ erts_free(ERTS_ALC_T_LOADER_TMP, mods);
}
erts_end_staging_code_ix();
/*
@@ -653,8 +677,9 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
}
else {
if (modp->curr.num_breakpoints > 0 ||
- modp->curr.num_traced_exports > 0) {
- /* we have tracing, retry single threaded */
+ modp->curr.num_traced_exports > 0 ||
+ IF_HIPE(hipe_need_blocking(modp))) {
+ /* tracing or hipe need to go single threaded */
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
erts_smp_thr_progress_block();
is_blocking = 1;
@@ -668,7 +693,14 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
success = 1;
}
}
- return staging_epilogue(BIF_P, success, res, is_blocking, NULL, 0);
+ {
+ struct m mod;
+ Eterm retval;
+ mod.module = BIF_ARG_1;
+ mod.modp = modp;
+ retval = staging_epilogue(BIF_P, success, res, is_blocking, &mod, 1, 0);
+ return retval;
+ }
}
BIF_RETTYPE module_loaded_1(BIF_ALIST_1)
@@ -809,6 +841,9 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
}
modp->curr.code_hdr->on_load_function_ptr = NULL;
set_default_trace_pattern(BIF_ARG_1);
+ #ifdef HIPE
+ hipe_redirect_to_module(modp);
+ #endif
} else if (BIF_ARG_2 == am_false) {
int i;
@@ -1037,6 +1072,10 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls)
char* mod_start;
Uint mod_size;
Eterm* sp;
+#ifdef HIPE
+ void *nat_start = NULL;
+ Uint nat_size = 0;
+#endif
*redsp += 1;
@@ -1066,6 +1105,20 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls)
}
}
+#ifdef HIPE
+ /*
+ * Check all continuation pointers stored on the native stack if the module
+ * has native code.
+ */
+ if (modp->old.hipe_code) {
+ nat_start = modp->old.hipe_code->text_segment;
+ nat_size = modp->old.hipe_code->text_segment_size;
+ if (nat_size && nstack_any_cps_in_segment(rp, nat_start, nat_size)) {
+ return am_true;
+ }
+ }
+#endif
+
/*
* Check all continuation pointers stored in stackdump
* and clear exception stackdump if there is a pointer
@@ -1082,8 +1135,16 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls)
rp->ftrace = NIL;
} else {
int i;
+ char *area_start = mod_start;
+ Uint area_size = mod_size;
+#ifdef HIPE
+ if (rp->freason & EXF_NATIVE) {
+ area_start = nat_start;
+ area_size = nat_size;
+ }
+#endif
for (i = 0; i < s->depth; i++) {
- if (ErtsInArea(s->trace[i], mod_start, mod_size)) {
+ if (ErtsInArea(s->trace[i], area_start, area_size)) {
rp->freason = EXC_NULL;
rp->fvalue = NIL;
rp->ftrace = NIL;
@@ -1619,15 +1680,18 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
/*
* Unload any NIF library
*/
- if (modp->old.nif != NULL) {
+ if (modp->old.nif != NULL
+ || IF_HIPE(hipe_purge_need_blocking(modp))) {
/* ToDo: Do unload nif without blocking */
erts_rwunlock_old_code(code_ix);
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
erts_smp_thr_progress_block();
is_blocking = 1;
erts_rwlock_old_code(code_ix);
- erts_unload_nif(modp->old.nif);
- modp->old.nif = NULL;
+ if (modp->old.nif) {
+ erts_unload_nif(modp->old.nif);
+ modp->old.nif = NULL;
+ }
}
/*
@@ -1646,7 +1710,9 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
modp->old.code_length = 0;
modp->old.catches = BEAM_CATCHES_NIL;
erts_remove_from_ranges(code);
-
+#ifdef HIPE
+ hipe_purge_module(modp, is_blocking);
+#endif
ERTS_BIF_PREP_RET(ret, am_true);
}
@@ -1719,6 +1785,8 @@ delete_code(Module* modp)
(BeamInstr) BeamOp(op_i_generic_breakpoint)) {
ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
ASSERT(modp->curr.num_traced_exports > 0);
+ DBG_TRACE_MFA_P(&ep->info.mfa,
+ "export trace cleared, code_ix=%d", code_ix);
erts_clear_export_break(modp, &ep->info);
}
else ASSERT(ep->beam[0] == (BeamInstr) em_call_error_handler
@@ -1727,17 +1795,15 @@ delete_code(Module* modp)
ep->addressv[code_ix] = ep->beam;
ep->beam[0] = (BeamInstr) em_call_error_handler;
ep->beam[1] = 0;
+ DBG_TRACE_MFA_P(&ep->info.mfa,
+ "export invalidation, code_ix=%d", code_ix);
}
}
ASSERT(modp->curr.num_breakpoints == 0);
ASSERT(modp->curr.num_traced_exports == 0);
modp->old = modp->curr;
- modp->curr.code_hdr = NULL;
- modp->curr.code_length = 0;
- modp->curr.catches = BEAM_CATCHES_NIL;
- modp->curr.nif = NULL;
-
+ erts_module_instance_init(&modp->curr);
}
@@ -1751,9 +1817,11 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module)
* if not, delete old code; error if old code already exists.
*/
- if (modp->curr.code_hdr && modp->old.code_hdr) {
- return am_not_purged;
- } else if (!modp->old.code_hdr) { /* Make the current version old. */
+ if (modp->curr.code_hdr) {
+ if (modp->old.code_hdr) {
+ return am_not_purged;
+ }
+ /* Make the current version old. */
delete_code(modp);
}
return NIL;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 3b02df8bac..84e2d42af6 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6121,6 +6121,7 @@ call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func)
Uint sz;
int i;
+ DBG_TRACE_MFA_P(mfa, "call_error_handler");
/*
* Search for the error_handler module.
*/
@@ -6676,9 +6677,6 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free)
funp->fe = fe;
funp->num_free = num_free;
funp->creator = p->common.id;
-#ifdef HIPE
- funp->native_address = fe->native_address;
-#endif
funp->arity = (int)fe->address[-1] - num_free;
for (i = 0; i < num_free; i++) {
*hp++ = reg[i];
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 13552ae88c..38574eaf28 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -44,6 +44,7 @@
#include "hipe_bif0.h"
#include "hipe_mode_switch.h"
#include "hipe_arch.h"
+#include "hipe_load.h"
#endif
ErlDrvBinary* erts_gzinflate_buffer(char*, int);
@@ -480,9 +481,12 @@ static void free_loader_state(Binary* magic);
static ErlHeapFragment* new_literal_fragment(Uint size);
static void free_literal_fragment(ErlHeapFragment*);
static void loader_state_dtor(Binary* magic);
+#ifdef HIPE
static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
Eterm group_leader, Eterm module,
- BeamCodeHeader* code, Uint size);
+ BeamCodeHeader* code_hdr, Uint size,
+ HipeModule *hipe_code);
+#endif
static int init_iff_file(LoaderState* stp, byte* code, Uint size);
static int scan_iff_file(LoaderState* stp, Uint* chunk_types,
Uint num_types, Uint num_mandatory);
@@ -537,8 +541,6 @@ static Eterm compilation_info_for_module(Process* p, BeamCodeHeader*);
static Eterm md5_of_module(Process* p, BeamCodeHeader*);
static Eterm has_native(BeamCodeHeader*);
static Eterm native_addresses(Process* p, BeamCodeHeader*);
-int patch_funentries(Eterm Patchlist);
-int patch(Eterm Addresses, Uint fe);
static int safe_mul(UWord a, UWord b, UWord* resp);
static int must_swap_floats;
@@ -842,9 +844,7 @@ erts_finish_loading(Binary* magic, Process* c_p,
erts_alloc(ERTS_ALC_T_PREPARED_CODE,
sizeof(struct erl_module_instance));
inst_p = mod_tab_p->on_load;
- inst_p->nif = 0;
- inst_p->num_breakpoints = 0;
- inst_p->num_traced_exports = 0;
+ erts_module_instance_init(inst_p);
}
inst_p->code_hdr = stp->hdr;
@@ -943,6 +943,13 @@ erts_module_for_prepared_code(Binary* magic)
LoaderState* stp;
if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) {
+#ifdef HIPE
+ HipeLoaderState *hipe_stp;
+ if ((hipe_stp = hipe_get_loader_state(magic))
+ && hipe_stp->text_segment != 0) {
+ return hipe_stp->module;
+ }
+#endif
return NIL;
}
stp = ERTS_MAGIC_BIN_DATA(magic);
@@ -1091,10 +1098,12 @@ loader_state_dtor(Binary* magic)
ASSERT(stp->genop_blocks == 0);
}
+#ifdef HIPE
static Eterm
stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
Eterm group_leader, Eterm module,
- BeamCodeHeader* code_hdr, Uint size)
+ BeamCodeHeader* code_hdr, Uint size,
+ HipeModule *hipe_code)
{
Module* modp;
Eterm retval;
@@ -1117,6 +1126,9 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
modp->curr.code_hdr = code_hdr;
modp->curr.code_length = size;
modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */
+ DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code "
+ "first_hipe_ref = %p", hipe_code->first_hipe_ref);
+ modp->curr.hipe_code = hipe_code;
/*
* Update ranges (used for finding a function from a PC value).
@@ -1125,6 +1137,7 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
erts_update_ranges((BeamInstr*)modp->curr.code_hdr, size);
return NIL;
}
+#endif
static int
init_iff_file(LoaderState* stp, byte* code, Uint size)
@@ -4776,9 +4789,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
}
ep = erts_export_put(stp->module, stp->export[i].function,
stp->export[i].arity);
- if (!on_load) {
- ep->addressv[erts_staging_code_ix()] = address;
- } else {
+ if (on_load) {
/*
* on_load: Don't make any of the exported functions
* callable yet. Keep any function in the current
@@ -4786,6 +4797,8 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
*/
ep->beam[1] = (BeamInstr) address;
}
+ else
+ ep->addressv[erts_staging_code_ix()] = address;
}
/*
@@ -5987,23 +6000,19 @@ code_module_md5_1(BIF_ALIST_1)
return res;
}
+#ifdef HIPE
#define WORDS_PER_FUNCTION (sizeof(ErtsCodeInfo) / sizeof(UWord) + 1)
static BeamInstr*
make_stub(ErtsCodeInfo* info, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode)
{
+ DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", erts_codeinfo_to_code(info));
ASSERT(WORDS_PER_FUNCTION == 6);
info->op = (BeamInstr) BeamOp(op_i_func_info_IaaI);
info->native = native;
info->mfa.module = mod;
info->mfa.function = func;
info->mfa.arity = arity;
-#ifdef HIPE
- if (native) {
- erts_codeinfo_to_code(info)[0] = BeamOpCode(op_move_return_n);
- hipe_mfa_save_orig_beam_op(mod, func, arity, erts_codeinfo_to_code(info));
- }
-#endif
erts_codeinfo_to_code(info)[0] = OpCode;
return erts_codeinfo_to_code(info)+1;
}
@@ -6068,9 +6077,7 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci)
{
unsigned int i;
unsigned int n = stp->num_exps;
-#ifdef HIPE
Lambda* lp;
-#endif
if (is_bif(ci->mfa.module, ci->mfa.function, ci->mfa.arity)) {
ci->native = 0;
@@ -6091,6 +6098,8 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci)
ci->mfa.function,
ci->mfa.arity);
ep->addressv[erts_staging_code_ix()] = erts_codeinfo_to_code(ci);
+ DBG_TRACE_MFA_P(&ci->mfa,"set beam stub at %p in export at %p (code_ix=%d)",
+ erts_codeinfo_to_code(ci), ep, erts_staging_code_ix());
return;
}
}
@@ -6100,7 +6109,6 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci)
* Search the lambda table to find out which.
*/
-#ifdef HIPE
n = stp->num_lambdas;
for (i = 0, lp = stp->lambdas; i < n; i++, lp++) {
ErlFunEntry* fe = stp->lambdas[i].fe;
@@ -6109,7 +6117,6 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci)
fe->address = erts_codeinfo_to_code(ci);
}
}
-#endif
return;
}
@@ -6118,10 +6125,9 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci)
[{Adr, Patchtyppe} | Addresses]
and the address of a fun_entry.
*/
-int
+static int
patch(Eterm Addresses, Uint fe)
{
-#ifdef HIPE
Eterm* listp;
Eterm tuple;
Eterm* tp;
@@ -6157,15 +6163,13 @@ patch(Eterm Addresses, Uint fe)
}
-#endif
return 1;
}
-int
+static int
patch_funentries(Eterm Patchlist)
{
-#ifdef HIPE
while (!is_nil(Patchlist)) {
Eterm Info;
Eterm MFA;
@@ -6254,23 +6258,23 @@ patch_funentries(Eterm Patchlist)
return 0;
}
-#endif
return 1; /* Signal that all went well */
}
-
/*
* Do a dummy load of a module. No threaded code will be loaded.
* Used for loading native code.
* Will also patch all references to fun_entries to point to
* the new fun_entries created.
*/
-
Eterm
-erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
+erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info)
{
Binary* magic;
+ Binary* hipe_magic;
LoaderState* stp;
+ HipeLoaderState* hipe_stp;
+ HipeModule *hipe_code;
BeamInstr Funcs;
BeamInstr Patchlist;
Eterm MD5Bin;
@@ -6293,8 +6297,12 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
*/
magic = erts_alloc_loader_state();
stp = ERTS_MAGIC_BIN_DATA(magic);
+ hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code));
- if (is_not_atom(Mod)) {
+ if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) ||
+ !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val,
+ hipe_stp = hipe_get_loader_state(hipe_magic)) ||
+ hipe_stp->module == NIL || hipe_stp->text_segment == 0) {
goto error;
}
if (is_not_tuple(Info)) {
@@ -6322,7 +6330,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
* Scan the Beam binary and read the interesting sections.
*/
- stp->module = Mod;
+ stp->module = hipe_stp->module;
stp->group_leader = p->group_leader;
stp->num_functions = n;
if (!init_iff_file(stp, bytes, size)) {
@@ -6431,12 +6439,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
* as the body until we know what kind of trap we should put there.
*/
code_hdr->functions[i] = (ErtsCodeInfo*)fp;
-#ifdef HIPE
op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */
-#else
- op = (Eterm) BeamOpCode(op_move_return_n);
-#endif
- fp = make_stub((ErtsCodeInfo*)fp, Mod, func, arity,
+ fp = make_stub((ErtsCodeInfo*)fp, hipe_stp->module, func, arity,
(Uint)native_address, op);
}
@@ -6477,11 +6481,20 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
}
/*
+ * Initialise HiPE module
+ */
+ hipe_code->text_segment = hipe_stp->text_segment;
+ hipe_code->text_segment_size = hipe_stp->text_segment_size;
+ hipe_code->data_segment = hipe_stp->data_segment;
+ hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs;
+ hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc;
+
+ /*
* Insert the module in the module table.
*/
- rval = stub_insert_new_code(p, 0, p->group_leader, Mod,
- code_hdr, code_size);
+ rval = stub_insert_new_code(p, 0, p->group_leader, hipe_stp->module,
+ code_hdr, code_size, hipe_code);
if (rval != NIL) {
goto error;
}
@@ -6497,18 +6510,69 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
}
if (patch_funentries(Patchlist)) {
+ Eterm mod = hipe_stp->module;
+ /* Prevent code from being freed */
+ hipe_stp->text_segment = 0;
+ hipe_stp->data_segment = 0;
+ hipe_stp->new_hipe_refs = NULL;
+ hipe_stp->new_hipe_sdesc = NULL;
+
erts_free_aligned_binary_bytes(temp_alloc);
free_loader_state(magic);
- return Mod;
+ hipe_free_loader_state(hipe_stp);
+
+ return mod;
}
error:
+ erts_free(ERTS_ALC_T_HIPE, hipe_code);
erts_free_aligned_binary_bytes(temp_alloc);
free_loader_state(magic);
BIF_ERROR(p, BADARG);
}
+int erts_commit_hipe_patch_load(Eterm hipe_magic_bin)
+{
+ Binary* hipe_magic;
+ HipeLoaderState* hipe_stp;
+ HipeModule *hipe_code;
+ Module* modp;
+
+ if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) ||
+ !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val,
+ hipe_stp = hipe_get_loader_state(hipe_magic)) ||
+ hipe_stp->module == NIL || hipe_stp->text_segment == 0) {
+ return 0;
+ }
+
+ modp = erts_get_module(hipe_stp->module, erts_active_code_ix());
+ if (!modp)
+ return 0;
+
+ /*
+ * Initialise HiPE module
+ */
+ hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code));
+ hipe_code->text_segment = hipe_stp->text_segment;
+ hipe_code->text_segment_size = hipe_stp->text_segment_size;
+ hipe_code->data_segment = hipe_stp->data_segment;
+ hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs;
+ hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc;
+
+ modp->curr.hipe_code = hipe_code;
+
+ /* Prevent code from being freed */
+ hipe_stp->text_segment = 0;
+ hipe_stp->data_segment = 0;
+ hipe_stp->new_hipe_refs = NULL;
+ hipe_stp->new_hipe_sdesc = NULL;
+
+ return 1;
+}
+
#undef WORDS_PER_FUNCTION
+#endif /* HIPE */
+
static int safe_mul(UWord a, UWord b, UWord* resp)
{
@@ -6522,3 +6586,46 @@ static int safe_mul(UWord a, UWord b, UWord* resp)
}
}
+#ifdef ENABLE_DBG_TRACE_MFA
+
+#define MFA_MAX 10
+Eterm dbg_trace_m[MFA_MAX];
+Eterm dbg_trace_f[MFA_MAX];
+Uint dbg_trace_a[MFA_MAX];
+unsigned int dbg_trace_ix = 0;
+
+void dbg_set_traced_mfa(const char* m, const char* f, Uint a)
+{
+ unsigned i = dbg_trace_ix++;
+ ASSERT(i < MFA_MAX);
+ dbg_trace_m[i] = am_atom_put(m, strlen(m));
+ dbg_trace_f[i] = am_atom_put(f, strlen(f));
+ dbg_trace_a[i] = a;
+}
+
+int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a)
+{
+ unsigned int i;
+ for (i = 0; i < dbg_trace_ix; ++i) {
+ if (m == dbg_trace_m[i] &&
+ (!f || (f == dbg_trace_f[i] && a == dbg_trace_a[i]))) {
+
+ return i+1;
+ }
+ }
+ return 0;
+}
+
+void dbg_vtrace_mfa(unsigned ix, const char* format, ...)
+{
+ va_list arglist;
+ va_start(arglist, format);
+ ASSERT(--ix < MFA_MAX);
+ erts_fprintf(stderr, "MFA TRACE %T:%T/%u: ",
+ dbg_trace_m[ix], dbg_trace_f[ix], (int)dbg_trace_a[ix]);
+
+ erts_vfprintf(stderr, format, arglist);
+ va_end(arglist);
+}
+
+#endif /* ENABLE_DBG_TRACE_MFA */
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index f67dfd7a81..6a3110d0f5 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -143,4 +143,34 @@ struct BeamCodeLineTab_ {
#define LOC_FILE(Loc) ((Loc) >> 24)
#define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1))
+
+/*
+ * MFA event debug "tracing" usage:
+ *
+ * #define ENABLE_DBG_TRACE_MFA
+ * call dbg_set_traced_mfa("mymod","myfunc",arity)
+ * for the function(s) to trace, in some init function.
+ *
+ * Run and get stderr printouts when interesting things happen to your MFA.
+ */
+#ifdef ENABLE_DBG_TRACE_MFA
+
+void dbg_set_traced_mfa(const char* m, const char* f, Uint a);
+int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a);
+void dbg_vtrace_mfa(unsigned ix, const char* format, ...);
+#define DBG_TRACE_MFA(M,F,A,FMT, ...) do {\
+ unsigned ix;\
+ if ((ix=dbg_is_traced_mfa(M,F,A))) \
+ dbg_vtrace_mfa(ix, FMT"\n", ##__VA_ARGS__);\
+ }while(0)
+
+#define DBG_TRACE_MFA_P(MFA, FMT, ...) \
+ DBG_TRACE_MFA((MFA)->module, (MFA)->function, (MFA)->arity, FMT, ##__VA_ARGS__)
+
+#else
+# define dbg_set_traced_mfa(M,F,A)
+# define DBG_TRACE_MFA(M,F,A,FMT, ...)
+# define DBG_TRACE_MFA_P(MFA,FMT, ...)
+#endif /* ENABLE_DBG_TRACE_MFA */
+
#endif /* _BEAM_LOAD_H */
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 3c2c9def3b..56aa78a08a 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -373,10 +373,16 @@ set_default_exec_alloc_opts(struct au_init *ip)
ip->init.util.rmbcmt = 0;
ip->init.util.acul = 0;
+# ifdef ERTS_HAVE_EXEC_MMAPPER
ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc;
ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc;
ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc;
ip->init.util.mseg_mmapper = &erts_exec_mmapper;
+# else
+ ip->init.util.mseg_alloc = &erts_alcu_exec_mseg_alloc;
+ ip->init.util.mseg_realloc = &erts_alcu_exec_mseg_realloc;
+ ip->init.util.mseg_dealloc = &erts_alcu_exec_mseg_dealloc;
+# endif
}
#endif /* ERTS_ALC_A_EXEC */
@@ -1571,7 +1577,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
break;
case 'X':
if (has_prefix("scs", argv[i]+3)) {
-#ifdef ERTS_ALC_A_EXEC
+#ifdef ERTS_HAVE_EXEC_MMAPPER
init->mseg.exec_mmap.scs =
#endif
get_mb_value(argv[i]+6, argv, &i);
@@ -2852,7 +2858,7 @@ erts_allocator_info(int to, void *arg)
erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n");
erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis);
#endif
-#ifdef ERTS_ALC_A_EXEC
+#ifdef ERTS_HAVE_EXEC_MMAPPER
erts_print(to, arg, "=allocator:erts_mmap.exec_mmap\n");
erts_mmap_info(&erts_exec_mmapper, &to, arg, NULL, NULL, &emis);
#endif
@@ -3010,7 +3016,7 @@ erts_allocator_options(void *proc)
#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
terms[length++] = ERTS_MAKE_AM("literal_mmap");
#endif
-#ifdef ERTS_ALC_A_EXEC
+#ifdef ERTS_HAVE_EXEC_MMAPPER
terms[length++] = ERTS_MAKE_AM("exec_mmap");
#endif
features = length ? erts_bld_list(hpp, szp, length, terms) : NIL;
@@ -3102,7 +3108,7 @@ reply_alloc_info(void *vair)
# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
struct erts_mmap_info_struct mmap_info_literal;
# endif
-# ifdef ERTS_ALC_A_EXEC
+# ifdef ERTS_HAVE_EXEC_MMAPPER
struct erts_mmap_info_struct mmap_info_exec;
# endif
#endif
@@ -3232,7 +3238,7 @@ reply_alloc_info(void *vair)
erts_bld_atom(hpp,szp,"literal_mmap"),
ainfo);
# endif
-# ifdef ERTS_ALC_A_EXEC
+# ifdef ERTS_HAVE_EXEC_MMAPPER
ai_list = erts_bld_cons(hpp, szp,
ainfo, ai_list);
ainfo = (air->only_sz ? NIL :
@@ -4129,12 +4135,20 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr)
ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
void *dptr;
Uint size;
+ int free_pattern = n;
ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX);
dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE);
- sys_memset((void *) dptr, n, size + FENCE_SZ);
+#ifdef ERTS_ALC_A_EXEC
+# if defined(__i386__) || defined(__x86_64__)
+ if (ERTS_ALC_T2A(ERTS_ALC_N2T(n)) == ERTS_ALC_A_EXEC) {
+ free_pattern = 0x0f; /* Illegal instruction */
+ }
+# endif
+#endif
+ sys_memset((void *) dptr, free_pattern, size + FENCE_SZ);
(*real_af->free)(n, real_af->extra, dptr);
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 2995f2f822..77be48035d 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -907,7 +907,9 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
#elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
-/* Used by literal allocator that has its own mmapper (super carrier) */
+/* For allocators that have their own mmapper (super carrier),
+ * like literal_alloc and exec_alloc on amd64
+ */
void*
erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
{
@@ -948,6 +950,50 @@ erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size,
}
#endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */
+#if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER)
+
+/*
+ * For exec_alloc on non-amd64 that just need memory with PROT_EXEC
+ */
+void*
+erts_alcu_exec_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags)
+{
+ void* res = erts_alcu_mseg_alloc(allctr, size_p, flags);
+
+ if (res) {
+ int r = mprotect(res, *size_p, PROT_EXEC | PROT_READ | PROT_WRITE);
+ ASSERT(r == 0); (void)r;
+ }
+ return res;
+}
+
+void*
+erts_alcu_exec_mseg_realloc(Allctr_t *allctr, void *seg,
+ Uint old_size, Uint *new_size_p)
+{
+ void *res;
+
+ if (seg && old_size) {
+ int r = mprotect(seg, old_size, PROT_READ | PROT_WRITE);
+ ASSERT(r == 0); (void)r;
+ }
+ res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p);
+ if (res) {
+ int r = mprotect(res, *new_size_p, PROT_EXEC | PROT_READ | PROT_WRITE);
+ ASSERT(r == 0); (void)r;
+ }
+ return res;
+}
+
+void
+erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags)
+{
+ int r = mprotect(seg, size, PROT_READ | PROT_WRITE);
+ ASSERT(r == 0); (void)r;
+ erts_alcu_mseg_dealloc(allctr, seg, size, flags);
+}
+#endif /* ERTS_ALC_A_EXEC && !ERTS_HAVE_EXEC_MMAPPER */
+
#endif /* HAVE_ERTS_MSEG */
void*
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index f50f09907a..2958bdf8d1 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -210,6 +210,12 @@ void* erts_alcu_mmapper_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
void erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
# endif
+
+# if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER)
+void* erts_alcu_exec_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags);
+void* erts_alcu_exec_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p);
+void erts_alcu_exec_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags);
+# endif
#endif /* HAVE_ERTS_MSEG */
void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign);
diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h
index 73c3e19c1c..6ba055d92c 100644
--- a/erts/emulator/beam/erl_fun.h
+++ b/erts/emulator/beam/erl_fun.h
@@ -57,9 +57,6 @@ typedef struct erl_fun_thing {
Eterm thing_word; /* Subtag FUN_SUBTAG. */
ErlFunEntry* fe; /* Pointer to fun entry. */
struct erl_off_heap_header* next;
-#ifdef HIPE
- UWord* native_address; /* Native code for the fun. */
-#endif
Uint arity; /* The arity of the fun. */
Uint num_free; /* Number of free variables (in env). */
/* -- The following may be compound Erlang terms ---------------------- */
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 060fde074c..94568e79a9 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -146,6 +146,8 @@ export_alloc(struct export_entry* tmpl_e)
blob->entryv[ix].ep = &blob->exp;
}
ix = 0;
+
+ DBG_TRACE_MFA_P(&obj->info.mfa, "export allocation at %p", obj);
}
else { /* Existing entry in another table, use free entry in blob */
blob = entry_to_blob(tmpl_e);
@@ -164,9 +166,12 @@ export_free(struct export_entry* obj)
obj->slot.index = -1;
for (i=0; i < ERTS_NUM_CODE_IX; i++) {
if (blob->entryv[i].slot.index >= 0) {
+ DBG_TRACE_MFA_P(&blob->exp.info.mfa, "export entry slot %u freed for %p",
+ (obj - blob->entryv), &blob->exp);
return;
}
}
+ DBG_TRACE_MFA_P(&blob->exp.info.mfa, "export blob deallocation at %p", &blob->exp);
erts_free(ERTS_ALC_T_EXPORT, blob);
erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob));
}
diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index c297bb26b3..98df8a0726 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -3750,7 +3750,6 @@ dec_term_atom_common:
if (funp->fe->native_address == NULL) {
hipe_set_closure_stub(funp->fe, num_free);
}
- funp->native_address = funp->fe->native_address;
#endif
hp = factory->hp;
@@ -3822,9 +3821,6 @@ dec_term_atom_common:
funp->fe = erts_put_fun_entry(module, old_uniq, old_index);
funp->arity = funp->fe->address[-1] - num_free;
-#ifdef HIPE
- funp->native_address = funp->fe->native_address;
-#endif
hp = factory->hp;
/* Environment */
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 7ee046ac12..537aaf3177 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -1048,6 +1048,7 @@ void erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa);
Eterm erts_module_info_0(Process* p, Eterm module);
Eterm erts_module_info_1(Process* p, Eterm module, Eterm what);
Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info);
+int erts_commit_hipe_patch_load(Eterm hipe_magic_bin);
/* beam_ranges.c */
void erts_init_ranges(void);
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index 4f36377450..3e622c0f3a 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -26,6 +26,7 @@
#include "erl_vm.h"
#include "global.h"
#include "module.h"
+#include "beam_catches.h"
#ifdef DEBUG
# define IF_DEBUG(x) x
@@ -67,6 +68,18 @@ static int module_cmp(Module* tmpl, Module* obj)
return tmpl->module != obj->module;
}
+void erts_module_instance_init(struct erl_module_instance* modi)
+{
+ modi->code_hdr = 0;
+ modi->code_length = 0;
+ modi->catches = BEAM_CATCHES_NIL;
+ modi->nif = NULL;
+ modi->num_breakpoints = 0;
+ modi->num_traced_exports = 0;
+#ifdef HIPE
+ modi->hipe_code = NULL;
+#endif
+}
static Module* module_alloc(Module* tmpl)
{
@@ -74,18 +87,11 @@ static Module* module_alloc(Module* tmpl)
erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module));
obj->module = tmpl->module;
- obj->curr.code_hdr = 0;
- obj->old.code_hdr = 0;
- obj->curr.code_length = 0;
- obj->old.code_length = 0;
obj->slot.index = -1;
- obj->curr.nif = NULL;
- obj->old.nif = NULL;
- obj->curr.num_breakpoints = 0;
- obj->old.num_breakpoints = 0;
- obj->curr.num_traced_exports = 0;
- obj->old.num_traced_exports = 0;
+ erts_module_instance_init(&obj->curr);
+ erts_module_instance_init(&obj->old);
obj->on_load = 0;
+ DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc");
return obj;
}
@@ -119,6 +125,7 @@ void init_module_table(void)
erts_smp_atomic_init_nob(&tot_module_bytes, 0);
}
+
Module*
erts_get_module(Eterm mod, ErtsCodeIndex code_ix)
{
@@ -139,19 +146,14 @@ erts_get_module(Eterm mod, ErtsCodeIndex code_ix)
}
}
-Module*
-erts_put_module(Eterm mod)
+
+static Module* put_module(Eterm mod, IndexTable* mod_tab)
{
Module e;
- IndexTable* mod_tab;
int oldsz, newsz;
Module* res;
ASSERT(is_atom(mod));
- ERTS_SMP_LC_ASSERT(erts_initialized == 0
- || erts_has_code_write_permission());
-
- mod_tab = &module_tables[erts_staging_code_ix()];
e.module = atom_val(mod);
oldsz = index_table_sz(mod_tab);
res = (Module*) index_put_entry(mod_tab, (void*) &e);
@@ -160,6 +162,24 @@ erts_put_module(Eterm mod)
return res;
}
+Module*
+erts_put_module(Eterm mod)
+{
+ ERTS_SMP_LC_ASSERT(erts_initialized == 0
+ || erts_has_code_write_permission());
+
+ return put_module(mod, &module_tables[erts_staging_code_ix()]);
+}
+
+Module*
+erts_put_active_module(Eterm mod)
+{
+ ASSERT(is_atom(mod));
+ //SVERK Why not? ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+ return put_module(mod, &module_tables[erts_active_code_ix()]);
+}
+
Module *module_code(int i, ErtsCodeIndex code_ix)
{
return (Module*) erts_index_lookup(&module_tables[code_ix], i);
@@ -181,6 +201,13 @@ static ErtsCodeIndex dbg_load_code_ix = 0;
static int entries_at_start_staging = 0;
+static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod)
+{
+ dst_mod->curr = src_mod->curr;
+ dst_mod->old = src_mod->old;
+ dst_mod->on_load = src_mod->on_load;
+}
+
void module_start_staging(void)
{
IndexTable* src = &module_tables[erts_active_code_ix()];
@@ -199,10 +226,7 @@ void module_start_staging(void)
src_mod = (Module*) erts_index_lookup(src, i);
dst_mod = (Module*) erts_index_lookup(dst, i);
ASSERT(src_mod->module == dst_mod->module);
-
- dst_mod->curr = src_mod->curr;
- dst_mod->old = src_mod->old;
- dst_mod->on_load = src_mod->on_load;
+ copy_module(dst_mod, src_mod);
}
/*
@@ -214,9 +238,7 @@ void module_start_staging(void)
dst_mod = (Module*) index_put_entry(dst, src_mod);
ASSERT(dst_mod != src_mod);
- dst_mod->curr = src_mod->curr;
- dst_mod->old = src_mod->old;
- dst_mod->on_load = src_mod->on_load;
+ copy_module(dst_mod, src_mod);
}
newsz = index_table_sz(dst);
erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz));
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 1c1afc8461..1d6d9dd463 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -23,6 +23,10 @@
#include "index.h"
+#ifdef HIPE
+#include "hipe_module.h"
+#endif
+
struct erl_module_instance {
BeamCodeHeader* code_hdr;
int code_length; /* Length of loaded code in bytes. */
@@ -30,6 +34,9 @@ struct erl_module_instance {
struct erl_module_nif* nif;
int num_breakpoints;
int num_traced_exports;
+#ifdef HIPE
+ HipeModule *hipe_code;
+#endif
};
typedef struct erl_module {
@@ -42,8 +49,10 @@ typedef struct erl_module {
struct erl_module_instance* on_load;
} Module;
+void erts_module_instance_init(struct erl_module_instance* modi);
Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix);
Module* erts_put_module(Eterm mod);
+Module* erts_put_active_module(Eterm mod); /* only while blocked */
void init_module_table(void);
void module_start_staging(void);
diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index 62739d2a78..e3cff4a4ba 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -73,8 +73,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
{
Sint rel32;
- if (trampoline)
- return -1;
+ ASSERT(trampoline == NULL);
+
rel32 = (Sint)destAddress - (Sint)callAddress - 4;
if ((Sint)(Sint32)rel32 != rel32)
return -1;
@@ -83,29 +83,6 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
return 0;
}
-#if 0 /* change to non-zero to get allocation statistics at exit() */
-static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost;
-static unsigned int atexit_done;
-
-static void alloc_code_stats(void)
-{
- printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n",
- total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost);
-}
-
-static void atexit_alloc_code_stats(void)
-{
- if (!atexit_done) {
- atexit_done = 1;
- (void)atexit(alloc_code_stats);
- }
-}
-
-#define ALLOC_CODE_STATS(X) do{X;}while(0)
-#else
-#define ALLOC_CODE_STATS(X) do{}while(0)
-#endif
-
/*
* Memory allocator for executable code.
*
@@ -116,9 +93,6 @@ static void atexit_alloc_code_stats(void)
*/
static void *alloc_code(unsigned int alloc_bytes)
{
- ALLOC_CODE_STATS(++nr_allocs);
- ALLOC_CODE_STATS(total_alloc += alloc_bytes);
-
return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes);
}
@@ -130,6 +104,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
return alloc_code(nrbytes);
}
+void hipe_free_code(void* code, unsigned int bytes)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, code);
+}
+
/* Make stub for native code calling exported beam function.
*/
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
@@ -234,6 +213,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
return code;
}
+void hipe_free_native_stub(void* stub)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
+}
+
void hipe_arch_print_pcb(struct hipe_process_state *p)
{
#define U(n,x) \
diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h
index 6f959815bb..059b8e7f29 100644
--- a/erts/emulator/hipe/hipe_arch.h
+++ b/erts/emulator/hipe/hipe_arch.h
@@ -30,23 +30,31 @@ extern void hipe_patch_load_fe(Uint *address, Uint value);
extern int hipe_patch_insn(void *address, Uint value, Eterm type);
extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline);
-extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p);
-extern void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity);
+extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, struct process *p);
+extern void hipe_free_code(void*, unsigned int);
+extern void *hipe_make_native_stub(void *exp, unsigned int beamArity);
+extern void hipe_free_native_stub(void*);
+
#if defined(__sparc__)
#include "hipe_sparc.h"
+#include "hipe_sparc_asm.h"
#endif
#if defined(__i386__)
#include "hipe_x86.h"
+#include "hipe_x86_asm.h"
#endif
#if defined(__x86_64__)
#include "hipe_amd64.h"
+#include "hipe_amd64_asm.h"
#endif
#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
#include "hipe_ppc.h"
+#include "hipe_ppc_asm.h"
#endif
#if defined(__arm__)
#include "hipe_arm.h"
+#include "hipe_arm_asm.h"
#endif
#if !defined(AEXTERN)
diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c
index f8ef468341..b61939724c 100644
--- a/erts/emulator/hipe/hipe_arm.c
+++ b/erts/emulator/hipe/hipe_arm.c
@@ -25,7 +25,6 @@
#endif
#include "global.h"
#include "erl_binary.h"
-#include <sys/mman.h>
#include "hipe_arch.h"
#include "hipe_native_bif.h" /* nbif_callemu() */
@@ -57,30 +56,6 @@ void hipe_flush_icache_word(void *address)
hipe_flush_icache_range(address, 4);
}
-/*
- * Management of 32MB code segments for regular code and trampolines.
- */
-
-#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */
-
-static struct segment {
- unsigned int *base; /* [base,base+32MB[ */
- unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */
- unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */
- /* On ARM we always allocate a trampoline at base+32MB-8 for
- nbif_callemu, so tramp_pos <= base+32MB-8. */
-} curseg;
-
-#define in_area(ptr,start,nbytes) \
- ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
-
-static void *new_code_mapping(void)
-{
- return mmap(0, SEGMENT_NRBYTES,
- PROT_EXEC|PROT_READ|PROT_WRITE,
- MAP_PRIVATE|MAP_ANONYMOUS,
- -1, 0);
-}
static int check_callees(Eterm callees)
{
@@ -107,126 +82,53 @@ static int check_callees(Eterm callees)
return arity;
}
-static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec)
-{
- unsigned int *base, *address, *tramp_pos, nrfreewords;
- int trampnr;
- Eterm mfa, m, f;
- unsigned int a, *trampoline;
-
- m = NIL; f = NIL; a = 0; /* silence stupid compiler warning */
- tramp_pos = curseg.tramp_pos;
- address = curseg.code_pos;
- nrfreewords = tramp_pos - address;
- if (nrwords > nrfreewords)
- return NULL;
- curseg.code_pos = address + nrwords;
- nrfreewords -= nrwords;
+#define TRAMPOLINE_WORDS 2
- base = curseg.base;
- for (trampnr = 1; trampnr <= nrcallees; ++trampnr) {
- mfa = tuple_val(callees)[trampnr];
- if (is_atom(mfa))
- trampoline = hipe_primop_get_trampoline(mfa);
- else {
- m = tuple_val(mfa)[1];
- f = tuple_val(mfa)[2];
- a = unsigned_val(tuple_val(mfa)[3]);
- trampoline = hipe_mfa_get_trampoline(m, f, a);
- }
- if (!in_area(trampoline, base, SEGMENT_NRBYTES)) {
- if (nrfreewords < 2)
- return NULL;
- nrfreewords -= 2;
- tramp_pos = trampoline = tramp_pos - 2;
- trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */
- trampoline[1] = 0; /* callee's address */
- hipe_flush_icache_range(trampoline, 2*sizeof(int));
- if (is_atom(mfa))
- hipe_primop_set_trampoline(mfa, trampoline);
- else
- hipe_mfa_set_trampoline(m, f, a, trampoline);
- }
- trampvec[trampnr-1] = trampoline;
+static void generate_trampolines(Uint32* address,
+ int nrcallees, Eterm callees,
+ Uint32** trampvec)
+{
+ Uint32* trampoline = address;
+ int i;
+
+ for (i = 0; i < nrcallees; ++i) {
+ trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */
+ trampoline[1] = 0; /* callee's address */
+ trampvec[i] = trampoline;
+ trampoline += TRAMPOLINE_WORDS;
}
- curseg.tramp_pos = tramp_pos;
- return address;
+ hipe_flush_icache_range(address, nrcallees*2*sizeof(Uint32));
}
void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
{
- Uint nrwords;
+ Uint code_words;
int nrcallees;
Eterm trampvecbin;
- unsigned int **trampvec;
- unsigned int *address;
- unsigned int *base;
- struct segment oldseg;
+ Uint32 **trampvec;
+ Uint32 *address;
if (nrbytes & 0x3)
return NULL;
- nrwords = nrbytes >> 2;
+ code_words = nrbytes / sizeof(Uint32);
nrcallees = check_callees(callees);
if (nrcallees < 0)
return NULL;
- trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*));
- trampvec = (unsigned int**)binary_bytes(trampvecbin);
+ trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*));
+ trampvec = (Uint32**)binary_bytes(trampvecbin);
- address = try_alloc(nrwords, nrcallees, callees, trampvec);
- if (!address) {
- base = new_code_mapping();
- if (base == MAP_FAILED)
- return NULL;
- oldseg = curseg;
- curseg.base = base;
- curseg.code_pos = base;
- curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES);
- curseg.tramp_pos -= 2;
- curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */
- curseg.tramp_pos[1] = (unsigned int)&nbif_callemu;
+ address = erts_alloc(ERTS_ALC_T_HIPE_EXEC,
+ (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32));
- address = try_alloc(nrwords, nrcallees, callees, trampvec);
- if (!address) {
- munmap(base, SEGMENT_NRBYTES);
- curseg = oldseg;
- return NULL;
- }
- /* commit to new segment, ignore leftover space in old segment */
- }
+ generate_trampolines(address + code_words, nrcallees, callees, trampvec);
*trampolines = trampvecbin;
return address;
}
-static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu)
+void hipe_free_code(void* code, unsigned int bytes)
{
- unsigned int *address;
- unsigned int *base;
- struct segment oldseg;
-
- address = try_alloc(nrwords, 0, NIL, NULL);
- if (!address) {
- base = new_code_mapping();
- if (base == MAP_FAILED)
- return NULL;
- oldseg = curseg;
- curseg.base = base;
- curseg.code_pos = base;
- curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES);
- curseg.tramp_pos -= 2;
- curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */
- curseg.tramp_pos[1] = (unsigned int)&nbif_callemu;
-
- address = try_alloc(nrwords, 0, NIL, NULL);
- if (!address) {
- munmap(base, SEGMENT_NRBYTES);
- curseg = oldseg;
- return NULL;
- }
- /* commit to new segment, ignore leftover space in old segment */
- }
- *tramp_callemu = (unsigned int*)((char*)curseg.base + SEGMENT_NRBYTES) - 2;
- return address;
+ erts_free(ERTS_ALC_T_HIPE_EXEC, code);
}
/*
@@ -266,8 +168,8 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type)
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
{
unsigned int *code;
- unsigned int *tramp_callemu;
int callemu_offset;
+ int is_short_jmp;
/*
* Native code calls BEAM via a stub looking as follows:
@@ -277,36 +179,57 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
* b nbif_callemu
* .long callee_exp
*
+ * or if nbif_callemu is too far away:
+ *
+ * mov r0, #beamArity
+ * ldr r8, [pc,#0] // callee_exp
+ * ldr pc, [pc,#0] // nbif_callemu
+ * .long callee_exp
+ * .long nbif_callemu
+ *
* I'm using r0 and r8 since they aren't used for
- * parameter passing in native code. The branch to
- * nbif_callemu may need to go via a trampoline.
- * (Trampolines are allowed to modify r12, but they don't.)
+ * parameter passing in native code.
*/
- code = alloc_stub(4, &tramp_callemu);
+ code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 5*sizeof(Uint32));
if (!code)
return NULL;
callemu_offset = ((int)&nbif_callemu - ((int)&code[2] + 8)) >> 2;
- if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) {
- callemu_offset = ((int)tramp_callemu - ((int)&code[2] + 8)) >> 2;
- if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF))
- abort();
+ is_short_jmp = (callemu_offset >= -0x00800000 &&
+ callemu_offset <= 0x007FFFFF);
+#ifdef DEBUG
+ if (is_short_jmp && (callemu_offset % 3)==0) {
+ is_short_jmp = 0;
}
+#endif
/* mov r0, #beamArity */
code[0] = 0xE3A00000 | (beamArity & 0xFF);
/* ldr r8, [pc,#0] // callee_exp */
code[1] = 0xE59F8000;
- /* b nbif_callemu */
- code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF);
+ if (is_short_jmp) {
+ /* b nbif_callemu */
+ code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF);
+ }
+ else {
+ /* ldr pc, [pc,#0] // nbif_callemu */
+ code[2] = 0xE59FF000;
+ /* .long nbif_callemu */
+ code[4] = (unsigned int)&nbif_callemu;
+ }
/* .long callee_exp */
code[3] = (unsigned int)callee_exp;
- hipe_flush_icache_range(code, 4*sizeof(int));
+ hipe_flush_icache_range(code, 5*sizeof(Uint32));
return code;
}
+void hipe_free_native_stub(void* stub)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
+}
+
static void patch_b(Uint32 *address, Sint32 offset, Uint32 AA)
{
Uint32 oldI = *address;
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 9faa6e1649..453a452590 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -44,6 +44,7 @@
#include "hipe_mode_switch.h"
#include "hipe_native_bif.h"
#include "hipe_bif0.h"
+#include "hipe_load.h"
/* We need hipe_literals.h for HIPE_SYSTEM_CRC, but it redefines
a few constants. #undef them here to avoid warnings. */
#undef F_TIMO
@@ -54,6 +55,7 @@
#define BeamOpCode(Op) ((Uint)BeamOp(Op))
+
int term_to_Sint32(Eterm term, Sint *sp)
{
Sint val;
@@ -374,15 +376,31 @@ BIF_RETTYPE hipe_bifs_ref_set_2(BIF_ALIST_2)
}
/*
+ * BIFs for loading code.
+ */
+
+static HipeLoaderState *get_loader_state(Eterm term)
+{
+ ProcBin *pb;
+
+ if (!ERTS_TERM_IS_MAGIC_BINARY(term)) return NULL;
+
+ pb = (ProcBin*) binary_val(term);
+ return hipe_get_loader_state(pb->val);
+}
+
+
+/*
* Allocate memory and copy machine code to it.
*/
-BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
+BIF_RETTYPE hipe_bifs_enter_code_3(BIF_ALIST_3)
{
Uint nrbytes;
void *bytes;
void *address;
Eterm trampolines;
Eterm *hp;
+ HipeLoaderState *stp;
#ifndef DEBUG
ERTS_DECLARE_DUMMY(Uint bitoffs);
ERTS_DECLARE_DUMMY(Uint bitsize);
@@ -391,7 +409,8 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
Uint bitsize;
#endif
- if (is_not_binary(BIF_ARG_1))
+ if (is_not_binary(BIF_ARG_1) ||
+ (!(stp = get_loader_state(BIF_ARG_3))))
BIF_ERROR(BIF_P, BADARG);
nrbytes = binary_size(BIF_ARG_1);
ERTS_GET_BINARY_BYTES(BIF_ARG_1, bytes, bitoffs, bitsize);
@@ -406,11 +425,15 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
nrcallees = arityval(tuple_val(BIF_ARG_2)[0]);
else
nrcallees = 0;
+ // XXX: Is there any reason to not just BIF_ERROR, so that the runtime
+ // survives?
erts_exit(ERTS_ERROR_EXIT, "%s: failed to allocate %lu bytes and %lu trampolines\r\n",
__func__, (unsigned long)nrbytes, (unsigned long)nrcallees);
}
memcpy(address, bytes, nrbytes);
hipe_flush_icache_range(address, nrbytes);
+ stp->text_segment = address;
+ stp->text_segment_size = nrbytes;
hp = HAlloc(BIF_P, 3);
hp[0] = make_arityval(2);
hp[1] = address_to_term(address, BIF_P);
@@ -423,25 +446,34 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2)
/*
* Allocate memory for arbitrary non-Erlang data.
*/
-BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2)
+BIF_RETTYPE hipe_bifs_alloc_data_3(BIF_ALIST_3)
{
- Uint align, nrbytes;
- void *block;
+ Uint align;
+ HipeLoaderState *stp;
if (is_not_small(BIF_ARG_1) || is_not_small(BIF_ARG_2) ||
+ (!(stp = get_loader_state(BIF_ARG_3))) ||
(align = unsigned_val(BIF_ARG_1), !IS_POWER_OF_TWO(align)))
BIF_ERROR(BIF_P, BADARG);
- nrbytes = unsigned_val(BIF_ARG_2);
- if (nrbytes == 0)
+
+ if (stp->data_segment_size || stp->data_segment)
+ BIF_ERROR(BIF_P, BADARG);
+
+ stp->data_segment_size = unsigned_val(BIF_ARG_2);
+ if (stp->data_segment_size == 0)
BIF_RET(make_small(0));
- block = erts_alloc(ERTS_ALC_T_HIPE, nrbytes);
- if ((unsigned long)block & (align-1)) {
- fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte aligned\r\n",
- __FUNCTION__, (unsigned long)nrbytes, block, (unsigned long)align);
- erts_free(ERTS_ALC_T_HIPE, block);
+ stp->data_segment = erts_alloc(ERTS_ALC_T_HIPE, stp->data_segment_size);
+ if ((unsigned long)stp->data_segment & (align-1)) {
+ fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte "
+ "aligned\r\n",
+ __FUNCTION__, (unsigned long)stp->data_segment_size,
+ stp->data_segment, (unsigned long)align);
+ erts_free(ERTS_ALC_T_HIPE, stp->data_segment);
+ stp->data_segment = NULL;
+ stp->data_segment_size = 0;
BIF_ERROR(BIF_P, EXC_NOTSUP);
}
- BIF_RET(address_to_term(block, BIF_P));
+ BIF_RET(address_to_term(stp->data_segment, BIF_P));
}
/*
@@ -537,13 +569,13 @@ BIF_RETTYPE hipe_bifs_merge_term_1(BIF_ALIST_1)
BIF_RET(val);
}
-struct mfa_t {
+struct hipe_mfa {
Eterm mod;
Eterm fun;
Uint ari;
};
-static int term_to_mfa(Eterm term, struct mfa_t *mfa)
+static int term_to_mfa(Eterm term, struct hipe_mfa *mfa)
{
Eterm mod, fun, a;
Uint ari;
@@ -601,7 +633,7 @@ static Uint *hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity)
Uint *hipe_bifs_find_pc_from_mfa(Eterm term)
{
- struct mfa_t mfa;
+ struct hipe_mfa mfa;
if (!term_to_mfa(term, &mfa))
return NULL;
@@ -616,12 +648,20 @@ BIF_RETTYPE hipe_bifs_fun_to_address_1(BIF_ALIST_1)
BIF_RET(address_to_term(pc, BIF_P));
}
+BIF_RETTYPE hipe_bifs_commit_patch_load_1(BIF_ALIST_1)
+{
+ if (!erts_commit_hipe_patch_load(BIF_ARG_1))
+ BIF_ERROR(BIF_P, BADARG);
+
+ BIF_RET(am_ok);
+}
+
BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
{
Eterm *pc;
void *address;
int is_closure;
- struct mfa_t mfa;
+ struct hipe_mfa mfa;
switch (BIF_ARG_3) {
case am_false:
@@ -644,28 +684,22 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
pc = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari);
if (pc) {
- hipe_mfa_save_orig_beam_op(mfa.mod, mfa.fun, mfa.ari, pc);
-#if HIPE
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mfa.mod, mfa.fun, mfa.ari);
- printf(": planting call trap to %p at BEAM pc %p\r\n", address, pc);
-#endif
+ DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", pc, address);
hipe_set_call_trap(pc, address, is_closure);
BIF_RET(am_true);
-#endif
}
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mfa.mod, mfa.fun, mfa.ari);
- printf(": no BEAM pc found\r\n");
-#endif
+ DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "failed set call trap to %p, no beam code found", address);
BIF_RET(am_false);
}
-BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
+BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2)
{
- struct sdesc *sdesc;
+ struct hipe_sdesc *sdesc;
+ HipeLoaderState* stp;
+
+ stp = get_loader_state(BIF_ARG_2);
+ if (!stp)
+ BIF_ERROR(BIF_P, BADARG);
sdesc = hipe_decode_sdesc(BIF_ARG_1);
if (!sdesc) {
@@ -676,6 +710,13 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
fprintf(stderr, "%s: duplicate entry!\r\n", __FUNCTION__);
BIF_ERROR(BIF_P, BADARG);
}
+
+ /*
+ * Link into list of sdesc's in same module instance
+ */
+ sdesc->next_in_modi = stp->new_hipe_sdesc;
+ stp->new_hipe_sdesc = sdesc;
+
BIF_RET(NIL);
}
@@ -778,9 +819,6 @@ BIF_RETTYPE hipe_bifs_bif_address_3(BIF_ALIST_3)
struct primop {
HashBucket bucket; /* bucket.hvalue == atom_val(name) */
const void *address;
-#if defined(__arm__)
- void *trampoline;
-#endif
};
static struct primop primops[] = {
@@ -839,29 +877,6 @@ static struct primop *primop_table_get(Eterm name)
return hash_get(&primop_table, &tmpl);
}
-#if defined(__arm__)
-static struct primop *primop_table_put(Eterm name)
-{
- struct primop tmpl;
-
- init_primop_table();
- tmpl.bucket.hvalue = atom_val(name);
- return hash_put(&primop_table, &tmpl);
-}
-
-void *hipe_primop_get_trampoline(Eterm name)
-{
- struct primop *primop = primop_table_get(name);
- return primop ? primop->trampoline : NULL;
-}
-
-void hipe_primop_set_trampoline(Eterm name, void *trampoline)
-{
- struct primop *primop = primop_table_put(name);
- primop->trampoline = trampoline;
-}
-#endif
-
/*
* hipe_bifs_primop_address(Atom) -> address or false
*/
@@ -970,7 +985,7 @@ BIF_RETTYPE hipe_bifs_get_fe_2(BIF_ALIST_2)
atom_buf[0] = '\0';
strncat(atom_buf, (char*)atom_tab(i)->name, atom_tab(i)->len);
- printf("no fun entry for %s %ld:%ld\n", atom_buf, uniq, index);
+ printf("no fun entry for %s %ld:%ld\n", atom_buf, (unsigned long)uniq, (unsigned long)index);
BIF_ERROR(BIF_P, BADARG);
}
BIF_RET(address_to_term((void *)fe, BIF_P));
@@ -997,30 +1012,35 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2)
BIF_RET(am_true);
}
+struct hipe_ref_head {
+ struct hipe_ref_head* next;
+ struct hipe_ref_head* prev;
+};
+
/*
- * MFA info hash table:
+ * An exported function called from or implemented by native code
* - maps MFA to native code entry point
- * - the MFAs it calls (refers_to)
- * - the references to it (referred_from)
+ * - all references to it (callers)
* - maps MFA to most recent trampoline [if powerpc or arm]
*/
struct hipe_mfa_info {
+ HashBucket mod2mfa;
struct {
unsigned long hvalue;
struct hipe_mfa_info *next;
} bucket;
Eterm m; /* atom */
Eterm f; /* atom */
- unsigned int a;
+ unsigned int a : sizeof(int)*8 - 1;
+ unsigned int is_stub : 1; /* if beam or not (yet) loaded */
void *remote_address;
- void *local_address;
- Eterm *beam_code;
- Uint orig_beam_op;
- struct hipe_mfa_info_list *refers_to;
- struct ref *referred_from;
-#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
- void *trampoline;
+ void *new_address;
+ struct hipe_ref_head callers; /* sentinel in list of hipe_ref's */
+ struct hipe_mfa_info* next_in_mod;
+#ifdef DEBUG
+ Export* dbg_export;
#endif
+
};
static struct {
@@ -1038,6 +1058,82 @@ static struct {
erts_smp_rwmtx_t lock;
} hipe_mfa_info_table;
+Hash mod2mfa_tab; /* map from module atom to list of hipe_mfa_info */
+
+static HashValue mod2mfa_hash(struct hipe_mfa_info* mfa)
+{
+ return mfa->mod2mfa.hvalue;
+}
+
+static int mod2mfa_cmp(HashBucket* tmpl, struct hipe_mfa_info* mfa)
+{
+ return tmpl->hvalue != mfa->mod2mfa.hvalue;
+}
+
+static struct hipe_mfa_info* mod2mfa_alloc(struct hipe_mfa_info* tmpl)
+{
+ return tmpl; /* hash_put always use mfa itself at template */
+}
+
+static void mod2mfa_free(struct hipe_mfa_info* mfa)
+{
+}
+
+static void mod2mfa_tab_init(void)
+{
+ HashFunctions f;
+ static int init_done = 0;
+
+ if (init_done)
+ return;
+ init_done = 1;
+
+ f.hash = (H_FUN) mod2mfa_hash;
+ f.cmp = (HCMP_FUN) mod2mfa_cmp;
+ f.alloc = (HALLOC_FUN) mod2mfa_alloc;
+ f.free = (HFREE_FUN) mod2mfa_free;
+ f.meta_alloc = (HMALLOC_FUN) erts_alloc;
+ f.meta_free = (HMFREE_FUN) erts_free;
+ f.meta_print = (HMPRINT_FUN) erts_print;
+
+ hash_init(ERTS_ALC_T_HIPE, &mod2mfa_tab, "mod2mfa_tab", 50, f);
+}
+
+static struct hipe_mfa_info* mod2mfa_get(Module* modp)
+{
+ HashBucket tmpl;
+ tmpl.hvalue = modp->module;
+ return hash_get(&mod2mfa_tab, &tmpl);
+}
+
+static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa)
+{
+ mfa->mod2mfa.hvalue = atom_val(mfa->m);
+ return hash_put(&mod2mfa_tab, mfa);
+}
+
+
+
+/*
+ * An external native call site M:F(...)
+ * to be patched when the callee changes.
+ */
+struct hipe_ref {
+ struct hipe_ref_head head; /* list of refs to same calleee */
+ void *address;
+#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+ void *trampoline;
+#endif
+ unsigned int flags;
+ struct hipe_ref* next_from_modi; /* list of refs from same module instance */
+#if defined(DEBUG)
+ struct hipe_mfa_info* callee;
+ Eterm caller_m, caller_f, caller_a;
+#endif
+};
+#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */
+
+
static inline void hipe_mfa_info_table_init_lock(void)
{
erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock");
@@ -1063,6 +1159,16 @@ static inline void hipe_mfa_info_table_rwunlock(void)
erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock);
}
+static ERTS_INLINE
+struct hipe_mfa_info* mod2mfa_get_safe(Module* modp)
+{
+ struct hipe_mfa_info* mfa;
+ hipe_mfa_info_table_rlock();
+ mfa = mod2mfa_get(modp);
+ hipe_mfa_info_table_runlock();
+ return mfa;
+}
+
#define HIPE_MFA_HASH(M,F,A) (atom_val(M) ^ atom_val(F) ^ (A))
static struct hipe_mfa_info **hipe_mfa_info_table_alloc_bucket(unsigned int size)
@@ -1108,14 +1214,14 @@ static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigne
res->m = m;
res->f = f;
res->a = arity;
+ res->is_stub = 0;
res->remote_address = NULL;
- res->local_address = NULL;
- res->beam_code = NULL;
- res->orig_beam_op = 0;
- res->refers_to = NULL;
- res->referred_from = NULL;
-#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
- res->trampoline = NULL;
+ res->new_address = NULL;
+ res->callers.next = &res->callers;
+ res->callers.prev = &res->callers;
+ res->next_in_mod = NULL;
+#ifdef DEBUG
+ res->dbg_export = NULL;
#endif
return res;
@@ -1133,6 +1239,8 @@ void hipe_mfa_info_table_init(void)
hipe_mfa_info_table.bucket = hipe_mfa_info_table_alloc_bucket(size);
hipe_mfa_info_table_init_lock();
+
+ mod2mfa_tab_init();
}
static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eterm f, unsigned int arity)
@@ -1154,21 +1262,12 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter
return NULL;
}
-#if 0 /* XXX: unused */
-void *hipe_mfa_find_na(Eterm m, Eterm f, unsigned int arity)
-{
- const struct hipe_mfa_info *p;
-
- p = hipe_mfa_info_table_get(m, f, arity);
- return p ? p->address : NULL;
-}
-#endif
-
static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned int arity)
{
unsigned long h;
unsigned int i;
struct hipe_mfa_info *p;
+ struct hipe_mfa_info *first_in_mod;
unsigned int size;
h = HIPE_MFA_HASH(m, f, arity);
@@ -1189,216 +1288,137 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f,
size = 1 << hipe_mfa_info_table.log2size;
if (hipe_mfa_info_table.used > (4*size/5)) /* rehash at 80% */
hipe_mfa_info_table_grow();
- return p;
-}
-static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address, int is_exported)
-{
- struct hipe_mfa_info *p;
+ first_in_mod = mod2mfa_put(p);
+ if (p != first_in_mod) {
+ p->next_in_mod = first_in_mod->next_in_mod;
+ first_in_mod->next_in_mod = p;
+ }
+ else {
+ p->next_in_mod = NULL;
+ }
- hipe_mfa_info_table_rwlock();
- p = hipe_mfa_info_table_put_rwlocked(m, f, arity);
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(m, f, arity);
- printf(": changing address from %p to %p\r\n", p->local_address, address);
-#endif
- p->local_address = address;
- if (is_exported)
- p->remote_address = address;
- hipe_mfa_info_table_rwunlock();
+ DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p);
+
+ return p;
}
-#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
-void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int arity)
+static void remove_mfa_info(struct hipe_mfa_info* rm)
{
+ unsigned int i;
struct hipe_mfa_info *p;
- void *trampoline;
-
- hipe_mfa_info_table_rlock();
- p = hipe_mfa_info_table_get_locked(m, f, arity);
- trampoline = p ? p->trampoline : NULL;
- hipe_mfa_info_table_runlock();
- return trampoline;
+ struct hipe_mfa_info **prevp;
+
+ i = rm->bucket.hvalue & hipe_mfa_info_table.mask;
+ prevp = &hipe_mfa_info_table.bucket[i];
+ for (;;) {
+ p = *prevp;
+ ASSERT(p);
+ if (p == rm) {
+ *prevp = p->bucket.next;
+ ASSERT(hipe_mfa_info_table.used > 0);
+ hipe_mfa_info_table.used--;
+ return;
+ }
+ prevp = &p->bucket.next;
+ }
}
-void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int arity, void *trampoline)
+static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address)
{
struct hipe_mfa_info *p;
hipe_mfa_info_table_rwlock();
p = hipe_mfa_info_table_put_rwlocked(m, f, arity);
- p->trampoline = trampoline;
+ DBG_TRACE_MFA(m,f,arity,"set native address in hipe_mfa_info at %p", p);
+ p->new_address = address;
+
hipe_mfa_info_table_rwunlock();
}
-#endif
BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3)
{
- struct mfa_t mfa;
+ struct hipe_mfa mfa;
void *address;
- int is_exported;
- if (!term_to_mfa(BIF_ARG_1, &mfa))
- BIF_ERROR(BIF_P, BADARG);
- address = term_to_address(BIF_ARG_2);
- if (!address)
- BIF_ERROR(BIF_P, BADARG);
- if (BIF_ARG_3 == am_true)
- is_exported = 1;
- else if (BIF_ARG_3 == am_false)
- is_exported = 0;
- else
+ switch (BIF_ARG_3) {
+ case am_true: /* is_exported */
+ if (!term_to_mfa(BIF_ARG_1, &mfa))
+ BIF_ERROR(BIF_P, BADARG);
+ address = term_to_address(BIF_ARG_2);
+ if (!address)
+ BIF_ERROR(BIF_P, BADARG);
+ hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address);
+ break;
+ case am_false:
+ break; /* ignore local functions */
+ default:
BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address, is_exported);
- BIF_RET(NIL);
-}
-
-BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1)
-{
- Eterm lst;
- struct mfa_t mfa;
- struct hipe_mfa_info *p;
-
- hipe_mfa_info_table_rwlock();
- lst = BIF_ARG_1;
- while (is_list(lst)) {
- if (!term_to_mfa(CAR(list_val(lst)), &mfa))
- break;
- lst = CDR(list_val(lst));
- p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
- if (p) {
- p->remote_address = NULL;
- p->local_address = NULL;
- if (p->beam_code) {
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mfa.mod, mfa.fun, mfa.ari);
- printf(": removing call trap from BEAM pc %p (new op %#lx)\r\n",
- p->beam_code, p->orig_beam_op);
-#endif
- p->beam_code[0] = p->orig_beam_op;
- p->beam_code = NULL;
- p->orig_beam_op = 0;
- } else {
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mfa.mod, mfa.fun, mfa.ari);
- printf(": no call trap to remove\r\n");
-#endif
- }
- }
}
- hipe_mfa_info_table_rwunlock();
- if (is_not_nil(lst))
- BIF_ERROR(BIF_P, BADARG);
BIF_RET(NIL);
}
-void hipe_mfa_save_orig_beam_op(Eterm mod, Eterm fun, unsigned int ari, Eterm *pc)
+
+/* Ask if we need to block all threads
+ * while loading/deleting code for this module?
+ */
+int hipe_need_blocking(Module* modp)
{
- Uint orig_beam_op;
struct hipe_mfa_info *p;
- orig_beam_op = pc[0];
- if (orig_beam_op != BeamOpCode(op_hipe_trap_call_closure) &&
- orig_beam_op != BeamOpCode(op_hipe_trap_call)) {
- hipe_mfa_info_table_rwlock();
- p = hipe_mfa_info_table_put_rwlocked(mod, fun, ari);
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mod, fun, ari);
- printf(": saving orig op %#lx from BEAM pc %p\r\n", orig_beam_op, pc);
-#endif
- p->beam_code = pc;
- p->orig_beam_op = orig_beam_op;
- hipe_mfa_info_table_rwunlock();
- } else {
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mod, fun, ari);
- printf(": orig op %#lx already saved\r\n", orig_beam_op);
-#endif
+ /* Need to block if we have at least one native caller to this module
+ * or native code to make unaccessible.
+ */
+ hipe_mfa_info_table_rlock();
+ for (p = mod2mfa_get(modp); p; p = p->next_in_mod) {
+ ASSERT(!p->new_address);
+ if (p->callers.next != &p->callers || !p->is_stub) {
+ break;
+ }
}
+ hipe_mfa_info_table_runlock();
+ return (p != NULL);
}
-static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote)
-{
- Export *export_entry;
- void *StubAddress;
-
- ASSERT(is_remote);
-
- export_entry = erts_export_get_or_make_stub(m, f, arity);
- StubAddress = hipe_make_native_stub(export_entry, arity);
- if (!StubAddress)
- erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
- return StubAddress;
-}
-
-static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info **pp)
+static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a)
{
struct hipe_mfa_info *p;
- void *address;
p = hipe_mfa_info_table_get_locked(m, f, a);
- if (p) {
- /* find address, predicting for a runtime apply call */
- address = p->remote_address;
- if (!is_remote)
- address = p->local_address;
- if (address)
- return address;
-
- /* bummer, install stub, checking if one already existed */
- address = p->remote_address;
- if (address)
- return address;
- }
- /* Caller must take the slow path with the write lock held, but allow
- it to avoid some work if it already holds the write lock. */
- if (pp)
- *pp = p;
- return NULL;
+ return p ? p->remote_address : NULL;
}
-static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info *p)
+static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a)
{
- void *address;
+ struct hipe_mfa_info *p = hipe_mfa_info_table_put_rwlocked(m, f, a);
- if (!p)
- p = hipe_mfa_info_table_put_rwlocked(m, f, a);
- address = hipe_make_stub(m, f, a, is_remote);
- /* XXX: how to tell if a BEAM MFA is exported or not? */
- p->remote_address = address;
- return address;
-}
+ if (!p->remote_address) {
+ Export* export_entry = erts_export_get_or_make_stub(m, f, a);
+ void* stubAddress = hipe_make_native_stub(export_entry, a);
+ if (!stubAddress)
+ erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
-static void *hipe_get_na_nofail_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote)
-{
- struct hipe_mfa_info *p;
- void *address;
-
- address = hipe_get_na_try_locked(m, f, a, is_remote, &p);
- if (address)
- return address;
-
- address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, p);
- return address;
+ p->remote_address = stubAddress;
+ p->is_stub = 1;
+#ifdef DEBUG
+ p->dbg_export = export_entry;
+#endif
+ }
+ return p->remote_address;
}
-static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a, int is_remote)
+static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a)
{
void *address;
hipe_mfa_info_table_rlock();
- address = hipe_get_na_try_locked(m, f, a, is_remote, NULL);
+ address = hipe_get_na_try_locked(m, f, a);
hipe_mfa_info_table_runlock();
if (address)
return address;
hipe_mfa_info_table_rwlock();
- address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, NULL);
+ address = hipe_get_na_slow_rwlocked(m, f, a);
hipe_mfa_info_table_rwunlock();
return address;
}
@@ -1408,7 +1428,7 @@ void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a)
{
if (is_not_atom(m) || is_not_atom(f) || a > 255)
return NULL;
- return hipe_get_na_nofail(m, f, a, 1);
+ return hipe_get_na_nofail(m, f, a);
}
/* primop, but called like a BIF for error handling purposes */
@@ -1420,25 +1440,19 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3)
if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2))
BIF_ERROR(BIF_P, BADARG);
arity = unsigned_val(BIF_ARG_3); /* no error check */
- address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity, 1);
+ address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity);
BIF_RET((Eterm)address); /* semi-Ok */
}
-BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2)
+BIF_RETTYPE hipe_bifs_find_na_or_make_stub_1(BIF_ALIST_1)
{
- struct mfa_t mfa;
+ struct hipe_mfa mfa;
void *address;
- int is_remote;
if (!term_to_mfa(BIF_ARG_1, &mfa))
BIF_ERROR(BIF_P, BADARG);
- if (BIF_ARG_2 == am_true)
- is_remote = 1;
- else if (BIF_ARG_2 == am_false)
- is_remote = 0;
- else
- BIF_ERROR(BIF_P, BADARG);
- address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari, is_remote);
+
+ address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari);
BIF_RET(address_to_term(address, BIF_P));
}
@@ -1460,7 +1474,7 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2)
f = ep->info.mfa.function;
} else
goto badfun;
- address = hipe_get_na_nofail(m, f, BIF_ARG_2, 1);
+ address = hipe_get_na_nofail(m, f, BIF_ARG_2);
BIF_RET((Eterm)address);
badfun:
@@ -1471,73 +1485,31 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2)
int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a)
{
- struct hipe_mfa_info *mfa;
- long mfa_offset, ra_offset;
- struct hipe_mfa_info **bucket;
- unsigned int i, nrbuckets;
+ const struct hipe_sdesc* sdesc = hipe_find_sdesc((unsigned long)ra);
- /* Note about locking: the table is only updated from the
- loader, which runs with the rest of the system suspended. */
- /* XXX: alas not true; see comment at hipe_mfa_info_table.lock */
- hipe_mfa_info_table_rlock();
- bucket = hipe_mfa_info_table.bucket;
- nrbuckets = 1 << hipe_mfa_info_table.log2size;
- mfa = NULL;
- mfa_offset = LONG_MAX;
- for (i = 0; i < nrbuckets; ++i) {
- struct hipe_mfa_info *b = bucket[i];
- while (b != NULL) {
- ra_offset = (char*)ra - (char*)b->local_address;
- if (ra_offset > 0 && ra_offset < mfa_offset) {
- mfa_offset = ra_offset;
- mfa = b;
- }
- b = b->bucket.next;
- }
- }
- if (mfa) {
- *m = mfa->m;
- *f = mfa->f;
- *a = mfa->a;
- }
- hipe_mfa_info_table_runlock();
- return mfa ? 1 : 0;
-}
+ if (!sdesc || sdesc->m_aix == atom_val(am_Empty))
+ return 0;
-/*
- * Patch Reference Handling.
- */
-struct hipe_mfa_info_list {
- struct hipe_mfa_info *mfa;
- struct hipe_mfa_info_list *next;
-};
+ *m = make_atom(sdesc->m_aix);
+ *f = make_atom(sdesc->f_aix);
+ *a = sdesc->a;
+ return 1;
+}
-struct ref {
- struct hipe_mfa_info *caller_mfa;
- void *address;
- void *trampoline;
- unsigned int flags;
- struct ref *next;
-};
-#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */
-#define REF_FLAG_IS_REMOTE 2 /* bit 1: 0 == local, 1 == remote */
-#define REF_FLAG_PENDING_REDIRECT 4 /* bit 2: 1 == pending redirect */
-#define REF_FLAG_PENDING_REMOVE 8 /* bit 3: 1 == pending remove */
-/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,'remote'|'local'})
+/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,LoaderState})
*/
BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
{
- struct mfa_t callee;
+ struct hipe_mfa callee;
Eterm *tuple;
- struct mfa_t caller;
+ struct hipe_mfa caller;
void *address;
void *trampoline;
unsigned int flags;
struct hipe_mfa_info *callee_mfa;
- struct hipe_mfa_info *caller_mfa;
- struct hipe_mfa_info_list *refers_to;
- struct ref *ref;
+ struct hipe_ref *ref;
+ HipeLoaderState* stp;
if (!term_to_mfa(BIF_ARG_1, &callee))
goto badarg;
@@ -1568,63 +1540,90 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
if (!trampoline)
goto badarg;
}
- switch (tuple[5]) {
- case am_local:
- break;
- case am_remote:
- flags |= REF_FLAG_IS_REMOTE;
- break;
- default:
- goto badarg;
- }
+ stp = get_loader_state(tuple[5]);
+ if (!stp)
+ goto badarg;
+
hipe_mfa_info_table_rwlock();
callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari);
- caller_mfa = hipe_mfa_info_table_put_rwlocked(caller.mod, caller.fun, caller.ari);
- refers_to = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*refers_to));
- refers_to->mfa = callee_mfa;
- refers_to->next = caller_mfa->refers_to;
- caller_mfa->refers_to = refers_to;
-
- ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*ref));
- ref->caller_mfa = caller_mfa;
+ ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref));
ref->address = address;
+#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
ref->trampoline = trampoline;
+#endif
ref->flags = flags;
- ref->next = callee_mfa->referred_from;
- callee_mfa->referred_from = ref;
+
+ /*
+ * Link into list of refs to same callee
+ */
+ ASSERT(callee_mfa->callers.next->prev == &callee_mfa->callers);
+ ASSERT(callee_mfa->callers.prev->next == &callee_mfa->callers);
+ ref->head.next = callee_mfa->callers.next;
+ ref->head.prev = &callee_mfa->callers;
+ ref->head.next->prev = &ref->head;
+ ref->head.prev->next = &ref->head;
+
+ /*
+ * Link into list of refs from same module instance
+ */
+ ref->next_from_modi = stp->new_hipe_refs;
+ stp->new_hipe_refs = ref;
+
+#if defined(DEBUG)
+ ref->callee = callee_mfa;
+ ref->caller_m = caller.mod;
+ ref->caller_f = caller.fun;
+ ref->caller_a = caller.ari;
+#endif
hipe_mfa_info_table_rwunlock();
+ DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p)",
+ ref, callee.mod, callee.fun, callee.ari, ref->address);
+ DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p)",
+ ref, caller.mod, caller.fun, caller.ari, ref->address);
BIF_RET(NIL);
badarg:
BIF_ERROR(BIF_P, BADARG);
}
-/* Given a CalleeMFA, mark each ref to it as pending-redirect.
- * This ensures that remove_refs_from() won't remove them: any
- * removal is instead done at the end of redirect_referred_from().
- */
-BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */
+
+static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me)
{
- struct mfa_t mfa;
- const struct hipe_mfa_info *p;
- struct ref *ref;
+ struct hipe_mfa_info* p;
- if (!term_to_mfa(BIF_ARG_1, &mfa))
- BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_info_table_rwlock();
- p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
- if (p)
- for (ref = p->referred_from; ref != NULL; ref = ref->next)
- ref->flags |= REF_FLAG_PENDING_REDIRECT;
- hipe_mfa_info_table_rwunlock();
- BIF_RET(NIL);
+ p = hash_get(&mod2mfa_tab, unlink_me);
+ ASSERT(p);
+ if (p == unlink_me) {
+ hash_erase(&mod2mfa_tab, p);
+ if (p->next_in_mod)
+ mod2mfa_put(p->next_in_mod);
+ }
+ else {
+ struct hipe_mfa_info** prevp;
+
+ do {
+ prevp = &p->next_in_mod;
+ p = *prevp;
+ ASSERT(p && p->m == unlink_me->m);
+ } while (p != unlink_me);
+
+ *prevp = p->next_in_mod;
+ }
+}
+
+static void purge_mfa(struct hipe_mfa_info* p)
+{
+ ASSERT(p->is_stub);
+ remove_mfa_info(p);
+ hipe_free_native_stub(p->remote_address);
+ erts_free(ERTS_ALC_T_HIPE, p);
}
/* Called by init:restart after unloading all hipe compiled modules
- * to work around bug causing execution of deallocated beam code.
- * Can be removed when delete/purge of native modules works better.
+ * to work around old bug that caused execution of deallocated beam code.
+ * Can be removed now when delete/purge of native modules works better.
* Test: Do init:restart in debug compiled vm with hipe compiled kernel.
*/
static void hipe_purge_all_refs(void)
@@ -1634,126 +1633,249 @@ static void hipe_purge_all_refs(void)
hipe_mfa_info_table_rwlock();
+ ASSERT(hipe_mfa_info_table.used == 0);
bucket = hipe_mfa_info_table.bucket;
nrbuckets = 1 << hipe_mfa_info_table.log2size;
for (i = 0; i < nrbuckets; ++i) {
+ ASSERT(bucket[i] == NULL);
while (bucket[i] != NULL) {
struct hipe_mfa_info* mfa = bucket[i];
bucket[i] = mfa->bucket.next;
- while (mfa->refers_to) {
- struct hipe_mfa_info_list *to = mfa->refers_to;
- mfa->refers_to = to->next;
- erts_free(ERTS_ALC_T_HIPE, to);
- }
- while (mfa->referred_from) {
- struct ref* from = mfa->referred_from;
- mfa->referred_from = from->next;
- erts_free(ERTS_ALC_T_HIPE, from);
- }
- erts_free(ERTS_ALC_T_HIPE, mfa);
+ hash_erase(&mod2mfa_tab, mfa);
+ erts_free(ERTS_ALC_T_HIPE, mfa);
}
}
+ hipe_mfa_info_table.used = 0;
hipe_mfa_info_table_rwunlock();
}
BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1)
{
- struct mfa_t mfa;
- struct hipe_mfa_info *caller_mfa, *callee_mfa;
- struct hipe_mfa_info_list *refers_to, *tmp_refers_to;
- struct ref **prev, *ref;
-
if (BIF_ARG_1 == am_all) {
hipe_purge_all_refs();
BIF_RET(am_ok);
}
- if (!term_to_mfa(BIF_ARG_1, &mfa))
- BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_info_table_rwlock();
- caller_mfa = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
- if (caller_mfa) {
- refers_to = caller_mfa->refers_to;
- while (refers_to) {
- callee_mfa = refers_to->mfa;
- prev = &callee_mfa->referred_from;
- ref = *prev;
- while (ref) {
- if (ref->caller_mfa == caller_mfa) {
- if (ref->flags & REF_FLAG_PENDING_REDIRECT) {
- ref->flags |= REF_FLAG_PENDING_REMOVE;
- prev = &ref->next;
- ref = ref->next;
- } else {
- struct ref *tmp = ref;
- ref = ref->next;
- *prev = ref;
- erts_free(ERTS_ALC_T_HIPE, tmp);
- }
- } else {
- prev = &ref->next;
- ref = ref->next;
- }
- }
- tmp_refers_to = refers_to;
- refers_to = refers_to->next;
- erts_free(ERTS_ALC_T_HIPE, tmp_refers_to);
- }
- caller_mfa->refers_to = NULL;
+ ASSERT(!"hipe_bifs_remove_refs_from_1() called");
+ BIF_ERROR(BIF_P, BADARG);
+}
+
+int hipe_purge_need_blocking(Module* modp)
+{
+ /* SVERK: Verify if this is really necessary */
+ if (modp->old.hipe_code) {
+ if (modp->old.hipe_code->first_hipe_ref ||
+ modp->old.hipe_code->first_hipe_sdesc)
+ return 1;
+ }
+ if (!modp->curr.code_hdr) {
+ return mod2mfa_get_safe(modp) != NULL;
+ }
+ return 0;
+}
+
+void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module,
+ int is_blocking)
+{
+ struct hipe_ref* ref = first_ref;
+
+ ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+
+ while (ref) {
+ struct hipe_ref* free_ref = ref;
+
+ DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref,
+ ref->callee->m, ref->callee->f, ref->callee->a);
+ DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref,
+ ref->caller_m, ref->caller_f, ref->caller_a);
+ ASSERT(ref->caller_m == caller_module);
+
+ /*
+ * Unlink from other refs to same callee
+ */
+ ASSERT(ref->head.next->prev == &ref->head);
+ ASSERT(ref->head.prev->next == &ref->head);
+ ASSERT(ref->head.next != &ref->head);
+ ASSERT(ref->head.prev != &ref->head);
+ ref->head.next->prev = ref->head.prev;
+ ref->head.prev->next = ref->head.next;
+
+ /*
+ * Was this the last ref to that callee?
+ */
+ if (ref->head.next == ref->head.prev) {
+ struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers);
+ if (p->is_stub) {
+ if (!is_blocking)
+ hipe_mfa_info_table_rwlock();
+ unlink_mfa_from_mod(p);
+ purge_mfa(p);
+ if (!is_blocking)
+ hipe_mfa_info_table_rwunlock();
+ }
+ }
+
+ ref = ref->next_from_modi;
+ erts_free(ERTS_ALC_T_HIPE, free_ref);
+ }
+}
+
+void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module,
+ int is_blocking)
+{
+ struct hipe_sdesc* sdesc = first_sdesc;
+
+ ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+
+ ERTS_SMP_LC_ASSERT(is_blocking); /*XXX Fix safe sdesc destruction */
+
+ while (sdesc) {
+ struct hipe_sdesc* free_sdesc = sdesc;
+
+ DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue);
+ ASSERT(make_atom(sdesc->m_aix) == module);
+
+ sdesc = sdesc->next_in_modi;
+ hipe_destruct_sdesc(free_sdesc);
+ }
+}
+
+
+void hipe_purge_module(Module* modp, int is_blocking)
+{
+ ASSERT(modp);
+
+ ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking());
+
+ DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module");
+
+ if (modp->old.hipe_code) {
+ /*
+ * Remove all hipe_ref's (external calls) from the old module instance
+ */
+ if (modp->old.hipe_code->first_hipe_ref) {
+ ERTS_SMP_LC_ASSERT(is_blocking);
+
+ hipe_purge_refs(modp->old.hipe_code->first_hipe_ref,
+ make_atom(modp->module), is_blocking);
+ modp->old.hipe_code->first_hipe_ref = NULL;
+ }
+
+ /*
+ * Remove all hipe_sdesc's for the old module instance
+ */
+ if (modp->old.hipe_code->first_hipe_sdesc) {
+ ERTS_SMP_LC_ASSERT(is_blocking);
+
+ hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc,
+ make_atom(modp->module), is_blocking);
+ modp->old.hipe_code->first_hipe_sdesc = NULL;
+ }
+
+ hipe_free_module(modp->old.hipe_code);
+ modp->old.hipe_code = NULL;
+ }
+
+
+ /*
+ * Remove unreferred hipe_mfa_info's
+ * when all module instances are removed (like in init:restart)
+ */
+ if (is_blocking && modp->curr.code_hdr == NULL) {
+ struct hipe_mfa_info* was_first = mod2mfa_get(modp);
+ struct hipe_mfa_info* is_first = was_first;
+ struct hipe_mfa_info** prevp = &is_first;
+ struct hipe_mfa_info *p;
+
+ if (was_first) {
+ for (p = was_first ; p; p = *prevp) {
+ if (p->callers.next == &p->callers) {
+ *prevp = p->next_in_mod;
+ if (p != was_first)
+ purge_mfa(p);
+ }
+ else
+ prevp = &p->next_in_mod;
+ }
+ if (was_first != is_first) {
+ hash_erase(&mod2mfa_tab, was_first);
+ purge_mfa(was_first);
+ if (is_first)
+ mod2mfa_put(is_first);
+ }
+ }
}
- hipe_mfa_info_table_rwunlock();
- BIF_RET(am_ok);
}
-/* redirect_referred_from(CalleeMFA)
- * Redirect all pending-redirect refs in CalleeMFA's referred_from.
- * Then remove any pending-redirect && pending-remove refs from CalleeMFA's referred_from.
+/*
+ * Redirect all existing native calls to this module
*/
-BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1)
+void hipe_redirect_to_module(Module* modp)
{
- struct mfa_t mfa;
struct hipe_mfa_info *p;
- struct ref **prev, *ref;
- int is_remote, res;
- void *new_address;
+ struct hipe_ref_head* refh;
+
+ ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+ for (p = mod2mfa_get(modp); p; p = p->next_in_mod) {
+ if (p->new_address) {
+ if (p->is_stub) {
+ hipe_free_native_stub(p->remote_address);
+ p->is_stub = 0;
+ }
+ DBG_TRACE_MFA(p->m, p->f, p->a, "Commit new_address %p", p->new_address);
+ p->remote_address = p->new_address;
+ p->new_address = NULL;
+#ifdef DEBUG
+ p->dbg_export = NULL;
+#endif
+ }
+ else if (!p->is_stub) {
+ Export* exp = erts_export_get_or_make_stub(p->m, p->f, p->a);
+ p->remote_address = hipe_make_native_stub(exp, p->a);
+ DBG_TRACE_MFA(p->m, p->f, p->a, "Commit stub %p", p->remote_address);
+ if (!p->remote_address)
+ erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
+ p->is_stub = 1;
+#ifdef DEBUG
+ p->dbg_export = exp;
+#endif
+ }
+ else {
+ DBG_TRACE_MFA(p->m, p->f, p->a, "Commit no-op, already stub");
+ ASSERT(p->remote_address && p->dbg_export);
+ }
- if (!term_to_mfa(BIF_ARG_1, &mfa))
- BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_info_table_rwlock();
- p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
- if (p) {
- prev = &p->referred_from;
- ref = *prev;
- while (ref) {
- if (ref->flags & REF_FLAG_PENDING_REDIRECT) {
- is_remote = ref->flags & REF_FLAG_IS_REMOTE;
- new_address = hipe_get_na_nofail_rwlocked(p->m, p->f, p->a, is_remote);
- if (ref->flags & REF_FLAG_IS_LOAD_MFA)
- res = hipe_patch_insn(ref->address, (Uint)new_address, am_load_mfa);
- else
- res = hipe_patch_call(ref->address, new_address, ref->trampoline);
- if (res)
- fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__);
- ref->flags &= ~REF_FLAG_PENDING_REDIRECT;
- if (ref->flags & REF_FLAG_PENDING_REMOVE) {
- struct ref *tmp = ref;
- ref = ref->next;
- *prev = ref;
- erts_free(ERTS_ALC_T_HIPE, tmp);
- } else {
- prev = &ref->next;
- ref = ref->next;
- }
- } else {
- prev = &ref->next;
- ref = ref->next;
- }
+ DBG_TRACE_MFA(p->m,p->f,p->a,"START REDIRECT towards hipe_mfa_info at %p", p);
+ for (refh = p->callers.next; refh != &p->callers; refh = refh->next) {
+ struct hipe_ref* ref = (struct hipe_ref*) refh;
+ int res;
+
+ DBG_TRACE_MFA(p->m,p->f,p->a, " REDIRECT ref at %p FROM %T:%T/%u (%p -> %p)",
+ ref, ref->caller_m, ref->caller_f, ref->caller_a,
+ ref->address, p->remote_address);
+
+ DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a,
+ " REDIRECT ref at %p TO %T:%T/%u (%p -> %p)",
+ ref, p->m,p->f,p->a, ref->address, p->remote_address);
+
+ if (ref->flags & REF_FLAG_IS_LOAD_MFA)
+ res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa);
+ else {
+#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
+ void* trampoline = ref->trampoline;
+#else
+ void* trampoline = NULL;
+#endif
+ res = hipe_patch_call(ref->address, p->remote_address, trampoline);
+ }
+ if (res)
+ fprintf(stderr, "%s: patch failed", __FUNCTION__);
}
+ DBG_TRACE_MFA(p->m,p->f,p->a,"DONE REDIRECT towards hipe_mfa_info at %p", p);
}
- hipe_mfa_info_table_rwunlock();
- BIF_RET(NIL);
}
BIF_RETTYPE hipe_bifs_check_crc_1(BIF_ALIST_1)
@@ -1807,89 +1929,6 @@ void hipe_patch_address(Uint *address, Eterm patchtype, Uint value)
}
}
-struct modinfo {
- HashBucket bucket; /* bucket.hvalue == atom_val(the module name) */
- unsigned int code_size;
-};
-
-static Hash modinfo_table;
-
-static HashValue modinfo_hash(void *tmpl)
-{
- Eterm mod = (Eterm)tmpl;
- return atom_val(mod);
-}
-
-static int modinfo_cmp(void *tmpl, void *bucket)
-{
- /* bucket->hvalue == modinfo_hash(tmpl), so just return 0 (match) */
- return 0;
-}
-
-static void *modinfo_alloc(void *tmpl)
-{
- struct modinfo *p;
-
- p = (struct modinfo*)erts_alloc(ERTS_ALC_T_HIPE, sizeof(*p));
- p->code_size = 0;
- return &p->bucket;
-}
-
-static void init_modinfo_table(void)
-{
- HashFunctions f;
- static int init_done = 0;
-
- if (init_done)
- return;
- init_done = 1;
- f.hash = (H_FUN) modinfo_hash;
- f.cmp = (HCMP_FUN) modinfo_cmp;
- f.alloc = (HALLOC_FUN) modinfo_alloc;
- f.free = (HFREE_FUN) NULL;
- f.meta_alloc = (HMALLOC_FUN) erts_alloc;
- f.meta_free = (HMFREE_FUN) erts_free;
- f.meta_print = (HMPRINT_FUN) erts_print;
- hash_init(ERTS_ALC_T_HIPE, &modinfo_table, "modinfo_table", 11, f);
-}
-
-BIF_RETTYPE hipe_bifs_update_code_size_3(BIF_ALIST_3)
-{
- struct modinfo *p;
- Sint code_size;
-
- init_modinfo_table();
-
- if (is_not_atom(BIF_ARG_1) ||
- is_not_small(BIF_ARG_3) ||
- (code_size = signed_val(BIF_ARG_3)) < 0)
- BIF_ERROR(BIF_P, BADARG);
-
- p = (struct modinfo*)hash_put(&modinfo_table, (void*)BIF_ARG_1);
-
- if (is_nil(BIF_ARG_2)) /* some MFAs, not whole module */
- p->code_size += code_size;
- else /* whole module */
- p->code_size = code_size;
- BIF_RET(NIL);
-}
-
-BIF_RETTYPE hipe_bifs_code_size_1(BIF_ALIST_1)
-{
- struct modinfo *p;
- unsigned int code_size;
-
- init_modinfo_table();
-
- if (is_not_atom(BIF_ARG_1))
- BIF_ERROR(BIF_P, BADARG);
-
- p = (struct modinfo*)hash_get(&modinfo_table, (void*)BIF_ARG_1);
-
- code_size = p ? p->code_size : 0;
- BIF_RET(make_small(code_size));
-}
-
BIF_RETTYPE hipe_bifs_patch_insn_3(BIF_ALIST_3)
{
Uint *address, value;
@@ -1925,3 +1964,23 @@ BIF_RETTYPE hipe_bifs_patch_call_3(BIF_ALIST_3)
BIF_ERROR(BIF_P, BADARG);
BIF_RET(NIL);
}
+
+BIF_RETTYPE hipe_bifs_alloc_loader_state_1(BIF_ALIST_1)
+{
+ Binary *magic;
+ Eterm *hp;
+ Eterm res;
+
+ if (is_not_atom(BIF_ARG_1))
+ BIF_ERROR(BIF_P, BADARG);
+
+ magic = hipe_alloc_loader_state(BIF_ARG_1);
+
+ if (!magic)
+ BIF_ERROR(BIF_P, BADARG);
+
+ hp = HAlloc(BIF_P, PROC_BIN_SIZE);
+ res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic);
+ erts_refc_dec(&magic->refc, 1);
+ BIF_RET(res);
+}
diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h
index c9a8216368..4a59bacc6e 100644
--- a/erts/emulator/hipe/hipe_bif0.h
+++ b/erts/emulator/hipe/hipe_bif0.h
@@ -32,17 +32,14 @@ extern void hipe_mfa_info_table_init(void);
extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a);
extern BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3);
extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a);
-#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
-extern void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int a);
-extern void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int a, void *trampoline);
-#endif
-#if defined(__arm__)
-extern void *hipe_primop_get_trampoline(Eterm name);
-extern void hipe_primop_set_trampoline(Eterm name, void *trampoline);
-#endif
/* needed in beam_load.c */
-void hipe_mfa_save_orig_beam_op(Eterm m, Eterm f, unsigned int a, Eterm *pc);
+int hipe_need_blocking(Module*);
+int hipe_purge_need_blocking(Module*);
+void hipe_purge_refs(struct hipe_ref*, Eterm, int is_blocking);
+void hipe_purge_sdescs(struct hipe_sdesc*, Eterm, int is_blocking);
+void hipe_purge_module(Module*, int is_blocking);
+void hipe_redirect_to_module(Module* modp);
/* these are also needed in hipe_amd64.c */
extern void *term_to_address(Eterm);
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 99237aae05..264ea2c34a 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -44,22 +44,20 @@ bif hipe_bifs:ref/1
bif hipe_bifs:ref_get/1
bif hipe_bifs:ref_set/2
-bif hipe_bifs:enter_code/2
-bif hipe_bifs:alloc_data/2
+bif hipe_bifs:enter_code/3
+bif hipe_bifs:alloc_data/3
bif hipe_bifs:constants_size/0
bif hipe_bifs:merge_term/1
bif hipe_bifs:fun_to_address/1
+bif hipe_bifs:commit_patch_load/1
bif hipe_bifs:set_native_address/3
#bif hipe_bifs:address_to_fun/1
bif hipe_bifs:set_funinfo_native_address/3
-bif hipe_bifs:invalidate_funinfo_native_addresses/1
+#bif hipe_bifs:invalidate_funinfo_native_addresses/1
-bif hipe_bifs:update_code_size/3
-bif hipe_bifs:code_size/1
-
-bif hipe_bifs:enter_sdesc/1
+bif hipe_bifs:enter_sdesc/2
bif hipe_bifs:bif_address/3
bif hipe_bifs:primop_address/1
@@ -72,7 +70,7 @@ bif hipe_bifs:term_to_word/1
bif hipe_bifs:get_fe/2
bif hipe_bifs:set_native_address_in_fe/2
-bif hipe_bifs:find_na_or_make_stub/2
+bif hipe_bifs:find_na_or_make_stub/1
bif hipe_bifs:check_crc/1
bif hipe_bifs:system_crc/0
@@ -84,9 +82,9 @@ bif hipe_bifs:patch_insn/3
bif hipe_bifs:patch_call/3
bif hipe_bifs:add_ref/2
-bif hipe_bifs:mark_referred_from/1
bif hipe_bifs:remove_refs_from/1
-bif hipe_bifs:redirect_referred_from/1
+
+bif hipe_bifs:alloc_loader_state/1
# atoms used by add_ref/2
atom call
diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c
index 68c65dea27..e6ce7ce628 100644
--- a/erts/emulator/hipe/hipe_gc.c
+++ b/erts/emulator/hipe/hipe_gc.c
@@ -38,7 +38,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop)
/* known nstack walk state */
Eterm *nsp;
Eterm *nsp_end;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
unsigned int sdesc_size;
unsigned long ra;
unsigned int i;
@@ -123,7 +123,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop)
/* known nstack walk state */
Eterm *nsp;
Eterm *nsp_end;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
unsigned int sdesc_size;
unsigned long ra;
unsigned int i;
@@ -244,7 +244,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area,
/* known nstack walk state */
Eterm *nsp;
Eterm *nsp_end;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
/* arch-specific nstack walk state */
struct nstack_walk_state walk_state;
@@ -311,7 +311,7 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size)
{
Eterm *nsp;
Eterm *nsp_end;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
/* arch-specific nstack walk state */
struct nstack_walk_state walk_state;
@@ -356,3 +356,36 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size)
}
return 0;
}
+
+int
+nstack_any_cps_in_segment(Process *p, char* seg_start, Uint seg_size)
+{
+ Eterm *nsp;
+ Eterm *nsp_end;
+ const struct hipe_sdesc *sdesc;
+ /* arch-specific nstack walk state */
+ struct nstack_walk_state walk_state;
+
+ if (!p->hipe.nstack || !nstack_walk_init_check(p))
+ return 0;
+ ASSERT(p->hipe.nsp && p->hipe.nstend);
+ nsp = nstack_walk_nsp_begin(p);
+ nsp_end = nstack_walk_nsp_end(p);
+ sdesc = nstack_walk_init_sdesc_ignore_trap(p, &walk_state);
+
+ /* Check the topmost frame */
+ if (ErtsInArea(sdesc->bucket.hvalue, seg_start, seg_size))
+ return 1;
+
+ while (!nstack_walk_nsp_reached_end(nsp, nsp_end)) {
+ unsigned sdesc_size = nstack_walk_frame_size(sdesc);
+ unsigned long ra = nstack_walk_frame_ra(nsp, sdesc);
+ if (ra == (unsigned long)nbif_stack_trap_ra)
+ ra = (unsigned long)p->hipe.ngra;
+ if (ErtsInArea(ra, seg_start, seg_size))
+ return 1;
+ sdesc = hipe_find_sdesc(ra);
+ nsp = nstack_walk_next_frame(nsp, sdesc_size);
+ }
+ return 0;
+}
diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c
new file mode 100644
index 0000000000..2998ed87a2
--- /dev/null
+++ b/erts/emulator/hipe/hipe_load.c
@@ -0,0 +1,105 @@
+/*
+ * %CopyrightBegin%
+
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * hipe_load.c
+ *
+ * HiPE atomic code loader
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include "sys.h"
+#include "global.h"
+#include "erl_binary.h"
+#include "hipe_load.h"
+#include "hipe_bif0.h"
+
+void hipe_free_loader_state(HipeLoaderState *stp)
+{
+ if (stp->module == NIL) return;
+
+ // TODO: Needs to be freed separately. We'd like have a unified executable
+ // code allocator, so postpone this for now.
+ /* if (stp->text_segment) */
+ /* erts_free(ERTS_ALC_T_HIPE, stp->text_segment); */
+ stp->text_segment = NULL;
+ stp->text_segment_size = 0;
+
+ if (stp->data_segment)
+ erts_free(ERTS_ALC_T_HIPE, stp->data_segment);
+ stp->data_segment = NULL;
+ stp->data_segment_size = 0;
+
+ if (stp->new_hipe_refs) {
+ hipe_purge_refs(stp->new_hipe_refs, stp->module, 0);
+ stp->new_hipe_refs = NULL;
+ }
+ if (stp->new_hipe_sdesc) {
+ hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module, 0);
+ stp->new_hipe_sdesc = NULL;
+ }
+
+ stp->module = NIL;
+}
+
+static void
+hipe_loader_state_dtor(Binary* magic)
+{
+ HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic);
+
+ ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(magic) == hipe_loader_state_dtor);
+
+ hipe_free_loader_state(stp);
+}
+
+Binary *hipe_alloc_loader_state(Eterm module)
+{
+ HipeLoaderState *stp;
+ Binary *magic;
+
+ if (is_not_atom(module)) return NULL;
+
+ magic = erts_create_magic_binary(sizeof(HipeLoaderState),
+ hipe_loader_state_dtor);
+ erts_refc_inc(&magic->refc, 1);
+ stp = ERTS_MAGIC_BIN_DATA(magic);
+
+ stp->module = module;
+ stp->text_segment = NULL;
+ stp->text_segment_size = 0;
+ stp->data_segment = NULL;
+ stp->data_segment_size = 0;
+
+ stp->new_hipe_refs = NULL;
+ stp->new_hipe_sdesc = NULL;
+
+ return magic;
+}
+
+HipeLoaderState *
+hipe_get_loader_state(Binary *magic)
+{
+ if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor)
+ return NULL;
+
+ return (HipeLoaderState*) ERTS_MAGIC_BIN_DATA(magic);
+}
diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h
new file mode 100644
index 0000000000..40c8a8aa2a
--- /dev/null
+++ b/erts/emulator/hipe/hipe_load.h
@@ -0,0 +1,48 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * hipe_load.h
+ *
+ * HiPE atomic code loader
+ */
+#ifndef HIPE_LOAD_H
+#define HIPE_LOAD_H
+
+#include "global.h"
+
+typedef struct hipe_loader_state {
+ Eterm module; /* Module name, atom */
+
+ void *text_segment;
+ Uint text_segment_size;
+
+ void *data_segment;
+ Uint data_segment_size;
+
+ struct hipe_ref* new_hipe_refs;
+ struct hipe_sdesc* new_hipe_sdesc;
+
+} HipeLoaderState;
+
+extern Binary *hipe_alloc_loader_state(Eterm module);
+extern void hipe_free_loader_state(HipeLoaderState*);
+extern HipeLoaderState *hipe_get_loader_state(Binary *binary);
+
+#endif /* HIPE_LOAD_H */
diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c
index b9d4226705..4573980e1e 100644
--- a/erts/emulator/hipe/hipe_mkliterals.c
+++ b/erts/emulator/hipe/hipe_mkliterals.c
@@ -435,9 +435,6 @@ static const struct rts_param rts_params[] = {
presence or absence of struct erl_fun_thing's "next" field. */
{ 5, "EFT_CREATOR", 1, offsetof(struct erl_fun_thing, creator) },
{ 6, "EFT_FE", 1, offsetof(struct erl_fun_thing, fe) },
-#ifdef HIPE
- { 7, "EFT_NATIVE_ADDRESS", 1, offsetof(struct erl_fun_thing, native_address) },
-#endif
{ 8, "EFT_ARITY", 1, offsetof(struct erl_fun_thing, arity) },
{ 9, "EFT_NUM_FREE", 1, offsetof(struct erl_fun_thing, num_free) },
{ 10, "EFT_ENV", 1, offsetof(struct erl_fun_thing, env[0]) },
diff --git a/erts/emulator/hipe/hipe_module.c b/erts/emulator/hipe/hipe_module.c
new file mode 100644
index 0000000000..469f077dd2
--- /dev/null
+++ b/erts/emulator/hipe/hipe_module.c
@@ -0,0 +1,35 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include "sys.h"
+#include "hipe_arch.h"
+#include "hipe_module.h"
+
+void hipe_free_module(HipeModule *mod)
+{
+ hipe_free_code(mod->text_segment, mod->text_segment_size);
+ if (mod->data_segment) /* Some modules lack data segments */
+ erts_free(ERTS_ALC_T_HIPE, mod->data_segment);
+
+ erts_free(ERTS_ALC_T_HIPE, mod);
+}
diff --git a/erts/emulator/hipe/hipe_module.h b/erts/emulator/hipe/hipe_module.h
new file mode 100644
index 0000000000..b489f567cb
--- /dev/null
+++ b/erts/emulator/hipe/hipe_module.h
@@ -0,0 +1,45 @@
+/*
+ * %CopyrightBegin%
+
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+/*
+ * hipe_module.h
+ *
+ *
+ */
+#ifndef HIPE_MODULE_H
+#define HIPE_MODULE_H
+
+/* Forward-declare type to resolve circular dependency with module.h */
+typedef struct hipe_module HipeModule;
+
+#include "global.h"
+
+struct hipe_module {
+ void *text_segment;
+ Uint text_segment_size;
+ void *data_segment;
+
+ struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */
+ struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */
+};
+
+extern void hipe_free_module(HipeModule *mod);
+
+#endif /* HIPE_MODULE_H */
diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c
index 9b2048c457..4413748936 100644
--- a/erts/emulator/hipe/hipe_ppc.c
+++ b/erts/emulator/hipe/hipe_ppc.c
@@ -25,7 +25,6 @@
#endif
#include "global.h"
#include "erl_binary.h"
-#include <sys/mman.h>
#include "hipe_arch.h"
#include "hipe_native_bif.h" /* nbif_callemu() */
@@ -68,34 +67,6 @@ void hipe_flush_icache_range(void *address, unsigned int nbytes)
asm volatile("sync\n\tisync");
}
-/*
- * Management of 32MB code segments for regular code and trampolines.
- */
-
-#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */
-
-static struct segment {
- unsigned int *base; /* [base,base+32MB[ */
- unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */
- unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */
-} curseg;
-
-#define in_area(ptr,start,nbytes) \
- ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes))
-
-/* Darwin breakage */
-#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-
-static void *new_code_mapping(void)
-{
- return mmap(0, SEGMENT_NRBYTES,
- PROT_EXEC|PROT_READ|PROT_WRITE,
- MAP_PRIVATE|MAP_ANONYMOUS,
- -1, 0);
-}
-
static int check_callees(Eterm callees)
{
Eterm *tuple;
@@ -119,126 +90,71 @@ static int check_callees(Eterm callees)
return arity;
}
-static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec)
+
+static void generate_trampolines(Uint32* address,
+ int nrcallees, Eterm callees,
+ Uint32** trampvec)
{
- unsigned int *base, *address, *tramp_pos, nrfreewords;
- int trampnr;
+ Uint32* trampoline = address;
+ int i;
- tramp_pos = curseg.tramp_pos;
- address = curseg.code_pos;
- nrfreewords = tramp_pos - address;
- if (nrwords > nrfreewords)
- return NULL;
- curseg.code_pos = address + nrwords;
- nrfreewords -= nrwords;
-
- base = curseg.base;
- for (trampnr = 1; trampnr <= nrcallees; ++trampnr) {
- Eterm mfa = tuple_val(callees)[trampnr];
- Eterm m = tuple_val(mfa)[1];
- Eterm f = tuple_val(mfa)[2];
- unsigned int a = unsigned_val(tuple_val(mfa)[3]);
- unsigned int *trampoline = hipe_mfa_get_trampoline(m, f, a);
- if (!in_area(trampoline, base, SEGMENT_NRBYTES)) {
+ for (i = 0; i < nrcallees; ++i) {
#if defined(__powerpc64__)
- if (nrfreewords < 7)
- return NULL;
- nrfreewords -= 7;
- tramp_pos = trampoline = tramp_pos - 7;
- trampoline[0] = 0x3D600000; /* addis r11,r0,0 */
- trampoline[1] = 0x616B0000; /* ori r11,r11,0 */
- trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */
- trampoline[3] = 0x656B0000; /* oris r11,r11,0 */
- trampoline[4] = 0x616B0000; /* ori r11,r11,0 */
- trampoline[5] = 0x7D6903A6; /* mtctr r11 */
- trampoline[6] = 0x4E800420; /* bctr */
- hipe_flush_icache_range(trampoline, 7*sizeof(int));
+# define TRAMPOLINE_WORDS 7
+ trampoline[0] = 0x3D600000; /* addis r11,r0,0 */
+ trampoline[1] = 0x616B0000; /* ori r11,r11,0 */
+ trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */
+ trampoline[3] = 0x656B0000; /* oris r11,r11,0 */
+ trampoline[4] = 0x616B0000; /* ori r11,r11,0 */
+ trampoline[5] = 0x7D6903A6; /* mtctr r11 */
+ trampoline[6] = 0x4E800420; /* bctr */
#else
- if (nrfreewords < 4)
- return NULL;
- nrfreewords -= 4;
- tramp_pos = trampoline = tramp_pos - 4;
- trampoline[0] = 0x39600000; /* addi r11,r0,0 */
- trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */
- trampoline[2] = 0x7D6903A6; /* mtctr r11 */
- trampoline[3] = 0x4E800420; /* bctr */
- hipe_flush_icache_range(trampoline, 4*sizeof(int));
+# define TRAMPOLINE_WORDS 4
+ trampoline[0] = 0x39600000; /* addi r11,r0,0 */
+ trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */
+ trampoline[2] = 0x7D6903A6; /* mtctr r11 */
+ trampoline[3] = 0x4E800420; /* bctr */
#endif
- hipe_mfa_set_trampoline(m, f, a, trampoline);
- }
- trampvec[trampnr-1] = trampoline;
+ trampvec[i] = trampoline;
+ trampoline += TRAMPOLINE_WORDS;
}
- curseg.tramp_pos = tramp_pos;
- return address;
+ hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_WORDS*sizeof(Uint32));
}
void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
{
- Uint nrwords;
+ Uint code_words;
int nrcallees;
Eterm trampvecbin;
- unsigned int **trampvec;
- unsigned int *address;
- unsigned int *base;
- struct segment oldseg;
+ Uint32 **trampvec;
+ Uint32 *address;
if (nrbytes & 0x3)
return NULL;
- nrwords = nrbytes >> 2;
+ code_words = nrbytes / sizeof(Uint32);
nrcallees = check_callees(callees);
if (nrcallees < 0)
return NULL;
- trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*));
- trampvec = (unsigned int**)binary_bytes(trampvecbin);
-
- address = try_alloc(nrwords, nrcallees, callees, trampvec);
- if (!address) {
- base = new_code_mapping();
- if (base == MAP_FAILED)
- return NULL;
- oldseg = curseg;
- curseg.base = base;
- curseg.code_pos = base;
- curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES);
-
- address = try_alloc(nrwords, nrcallees, callees, trampvec);
- if (!address) {
- munmap(base, SEGMENT_NRBYTES);
- curseg = oldseg;
- return NULL;
- }
- /* commit to new segment, ignore leftover space in old segment */
- }
+ trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*));
+ trampvec = (Uint32**)binary_bytes(trampvecbin);
+
+ address = erts_alloc(ERTS_ALC_T_HIPE_EXEC,
+ (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32));
+
+ generate_trampolines(address + code_words, nrcallees, callees, trampvec);
*trampolines = trampvecbin;
return address;
}
-static unsigned int *alloc_stub(Uint nrwords)
+void hipe_free_code(void* code, unsigned int bytes)
{
- unsigned int *address;
- unsigned int *base;
- struct segment oldseg;
-
- address = try_alloc(nrwords, 0, NIL, NULL);
- if (!address) {
- base = new_code_mapping();
- if (base == MAP_FAILED)
- return NULL;
- oldseg = curseg;
- curseg.base = base;
- curseg.code_pos = base;
- curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES);
-
- address = try_alloc(nrwords, 0, NIL, NULL);
- if (!address) {
- munmap(base, SEGMENT_NRBYTES);
- curseg = oldseg;
- return NULL;
- }
- /* commit to new segment, ignore leftover space in old segment */
- }
- return address;
+ erts_free(ERTS_ALC_T_HIPE_EXEC, code);
+}
+
+void hipe_free_native_stub(void* stub)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
}
static void patch_imm16(Uint32 *address, unsigned int imm16)
@@ -288,12 +204,12 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type)
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
{
- unsigned int *code;
+ Uint32 *code;
if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL)
abort();
- code = alloc_stub(7);
+ code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 7*sizeof(Uint32));
if (!code)
return NULL;
@@ -312,7 +228,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
/* ba nbif_callemu */
code[6] = 0x48000002 | (unsigned long)&nbif_callemu;
- hipe_flush_icache_range(code, 7*sizeof(int));
+ hipe_flush_icache_range(code, 7*sizeof(Uint32));
return code;
}
@@ -360,7 +276,7 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type)
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
{
- unsigned int *code;
+ Uint32 *code;
/*
* Native code calls BEAM via a stub looking as follows:
@@ -383,7 +299,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL)
abort();
- code = alloc_stub(4);
+ code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 4*sizeof(Uint32));
if (!code)
return NULL;
@@ -396,7 +312,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
/* ba nbif_callemu */
code[3] = 0x48000002 | (unsigned long)&nbif_callemu;
- hipe_flush_icache_range(code, 4*sizeof(int));
+ hipe_flush_icache_range(code, 4*sizeof(Uint32));
return code;
}
diff --git a/erts/emulator/hipe/hipe_risc_gc.h b/erts/emulator/hipe/hipe_risc_gc.h
index 09568c140e..f019434f67 100644
--- a/erts/emulator/hipe/hipe_risc_gc.h
+++ b/erts/emulator/hipe/hipe_risc_gc.h
@@ -27,7 +27,7 @@
/* arch wrapper includes hipe_${arch}_asm.h to define NR_ARG_REGS */
struct nstack_walk_state {
- const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */
+ const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */
};
static inline int nstack_walk_init_check(const Process *p)
@@ -43,20 +43,20 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p)
return p->hipe.nsp + nstkarity;
}
-static inline const struct sdesc*
+static inline const struct hipe_sdesc*
nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state)
{
- const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra);
+ const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra);
state->sdesc0 = sdesc;
return sdesc;
}
-static inline const struct sdesc*
+static inline const struct hipe_sdesc*
nstack_walk_init_sdesc_ignore_trap(const Process *p,
struct nstack_walk_state *state)
{
unsigned long ra = (unsigned long)p->hipe.nra;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
if (ra == (unsigned long)&nbif_stack_trap_ra)
ra = (unsigned long)p->hipe.ngra;
sdesc = hipe_find_sdesc(ra);
@@ -64,7 +64,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p,
return sdesc;
}
-static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0)
+static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0)
{
Eterm *nsp = p->hipe.nsp;
p->hipe.nsp = nstack_walk_nsp_begin(p);
@@ -103,7 +103,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp
return nsp >= nsp_end;
}
-static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc)
+static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc)
{
return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc);
}
@@ -114,7 +114,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i)
}
static inline unsigned long
-nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc)
+nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc)
{
return nsp[sdesc_fsize(sdesc)];
}
diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h
index 0284265307..1369b392fe 100644
--- a/erts/emulator/hipe/hipe_risc_glue.h
+++ b/erts/emulator/hipe/hipe_risc_glue.h
@@ -66,13 +66,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y)
static __inline__ void hipe_arch_glue_init(void)
{
- static struct sdesc_with_exnra nbif_return_sdesc = {
- .exnra = (unsigned long)&nbif_fail,
- .sdesc = {
- .bucket = { .hvalue = (unsigned long)&nbif_return },
- .summary = (1<<8),
- },
- };
+ static struct hipe_sdesc_with_exnra nbif_return_sdesc;
+
+ nbif_return_sdesc.exnra = (unsigned long)nbif_fail;
+ nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return;
+ nbif_return_sdesc.sdesc.fsize = 0;
+ nbif_return_sdesc.sdesc.has_exnra = 1;
+ nbif_return_sdesc.sdesc.stk_nargs = 0;
+ nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty);
+ nbif_return_sdesc.sdesc.f_aix = atom_val(am_return);
+ nbif_return_sdesc.sdesc.a = 0;
+
hipe_init_sdesc_table(&nbif_return_sdesc.sdesc);
}
diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c
index dc98c96b8f..4001bedeb6 100644
--- a/erts/emulator/hipe/hipe_risc_stack.c
+++ b/erts/emulator/hipe/hipe_risc_stack.c
@@ -56,8 +56,8 @@ void hipe_print_nstack(Process *p)
{
Eterm *nsp;
Eterm *nsp_end;
- const struct sdesc *sdesc1;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc1;
+ const struct hipe_sdesc *sdesc;
unsigned long ra;
unsigned long exnra;
unsigned int mask;
@@ -175,7 +175,7 @@ void hipe_print_nstack(Process *p)
#define MINSTACK 128
#define NSKIPFRAMES 4
-void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc)
+void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc)
{
Eterm *nsp;
Eterm *nsp_end;
@@ -216,7 +216,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc)
void (*hipe_handle_stack_trap(Process *p))(void)
{
void (*ngra)(void) = p->hipe.ngra;
- const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra);
+ const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra);
hipe_update_stack_trap(p, sdesc);
return ngra;
}
@@ -237,7 +237,7 @@ void hipe_find_handler(Process *p)
unsigned long ra;
unsigned long exnra;
unsigned int arity;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
nsp = p->hipe.nsp;
nsp_end = p->hipe.nstend;
@@ -277,7 +277,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace)
Eterm *nsp_end;
unsigned long ra, prev_ra;
unsigned int arity;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
int i;
if (depth < 1)
@@ -292,7 +292,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace)
ra = (unsigned long)p->hipe.nra;
prev_ra = 0;
i = 0;
- for (;;) {
+ while (nsp < nsp_end) {
if (ra == (unsigned long)nbif_stack_trap_ra)
ra = (unsigned long)p->hipe.ngra;
if (ra != prev_ra) {
@@ -302,8 +302,6 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace)
break;
prev_ra = ra;
}
- if (nsp >= nsp_end)
- break;
sdesc = hipe_find_sdesc(ra);
nsp += arity + sdesc_fsize(sdesc);
arity = sdesc_arity(sdesc);
diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c
index 23020f34ee..876b20bb15 100644
--- a/erts/emulator/hipe/hipe_sparc.c
+++ b/erts/emulator/hipe/hipe_sparc.c
@@ -24,7 +24,6 @@
#include "config.h"
#endif
#include "global.h"
-#include <sys/mman.h>
#include "hipe_arch.h"
#include "hipe_native_bif.h" /* nbif_callemu() */
@@ -88,8 +87,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
{
Uint32 relDest, newI;
- if (trampoline)
- return -1;
+ ASSERT(trampoline == NULL);
+
relDest = (Uint32)((Sint32)destAddress - (Sint32)callAddress);
newI = (1 << 30) | (relDest >> 2);
*(Uint32*)callAddress = newI;
@@ -97,105 +96,9 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
return 0;
}
-/*
- * Memory allocator for executable code.
- *
- * This is required on x86 because some combinations
- * of Linux kernels and CPU generations default to
- * non-executable memory mappings, causing ordinary
- * malloc() memory to be non-executable.
- */
-static unsigned int code_bytes;
-static char *code_next;
-
-#if 0 /* change to non-zero to get allocation statistics at exit() */
-static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost;
-static unsigned int atexit_done;
-
-static void alloc_code_stats(void)
-{
- printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n",
- total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost);
-}
-
-static void atexit_alloc_code_stats(void)
-{
- if (!atexit_done) {
- atexit_done = 1;
- (void)atexit(alloc_code_stats);
- }
-}
-
-#define ALLOC_CODE_STATS(X) do{X;}while(0)
-#else
-#define ALLOC_CODE_STATS(X) do{}while(0)
-#endif
-
-static int morecore(unsigned int alloc_bytes)
-{
- unsigned int map_bytes;
- char *map_hint, *map_start;
-
- /* Page-align the amount to allocate. */
- map_bytes = (alloc_bytes + 4095) & ~4095;
-
- /* Round up small allocations. */
- if (map_bytes < 1024*1024)
- map_bytes = 1024*1024;
- else
- ALLOC_CODE_STATS(++nr_large);
-
- /* Create a new memory mapping, ensuring it is executable
- and in the low 2GB of the address space. Also attempt
- to make it adjacent to the previous mapping. */
- map_hint = code_next + code_bytes;
- if ((unsigned long)map_hint & 4095)
- abort();
- map_start = mmap(map_hint, map_bytes,
- PROT_EXEC|PROT_READ|PROT_WRITE,
- MAP_PRIVATE|MAP_ANONYMOUS
-#ifdef __x86_64__
- |MAP_32BIT
-#endif
- ,
- -1, 0);
- if (map_start == MAP_FAILED)
- return -1;
-
- ALLOC_CODE_STATS(total_mapped += map_bytes);
-
- /* Merge adjacent mappings, so the trailing portion of the previous
- mapping isn't lost. In practice this is quite successful. */
- if (map_start == map_hint) {
- ALLOC_CODE_STATS(++nr_joins);
- code_bytes += map_bytes;
- } else {
- ALLOC_CODE_STATS(++nr_splits);
- ALLOC_CODE_STATS(total_lost += code_bytes);
- code_next = map_start;
- code_bytes = map_bytes;
- }
-
- ALLOC_CODE_STATS(atexit_alloc_code_stats());
-
- return 0;
-}
-
static void *alloc_code(unsigned int alloc_bytes)
{
- void *res;
-
- /* Align function entries. */
- alloc_bytes = (alloc_bytes + 3) & ~3;
-
- if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0)
- return NULL;
- ALLOC_CODE_STATS(++nr_allocs);
- ALLOC_CODE_STATS(total_alloc += alloc_bytes);
- res = code_next;
- code_next += alloc_bytes;
- code_bytes -= alloc_bytes;
- return res;
+ return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes);
}
void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
@@ -206,6 +109,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
return alloc_code(nrbytes);
}
+void hipe_free_code(void* code, unsigned int nrbytes)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, code);
+}
+
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
{
unsigned int *code;
@@ -235,6 +143,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
return code;
}
+void hipe_free_native_stub(void* stub)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
+}
+
void hipe_arch_print_pcb(struct hipe_process_state *p)
{
#define U(n,x) \
diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c
index e2e6eb74b1..b80e44bc37 100644
--- a/erts/emulator/hipe/hipe_stack.c
+++ b/erts/emulator/hipe/hipe_stack.c
@@ -43,10 +43,10 @@
*/
struct hipe_sdesc_table hipe_sdesc_table;
-static struct sdesc **alloc_bucket(unsigned int size)
+static struct hipe_sdesc **alloc_bucket(unsigned int size)
{
- unsigned long nbytes = size * sizeof(struct sdesc*);
- struct sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes);
+ unsigned long nbytes = size * sizeof(struct hipe_sdesc*);
+ struct hipe_sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes);
sys_memzero(bucket, nbytes);
return bucket;
}
@@ -54,7 +54,7 @@ static struct sdesc **alloc_bucket(unsigned int size)
static void hipe_grow_sdesc_table(void)
{
unsigned int old_size, new_size, new_mask;
- struct sdesc **old_bucket, **new_bucket;
+ struct hipe_sdesc **old_bucket, **new_bucket;
unsigned int i;
old_size = 1 << hipe_sdesc_table.log2size;
@@ -66,9 +66,9 @@ static void hipe_grow_sdesc_table(void)
new_bucket = alloc_bucket(new_size);
hipe_sdesc_table.bucket = new_bucket;
for (i = 0; i < old_size; ++i) {
- struct sdesc *b = old_bucket[i];
+ struct hipe_sdesc *b = old_bucket[i];
while (b != NULL) {
- struct sdesc *next = b->bucket.next;
+ struct hipe_sdesc *next = b->bucket.next;
unsigned int j = (b->bucket.hvalue >> HIPE_RA_LSR_COUNT) & new_mask;
b->bucket.next = new_bucket[j];
new_bucket[j] = b;
@@ -78,11 +78,11 @@ static void hipe_grow_sdesc_table(void)
erts_free(ERTS_ALC_T_HIPE, old_bucket);
}
-struct sdesc *hipe_put_sdesc(struct sdesc *sdesc)
+struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc)
{
unsigned long ra;
unsigned int i;
- struct sdesc *chain;
+ struct hipe_sdesc *chain;
unsigned int size;
ra = sdesc->bucket.hvalue;
@@ -102,7 +102,29 @@ struct sdesc *hipe_put_sdesc(struct sdesc *sdesc)
return sdesc;
}
-void hipe_init_sdesc_table(struct sdesc *sdesc)
+void hipe_destruct_sdesc(struct hipe_sdesc *sdesc)
+{
+ unsigned int i;
+ struct hipe_sdesc** prevp;
+ void* free_me;
+
+ i = (sdesc->bucket.hvalue >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask;
+ prevp = &hipe_sdesc_table.bucket[i];
+
+ for (; *prevp != sdesc; prevp = &(*prevp)->bucket.next)
+ ASSERT(*prevp);
+
+ *prevp = sdesc->bucket.next;
+ hipe_sdesc_table.used -= 1;
+
+ if (sdesc->has_exnra)
+ free_me = ErtsContainerStruct(sdesc, struct hipe_sdesc_with_exnra, sdesc);
+ else
+ free_me = sdesc;
+ erts_free(ERTS_ALC_T_HIPE, free_me);
+}
+
+void hipe_init_sdesc_table(struct hipe_sdesc *sdesc)
{
unsigned int log2size, size;
@@ -121,31 +143,46 @@ void hipe_init_sdesc_table(struct sdesc *sdesc)
* representation. If different representations are needed in
* the future, this code has to be made target dependent.
*/
-struct sdesc *hipe_decode_sdesc(Eterm arg)
+struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
{
Uint ra, exnra;
Eterm *live;
- Uint fsize, arity, nlive, i, nslots, off;
+ Uint fsize, nargs, stk_nargs, nlive, i, nslots, off;
Uint livebitswords, sdescbytes;
void *p;
- struct sdesc *sdesc;
-
- if (is_not_tuple(arg) ||
- (tuple_val(arg))[0] != make_arityval(6) ||
- term_to_Uint((tuple_val(arg))[1], &ra) == 0 ||
- term_to_Uint((tuple_val(arg))[2], &exnra) == 0 ||
- is_not_small((tuple_val(arg))[3]) ||
- (fsize = unsigned_val((tuple_val(arg))[3])) > 65535 ||
- is_not_small((tuple_val(arg))[4]) ||
- (arity = unsigned_val((tuple_val(arg))[4])) > 255 ||
- is_not_tuple((tuple_val(arg))[5]))
+ struct hipe_sdesc *sdesc;
+ Eterm* mfa_tpl;
+ Eterm* tp;
+
+ if (is_not_tuple(arg))
+ return 0;
+
+ tp = tuple_val(arg);
+ if (tp[0] != make_arityval(6) ||
+ term_to_Uint(tp[1], &ra) == 0 ||
+ term_to_Uint(tp[2], &exnra) == 0 ||
+ is_not_small(tp[3]) ||
+ (fsize = unsigned_val(tp[3])) > 65535 ||
+ is_not_small(tp[4]) ||
+ (stk_nargs = unsigned_val(tp[4])) > 255 ||
+ is_not_tuple(tp[5]) ||
+ is_not_tuple(tp[6]) ||
+ (mfa_tpl = tuple_val(tp[6]))[0] != make_arityval(3) ||
+ is_not_atom(mfa_tpl[1]) ||
+ is_not_atom(mfa_tpl[2]) ||
+ is_not_small(mfa_tpl[3]) ||
+ (nargs = unsigned_val(mfa_tpl[3])) > 255)
return 0;
+
+ if (stk_nargs > nargs)
+ return 0;
+
/* Get tuple with live slots */
- live = tuple_val((tuple_val(arg))[5]) + 1;
+ live = tuple_val(tp[5]) + 1;
/* Get number of live slots */
nlive = arityval(live[-1]);
- /* Calculate size of frame = locals + ra + arguments */
- nslots = fsize + 1 + arity;
+ /* Calculate size of frame = locals + ra + stack arguments */
+ nslots = fsize + 1 + stk_nargs;
/* Check that only valid slots are given. */
for (i = 0; i < nlive; ++i) {
if (is_not_small(live[i]) ||
@@ -155,27 +192,34 @@ struct sdesc *hipe_decode_sdesc(Eterm arg)
}
/* Calculate number of words for the live bitmap. */
- livebitswords = (fsize + arity + 1 + 31) / 32;
+ livebitswords = (fsize + stk_nargs + 1 + 31) / 32;
/* Calculate number of bytes needed for the stack descriptor. */
sdescbytes =
(exnra
- ? offsetof(struct sdesc_with_exnra, sdesc.livebits)
- : offsetof(struct sdesc, livebits))
+ ? offsetof(struct hipe_sdesc_with_exnra, sdesc.livebits)
+ : offsetof(struct hipe_sdesc, livebits))
+ livebitswords * sizeof(int);
p = erts_alloc(ERTS_ALC_T_HIPE, sdescbytes);
/* If we have an exception handler use the
special sdesc_with_exnra structure. */
if (exnra) {
- struct sdesc_with_exnra *sdesc_we = p;
+ struct hipe_sdesc_with_exnra *sdesc_we = p;
sdesc_we->exnra = exnra;
sdesc = &(sdesc_we->sdesc);
} else
sdesc = p;
+ sdesc->m_aix = atom_val(mfa_tpl[1]);
+ sdesc->f_aix = atom_val(mfa_tpl[2]);
+ sdesc->a = nargs;
+
+
/* Initialise head of sdesc. */
sdesc->bucket.next = 0;
sdesc->bucket.hvalue = ra;
- sdesc->summary = (fsize << 9) | (exnra ? (1<<8) : 0) | arity;
+ sdesc->fsize = fsize;
+ sdesc->has_exnra = (exnra ? 1 : 0);
+ sdesc->stk_nargs = stk_nargs;
/* Clear all live-bits */
for (i = 0; i < livebitswords; ++i)
sdesc->livebits[i] = 0;
@@ -184,13 +228,5 @@ struct sdesc *hipe_decode_sdesc(Eterm arg)
off = unsigned_val(live[i]);
sdesc->livebits[off / 32] |= (1 << (off & 31));
}
-#ifdef DEBUG
- {
- Eterm mfa_tpl = tuple_val(arg)[6];
- sdesc->dbg_M = tuple_val(mfa_tpl)[1];
- sdesc->dbg_F = tuple_val(mfa_tpl)[2];
- sdesc->dbg_A = tuple_val(mfa_tpl)[3];
- }
-#endif
return sdesc;
}
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index afa0ed4256..7e30358767 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -30,40 +30,43 @@
#include <stddef.h> /* offsetof() */
-struct sdesc {
+struct hipe_sdesc {
struct {
unsigned long hvalue; /* return address */
- struct sdesc *next; /* hash collision chain */
+ struct hipe_sdesc *next; /* hash collision chain */
} bucket;
- unsigned int summary; /* frame size, exn handler presence flag, arity */
-#ifdef DEBUG
- Eterm dbg_M, dbg_F;
- unsigned dbg_A;
-#endif
- unsigned int livebits[1]; /* size depends on arch & data in summary field */
+ unsigned int fsize : 23; /* frame size */
+ unsigned int has_exnra : 1; /* exn handler presence flag */
+ unsigned int stk_nargs : 8; /* arguments on stack */
+ Uint32 m_aix;
+ Uint32 f_aix;
+ Uint32 a;
+ struct hipe_sdesc* next_in_modi;
+ Uint32 livebits[1]; /* size depends on arch & data in summary field */
};
-struct sdesc_with_exnra {
+struct hipe_sdesc_with_exnra {
unsigned long exnra;
- struct sdesc sdesc;
+ struct hipe_sdesc sdesc;
};
-static __inline__ unsigned int sdesc_fsize(const struct sdesc *sdesc)
+static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc)
{
- return sdesc->summary >> 9;
+ return sdesc->fsize;
}
-static __inline__ unsigned int sdesc_arity(const struct sdesc *sdesc)
+/* Nr of arguments pushed on stack */
+static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc)
{
- return sdesc->summary & 0xFF;
+ return sdesc->stk_nargs;
}
-static __inline__ unsigned long sdesc_exnra(const struct sdesc *sdesc)
+static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc)
{
- if ((sdesc->summary & (1<<8))) {
+ if (sdesc->has_exnra) {
const char *tmp;
- tmp = (const char*)sdesc - offsetof(struct sdesc_with_exnra, sdesc);
- return ((const struct sdesc_with_exnra*)tmp)->exnra;
+ tmp = (const char*)sdesc - offsetof(struct hipe_sdesc_with_exnra, sdesc);
+ return ((const struct hipe_sdesc_with_exnra*)tmp)->exnra;
}
return 0;
}
@@ -72,13 +75,14 @@ struct hipe_sdesc_table {
unsigned int log2size;
unsigned int mask; /* INV: mask == (1 << log2size)-1 */
unsigned int used;
- struct sdesc **bucket;
+ struct hipe_sdesc **bucket;
};
extern struct hipe_sdesc_table hipe_sdesc_table;
-extern struct sdesc *hipe_put_sdesc(struct sdesc*);
-extern void hipe_init_sdesc_table(struct sdesc*);
-extern struct sdesc *hipe_decode_sdesc(Eterm);
+extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*);
+extern void hipe_destruct_sdesc(struct hipe_sdesc*);
+extern void hipe_init_sdesc_table(struct hipe_sdesc*);
+extern struct hipe_sdesc *hipe_decode_sdesc(Eterm);
#if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
#define __builtin_expect(x, expected_value) (x)
@@ -86,10 +90,10 @@ extern struct sdesc *hipe_decode_sdesc(Eterm);
#define likely(x) __builtin_expect((x),1)
#define unlikely(x) __builtin_expect((x),0)
-static __inline__ const struct sdesc *hipe_find_sdesc(unsigned long ra)
+static __inline__ const struct hipe_sdesc *hipe_find_sdesc(unsigned long ra)
{
unsigned int i = (ra >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask;
- const struct sdesc *sdesc = hipe_sdesc_table.bucket[i];
+ const struct hipe_sdesc *sdesc = hipe_sdesc_table.bucket[i];
if (likely(sdesc->bucket.hvalue == ra))
return sdesc;
do {
@@ -103,7 +107,7 @@ AEXTERN(void,nbif_stack_trap_ra,(void));
extern void hipe_print_nstack(Process*);
extern void hipe_find_handler(Process*);
extern void (*hipe_handle_stack_trap(Process*))(void);
-extern void hipe_update_stack_trap(Process*, const struct sdesc*);
+extern void hipe_update_stack_trap(Process*, const struct hipe_sdesc*);
extern int hipe_fill_stacktrace(Process*, int, Eterm**);
#if 0 && defined(HIPE_NSTACK_GROWS_UP)
@@ -134,5 +138,7 @@ extern void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop
extern Eterm *sweep_literals_nstack(Process *p, Eterm *n_htop, char *area,
Uint area_size);
extern int nstack_any_heap_ref_ptrs(Process *, char* mod_start, Uint mod_size);
+extern int nstack_any_cps_in_segment(Process *, char* seg_start, Uint seg_size);
+
#endif /* HIPE_STACK_H */
diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c
index 5f6c8c200e..c7e24673ac 100644
--- a/erts/emulator/hipe/hipe_x86.c
+++ b/erts/emulator/hipe/hipe_x86.c
@@ -24,7 +24,6 @@
#include "config.h"
#endif
#include "global.h"
-#include <sys/mman.h>
#include "hipe_arch.h"
#include "hipe_native_bif.h" /* nbif_callemu() */
@@ -62,118 +61,17 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline)
{
Uint rel32;
- if (trampoline)
- return -1;
+ ASSERT(trampoline == NULL);
+
rel32 = (Uint)destAddress - (Uint)callAddress - 4;
*(Uint32*)callAddress = rel32;
hipe_flush_icache_word(callAddress);
return 0;
}
-/*
- * Memory allocator for executable code.
- *
- * This is required on x86 because some combinations
- * of Linux kernels and CPU generations default to
- * non-executable memory mappings, causing ordinary
- * malloc() memory to be non-executable.
- */
-static unsigned int code_bytes;
-static char *code_next;
-
-#if 0 /* change to non-zero to get allocation statistics at exit() */
-static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost;
-static unsigned int atexit_done;
-
-static void alloc_code_stats(void)
-{
- printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n",
- total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost);
-}
-
-static void atexit_alloc_code_stats(void)
-{
- if (!atexit_done) {
- atexit_done = 1;
- (void)atexit(alloc_code_stats);
- }
-}
-
-#define ALLOC_CODE_STATS(X) do{X;}while(0)
-#else
-#define ALLOC_CODE_STATS(X) do{}while(0)
-#endif
-
-/* FreeBSD 6.1 and Darwin breakage */
-#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
-#define MAP_ANONYMOUS MAP_ANON
-#endif
-
-static int morecore(unsigned int alloc_bytes)
-{
- unsigned int map_bytes;
- char *map_hint, *map_start;
-
- /* Page-align the amount to allocate. */
- map_bytes = (alloc_bytes + 4095) & ~4095;
-
- /* Round up small allocations. */
- if (map_bytes < 1024*1024)
- map_bytes = 1024*1024;
- else
- ALLOC_CODE_STATS(++nr_large);
-
- /* Create a new memory mapping, ensuring it is executable
- and in the low 2GB of the address space. Also attempt
- to make it adjacent to the previous mapping. */
- map_hint = code_next + code_bytes;
- if ((unsigned long)map_hint & 4095)
- abort();
- map_start = mmap(map_hint, map_bytes,
- PROT_EXEC|PROT_READ|PROT_WRITE,
- MAP_PRIVATE|MAP_ANONYMOUS
-#ifdef __x86_64__
- |MAP_32BIT
-#endif
- ,
- -1, 0);
- if (map_start == MAP_FAILED)
- return -1;
-
- ALLOC_CODE_STATS(total_mapped += map_bytes);
-
- /* Merge adjacent mappings, so the trailing portion of the previous
- mapping isn't lost. In practice this is quite successful. */
- if (map_start == map_hint) {
- ALLOC_CODE_STATS(++nr_joins);
- code_bytes += map_bytes;
- } else {
- ALLOC_CODE_STATS(++nr_splits);
- ALLOC_CODE_STATS(total_lost += code_bytes);
- code_next = map_start;
- code_bytes = map_bytes;
- }
-
- ALLOC_CODE_STATS(atexit_alloc_code_stats());
-
- return 0;
-}
-
static void *alloc_code(unsigned int alloc_bytes)
{
- void *res;
-
- /* Align function entries. */
- alloc_bytes = (alloc_bytes + 3) & ~3;
-
- if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0)
- return NULL;
- ALLOC_CODE_STATS(++nr_allocs);
- ALLOC_CODE_STATS(total_alloc += alloc_bytes);
- res = code_next;
- code_next += alloc_bytes;
- code_bytes -= alloc_bytes;
- return res;
+ return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes);
}
void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p)
@@ -184,6 +82,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
return alloc_code(nrbytes);
}
+void hipe_free_code(void* code, unsigned int bytes)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, code);
+}
+
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
{
/*
@@ -264,6 +167,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
return code;
}
+void hipe_free_native_stub(void* stub)
+{
+ erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
+}
+
void hipe_arch_print_pcb(struct hipe_process_state *p)
{
#define U(n,x) \
diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h
index 00fe03d8f9..a703e24b8c 100644
--- a/erts/emulator/hipe/hipe_x86_gc.h
+++ b/erts/emulator/hipe/hipe_x86_gc.h
@@ -30,9 +30,9 @@
struct nstack_walk_state {
#ifdef SKIP_YOUNGEST_FRAME
- const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */
+ const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */
#else
- struct sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */
+ struct hipe_sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */
#endif
};
@@ -57,37 +57,36 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p)
#endif
}
-static inline const struct sdesc*
+static inline const struct hipe_sdesc*
nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state)
{
#ifdef SKIP_YOUNGEST_FRAME
- const struct sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]);
+ const struct hipe_sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]);
state->sdesc0 = sdesc;
return sdesc;
#else
- unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS;
- if ((int)nstkarity < 0)
- nstkarity = 0;
- state->sdesc0[0].summary = (0 << 9) | (0 << 8) | nstkarity;
+ state->sdesc0[0].bucket.hvalue = 0; /* for nstack_any_cps_in_segment */
+ state->sdesc0[0].fsize = 0;
+ state->sdesc0[0].has_exnra = 0;
+ state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 :
+ p->hipe.narity - NR_ARG_REGS);
state->sdesc0[0].livebits[0] = 0;
-# ifdef DEBUG
- state->sdesc0[0].dbg_M = 0;
- state->sdesc0[0].dbg_F = am_undefined;
- state->sdesc0[0].dbg_A = 0;
-# endif
+ state->sdesc0[0].m_aix = 0;
+ state->sdesc0[0].f_aix = atom_val(am_undefined);
+ state->sdesc0[0].a = 0;
/* XXX: this appears to prevent a gcc-4.1.1 bug on x86 */
__asm__ __volatile__("" : : "m"(*state) : "memory");
return &state->sdesc0[0];
#endif
}
-static inline const struct sdesc*
+static inline const struct hipe_sdesc*
nstack_walk_init_sdesc_ignore_trap(const Process *p,
struct nstack_walk_state *state)
{
#ifdef SKIP_YOUNGEST_FRAME
unsigned long ra = p->hipe.nsp[0];
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
if (ra == (unsigned long)nbif_stack_trap_ra)
ra = (unsigned long)p->hipe.ngra;
sdesc = hipe_find_sdesc(ra);
@@ -98,7 +97,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p,
#endif
}
-static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0)
+static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0)
{
#ifdef SKIP_YOUNGEST_FRAME
Eterm *nsp = p->hipe.nsp;
@@ -137,7 +136,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp
return nsp >= nsp_end;
}
-static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc)
+static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc)
{
return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc);
}
@@ -148,7 +147,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i)
}
static inline unsigned long
-nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc)
+nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc)
{
return nsp[sdesc_fsize(sdesc)];
}
diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h
index 818d7444e2..de2b061706 100644
--- a/erts/emulator/hipe/hipe_x86_glue.h
+++ b/erts/emulator/hipe/hipe_x86_glue.h
@@ -58,16 +58,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y)
static __inline__ void hipe_arch_glue_init(void)
{
- static struct sdesc_with_exnra nbif_return_sdesc = {
- .exnra = (unsigned long)nbif_fail,
- .sdesc = {
- .bucket = { .hvalue = (unsigned long)nbif_return },
- .summary = (1<<8),
- #ifdef DEBUG
- .dbg_F = am_return,
- #endif
- },
- };
+ static struct hipe_sdesc_with_exnra nbif_return_sdesc;
+
+ nbif_return_sdesc.exnra = (unsigned long)nbif_fail;
+ nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return;
+ nbif_return_sdesc.sdesc.fsize = 0;
+ nbif_return_sdesc.sdesc.has_exnra = 1;
+ nbif_return_sdesc.sdesc.stk_nargs = 0;
+ nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty);
+ nbif_return_sdesc.sdesc.f_aix = atom_val(am_return);
+ nbif_return_sdesc.sdesc.a = 0;
+
hipe_init_sdesc_table(&nbif_return_sdesc.sdesc);
}
diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c
index f1559b1451..31582b3a2e 100644
--- a/erts/emulator/hipe/hipe_x86_stack.c
+++ b/erts/emulator/hipe/hipe_x86_stack.c
@@ -52,15 +52,14 @@ void hipe_print_nstack(Process *p)
{
Eterm *nsp;
Eterm *nsp_end;
- struct sdesc sdesc0;
- const struct sdesc *sdesc1;
- const struct sdesc *sdesc;
+ struct hipe_sdesc sdesc0;
+ const struct hipe_sdesc *sdesc1;
+ const struct hipe_sdesc *sdesc;
unsigned long ra;
unsigned long exnra;
unsigned int mask;
unsigned int sdesc_size;
unsigned int i;
- unsigned int nstkarity;
static const char dashes[2*sizeof(long)+5] = {
[0 ... 2*sizeof(long)+3] = '-'
};
@@ -68,10 +67,10 @@ void hipe_print_nstack(Process *p)
nsp = p->hipe.nsp;
nsp_end = p->hipe.nstend;
- nstkarity = p->hipe.narity - NR_ARG_REGS;
- if ((int)nstkarity < 0)
- nstkarity = 0;
- sdesc0.summary = nstkarity;
+ sdesc0.fsize = 0;
+ sdesc0.has_exnra = 0;
+ sdesc0.stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 :
+ p->hipe.narity - NR_ARG_REGS);
sdesc0.livebits[0] = ~1;
sdesc = &sdesc0;
@@ -158,7 +157,7 @@ void hipe_print_nstack(Process *p)
#define MINSTACK 128
#define NSKIPFRAMES 4
-void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc)
+void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc)
{
Eterm *nsp;
Eterm *nsp_end;
@@ -199,7 +198,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc)
void (*hipe_handle_stack_trap(Process *p))(void)
{
void (*ngra)(void) = p->hipe.ngra;
- const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra);
+ const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra);
hipe_update_stack_trap(p, sdesc);
return ngra;
}
@@ -220,7 +219,7 @@ void hipe_find_handler(Process *p)
unsigned long ra;
unsigned long exnra;
unsigned int arity;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
unsigned int nstkarity;
nsp = p->hipe.nsp;
@@ -262,7 +261,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace)
Eterm *nsp_end;
unsigned long ra, prev_ra;
unsigned int arity;
- const struct sdesc *sdesc;
+ const struct hipe_sdesc *sdesc;
unsigned int nstkarity;
int i;
diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index 7bbb406f29..3ab03971a5 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -21,6 +21,7 @@
# include "config.h"
#endif
+#define ERTS_WANT_MEM_MAPPERS
#include "sys.h"
#include "erl_process.h"
#include "erl_smp.h"
@@ -358,12 +359,11 @@ char* erts_literals_start;
UWord erts_literals_size;
#endif
-#ifdef ERTS_ALC_A_EXEC
+#ifdef ERTS_HAVE_EXEC_MMAPPER
ErtsMemMapper erts_exec_mmapper;
#endif
-
#define ERTS_MMAP_SIZE_SC_SA_INC(SZ) \
do { \
mm->size.supercarrier.used.total += (SZ); \
diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h
index fa51b663fa..92e9eb9e41 100644
--- a/erts/emulator/sys/common/erl_mmap.h
+++ b/erts/emulator/sys/common/erl_mmap.h
@@ -159,9 +159,10 @@ Eterm erts_mmap_info_options(ErtsMemMapper*,
extern ErtsMemMapper erts_dflt_mmapper;
# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION)
extern ErtsMemMapper erts_literal_mmapper;
-# endif
-# ifdef ERTS_ALC_A_EXEC
+# ifdef ERTS_ALC_A_EXEC
+# define ERTS_HAVE_EXEC_MMAPPER
extern ErtsMemMapper erts_exec_mmapper;
+# endif
# endif
#endif /* ERTS_WANT_MEM_MAPPERS */
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index f3306a888c..e76082177e 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -1414,7 +1414,7 @@ erts_mseg_init(ErtsMsegInit_t *init)
erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms");
-#ifdef ERTS_ALC_A_EXEC
+#ifdef ERTS_HAVE_EXEC_MMAPPER
/* Initialize erts_exec_mapper *FIRST*, to increase probability
* of getting low memory for HiPE AMD64's small code model.
*/
diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 8427bb134d..465ddfa7a8 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -23,8 +23,8 @@
versions/1,new_binary_types/1, call_purged_fun_code_gone/1,
call_purged_fun_code_reload/1, call_purged_fun_code_there/1,
multi_proc_purge/1, t_check_old_code/1,
- external_fun/1,get_chunk/1,module_md5/1,make_stub/1,
- make_stub_many_funs/1,constant_pools/1,constant_refc_binaries/1,
+ external_fun/1,get_chunk/1,module_md5/1,
+ constant_pools/1,constant_refc_binaries/1,
false_dependency/1,coverage/1,fun_confusion/1,
t_copy_literals/1, t_copy_literals_frags/1]).
@@ -37,7 +37,7 @@ all() ->
[versions, new_binary_types, call_purged_fun_code_gone,
call_purged_fun_code_reload, call_purged_fun_code_there,
multi_proc_purge, t_check_old_code, external_fun, get_chunk,
- module_md5, make_stub, make_stub_many_funs,
+ module_md5,
constant_pools, constant_refc_binaries, false_dependency,
coverage, fun_confusion, t_copy_literals, t_copy_literals_frags].
@@ -442,67 +442,6 @@ module_md5_ok(Code) ->
end.
-make_stub(Config) when is_list(Config) ->
- catch erlang:purge_module(my_code_test),
- MD5 = erlang:md5(<<>>),
-
- Data = proplists:get_value(data_dir, Config),
- File = filename:join(Data, "my_code_test"),
- {ok,my_code_test,Code} = compile:file(File, [binary]),
-
- my_code_test = code:make_stub_module(my_code_test, Code, {[],[],MD5}),
- true = erlang:delete_module(my_code_test),
- true = erlang:purge_module(my_code_test),
-
- my_code_test = code:make_stub_module(my_code_test,
- make_unaligned_sub_binary(Code),
- {[],[],MD5}),
- true = erlang:delete_module(my_code_test),
- true = erlang:purge_module(my_code_test),
-
- my_code_test = code:make_stub_module(my_code_test, zlib:gzip(Code),
- {[],[],MD5}),
- true = erlang:delete_module(my_code_test),
- true = erlang:purge_module(my_code_test),
-
- %% Should fail.
- {'EXIT',{badarg,_}} =
- (catch code:make_stub_module(my_code_test, <<"bad">>, {[],[],MD5})),
- {'EXIT',{badarg,_}} =
- (catch code:make_stub_module(my_code_test,
- bit_sized_binary(Code),
- {[],[],MD5})),
- {'EXIT',{badarg,_}} =
- (catch code:make_stub_module(my_code_test_with_wrong_name,
- Code, {[],[],MD5})),
- ok.
-
-make_stub_many_funs(Config) when is_list(Config) ->
- catch erlang:purge_module(many_funs),
- MD5 = erlang:md5(<<>>),
-
- Data = proplists:get_value(data_dir, Config),
- File = filename:join(Data, "many_funs"),
- {ok,many_funs,Code} = compile:file(File, [binary]),
-
- many_funs = code:make_stub_module(many_funs, Code, {[],[],MD5}),
- true = erlang:delete_module(many_funs),
- true = erlang:purge_module(many_funs),
- many_funs = code:make_stub_module(many_funs,
- make_unaligned_sub_binary(Code),
- {[],[],MD5}),
- true = erlang:delete_module(many_funs),
- true = erlang:purge_module(many_funs),
-
- %% Should fail.
- {'EXIT',{badarg,_}} =
- (catch code:make_stub_module(many_funs, <<"bad">>, {[],[],MD5})),
- {'EXIT',{badarg,_}} =
- (catch code:make_stub_module(many_funs,
- bit_sized_binary(Code),
- {[],[],MD5})),
- ok.
-
constant_pools(Config) when is_list(Config) ->
Data = proplists:get_value(data_dir, Config),
File = filename:join(Data, "literals"),
diff --git a/erts/emulator/test/hipe_SUITE.erl b/erts/emulator/test/hipe_SUITE.erl
index a556b4ddc0..0b44dd7fb7 100644
--- a/erts/emulator/test/hipe_SUITE.erl
+++ b/erts/emulator/test/hipe_SUITE.erl
@@ -19,12 +19,17 @@
%%
-module(hipe_SUITE).
--export([all/0, t_copy_literals/1]).
+-export([all/0
+ ,t_copy_literals/1
+ ,t_purge/1
+ ]).
all() ->
case erlang:system_info(hipe_architecture) of
undefined -> {skip, "HiPE is disabled"};
- _ -> [t_copy_literals]
+ _ -> [t_copy_literals
+ ,t_purge
+ ]
end.
t_copy_literals(doc) ->
@@ -65,3 +70,51 @@ t_copy_literals(Config) when is_list(Config) ->
true = erlang:delete_module(ref_cell),
true = erlang:purge_module(ref_cell),
ok.
+
+t_purge(doc) -> "Checks that native code is properly found and purged";
+t_purge(Config) when is_list(Config) ->
+ Data = proplists:get_value(data_dir, Config),
+ Priv = proplists:get_value(priv_dir, Config),
+ SrcFile = filename:join(Data, "ref_cell"),
+ BeamFile = filename:join(Priv, "ref_cell"),
+ {ok,ref_cell} = c:c(SrcFile, [{outdir,Priv},native]),
+ true = code:is_module_native(ref_cell),
+
+ PA = ref_cell:start_link(),
+
+ %% Unload, PA should still be running
+ true = erlang:delete_module(ref_cell),
+ %% Can't use ref_cel:call/2, it's in old code!
+ call(PA, {put_res_of, fun()-> hej end}),
+ hej = call(PA, get),
+
+ %% Load same module again
+ code:load_abs(BeamFile),
+ true = code:is_module_native(ref_cell),
+ PB = ref_cell:start_link(),
+
+ %% Purge old code, PA should be killed, PB should survive
+ unlink(PA),
+ ARef = monitor(process, PA),
+ true = erlang:purge_module(ref_cell),
+ receive {'DOWN', ARef, process, PA, killed} -> ok
+ after 1 -> ct:fail("PA was not killed")
+ end,
+
+ %% Unload, PB should still be running
+ true = erlang:delete_module(ref_cell),
+ call(PB, {put_res_of, fun()-> svejs end}),
+ svejs = call(PB, get),
+
+ unlink(PB),
+ BRef = monitor(process, PB),
+ true = erlang:purge_module(ref_cell),
+ receive {'DOWN', BRef, process, PB, killed} -> ok
+ after 1 -> ct:fail("PB was not killed")
+ end,
+
+ ok.
+
+call(Pid, Call) ->
+ Pid ! {Call, self()},
+ receive {Pid, Res} -> Res end.
diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl
index c9cc1cfe25..42a038f2d1 100644
--- a/lib/hipe/cerl/erl_bif_types.erl
+++ b/lib/hipe/cerl/erl_bif_types.erl
@@ -1004,8 +1004,8 @@ type(erlang, tuple_to_list, 1, Xs, Opaques) ->
%%-- hipe_bifs ----------------------------------------------------------------
type(hipe_bifs, add_ref, 2, Xs, Opaques) ->
strict(hipe_bifs, add_ref, 2, Xs, fun (_) -> t_nil() end, Opaques);
-type(hipe_bifs, alloc_data, 2, Xs, Opaques) ->
- strict(hipe_bifs, alloc_data, 2, Xs,
+type(hipe_bifs, alloc_data, 3, Xs, Opaques) ->
+ strict(hipe_bifs, alloc_data, 3, Xs,
fun (_) -> t_integer() end, Opaques); % address
type(hipe_bifs, array, 2, Xs, Opaques) ->
strict(hipe_bifs, array, 2, Xs, fun (_) -> t_immarray() end, Opaques);
@@ -1052,16 +1052,16 @@ type(hipe_bifs, call_count_on, 1, Xs, Opaques) ->
fun (_) -> t_sup(t_atom('true'), t_nil()) end, Opaques);
type(hipe_bifs, check_crc, 1, Xs, Opaques) ->
strict(hipe_bifs, check_crc, 1, Xs, fun (_) -> t_boolean() end, Opaques);
-type(hipe_bifs, enter_code, 2, Xs, Opaques) ->
- strict(hipe_bifs, enter_code, 2, Xs,
+type(hipe_bifs, enter_code, 3, Xs, Opaques) ->
+ strict(hipe_bifs, enter_code, 3, Xs,
fun (_) -> t_tuple([t_integer(),
%% XXX: The tuple below contains integers and
%% is of size same as the length of the MFA list
t_sup(t_nil(), t_binary())]) end, Opaques);
-type(hipe_bifs, enter_sdesc, 1, Xs, Opaques) ->
- strict(hipe_bifs, enter_sdesc, 1, Xs, fun (_) -> t_nil() end, Opaques);
-type(hipe_bifs, find_na_or_make_stub, 2, Xs, Opaques) ->
- strict(hipe_bifs, find_na_or_make_stub, 2, Xs,
+type(hipe_bifs, enter_sdesc, 2, Xs, Opaques) ->
+ strict(hipe_bifs, enter_sdesc, 2, Xs, fun (_) -> t_nil() end, Opaques);
+type(hipe_bifs, find_na_or_make_stub, 1, Xs, Opaques) ->
+ strict(hipe_bifs, find_na_or_make_stub, 1, Xs,
fun (_) -> t_integer() end, Opaques); % address
type(hipe_bifs, fun_to_address, 1, Xs, Opaques) ->
strict(hipe_bifs, fun_to_address, 1, Xs,
@@ -1071,12 +1071,6 @@ type(hipe_bifs, get_fe, 2, Xs, Opaques) ->
type(hipe_bifs, get_rts_param, 1, Xs, Opaques) ->
strict(hipe_bifs, get_rts_param, 1, Xs,
fun (_) -> t_sup(t_integer(), t_nil()) end, Opaques);
-type(hipe_bifs, invalidate_funinfo_native_addresses, 1, Xs, Opaques) ->
- strict(hipe_bifs, invalidate_funinfo_native_addresses, 1, Xs,
- fun (_) -> t_nil() end, Opaques);
-type(hipe_bifs, mark_referred_from, 1, Xs, Opaques) ->
- strict(hipe_bifs, mark_referred_from, 1, Xs,
- fun (_) -> t_nil() end, Opaques);
type(hipe_bifs, merge_term, 1, Xs, Opaques) ->
strict(hipe_bifs, merge_term, 1, Xs, fun ([X]) -> X end, Opaques);
type(hipe_bifs, nstack_used_size, 0, _, _Opaques) ->
@@ -1088,9 +1082,6 @@ type(hipe_bifs, patch_insn, 3, Xs, Opaques) ->
type(hipe_bifs, primop_address, 1, Xs, Opaques) ->
strict(hipe_bifs, primop_address, 1, Xs,
fun (_) -> t_sup(t_integer(), t_atom('false')) end, Opaques);
-type(hipe_bifs, redirect_referred_from, 1, Xs, Opaques) ->
- strict(hipe_bifs, redirect_referred_from, 1, Xs,
- fun (_) -> t_nil() end, Opaques);
type(hipe_bifs, ref, 1, Xs, Opaques) ->
strict(hipe_bifs, ref, 1, Xs, fun (_) -> t_immarray() end, Opaques);
type(hipe_bifs, ref_get, 1, Xs, Opaques) ->
@@ -1103,6 +1094,9 @@ type(hipe_bifs, remove_refs_from, 1, Xs, Opaques) ->
type(hipe_bifs, set_funinfo_native_address, 3, Xs, Opaques) ->
strict(hipe_bifs, set_funinfo_native_address, 3, Xs,
fun (_) -> t_nil() end, Opaques);
+type(hipe_bifs, commit_patch_load, 1, Xs, Opaques) ->
+ strict(hipe_bifs, commit_patch_load, 1, Xs,
+ fun (_) -> t_atom() end, Opaques);
type(hipe_bifs, set_native_address, 3, Xs, Opaques) ->
strict(hipe_bifs, set_native_address, 3, Xs,
fun (_) -> t_nil() end, Opaques);
@@ -1114,15 +1108,14 @@ type(hipe_bifs, system_crc, 0, _, _Opaques) ->
type(hipe_bifs, term_to_word, 1, Xs, Opaques) ->
strict(hipe_bifs, term_to_word, 1, Xs,
fun (_) -> t_integer() end, Opaques);
-type(hipe_bifs, update_code_size, 3, Xs, Opaques) ->
- strict(hipe_bifs, update_code_size, 3, Xs,
- fun (_) -> t_nil() end, Opaques);
type(hipe_bifs, write_u8, 2, Xs, Opaques) ->
strict(hipe_bifs, write_u8, 2, Xs, fun (_) -> t_nil() end, Opaques);
type(hipe_bifs, write_u32, 2, Xs, Opaques) ->
strict(hipe_bifs, write_u32, 2, Xs, fun (_) -> t_nil() end, Opaques);
type(hipe_bifs, write_u64, 2, Xs, Opaques) ->
strict(hipe_bifs, write_u64, 2, Xs, fun (_) -> t_nil() end, Opaques);
+type(hipe_bifs, alloc_loader_state, 1, Xs, Opaques) ->
+ strict(hipe_bifs, alloc_loader_state, 1, Xs, fun (_) -> t_binary() end, Opaques);
%%-- lists --------------------------------------------------------------------
type(lists, all, 2, Xs, Opaques) ->
strict(lists, all, 2, Xs,
@@ -2470,9 +2463,10 @@ arg_types(hipe_bifs, add_ref, 2) ->
t_integer(),
t_sup(t_atom('call'), t_atom('load_mfa')),
t_trampoline(),
- t_sup(t_atom('remote'), t_atom('local'))])];
-arg_types(hipe_bifs, alloc_data, 2) ->
- [t_integer(), t_integer()];
+ t_atom(),
+ t_binary()])];
+arg_types(hipe_bifs, alloc_data, 3) ->
+ [t_integer(), t_integer(), t_binary()];
arg_types(hipe_bifs, array, 2) ->
[t_non_neg_fixnum(), t_immediate()];
arg_types(hipe_bifs, array_length, 1) ->
@@ -2507,22 +2501,19 @@ arg_types(hipe_bifs, call_count_on, 1) ->
[t_mfa()];
arg_types(hipe_bifs, check_crc, 1) ->
[t_crc32()];
-arg_types(hipe_bifs, enter_code, 2) ->
- [t_binary(), t_sup(t_nil(), t_tuple())];
-arg_types(hipe_bifs, enter_sdesc, 1) ->
- [t_tuple([t_integer(), t_integer(), t_integer(), t_integer(), t_integer(), t_mfa()])];
-arg_types(hipe_bifs, find_na_or_make_stub, 2) ->
- [t_mfa(), t_boolean()];
+arg_types(hipe_bifs, enter_code, 3) ->
+ [t_binary(), t_sup(t_nil(), t_tuple()), t_binary()];
+arg_types(hipe_bifs, enter_sdesc, 2) ->
+ [t_tuple([t_integer(), t_integer(), t_integer(), t_integer(), t_integer(), t_mfa(), t_atom()]),
+ t_binary()];
+arg_types(hipe_bifs, find_na_or_make_stub, 1) ->
+ [t_mfa()];
arg_types(hipe_bifs, fun_to_address, 1) ->
[t_mfa()];
arg_types(hipe_bifs, get_fe, 2) ->
[t_atom(), t_tuple([t_integer(), t_integer(), t_integer()])];
arg_types(hipe_bifs, get_rts_param, 1) ->
[t_fixnum()];
-arg_types(hipe_bifs, invalidate_funinfo_native_addresses, 1) ->
- [t_list(t_mfa())];
-arg_types(hipe_bifs, mark_referred_from, 1) ->
- [t_mfa()];
arg_types(hipe_bifs, merge_term, 1) ->
[t_any()];
arg_types(hipe_bifs, nstack_used_size, 0) ->
@@ -2533,8 +2524,6 @@ arg_types(hipe_bifs, patch_insn, 3) ->
[t_integer(), t_integer(), t_insn_type()];
arg_types(hipe_bifs, primop_address, 1) ->
[t_atom()];
-arg_types(hipe_bifs, redirect_referred_from, 1) ->
- [t_mfa()];
arg_types(hipe_bifs, ref, 1) ->
[t_immediate()];
arg_types(hipe_bifs, ref_get, 1) ->
@@ -2545,6 +2534,8 @@ arg_types(hipe_bifs, remove_refs_from, 1) ->
[t_sup([t_mfa(), t_atom('all')])];
arg_types(hipe_bifs, set_funinfo_native_address, 3) ->
arg_types(hipe_bifs, set_native_address, 3);
+arg_types(hipe_bifs, commit_patch_load, 1) ->
+ [t_binary()];
arg_types(hipe_bifs, set_native_address, 3) ->
[t_mfa(), t_integer(), t_boolean()];
arg_types(hipe_bifs, set_native_address_in_fe, 2) ->
@@ -2553,14 +2544,15 @@ arg_types(hipe_bifs, system_crc, 0) ->
[];
arg_types(hipe_bifs, term_to_word, 1) ->
[t_any()];
-arg_types(hipe_bifs, update_code_size, 3) ->
- [t_atom(), t_sup(t_nil(), t_binary()), t_integer()];
arg_types(hipe_bifs, write_u8, 2) ->
[t_integer(), t_byte()];
arg_types(hipe_bifs, write_u32, 2) ->
[t_integer(), t_integer()];
arg_types(hipe_bifs, write_u64, 2) ->
[t_integer(), t_integer()];
+arg_types(hipe_bifs, alloc_loader_state, 1) ->
+ [t_atom()];
+
%%------- lists ---------------------------------------------------------------
arg_types(lists, all, 2) ->
[t_fun([t_any()], t_boolean()), t_list()];
diff --git a/lib/hipe/llvm/hipe_llvm_main.erl b/lib/hipe/llvm/hipe_llvm_main.erl
index 476d6fb49c..164ccf20ef 100644
--- a/lib/hipe/llvm/hipe_llvm_main.erl
+++ b/lib/hipe/llvm/hipe_llvm_main.erl
@@ -257,15 +257,11 @@ fix_relocations(Relocs, RelocsDict, MFA) ->
fix_reloc(#elf_rel{symbol=#elf_sym{name=Name, section=undefined, type=notype},
offset=Offset, type=?PCREL_T, addend=?PCREL_A},
- RelocsDict, {ModName,_,_}) when Name =/= "" ->
+ RelocsDict, {_,_,_}) when Name =/= "" ->
case dict:fetch(Name, RelocsDict) of
- {call, {bif, BifName, _}} -> {?CALL_LOCAL, Offset, BifName};
- %% MFA calls to functions in the same module are of type 3, while all
- %% other MFA calls are of type 2.
- %% XXX: Does this code break hot code loading (by transforming external
- %% calls into local calls?)
- {call, {ModName,_F,_A}=CallMFA} -> {?CALL_LOCAL, Offset, CallMFA};
- {call, CallMFA} -> {?CALL_REMOTE, Offset, CallMFA}
+ {call, _, {bif, BifName, _}} -> {?CALL_LOCAL, Offset, BifName};
+ {call, not_remote, CallMFA} -> {?CALL_LOCAL, Offset, CallMFA};
+ {call, remote, CallMFA} -> {?CALL_REMOTE, Offset, CallMFA}
end;
fix_reloc(#elf_rel{symbol=#elf_sym{name=Name, section=undefined, type=notype},
offset=Offset, type=?ABS_T, addend=?ABS_A},
@@ -280,7 +276,8 @@ fix_reloc(#elf_rel{symbol=#elf_sym{name=Name, section=#elf_shdr{name=?TEXT},
offset=Offset, type=?PCREL_T, addend=?PCREL_A},
RelocsDict, MFA) when Name =/= "" ->
case dict:fetch(Name, RelocsDict) of
- {call, MFA} -> {?CALL_LOCAL, Offset, MFA}
+ {call, not_remote, MFA} -> {?CALL_LOCAL, Offset, MFA};
+ {call, remote, MFA} -> {?CALL_REMOTE, Offset, MFA}
end;
fix_reloc(#elf_rel{symbol=#elf_sym{name=Name, section=#elf_shdr{name=?RODATA},
type=object},
@@ -408,7 +405,7 @@ calls_with_stack_args(Dict) ->
calls_with_stack_args(dict:to_list(Dict), []).
calls_with_stack_args([], Calls) -> Calls;
-calls_with_stack_args([ {_Name, {call, {M, F, A}}} | Rest], Calls)
+calls_with_stack_args([ {_Name, {call, _, {M, F, A}}} | Rest], Calls)
when A > ?NR_ARG_REGS ->
Call =
case M of
diff --git a/lib/hipe/llvm/hipe_rtl_to_llvm.erl b/lib/hipe/llvm/hipe_rtl_to_llvm.erl
index 66b2e10fb8..55ab6a3db1 100644
--- a/lib/hipe/llvm/hipe_rtl_to_llvm.erl
+++ b/lib/hipe/llvm/hipe_rtl_to_llvm.erl
@@ -257,7 +257,7 @@ trans_alub_overflow(I, Sign, Relocs) ->
RtlDst = hipe_rtl:alub_dst(I),
TmpDst = mk_temp(),
Name = trans_alub_op(I, Sign),
- NewRelocs = relocs_store(Name, {call, {llvm, Name, 2}}, Relocs),
+ NewRelocs = relocs_store(Name, {call, remote, {llvm, Name, 2}}, Relocs),
WordTy = hipe_llvm:mk_int(?WORD_WIDTH),
ReturnType = hipe_llvm:mk_struct([WordTy, hipe_llvm:mk_int(1)]),
T1 = mk_temp(),
@@ -364,7 +364,7 @@ trans_call(I, Relocs) ->
{LoadedFixedRegs, I2} = load_fixed_regs(FixedRegs),
FinalArgs = fix_reg_args(LoadedFixedRegs) ++ CallArgs,
{Name, I3, Relocs2} =
- trans_call_name(RtlCallName, Relocs1, CallArgs, FinalArgs),
+ trans_call_name(RtlCallName, hipe_rtl:call_type(I), Relocs1, CallArgs, FinalArgs),
T1 = mk_temp(),
WordTy = hipe_llvm:mk_int(?WORD_WIDTH),
FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)),
@@ -430,17 +430,21 @@ expose_closure(CallName, CallArgs, Relocs) ->
{[], Relocs}
end.
-trans_call_name(RtlCallName, Relocs, CallArgs, FinalArgs) ->
+trans_call_name(RtlCallName, RtlCallType, Relocs, CallArgs, FinalArgs) ->
case RtlCallName of
PrimOp when is_atom(PrimOp) ->
LlvmName = trans_prim_op(PrimOp),
Relocs1 =
- relocs_store(LlvmName, {call, {bif, PrimOp, length(CallArgs)}}, Relocs),
+ relocs_store(LlvmName, {call, not_remote, {bif, PrimOp, length(CallArgs)}}, Relocs),
{"@" ++ LlvmName, [], Relocs1};
{M, F, A} when is_atom(M), is_atom(F), is_integer(A) ->
LlvmName = trans_mfa_name({M, F, A}),
+ ok = case RtlCallType of
+ not_remote -> ok;
+ remote -> ok
+ end,
Relocs1 =
- relocs_store(LlvmName, {call, {M, F, length(CallArgs)}}, Relocs),
+ relocs_store(LlvmName, {call, RtlCallType, {M, F, length(CallArgs)}}, Relocs),
{"@" ++ LlvmName, [], Relocs1};
Reg ->
case hipe_rtl:is_reg(Reg) of
@@ -501,7 +505,7 @@ trans_enter(I, Relocs) ->
{LoadedFixedRegs, I1} = load_fixed_regs(FixedRegs),
FinalArgs = fix_reg_args(LoadedFixedRegs) ++ CallArgs,
{Name, I2, NewRelocs} =
- trans_call_name(hipe_rtl:enter_fun(I), Relocs, CallArgs, FinalArgs),
+ trans_call_name(hipe_rtl:enter_fun(I), hipe_rtl:enter_type(I), Relocs, CallArgs, FinalArgs),
T1 = mk_temp(),
WordTy = hipe_llvm:mk_int(?WORD_WIDTH),
FunRetTy = hipe_llvm:mk_struct(lists:duplicate(?NR_PINNED_REGS + 1, WordTy)),
@@ -1457,9 +1461,9 @@ handle_relocations(Relocs, Data, Fun) ->
Relocs2 = lists:foldl(fun const_to_dict/2, Relocs1, ConstLabels),
%% Temporary Store inc_stack and llvm_fix_pinned_regs to Dictionary
%% TODO: Remove this
- Relocs3 = dict:store("inc_stack_0", {call, {bif, inc_stack_0, 0}}, Relocs2),
+ Relocs3 = dict:store("inc_stack_0", {call, remote, {bif, inc_stack_0, 0}}, Relocs2),
Relocs4 = dict:store("hipe_bifs.llvm_fix_pinned_regs.0",
- {call, {hipe_bifs, llvm_fix_pinned_regs, 0}}, Relocs3),
+ {call, remote, {hipe_bifs, llvm_fix_pinned_regs, 0}}, Relocs3),
BranchMetaData = [
hipe_llvm:mk_meta(?BRANCH_META_TAKEN, ["branch_weights", 99, 1])
, hipe_llvm:mk_meta(?BRANCH_META_NOT_TAKEN, ["branch_weights", 1, 99])
@@ -1477,9 +1481,10 @@ seperate_relocs([], CallAcc, AtomAcc, ClosureAcc, LabelAcc, JmpTableAcc) ->
{CallAcc, AtomAcc, ClosureAcc, LabelAcc, JmpTableAcc};
seperate_relocs([R|Rs], CallAcc, AtomAcc, ClosureAcc, LabelAcc, JmpTableAcc) ->
case R of
- {_, {call, _}} ->
+ {_, {call, _, _}} ->
seperate_relocs(Rs, [R | CallAcc], AtomAcc, ClosureAcc, LabelAcc,
JmpTableAcc);
+
{_, {atom, _}} ->
seperate_relocs(Rs, CallAcc, [R | AtomAcc], ClosureAcc, LabelAcc,
JmpTableAcc);
@@ -1554,13 +1559,13 @@ declare_closure_labels(ClosureLabels, Relocs, Fun) ->
hipe_llvm:mk_const_decl("@table_closures", "constant", TableType, List4),
{[ConstDecl], Relocs1}.
-%% @doc A call is treated as non external only in a case of a recursive
+%% @doc A call is treated as non external only in a case of a local recursive
%% function.
-is_external_call({_, {call, Fun}}, Fun) -> false;
+is_external_call({_, {call, _, MFA}}, MFA) -> false;
is_external_call(_, _) -> true.
%% @doc External declaration of a function.
-call_to_decl({Name, {call, MFA}}) ->
+call_to_decl({Name, {call, _, MFA}}) ->
{M, _F, A} = MFA,
CConv = "cc 11",
WordTy = hipe_llvm:mk_int(?WORD_WIDTH),
diff --git a/lib/hipe/rtl/hipe_rtl_primops.erl b/lib/hipe/rtl/hipe_rtl_primops.erl
index 835f489ec0..154c2a8c57 100644
--- a/lib/hipe/rtl/hipe_rtl_primops.erl
+++ b/lib/hipe/rtl/hipe_rtl_primops.erl
@@ -760,12 +760,9 @@ gen_fun_thing_skeleton(FunP, FunName={_Mod,_FunId,Arity}, NumFree,
%% And creates a fe (at load time).
FeVar = hipe_rtl:mk_new_reg(),
PidVar = hipe_rtl:mk_new_reg_gcsafe(),
- NativeVar = hipe_rtl:mk_new_reg(),
[hipe_rtl:mk_load_address(FeVar, {FunName, MagicNr, Index}, closure),
store_struct_field(FunP, ?EFT_FE, FeVar),
- load_struct_field(NativeVar, FeVar, ?EFE_NATIVE_ADDRESS),
- store_struct_field(FunP, ?EFT_NATIVE_ADDRESS, NativeVar),
store_struct_field(FunP, ?EFT_ARITY, hipe_rtl:mk_imm(Arity-NumFree)),
diff --git a/lib/hipe/rtl/hipe_tagscheme.erl b/lib/hipe/rtl/hipe_tagscheme.erl
index 8825a3ade3..8cf45772b5 100644
--- a/lib/hipe/rtl/hipe_tagscheme.erl
+++ b/lib/hipe/rtl/hipe_tagscheme.erl
@@ -68,9 +68,7 @@
-include("hipe_rtl.hrl").
-include("hipe_literals.hrl").
--ifdef(EFT_NATIVE_ADDRESS).
-export([if_fun_get_arity_and_address/5]).
--endif.
-undef(TAG_PRIMARY_BOXED).
-undef(TAG_IMMED2_MASK).
@@ -282,7 +280,6 @@ test_ref(X, TrueLab, FalseLab, Pred) ->
TrueLab, FalseLab, Pred)
].
--ifdef(EFT_NATIVE_ADDRESS).
test_closure(X, TrueLab, FalseLab, Pred) ->
Tmp = hipe_rtl:mk_new_reg_gcsafe(),
HalfTrueLab = hipe_rtl:mk_new_label(),
@@ -291,7 +288,6 @@ test_closure(X, TrueLab, FalseLab, Pred) ->
get_header(Tmp, X),
mask_and_compare(Tmp, ?TAG_HEADER_MASK, ?TAG_HEADER_FUN,
TrueLab, FalseLab, Pred)].
--endif.
test_fun(X, TrueLab, FalseLab, Pred) ->
Hdr = hipe_rtl:mk_new_reg_gcsafe(),
@@ -781,10 +777,9 @@ tag_fun(Res, X) ->
%% untag_fun(Res, X) ->
%% hipe_rtl:mk_alu(Res, X, 'sub', hipe_rtl:mk_imm(?TAG_PRIMARY_BOXED)).
--ifdef(EFT_NATIVE_ADDRESS).
if_fun_get_arity_and_address(ArityReg, AddressReg, FunP, BadFunLab, Pred) ->
%% EmuAddressPtrReg = hipe_rtl:mk_new_reg(),
- %% FEPtrReg = hipe_rtl:mk_new_reg(),
+ FEPtrReg = hipe_rtl:mk_new_reg(),
%% ArityReg = hipe_rtl:mk_new_reg(),
%% NumFreeReg = hipe_rtl:mk_new_reg(),
%% RealArityReg = hipe_rtl:mk_new_reg(),
@@ -797,11 +792,12 @@ if_fun_get_arity_and_address(ArityReg, AddressReg, FunP, BadFunLab, Pred) ->
hipe_rtl:mk_load(ArityReg, FunP,
hipe_rtl:mk_imm(-(?TAG_PRIMARY_BOXED)+
?EFT_ARITY)),
- hipe_rtl:mk_load(AddressReg, FunP,
+ hipe_rtl:mk_load(FEPtrReg, FunP,
hipe_rtl:mk_imm(-(?TAG_PRIMARY_BOXED)+
- ?EFT_NATIVE_ADDRESS))],
+ ?EFT_FE)),
+ hipe_rtl:mk_load(AddressReg, FEPtrReg,
+ hipe_rtl:mk_imm(?EFE_NATIVE_ADDRESS))],
IsFunCode ++ GetArityCode.
--endif.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%
diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl
index 835b7eb588..418b0c50e1 100644
--- a/lib/kernel/src/code_server.erl
+++ b/lib/kernel/src/code_server.erl
@@ -1130,19 +1130,18 @@ try_load_module_2(File, Mod, Bin, From, Architecture,
#state{moddb=Db}=St) ->
case catch hipe_unified_loader:load_native_code(Mod, Bin, Architecture) of
{module,Mod} = Module ->
- ets:insert(Db, [{{native,Mod},true},{Mod,File}]),
+ ets:insert(Db, {Mod,File}),
{reply,Module,St};
no_native ->
try_load_module_3(File, Mod, Bin, From, Architecture, St);
Error ->
error_msg("Native loading of ~ts failed: ~p\n", [File,Error]),
- {reply,ok,St}
+ {reply,{error,Error},St}
end.
-try_load_module_3(File, Mod, Bin, From, Architecture, St0) ->
+try_load_module_3(File, Mod, Bin, From, _Architecture, St0) ->
Action = fun({module,_}=Module, #state{moddb=Db}=S) ->
ets:insert(Db, {Mod,File}),
- post_beam_load([Mod], Architecture, S),
{reply,Module,S};
({error,on_load_failure}=Error, S) ->
{reply,Error,S};
@@ -1153,24 +1152,14 @@ try_load_module_3(File, Mod, Bin, From, Architecture, St0) ->
Res = erlang:load_module(Mod, Bin),
handle_on_load(Res, Action, Mod, From, St0).
-hipe_result_to_status(Result, #state{moddb=Db}) ->
+hipe_result_to_status(Result, #state{}) ->
case Result of
- {module,Mod} ->
- ets:insert(Db, [{{native,Mod},true}]),
+ {module,_} ->
Result;
_ ->
{error,Result}
end.
-post_beam_load(_, undefined, _) ->
- %% HiPE is disabled.
- ok;
-post_beam_load(Mods0, _Architecture, #state{moddb=Db}) ->
- %% post_beam_load/2 can potentially be very expensive because it
- %% blocks multi-scheduling. Therefore, we only want to call
- %% it with modules that are known to have native code loaded.
- Mods = [M || M <- Mods0, ets:member(Db, {native,M})],
- hipe_unified_loader:post_beam_load(Mods).
int_list([H|T]) when is_integer(H) -> int_list(T);
int_list([_|_]) -> false;
@@ -1313,15 +1302,12 @@ abort_if_sticky(L, Db) ->
[_|_] -> {error,Sticky}
end.
-do_finish_loading(Prepared, #state{moddb=Db}=St) ->
+do_finish_loading(Prepared, #state{moddb=Db}) ->
MagicBins = [B || {_,{B,_}} <- Prepared],
case erlang:finish_loading(MagicBins) of
ok ->
MFs = [{M,F} || {M,{_,F}} <- Prepared],
true = ets:insert(Db, MFs),
- Ms = [M || {M,_} <- MFs],
- Architecture = erlang:system_info(hipe_architecture),
- post_beam_load(Ms, Architecture, St),
ok;
{Reason,Ms} ->
{error,[{M,Reason} || M <- Ms]}
diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl
index 087cceb5d8..03116758de 100644
--- a/lib/kernel/src/hipe_unified_loader.erl
+++ b/lib/kernel/src/hipe_unified_loader.erl
@@ -41,10 +41,11 @@
% I think the real solution would be to let BIF erlang:load_module/2 redirect all
% hipe calls to the module and thereby remove post_beam_load.
+% SVERK: Can we remove -compile(no_native) now when post_beam_load is gone?
+
-export([chunk_name/1,
%% Only the code and code_server modules may call the entries below!
load_native_code/3,
- post_beam_load/1,
load_module/4,
load/3]).
@@ -101,15 +102,13 @@ word_size(Architecture) ->
load_native_code(_Mod, _Bin, undefined) ->
no_native;
load_native_code(Mod, Bin, Architecture) when is_atom(Mod), is_binary(Bin) ->
- %% patch_to_emu(Mod),
case code:get_chunk(Bin, chunk_name(Architecture)) of
undefined -> no_native;
NativeCode when is_binary(NativeCode) ->
erlang:system_flag(multi_scheduling, block_normal),
try
- OldReferencesToPatch = patch_to_emu_step1(Mod),
- case load_module(Mod, NativeCode, Bin, OldReferencesToPatch,
- Architecture) of
+ put(hipe_patch_closures, false),
+ case load_common(Mod, NativeCode, Bin, Architecture) of
bad_crc -> no_native;
Result -> Result
end
@@ -120,22 +119,6 @@ load_native_code(Mod, Bin, Architecture) when is_atom(Mod), is_binary(Bin) ->
%%========================================================================
--spec post_beam_load([module()]) -> 'ok'.
-
-post_beam_load([])->
- ok;
-post_beam_load([_|_]=Mods) ->
- erlang:system_flag(multi_scheduling, block_normal),
- try
- _ = [patch_to_emu(Mod) || Mod <- Mods],
- ok
- after
- erlang:system_flag(multi_scheduling, unblock_normal)
- end,
- ok.
-
-%%========================================================================
-
version_check(Version, Mod) when is_atom(Mod) ->
Ver = ?VERSION_STRING(),
case Version < Ver of
@@ -153,19 +136,12 @@ version_check(Version, Mod) when is_atom(Mod) ->
load_module(Mod, Bin, Beam, Architecture) ->
erlang:system_flag(multi_scheduling, block_normal),
try
- load_module_nosmp(Mod, Bin, Beam, Architecture)
+ put(hipe_patch_closures, false),
+ load_common(Mod, Bin, Beam, Architecture)
after
erlang:system_flag(multi_scheduling, unblock_normal)
end.
-load_module_nosmp(Mod, Bin, Beam, Architecture) ->
- load_module(Mod, Bin, Beam, [], Architecture).
-
-load_module(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
- ?debug_msg("************ Loading Module ~w ************\n",[Mod]),
- %% Loading a whole module, let the BEAM loader patch closures.
- put(hipe_patch_closures, false),
- load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture).
%%========================================================================
@@ -175,20 +151,17 @@ load_module(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
load(Mod, Bin, Architecture) ->
erlang:system_flag(multi_scheduling, block_normal),
try
- load_nosmp(Mod, Bin, Architecture)
+ ?debug_msg("********* Loading funs in module ~w *********\n",[Mod]),
+ %% Loading just some functions in a module; patch closures separately.
+ put(hipe_patch_closures, true),
+ load_common(Mod, Bin, [], Architecture)
after
erlang:system_flag(multi_scheduling, unblock_normal)
end.
-load_nosmp(Mod, Bin, Architecture) ->
- ?debug_msg("********* Loading funs in module ~w *********\n",[Mod]),
- %% Loading just some functions in a module; patch closures separately.
- put(hipe_patch_closures, true),
- load_common(Mod, Bin, [], [], Architecture).
-
%%------------------------------------------------------------------------
-load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
+load_common(Mod, Bin, Beam, Architecture) ->
%% Unpack the binary.
[{Version, CheckSum},
ConstAlign, ConstSize, ConstMap, LabelMap, ExportMap,
@@ -215,29 +188,31 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
put(closures_to_patch, []),
WordSize = word_size(Architecture),
WriteWord = write_word_fun(WordSize),
+ LoaderState = hipe_bifs:alloc_loader_state(Mod),
+ put(hipe_loader_state, LoaderState),
%% Create data segment
{ConstAddr,ConstMap2} =
- create_data_segment(ConstAlign, ConstSize, ConstMap, WriteWord),
+ create_data_segment(ConstAlign, ConstSize, ConstMap, WriteWord,
+ LoaderState),
%% Find callees for which we may need trampolines.
CalleeMFAs = find_callee_mfas(Refs, Architecture),
%% Write the code to memory.
{CodeAddress,Trampolines} =
- enter_code(CodeSize, CodeBinary, CalleeMFAs, Mod, Beam),
+ enter_code(CodeSize, CodeBinary, CalleeMFAs, LoaderState),
%% Construct CalleeMFA-to-trampoline mapping.
TrampolineMap = mk_trampoline_map(CalleeMFAs, Trampolines,
Architecture),
%% Patch references to code labels in data seg.
ok = patch_consts(LabelMap, ConstAddr, CodeAddress, WriteWord),
+
%% Find out which functions are being loaded (and where).
- %% Note: Addresses are sorted descending.
- {MFAs,Addresses} = exports(ExportMap, CodeAddress),
- %% Remove references to old versions of the module.
- ReferencesToPatch = get_refs_from(MFAs, []),
- %% io:format("References to patch: ~w~n", [ReferencesToPatch]),
- ok = remove_refs_from(MFAs),
+ %% Note: FunDefs are sorted descending address order.
+ FunDefs = exports(ExportMap, CodeAddress),
+
%% Patch all dynamic references in the code.
%% Function calls, Atoms, Constants, System calls
- ok = patch(Refs, CodeAddress, ConstMap2, Addresses, TrampolineMap),
+
+ ok = patch(Refs, CodeAddress, ConstMap2, FunDefs, TrampolineMap),
%% Tell the system where the loaded funs are.
%% (patches the BEAM code to redirect to native.)
@@ -250,25 +225,22 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) ->
lists:foreach(fun({FE, DestAddress}) ->
hipe_bifs:set_native_address_in_fe(FE, DestAddress)
end, erase(closures_to_patch)),
- export_funs(Addresses),
+ ok = hipe_bifs:commit_patch_load(LoaderState),
+ set_beam_call_traps(FunDefs),
ok;
BeamBinary when is_binary(BeamBinary) ->
%% Find all closures in the code.
[] = erase(closures_to_patch), %Clean up, assertion.
ClosurePatches = find_closure_patches(Refs),
AddressesOfClosuresToPatch =
- calculate_addresses(ClosurePatches, CodeAddress, Addresses),
- export_funs(Addresses),
- export_funs(Mod, MD5, BeamBinary,
- Addresses, AddressesOfClosuresToPatch)
+ calculate_addresses(ClosurePatches, CodeAddress, FunDefs),
+ export_funs(FunDefs),
+ make_beam_stub(Mod, LoaderState, MD5, BeamBinary, FunDefs,
+ AddressesOfClosuresToPatch)
end,
- %% Redirect references to the old module to the new module's BEAM stub.
- patch_to_emu_step2(OldReferencesToPatch),
- %% Patch referring functions to call the new function
- %% The call to export_funs/1 above updated the native addresses
- %% for the targets, so passing 'Addresses' is not needed.
- redirect(ReferencesToPatch),
+
%% Final clean up.
+ _ = erase(hipe_loader_state),
_ = erase(hipe_patch_closures),
_ = erase(hipe_assert_code_area),
?debug_msg("****************Loader Finished****************\n", []),
@@ -371,31 +343,31 @@ trampoline_map_lookup(Primop, Map) ->
is_exported :: boolean()}).
exports(ExportMap, BaseAddress) ->
- exports(ExportMap, BaseAddress, [], []).
+ exports(ExportMap, BaseAddress, []).
-exports([Offset,M,F,A,IsClosure,IsExported|Rest], BaseAddress, MFAs, Addresses) ->
+exports([Offset,M,F,A,IsClosure,IsExported|Rest], BaseAddress, FunDefs) ->
case IsExported andalso erlang:is_builtin(M, F, A) of
true ->
- exports(Rest, BaseAddress, MFAs, Addresses);
+ exports(Rest, BaseAddress, FunDefs);
_false ->
MFA = {M,F,A},
Address = BaseAddress + Offset,
FunDef = #fundef{address=Address, mfa=MFA, is_closure=IsClosure,
is_exported=IsExported},
- exports(Rest, BaseAddress, [MFA|MFAs], [FunDef|Addresses])
+ exports(Rest, BaseAddress, [FunDef|FunDefs])
end;
-exports([], _, MFAs, Addresses) ->
- {MFAs, Addresses}.
+exports([], _, FunDefs) ->
+ FunDefs.
mod({M,_F,_A}) -> M.
%%------------------------------------------------------------------------
-calculate_addresses(PatchOffsets, Base, Addresses) ->
+calculate_addresses(PatchOffsets, Base, FunDefs) ->
RemoteOrLocal = local, % closure code refs are local
[{Data,
offsets_to_addresses(Offsets, Base),
- get_native_address(DestMFA, Addresses, RemoteOrLocal)} ||
+ get_native_address(DestMFA, FunDefs, RemoteOrLocal)} ||
{{DestMFA,_,_}=Data,Offsets} <- PatchOffsets].
offsets_to_addresses(Os, Base) ->
@@ -424,9 +396,9 @@ find_closure_refs([], Refs) ->
%%------------------------------------------------------------------------
-export_funs([FunDef | Addresses]) ->
+set_beam_call_traps([FunDef | FunDefs]) ->
#fundef{address=Address, mfa=MFA, is_closure=IsClosure,
- is_exported=IsExported} = FunDef,
+ is_exported=_IsExported} = FunDef,
?IF_DEBUG({M,F,A} = MFA, no_debug),
?IF_DEBUG(
case IsClosure of
@@ -437,21 +409,38 @@ export_funs([FunDef | Addresses]) ->
?debug_msg("LINKING: ~w:~w/~w to closure (0x~.16b)\n",
[M,F,A, Address])
end, no_debug),
- hipe_bifs:set_funinfo_native_address(MFA, Address, IsExported),
hipe_bifs:set_native_address(MFA, Address, IsClosure),
- export_funs(Addresses);
+ set_beam_call_traps(FunDefs);
+set_beam_call_traps([]) ->
+ ok.
+
+export_funs([FunDef | FunDefs]) ->
+ #fundef{address=Address, mfa=MFA, is_closure=_IsClosure,
+ is_exported=IsExported} = FunDef,
+ ?IF_DEBUG({M,F,A} = MFA, no_debug),
+ ?IF_DEBUG(
+ case _IsClosure of
+ false ->
+ ?debug_msg("LINKING: ~w:~w/~w to (0x~.16b)\n",
+ [M,F,A, Address]);
+ true ->
+ ?debug_msg("LINKING: ~w:~w/~w to closure (0x~.16b)\n",
+ [M,F,A, Address])
+ end, no_debug),
+ hipe_bifs:set_funinfo_native_address(MFA, Address, IsExported),
+ export_funs(FunDefs);
export_funs([]) ->
ok.
-export_funs(Mod, MD5, Beam, Addresses, ClosuresToPatch) ->
- Fs = [{F,A,Address} || #fundef{address=Address, mfa={_M,F,A}} <- Addresses],
- Mod = code:make_stub_module(Mod, Beam, {Fs,ClosuresToPatch,MD5}),
+make_beam_stub(Mod, LoaderState, MD5, Beam, FunDefs, ClosuresToPatch) ->
+ Fs = [{F,A,Address} || #fundef{address=Address, mfa={_M,F,A}} <- FunDefs],
+ Mod = code:make_stub_module(LoaderState, Beam, {Fs,ClosuresToPatch,MD5}),
ok.
%%========================================================================
%% Patching
%% @spec patch(refs(), BaseAddress::integer(), ConstAndZone::term(),
-%% Addresses::term(), TrampolineMap::term()) -> 'ok'.
+%% FunDefs::term(), TrampolineMap::term()) -> 'ok'.
%% @type refs()=[{RefType::integer(), Reflist::reflist()} | refs()]
%%
%% @type reflist()= [{Data::term(), Offsets::offests()}|reflist()]
@@ -463,39 +452,39 @@ export_funs(Mod, MD5, Beam, Addresses, ClosuresToPatch) ->
%% (we use this to look up the address of a referred function only once).
%%
-patch([{Type,SortedRefs}|Rest], CodeAddress, ConstMap2, Addresses, TrampolineMap) ->
+patch([{Type,SortedRefs}|Rest], CodeAddress, ConstMap2, FunDefs, TrampolineMap) ->
?debug_msg("Patching ~w at [~w+offset] with ~w\n",
[Type,CodeAddress,SortedRefs]),
case ?EXT2PATCH_TYPE(Type) of
call_local ->
- patch_call(SortedRefs, CodeAddress, Addresses, 'local', TrampolineMap);
+ patch_call(SortedRefs, CodeAddress, FunDefs, 'local', TrampolineMap);
call_remote ->
- patch_call(SortedRefs, CodeAddress, Addresses, 'remote', TrampolineMap);
+ patch_call(SortedRefs, CodeAddress, FunDefs, 'remote', TrampolineMap);
Other ->
- patch_all(Other, SortedRefs, CodeAddress, {ConstMap2,CodeAddress}, Addresses)
+ patch_all(Other, SortedRefs, CodeAddress, {ConstMap2,CodeAddress}, FunDefs)
end,
- patch(Rest, CodeAddress, ConstMap2, Addresses, TrampolineMap);
+ patch(Rest, CodeAddress, ConstMap2, FunDefs, TrampolineMap);
patch([], _, _, _, _) -> ok.
%%----------------------------------------------------------------
%% Handle a 'call_local' or 'call_remote' patch.
%%
-patch_call([{DestMFA,Offsets}|SortedRefs], BaseAddress, Addresses, RemoteOrLocal, TrampolineMap) ->
+patch_call([{DestMFA,Offsets}|SortedRefs], BaseAddress, FunDefs, RemoteOrLocal, TrampolineMap) ->
case bif_address(DestMFA) of
false ->
- %% Previous code used mfa_to_address(DestMFA, Addresses)
+ %% Previous code used mfa_to_address(DestMFA, FunDefs)
%% here for local calls. That is wrong because even local
- %% destinations may not be present in Addresses: they may
+ %% destinations may not be present in FunDefs: they may
%% not have been compiled yet, or they may be BEAM-only
%% functions (e.g. module_info).
- DestAddress = get_native_address(DestMFA, Addresses, RemoteOrLocal),
+ DestAddress = get_native_address(DestMFA, FunDefs, RemoteOrLocal),
Trampoline = trampoline_map_get(DestMFA, TrampolineMap),
- patch_mfa_call_list(Offsets, BaseAddress, DestMFA, DestAddress, Addresses, RemoteOrLocal, Trampoline);
+ patch_mfa_call_list(Offsets, BaseAddress, DestMFA, DestAddress, FunDefs, RemoteOrLocal, Trampoline);
BifAddress when is_integer(BifAddress) ->
Trampoline = trampoline_map_lookup(DestMFA, TrampolineMap),
patch_bif_call_list(Offsets, BaseAddress, BifAddress, Trampoline)
end,
- patch_call(SortedRefs, BaseAddress, Addresses, RemoteOrLocal, TrampolineMap);
+ patch_call(SortedRefs, BaseAddress, FunDefs, RemoteOrLocal, TrampolineMap);
patch_call([], _, _, _, _) ->
ok.
@@ -506,49 +495,48 @@ patch_bif_call_list([Offset|Offsets], BaseAddress, BifAddress, Trampoline) ->
patch_bif_call_list(Offsets, BaseAddress, BifAddress, Trampoline);
patch_bif_call_list([], _, _, _) -> ok.
-patch_mfa_call_list([Offset|Offsets], BaseAddress, DestMFA, DestAddress, Addresses, RemoteOrLocal, Trampoline) ->
+patch_mfa_call_list([Offset|Offsets], BaseAddress, DestMFA, DestAddress, FunDefs, RemoteOrLocal, Trampoline) ->
CallAddress = BaseAddress+Offset,
- add_ref(DestMFA, CallAddress, Addresses, 'call', Trampoline, RemoteOrLocal),
+ add_ref(DestMFA, CallAddress, FunDefs, 'call', Trampoline, RemoteOrLocal),
?ASSERT(assert_local_patch(CallAddress)),
patch_call_insn(CallAddress, DestAddress, Trampoline),
- patch_mfa_call_list(Offsets, BaseAddress, DestMFA, DestAddress, Addresses, RemoteOrLocal, Trampoline);
+ patch_mfa_call_list(Offsets, BaseAddress, DestMFA, DestAddress, FunDefs, RemoteOrLocal, Trampoline);
patch_mfa_call_list([], _, _, _, _, _, _) -> ok.
patch_call_insn(CallAddress, DestAddress, Trampoline) ->
- %% This assertion is false when we're called from redirect/2.
- %% ?ASSERT(assert_local_patch(CallAddress)),
+ ?ASSERT(assert_local_patch(CallAddress)),
hipe_bifs:patch_call(CallAddress, DestAddress, Trampoline).
%% ____________________________________________________________________
%%
-patch_all(Type, [{Dest,Offsets}|Rest], BaseAddress, ConstAndZone, Addresses)->
- patch_all_offsets(Type, Dest, Offsets, BaseAddress, ConstAndZone, Addresses),
- patch_all(Type, Rest, BaseAddress, ConstAndZone, Addresses);
+patch_all(Type, [{Dest,Offsets}|Rest], BaseAddress, ConstAndZone, FunDefs)->
+ patch_all_offsets(Type, Dest, Offsets, BaseAddress, ConstAndZone, FunDefs),
+ patch_all(Type, Rest, BaseAddress, ConstAndZone, FunDefs);
patch_all(_, [], _, _, _) -> ok.
patch_all_offsets(Type, Data, [Offset|Offsets], BaseAddress,
- ConstAndZone, Addresses) ->
+ ConstAndZone, FunDefs) ->
?debug_msg("Patching ~w at [~w+~w] with ~w\n",
[Type,BaseAddress,Offset, Data]),
Address = BaseAddress + Offset,
- patch_offset(Type, Data, Address, ConstAndZone, Addresses),
+ patch_offset(Type, Data, Address, ConstAndZone, FunDefs),
?debug_msg("Patching done\n",[]),
- patch_all_offsets(Type, Data, Offsets, BaseAddress, ConstAndZone, Addresses);
+ patch_all_offsets(Type, Data, Offsets, BaseAddress, ConstAndZone, FunDefs);
patch_all_offsets(_, _, [], _, _, _) -> ok.
%%----------------------------------------------------------------
%% Handle any patch type except 'call_local' or 'call_remote'.
%%
-patch_offset(Type, Data, Address, ConstAndZone, Addresses) ->
+patch_offset(Type, Data, Address, ConstAndZone, FunDefs) ->
case Type of
load_address ->
- patch_load_address(Data, Address, ConstAndZone, Addresses);
+ patch_load_address(Data, Address, ConstAndZone, FunDefs);
load_atom ->
Atom = Data,
patch_atom(Address, Atom);
sdesc ->
- patch_sdesc(Data, Address, ConstAndZone, Addresses);
+ patch_sdesc(Data, Address, ConstAndZone, FunDefs);
x86_abs_pcrel ->
patch_instr(Address, Data, x86_abs_pcrel)
%% _ ->
@@ -561,37 +549,38 @@ patch_atom(Address, Atom) ->
patch_instr(Address, hipe_bifs:atom_to_word(Atom), atom).
patch_sdesc(?STACK_DESC(SymExnRA, FSize, Arity, Live),
- Address, {_ConstMap2,CodeAddress}, _Addresses) ->
+ Address, {_ConstMap2,CodeAddress}, FunDefs) ->
ExnRA =
case SymExnRA of
[] -> 0; % No catch
LabelOffset -> CodeAddress + LabelOffset
end,
?ASSERT(assert_local_patch(Address)),
- DBG_MFA = ?IF_DEBUG(address_to_mfa_lth(Address, _Addresses), {undefined,undefined,0}),
- hipe_bifs:enter_sdesc({Address, ExnRA, FSize, Arity, Live, DBG_MFA}).
+ MFA = address_to_mfa_lth(Address, FunDefs),
+ hipe_bifs:enter_sdesc({Address, ExnRA, FSize, Arity, Live, MFA},
+ get(hipe_loader_state)).
%%----------------------------------------------------------------
%% Handle a 'load_address'-type patch.
%%
-patch_load_address(Data, Address, ConstAndZone, Addresses) ->
+patch_load_address(Data, Address, ConstAndZone, FunDefs) ->
case Data of
{local_function,DestMFA} ->
- patch_load_mfa(Address, DestMFA, Addresses, 'local');
+ patch_load_mfa(Address, DestMFA, FunDefs, 'local');
{remote_function,DestMFA} ->
- patch_load_mfa(Address, DestMFA, Addresses, 'remote');
+ patch_load_mfa(Address, DestMFA, FunDefs, 'remote');
{constant,Name} ->
{ConstMap2,_CodeAddress} = ConstAndZone,
ConstAddress = find_const(Name, ConstMap2),
patch_instr(Address, ConstAddress, constant);
{closure,{DestMFA,Uniq,Index}} ->
- patch_closure(DestMFA, Uniq, Index, Address, Addresses);
+ patch_closure(DestMFA, Uniq, Index, Address, FunDefs);
{c_const,CConst} ->
patch_instr(Address, bif_address(CConst), c_const)
end.
-patch_closure(DestMFA, Uniq, Index, Address, Addresses) ->
+patch_closure(DestMFA, Uniq, Index, Address, FunDefs) ->
case get(hipe_patch_closures) of
false ->
[]; % This is taken care of when registering the module.
@@ -602,7 +591,7 @@ patch_closure(DestMFA, Uniq, Index, Address, Addresses) ->
%% address into the fun entry to ensure that the native code cannot
%% be called until it has been completely fixed up.
RemoteOrLocal = local, % closure code refs are local
- DestAddress = get_native_address(DestMFA, Addresses, RemoteOrLocal),
+ DestAddress = get_native_address(DestMFA, FunDefs, RemoteOrLocal),
BEAMAddress = hipe_bifs:fun_to_address(DestMFA),
FE = hipe_bifs:get_fe(mod(DestMFA), {Uniq, Index, BEAMAddress}),
put(closures_to_patch, [{FE,DestAddress}|get(closures_to_patch)]),
@@ -616,17 +605,17 @@ patch_closure(DestMFA, Uniq, Index, Address, Addresses) ->
%% Patch an instruction loading the address of an MFA.
%% RemoteOrLocal ::= 'remote' | 'local'
%%
-patch_load_mfa(CodeAddress, DestMFA, Addresses, RemoteOrLocal) ->
+patch_load_mfa(CodeAddress, DestMFA, FunDefs, RemoteOrLocal) ->
+ ?ASSERT(assert_local_patch(CodeAddress)),
DestAddress =
case bif_address(DestMFA) of
false ->
- NativeAddress = get_native_address(DestMFA, Addresses, RemoteOrLocal),
- add_ref(DestMFA, CodeAddress, Addresses, 'load_mfa', [], RemoteOrLocal),
+ NativeAddress = get_native_address(DestMFA, FunDefs, RemoteOrLocal),
+ add_ref(DestMFA, CodeAddress, FunDefs, 'load_mfa', [], RemoteOrLocal),
NativeAddress;
BifAddress when is_integer(BifAddress) ->
BifAddress
end,
- ?ASSERT(assert_local_patch(CodeAddress)),
patch_instr(CodeAddress, DestAddress, 'load_mfa').
%%----------------------------------------------------------------
@@ -702,9 +691,9 @@ bif_address(Name) when is_atom(Name) ->
%% memory, and produces a ConstMap2 mapping each constant's ConstNo to
%% its runtime address, tagged if the constant is a term.
%%
-create_data_segment(DataAlign, DataSize, DataList, WriteWord) ->
+create_data_segment(DataAlign, DataSize, DataList, WriteWord, LoaderState) ->
%%io:format("create_data_segment: \nDataAlign: ~p\nDataSize: ~p\nDataList: ~p\n",[DataAlign,DataSize,DataList]),
- DataAddress = hipe_bifs:alloc_data(DataAlign, DataSize),
+ DataAddress = hipe_bifs:alloc_data(DataAlign, DataSize, LoaderState),
enter_data(DataList, [], DataAddress, DataSize, WriteWord).
enter_data(List, ConstMap2, DataAddress, DataSize, WriteWord) ->
@@ -772,7 +761,7 @@ find_const(ConstNo, []) ->
%%----------------------------------------------------------------
%% Record that the code at address 'Address' has a reference
%% of type 'RefType' ('call' or 'load_mfa') to 'CalleeMFA'.
-%% 'Addresses' must be an address-descending list from exports/2.
+%% 'FunDefs' must be an address-descending list from exports/2.
%%
%% If 'RefType' is 'call', then 'Trampoline' may be the address
%% of a stub branching to 'CalleeMFA', where the stub is reachable
@@ -781,23 +770,18 @@ find_const(ConstNo, []) ->
%% RemoteOrLocal ::= 'remote' | 'local'.
%%
-%%
-%% -record(ref, {caller_mfa, address, ref_type, trampoline, remote_or_local}).
-%%
-
-add_ref(CalleeMFA, Address, Addresses, RefType, Trampoline, RemoteOrLocal) ->
- CallerMFA = address_to_mfa_lth(Address, Addresses),
- %% just a sanity assertion below
- true = case RemoteOrLocal of
- local ->
- {M1,_,_} = CalleeMFA,
- {M2,_,_} = CallerMFA,
- M1 =:= M2;
- remote ->
- true
- end,
- %% io:format("Adding ref ~w\n",[{CallerMFA, CalleeMFA, Address, RefType}]),
- hipe_bifs:add_ref(CalleeMFA, {CallerMFA,Address,RefType,Trampoline,RemoteOrLocal}).
+add_ref(CalleeMFA, Address, FunDefs, RefType, Trampoline, RemoteOrLocal) ->
+ CallerMFA = address_to_mfa_lth(Address, FunDefs),
+ case RemoteOrLocal of
+ local ->
+ %% just a sanity assertion
+ {M,_,_} = CalleeMFA,
+ {M,_,_} = CallerMFA;
+ remote ->
+ hipe_bifs:add_ref(CalleeMFA, {CallerMFA,Address,RefType,Trampoline,
+ get(hipe_loader_state)})
+ end,
+ ok.
% For FunDefs sorted from low to high addresses
address_to_mfa_lth(Address, FunDefs) ->
@@ -825,100 +809,26 @@ address_to_mfa_lth(_Address, [], Prev) ->
%% ?error_msg("Local adddress not found ~w\n",[Address]),
%% exit({?MODULE, local_address_not_found}).
-%%----------------------------------------------------------------
-%% Change callers of the given module to instead trap to BEAM.
-%% load_native_code/3 calls this just before loading native code.
-%%
-patch_to_emu(Mod) ->
- patch_to_emu_step2(patch_to_emu_step1(Mod)).
-
-%% Step 1 must occur before the loading of native code updates
-%% references information or creates a new BEAM stub module.
-patch_to_emu_step1(Mod) ->
- case is_loaded(Mod) of
- true ->
- %% Get exported functions
- MFAs = [{Mod,Fun,Arity} || {Fun,Arity} <- Mod:module_info(exports)],
- %% get_refs_from/2 only finds references from compiled static
- %% call sites to the module, but some native address entries
- %% were added as the result of dynamic apply calls. We must
- %% purge them too, but we have no explicit record of them.
- %% Therefore invalidate all native addresses for the module.
- hipe_bifs:invalidate_funinfo_native_addresses(MFAs),
- %% Find all call sites that call these MFAs. As a side-effect,
- %% create native stubs for any MFAs that are referred.
- ReferencesToPatch = get_refs_from(MFAs, []),
- ok = remove_refs_from(MFAs),
- ReferencesToPatch;
- false ->
- %% The first time we load the module, no redirection needs to be done.
- []
- end.
-
-%% Step 2 must occur after the new BEAM stub module is created.
-patch_to_emu_step2(ReferencesToPatch) ->
- redirect(ReferencesToPatch).
-
--spec is_loaded(Module::atom()) -> boolean().
-%% @doc Checks whether a module is loaded or not.
-is_loaded(M) when is_atom(M) ->
- try hipe_bifs:fun_to_address({M,module_info,0}) of
- I when is_integer(I) -> true
- catch _:_ -> false
- end.
-
-%%--------------------------------------------------------------------
-%% Given a list of MFAs, tag them with their referred_from references.
-%% The resulting {MFA,Refs} list is later passed to redirect/1, once
-%% the MFAs have been bound to (possibly new) native-code addresses.
-%%
-get_refs_from(MFAs, []) ->
- mark_referred_from(MFAs),
- MFAs.
-
-mark_referred_from(MFAs) ->
- lists:foreach(fun(MFA) -> hipe_bifs:mark_referred_from(MFA) end, MFAs).
-
-%%--------------------------------------------------------------------
-%% Given a list of MFAs with referred_from references, update their
-%% callers to refer to their new native-code addresses.
-%%
-%% The {MFA,Refs} list must come from get_refs_from/2.
-%%
-redirect(MFAs) ->
- lists:foreach(fun(MFA) -> hipe_bifs:redirect_referred_from(MFA) end, MFAs).
-
-%%--------------------------------------------------------------------
-%% Given a list of MFAs, remove all referred_from references having
-%% any of them as CallerMFA.
-%%
-%% This is the only place using refers_to. Whenever a reference is
-%% added from CallerMFA to CalleeMFA, CallerMFA is added to CalleeMFA's
-%% referred_from list, and CalleeMFA is added to CallerMFA's refers_to
-%% list. The refers_to list is used here to find the CalleeMFAs whose
-%% referred_from lists should be updated.
-%%
-remove_refs_from(MFAs) ->
- lists:foreach(fun(MFA) -> hipe_bifs:remove_refs_from(MFA) end, MFAs).
%%--------------------------------------------------------------------
%% To find the native code of an MFA we need to look in 3 places:
-%% 1. If it is compiled now look in the Addresses data structure.
+%% 1. If it is compiled now look in the FunDefs data structure.
%% 2. Then look in native_addresses from module info.
%% 3. Then (the function might have been singled compiled) look in
%% hipe_funinfo
%% If all else fails create a native stub for the MFA
-get_native_address(MFA, Addresses, RemoteOrLocal) ->
- case mfa_to_address(MFA, Addresses, RemoteOrLocal) of
+get_native_address(MFA, FunDefs, RemoteOrLocal) ->
+ case mfa_to_address(MFA, FunDefs, RemoteOrLocal) of
Adr when is_integer(Adr) -> Adr;
false ->
- IsRemote =
case RemoteOrLocal of
- remote -> true;
- local -> false
- end,
- hipe_bifs:find_na_or_make_stub(MFA, IsRemote)
+ remote ->
+ hipe_bifs:find_na_or_make_stub(MFA);
+ local ->
+ ?error_msg("Local function ~p not found\n",[MFA]),
+ exit({function_not_found,MFA})
+ end
end.
mfa_to_address(MFA, [#fundef{address=Adr, mfa=MFA,
@@ -960,10 +870,10 @@ assert_local_patch(Address) when is_integer(Address) ->
%% Beam: nil() | binary() (used as a flag)
-enter_code(CodeSize, CodeBinary, CalleeMFAs, Mod, Beam) ->
+enter_code(CodeSize, CodeBinary, CalleeMFAs, LoaderState) ->
true = byte_size(CodeBinary) =:= CodeSize,
- hipe_bifs:update_code_size(Mod, Beam, CodeSize),
- {CodeAddress,Trampolines} = hipe_bifs:enter_code(CodeBinary, CalleeMFAs),
+ {CodeAddress,Trampolines} = hipe_bifs:enter_code(CodeBinary, CalleeMFAs,
+ LoaderState),
?init_assert_patch(CodeAddress, byte_size(CodeBinary)),
{CodeAddress,Trampolines}.
diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl
index c5167efa56..96f8ff2c5f 100644
--- a/lib/kernel/test/code_SUITE.erl
+++ b/lib/kernel/test/code_SUITE.erl
@@ -483,23 +483,26 @@ load_binary(Config) when is_list(Config) ->
code:delete(code_b_test),
ok.
+
upgrade(Config) ->
DataDir = proplists:get_value(data_dir, Config),
- %%T = [beam, hipe],
- T = [beam],
-
- [upgrade_do(DataDir, Client, U1, U2, O1, O2)
- || Client<-T, U1<-T, U2<-T, O1<-T, O2<-T],
+ T = case erlang:system_info(hipe_architecture) of
+ undefined -> [beam];
+ _ -> [beam,hipe]
+ end,
+ [upgrade_do(DataDir, Client, T) || Client <- T],
ok.
-upgrade_do(DataDir, Client, U1, U2, O1, O2) ->
+upgrade_do(DataDir, Client, T) ->
compile_load(upgrade_client, DataDir, undefined, Client),
- upgrade_client:run(DataDir, U1, U2, O1, O2),
+ [upgrade_client:run(DataDir, U1, U2, O1, O2)
+ || U1<-T, U2<-T, O1<-T, O2<-T],
ok.
compile_load(Mod, Dir, Ver, CodeType) ->
+ erlang:display({"{{{{{{{{{{{{{{{{Loading",Mod,Ver,CodeType}),
Version = case Ver of
undefined ->
io:format("Compiling '~p' as ~p\n", [Mod, CodeType]),
@@ -516,9 +519,15 @@ compile_load(Mod, Dir, Ver, CodeType) ->
CompOpts = [binary, report] ++ Target ++ Version,
Src = filename:join(Dir, atom_to_list(Mod) ++ ".erl"),
+ T1 = erlang:now(),
{ok,Mod,Code} = compile:file(Src, CompOpts),
+ T2 = erlang:now(),
ObjFile = filename:basename(Src,".erl") ++ ".beam",
{module,Mod} = code:load_binary(Mod, ObjFile, Code),
+ T3 = erlang:now(),
+ io:format("Compile time ~p ms, Load time ~p ms\n",
+ [timer:now_diff(T2,T1) div 1000, timer:now_diff(T3,T2) div 1000]),
+ erlang:display({"}}}}}}}}}}}}}}}Loaded",Mod,Ver,CodeType}),
ok.
dir_req(Config) when is_list(Config) ->
@@ -810,8 +819,6 @@ check_funs({'$M_EXPR','$F_EXPR',_},
{code_server,start_link,1}]) -> 0;
check_funs({'$M_EXPR','$F_EXPR',_},
[{erlang,spawn_link,1},{code_server,start_link,1}]) -> 0;
-check_funs({'$M_EXPR',module_info,1},
- [{hipe_unified_loader,patch_to_emu_step1,1} | _]) -> 0;
check_funs({'$M_EXPR','$F_EXPR',2},
[{hipe_unified_loader,write_words,3} | _]) -> 0;
check_funs({'$M_EXPR','$F_EXPR',2},
@@ -823,11 +830,7 @@ check_funs({'$M_EXPR','$F_EXPR',2},
{hipe_unified_loader,sort_and_write,5} | _]) -> 0;
check_funs({'$M_EXPR','$F_EXPR',1},
[{lists,foreach,2},
- {hipe_unified_loader,patch_consts,3} | _]) -> 0;
-check_funs({'$M_EXPR','$F_EXPR',1},
- [{lists,foreach,2},
- {hipe_unified_loader,mark_referred_from,1},
- {hipe_unified_loader,get_refs_from,2}| _]) -> 0;
+ {hipe_unified_loader,patch_consts,4} | _]) -> 0;
check_funs({'$M_EXPR',warning_msg,2},
[{code_server,finish_on_load_report,2} | _]) -> 0;
check_funs({'$M_EXPR','$F_EXPR',1},
diff --git a/lib/kernel/test/code_SUITE_data/upgrade_client.erl b/lib/kernel/test/code_SUITE_data/upgrade_client.erl
index bb655e01d3..98d36f6014 100644
--- a/lib/kernel/test/code_SUITE_data/upgrade_client.erl
+++ b/lib/kernel/test/code_SUITE_data/upgrade_client.erl
@@ -9,6 +9,8 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
%% Load version 1 of upgradee
code_SUITE:compile_load(upgradee, Dir, 1, Upgradee1),
+ Tracer = start_tracing(),
+
?line 1 = upgradee:exp1(),
?line 1 = upgradee:exp1exp2(),
?line 1 = upgradee:exp1loc2(),
@@ -56,6 +58,15 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
?line {'EXIT',{undef,_}} = proxy_call(P, other, loc1),
?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
+ Env1 = "Env1",
+ put(loc1_fun, upgradee:get_local_fun(Env1)),
+ ?line {1,Env1} = (get(loc1_fun))(),
+
+ put(exp1exp2_fun, upgradee:get_exp1exp2_fun()),
+ ?line 1 = (get(exp1exp2_fun))(),
+
+ ?line 13 = check_tracing(Tracer),
+
%%
%% Load version 1 of other
%%
@@ -78,6 +89,8 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
?line {'EXIT',{undef,_}} = proxy_call(P, other, exp2),
?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
+ ?line 5 = check_tracing(Tracer),
+
%%
%% Load version 2 of upgradee
%%
@@ -130,6 +143,15 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
?line {'EXIT',{undef,_}} = proxy_call(P, other, exp2),
?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
+ ?line {1,Env1} = (get(loc1_fun))(),
+ Env2 = "Env2",
+ put(loc2_fun, upgradee:get_local_fun(Env2)),
+ ?line {2,Env2} = (get(loc2_fun))(),
+
+ ?line 2 = (get(exp1exp2_fun))(),
+
+ ?line 10 = check_tracing(Tracer),
+
%%
%% Load version 2 of other
%%
@@ -182,17 +204,26 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
?line {'EXIT',{undef,_}} = proxy_call(P, other, loc1loc2),
?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
+ ?line {1,Env1} = (get(loc1_fun))(),
+ ?line {2,Env2} = (get(loc2_fun))(),
+ ?line 2 = (get(exp1exp2_fun))(),
+
+ ?line 10 = check_tracing(Tracer),
%%
%% Upgrade proxy to version 2
%%
P ! upgrade_order,
-
%%
- io:format("Delete version 2 of 'upgradee'\n",[]),
+ io:format("Purge version 1 of 'upgradee'\n",[]),
%%
+ put(loc1_fun,undefined),
code:purge(upgradee),
+
+ %%
+ io:format("Delete version 2 of 'upgradee'\n",[]),
+ %%
code:delete(upgradee),
?line {'EXIT',{undef,_}} = (catch upgradee:exp2()),
@@ -239,17 +270,24 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) ->
?line {'EXIT',{undef,_}} = proxy_call(P, other, exp1loc2),
?line {'EXIT',{undef,_}} = proxy_call(P, other, loc1loc2),
?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2),
+
+ ?line {'EXIT',{undef,_}} = (catch (get(exp1exp2_fun))()),
+ ?line 14 = check_tracing(Tracer),
+
unlink(P),
exit(P, die_please),
io:format("Purge 'upgradee'\n",[]),
+ put(loc2_fun,undefined),
code:purge(upgradee),
io:format("Delete and purge 'other'\n",[]),
code:purge(other),
code:delete(other),
code:purge(other),
+
+ stop_tracing(Tracer),
ok.
proxy_call(Pid, CallType, Func) ->
@@ -257,3 +295,48 @@ proxy_call(Pid, CallType, Func) ->
receive
{Pid, call_result, Func, Ret} -> Ret
end.
+
+
+start_tracing() ->
+ Self = self(),
+ {Tracer,_} = spawn_opt(fun() -> tracer_loop(Self) end, [link,monitor]),
+ ?line 1 = erlang:trace_pattern({error_handler,undefined_function,3},
+ true, [global]),
+ ?line 1 = erlang:trace(Self, true, [call,{tracer,Tracer}]),
+ Tracer.
+
+
+tracer_loop(Receiver) ->
+ receive
+ die_please ->
+ ok;
+ {do_trace_delivered, Tracee} ->
+ _ = erlang:trace_delivered(Tracee),
+ tracer_loop(Receiver);
+
+ Msg ->
+ Receiver ! Msg,
+ tracer_loop(Receiver)
+ end.
+
+check_tracing(Tracer) ->
+ Tracer ! {do_trace_delivered, self()},
+ check_tracing_loop(0).
+
+check_tracing_loop(N) ->
+ Self = self(),
+ receive
+ {trace, _Pid, call, {_M, _F, _Args}} = Msg ->
+ io:format("Trace: ~p\n",[Msg]),
+ check_tracing_loop(N+1);
+ {trace_delivered, Self, _} ->
+ N
+ end.
+
+
+stop_tracing(Tracer) ->
+ erlang:trace(self(), false, [call]),
+ Tracer ! die_please,
+ receive
+ {'DOWN', _, process, Tracer, _} -> ok
+ end.
diff --git a/lib/kernel/test/code_SUITE_data/upgradee.erl b/lib/kernel/test/code_SUITE_data/upgradee.erl
index 62b1d95e30..8ca660c19c 100644
--- a/lib/kernel/test/code_SUITE_data/upgradee.erl
+++ b/lib/kernel/test/code_SUITE_data/upgradee.erl
@@ -8,6 +8,9 @@
-export([exp1/0]). % only exported in v1
-export([exp1loc2/0]). % exported in v1, local in v2
-export([exp1exp2/0]). % exported in v1 and v2
+-export([get_local_fun/1]).
+-export([get_exp1exp2_fun/0]).
+-export([exp1exp2_fun/0]).
exp1() -> ?VERSION.
loc1() -> ?VERSION.
@@ -20,6 +23,9 @@ loc1() -> ?VERSION.
-export([exp2/0]).
-export([loc1exp2/0]).
-export([exp1exp2/0]).
+-export([get_local_fun/1]).
+-export([get_exp1exp2_fun/0]).
+-export([exp1exp2_fun/0]).
exp2() -> ?VERSION.
loc2() -> ?VERSION.
@@ -31,6 +37,12 @@ exp1loc2() -> ?VERSION.
loc1exp2() -> ?VERSION.
loc1loc2() -> ?VERSION.
+get_local_fun(Env) -> fun() -> {?VERSION,Env} end.
+get_exp1exp2_fun() -> fun ?MODULE:exp1exp2_fun/0.
+
+exp1exp2_fun() ->
+ ?VERSION.
+
dispatch_loop() ->
receive
upgrade_order ->