aboutsummaryrefslogtreecommitdiffstats
path: root/erts
diff options
context:
space:
mode:
Diffstat (limited to 'erts')
-rw-r--r--erts/emulator/beam/beam_bif_load.c91
-rw-r--r--erts/emulator/beam/beam_emu.c1
-rw-r--r--erts/emulator/beam/beam_load.c94
-rw-r--r--erts/emulator/beam/beam_load.h20
-rw-r--r--erts/emulator/beam/export.c8
-rw-r--r--erts/emulator/beam/module.c69
-rw-r--r--erts/emulator/beam/module.h15
-rw-r--r--erts/emulator/hipe/hipe_amd64.c15
-rw-r--r--erts/emulator/hipe/hipe_arch.h10
-rw-r--r--erts/emulator/hipe/hipe_arm.c10
-rw-r--r--erts/emulator/hipe/hipe_bif0.c764
-rw-r--r--erts/emulator/hipe/hipe_bif0.h5
-rw-r--r--erts/emulator/hipe/hipe_bif0.tab6
-rw-r--r--erts/emulator/hipe/hipe_ppc.c10
-rw-r--r--erts/emulator/hipe/hipe_risc_glue.h20
-rw-r--r--erts/emulator/hipe/hipe_stack.c90
-rw-r--r--erts/emulator/hipe/hipe_stack.h18
-rw-r--r--erts/emulator/hipe/hipe_x86.c14
-rw-r--r--erts/emulator/hipe/hipe_x86_gc.h14
-rw-r--r--erts/emulator/hipe/hipe_x86_glue.h23
-rw-r--r--erts/emulator/hipe/hipe_x86_stack.c7
-rw-r--r--erts/emulator/test/code_SUITE.erl22
22 files changed, 813 insertions, 513 deletions
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 153fa205ed..237513095a 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -36,6 +36,12 @@
#include "erl_nif.h"
#include "erl_bits.h"
#include "erl_thr_progress.h"
+#ifdef HIPE
+# include "hipe_bif0.h"
+# define IF_HIPE(X) (X)
+#else
+# define IF_HIPE(X) (0)
+#endif
#ifdef HIPE
# include "hipe_stack.h"
@@ -169,6 +175,11 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3)
if (res == BIF_ARG_1) {
erts_end_staging_code_ix();
erts_commit_staging_code_ix();
+#ifdef HIPE
+ if (!modp)
+ modp = erts_get_module(BIF_ARG_1, erts_active_code_ix());
+ hipe_redirect_to_module(modp);
+#endif
}
else {
erts_abort_staging_code_ix();
@@ -241,7 +252,7 @@ struct m {
Uint exception;
};
-static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int);
+static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int, int);
#ifdef ERTS_SMP
static void smp_code_ix_commiter(void*);
@@ -377,8 +388,9 @@ finish_loading_1(BIF_ALIST_1)
for (i = 0; i < n; i++) {
if (p[i].modp->curr.num_breakpoints > 0 ||
p[i].modp->curr.num_traced_exports > 0 ||
- erts_is_default_trace_enabled()) {
- /* tracing involved, fallback with thread blocking */
+ erts_is_default_trace_enabled() ||
+ IF_HIPE(hipe_need_blocking(p[i].modp))) {
+ /* tracing or hipe need thread blocking */
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
erts_smp_thr_progress_block();
is_blocking = 1;
@@ -436,32 +448,36 @@ finish_loading_1(BIF_ALIST_1)
}
done:
- return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n);
+ return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n, 1);
}
static Eterm
staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
- struct m* loaded, int nloaded)
+ struct m* mods, int nmods, int free_mods)
{
#ifdef ERTS_SMP
if (is_blocking || !commit)
#endif
{
if (commit) {
+ int i;
erts_end_staging_code_ix();
erts_commit_staging_code_ix();
- if (loaded) {
- int i;
- for (i=0; i < nloaded; i++) {
- set_default_trace_pattern(loaded[i].module);
+
+ for (i=0; i < nmods; i++) {
+ if (mods[i].modp->curr.code_hdr) {
+ set_default_trace_pattern(mods[i].module);
}
+ #ifdef HIPE
+ hipe_redirect_to_module(mods[i].modp);
+ #endif
}
}
else {
erts_abort_staging_code_ix();
}
- if (loaded) {
- erts_free(ERTS_ALC_T_LOADER_TMP, loaded);
+ if (free_mods) {
+ erts_free(ERTS_ALC_T_LOADER_TMP, mods);
}
if (is_blocking) {
erts_smp_thr_progress_unblock();
@@ -474,8 +490,8 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking,
else {
ASSERT(is_value(res));
- if (loaded) {
- erts_free(ERTS_ALC_T_LOADER_TMP, loaded);
+ if (free_mods) {
+ erts_free(ERTS_ALC_T_LOADER_TMP, mods);
}
erts_end_staging_code_ix();
/*
@@ -653,8 +669,9 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
}
else {
if (modp->curr.num_breakpoints > 0 ||
- modp->curr.num_traced_exports > 0) {
- /* we have tracing, retry single threaded */
+ modp->curr.num_traced_exports > 0 ||
+ IF_HIPE(hipe_need_blocking(modp))) {
+ /* tracing or hipe need to go single threaded */
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
erts_smp_thr_progress_block();
is_blocking = 1;
@@ -668,7 +685,14 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
success = 1;
}
}
- return staging_epilogue(BIF_P, success, res, is_blocking, NULL, 0);
+ {
+ struct m mod;
+ Eterm retval;
+ mod.module = BIF_ARG_1;
+ mod.modp = modp;
+ retval = staging_epilogue(BIF_P, success, res, is_blocking, &mod, 1, 0);
+ return retval;
+ }
}
BIF_RETTYPE module_loaded_1(BIF_ALIST_1)
@@ -809,6 +833,9 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
}
modp->curr.code_hdr->on_load_function_ptr = NULL;
set_default_trace_pattern(BIF_ARG_1);
+ #ifdef HIPE
+ hipe_redirect_to_module(modp);
+ #endif
} else if (BIF_ARG_2 == am_false) {
int i;
@@ -1619,15 +1646,18 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
/*
* Unload any NIF library
*/
- if (modp->old.nif != NULL) {
+ if (modp->old.nif != NULL
+ || IF_HIPE(hipe_purge_need_blocking(modp))) {
/* ToDo: Do unload nif without blocking */
erts_rwunlock_old_code(code_ix);
erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
erts_smp_thr_progress_block();
is_blocking = 1;
erts_rwlock_old_code(code_ix);
- erts_unload_nif(modp->old.nif);
- modp->old.nif = NULL;
+ if (modp->old.nif) {
+ erts_unload_nif(modp->old.nif);
+ modp->old.nif = NULL;
+ }
}
/*
@@ -1646,7 +1676,9 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2)
modp->old.code_length = 0;
modp->old.catches = BEAM_CATCHES_NIL;
erts_remove_from_ranges(code);
-
+#ifdef HIPE
+ hipe_purge_module(modp);
+#endif
ERTS_BIF_PREP_RET(ret, am_true);
}
@@ -1719,6 +1751,8 @@ delete_code(Module* modp)
(BeamInstr) BeamOp(op_i_generic_breakpoint)) {
ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
ASSERT(modp->curr.num_traced_exports > 0);
+ DBG_TRACE_MFA(ep->code[0],ep->code[1],ep->code[2],
+ "export trace cleared, code_ix=%d", code_ix);
erts_clear_export_break(modp, ep->code+3);
}
else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler
@@ -1727,17 +1761,16 @@ delete_code(Module* modp)
ep->addressv[code_ix] = ep->code+3;
ep->code[3] = (BeamInstr) em_call_error_handler;
ep->code[4] = 0;
+ DBG_TRACE_MFA(ep->code[0],ep->code[1],ep->code[2],
+ "export invalidation, code_ix=%d", code_ix);
}
}
ASSERT(modp->curr.num_breakpoints == 0);
ASSERT(modp->curr.num_traced_exports == 0);
+ DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "delete_code old.first_hipe_ref=%p", modp->curr.first_hipe_ref);
modp->old = modp->curr;
- modp->curr.code_hdr = NULL;
- modp->curr.code_length = 0;
- modp->curr.catches = BEAM_CATCHES_NIL;
- modp->curr.nif = NULL;
-
+ erts_module_instance_init(&modp->curr);
}
@@ -1751,9 +1784,11 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module)
* if not, delete old code; error if old code already exists.
*/
- if (modp->curr.code_hdr && modp->old.code_hdr) {
- return am_not_purged;
- } else if (!modp->old.code_hdr) { /* Make the current version old. */
+ if (modp->curr.code_hdr) {
+ if (modp->old.code_hdr) {
+ return am_not_purged;
+ }
+ /* Make the current version old. */
delete_code(modp);
}
return NIL;
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 0ba06058a5..ccb0f786ec 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -6105,6 +6105,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func)
Uint sz;
int i;
+ DBG_TRACE_MFA(fi[0], fi[1], fi[2], "call_error_handler");
/*
* Search for the error_handler module.
*/
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index d69b18e22f..9b206f9a23 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -482,7 +482,8 @@ static void free_literal_fragment(ErlHeapFragment*);
static void loader_state_dtor(Binary* magic);
static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
Eterm group_leader, Eterm module,
- BeamCodeHeader* code, Uint size);
+ BeamCodeHeader* code, Uint size,
+ void* hipe_code_start, UWord hipe_code_size);
static int init_iff_file(LoaderState* stp, byte* code, Uint size);
static int scan_iff_file(LoaderState* stp, Uint* chunk_types,
Uint num_types, Uint num_mandatory);
@@ -842,9 +843,7 @@ erts_finish_loading(Binary* magic, Process* c_p,
erts_alloc(ERTS_ALC_T_PREPARED_CODE,
sizeof(struct erl_module_instance));
inst_p = mod_tab_p->on_load;
- inst_p->nif = 0;
- inst_p->num_breakpoints = 0;
- inst_p->num_traced_exports = 0;
+ erts_module_instance_init(inst_p);
}
inst_p->code_hdr = stp->hdr;
@@ -1094,7 +1093,8 @@ loader_state_dtor(Binary* magic)
static Eterm
stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
Eterm group_leader, Eterm module,
- BeamCodeHeader* code_hdr, Uint size)
+ BeamCodeHeader* code_hdr, Uint size,
+ void* hipe_code_start, UWord hipe_code_size)
{
Module* modp;
Eterm retval;
@@ -1117,6 +1117,17 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks,
modp->curr.code_hdr = code_hdr;
modp->curr.code_length = size;
modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */
+#if defined(HIPE)
+ DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code new_hipe_refs = %p", modp->new_hipe_refs);
+ modp->curr.first_hipe_ref = modp->new_hipe_refs;
+ modp->curr.first_hipe_sdesc = modp->new_hipe_sdesc;
+ modp->curr.hipe_code_start = hipe_code_start;
+ modp->new_hipe_refs = NULL;
+ modp->new_hipe_sdesc = NULL;
+# ifdef DEBUG
+ modp->curr.hipe_code_size = hipe_code_size;
+# endif
+#endif
/*
* Update ranges (used for finding a function from a PC value).
@@ -4773,9 +4784,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
}
ep = erts_export_put(stp->module, stp->export[i].function,
stp->export[i].arity);
- if (!on_load) {
- ep->addressv[erts_staging_code_ix()] = address;
- } else {
+ if (on_load) {
/*
* on_load: Don't make any of the exported functions
* callable yet. Keep any function in the current
@@ -4783,6 +4792,8 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p)
*/
ep->code[4] = (BeamInstr) address;
}
+ else
+ ep->addressv[erts_staging_code_ix()] = address;
}
/*
@@ -5989,17 +6000,12 @@ code_module_md5_1(BIF_ALIST_1)
static BeamInstr*
make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode)
{
+ DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", &fp[5]);
fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
fp[1] = native;
fp[2] = mod;
fp[3] = func;
fp[4] = arity;
-#ifdef HIPE
- if (native) {
- fp[5] = BeamOpCode(op_move_return_n);
- hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5);
- }
-#endif
fp[5] = OpCode;
return fp + WORDS_PER_FUNCTION;
}
@@ -6087,6 +6093,8 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp)
if (stp->export[i].function == function && stp->export[i].arity == arity) {
Export* ep = erts_export_put(mod, function, arity);
ep->addressv[erts_staging_code_ix()] = fp+5;
+ DBG_TRACE_MFA(mod,function,arity,"set beam stub at %p in export at %p (code_ix=%d)",
+ fp+5, ep, erts_staging_code_ix());
return;
}
}
@@ -6282,6 +6290,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
byte* temp_alloc = NULL;
byte* bytes;
Uint size;
+ UWord hipe_code_start = NULL, hipe_code_size = 0;
/*
* Must initialize stp->lambdas here because the error handling code
@@ -6297,7 +6306,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
goto error;
}
tp = tuple_val(Info);
- if (tp[0] != make_arityval(3)) {
+ if (tp[0] != make_arityval(5)) {
goto error;
}
Funcs = tp[1];
@@ -6314,6 +6323,15 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
}
size = binary_size(Beam);
+#ifdef HIPE
+ if (!term_to_Uint(tp[4], &hipe_code_start))
+ goto error;
+# ifdef DEBUG
+ if (!term_to_Uint(tp[5], &hipe_code_size))
+ goto error;
+# endif
+#endif
+
/*
* Scan the Beam binary and read the interesting sections.
*/
@@ -6476,7 +6494,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info)
*/
rval = stub_insert_new_code(p, 0, p->group_leader, Mod,
- code_hdr, code_size);
+ code_hdr, code_size,
+ (void*)hipe_code_start, hipe_code_size);
if (rval != NIL) {
goto error;
}
@@ -6517,3 +6536,46 @@ static int safe_mul(UWord a, UWord b, UWord* resp)
}
}
+#ifdef ENABLE_DBG_TRACE_MFA
+
+#define MFA_MAX 10
+Eterm dbg_trace_m[MFA_MAX];
+Eterm dbg_trace_f[MFA_MAX];
+Uint dbg_trace_a[MFA_MAX];
+unsigned int dbg_trace_ix = 0;
+
+void dbg_set_traced_mfa(const char* m, const char* f, Uint a)
+{
+ unsigned i = dbg_trace_ix++;
+ ASSERT(i < MFA_MAX);
+ dbg_trace_m[i] = am_atom_put(m, strlen(m));
+ dbg_trace_f[i] = am_atom_put(f, strlen(f));
+ dbg_trace_a[i] = a;
+}
+
+int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a)
+{
+ unsigned int i;
+ for (i = 0; i < dbg_trace_ix; ++i) {
+ if (m == dbg_trace_m[i] &&
+ (!f || (f == dbg_trace_f[i] && a == dbg_trace_a[i]))) {
+
+ return i+1;
+ }
+ }
+ return 0;
+}
+
+void dbg_vtrace_mfa(unsigned ix, const char* format, ...)
+{
+ va_list arglist;
+ va_start(arglist, format);
+ ASSERT(--ix < MFA_MAX);
+ erts_fprintf(stderr, "MFA TRACE %T:%T/%u: ",
+ dbg_trace_m[ix], dbg_trace_f[ix], (int)dbg_trace_a[ix]);
+
+ erts_vfprintf(stderr, format, arglist);
+ va_end(arglist);
+}
+
+#endif /* ENABLE_DBG_TRACE_MFA */
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index 6be4031822..5f0f34fcdd 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -151,4 +151,24 @@ struct BeamCodeLineTab_ {
#define LOC_FILE(Loc) ((Loc) >> 24)
#define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1))
+#ifdef DEBUG
+# define ENABLE_DBG_TRACE_MFA
+#endif
+
+#ifdef ENABLE_DBG_TRACE_MFA
+
+void dbg_set_traced_mfa(const char* m, const char* f, Uint a);
+int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a);
+void dbg_vtrace_mfa(unsigned ix, const char* format, ...);
+#define DBG_TRACE_MFA(M,F,A,FMT, ...) do {\
+ unsigned ix;\
+ if ((ix=dbg_is_traced_mfa(M,F,A))) \
+ dbg_vtrace_mfa(ix, FMT"\n", ##__VA_ARGS__);\
+ }while(0)
+
+#else
+# define dbg_set_traced_mfa(M,F,A)
+# define DBG_TRACE_MFA(M,F,A,FMT, ...)
+#endif /* ENABLE_DBG_TRACE_MFA */
+
#endif /* _BEAM_LOAD_H */
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 2a19211987..9da7fae1dc 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -145,6 +145,9 @@ export_alloc(struct export_entry* tmpl_e)
blob->entryv[ix].ep = &blob->exp;
}
ix = 0;
+
+ DBG_TRACE_MFA(obj->code[0], obj->code[1], obj->code[2],
+ "export allocation at %p", obj);
}
else { /* Existing entry in another table, use free entry in blob */
blob = entry_to_blob(tmpl_e);
@@ -163,9 +166,14 @@ export_free(struct export_entry* obj)
obj->slot.index = -1;
for (i=0; i < ERTS_NUM_CODE_IX; i++) {
if (blob->entryv[i].slot.index >= 0) {
+ DBG_TRACE_MFA(blob->exp.code[0], blob->exp.code[1], blob->exp.code[2],
+ "export entry slot %u freed for %p",
+ (obj - blob->entryv), &blob->exp);
return;
}
}
+ DBG_TRACE_MFA(blob->exp.code[0], blob->exp.code[1], blob->exp.code[2],
+ "export blob deallocation at %p", &blob->exp);
erts_free(ERTS_ALC_T_EXPORT, blob);
erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob));
}
diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c
index d4f6e17c56..93ce8fcf9f 100644
--- a/erts/emulator/beam/module.c
+++ b/erts/emulator/beam/module.c
@@ -26,6 +26,7 @@
#include "erl_vm.h"
#include "global.h"
#include "module.h"
+#include "beam_catches.h"
#ifdef DEBUG
# define IF_DEBUG(x) x
@@ -67,6 +68,23 @@ static int module_cmp(Module* tmpl, Module* obj)
return tmpl->module != obj->module;
}
+void erts_module_instance_init(struct erl_module_instance* modi)
+{
+ modi->code_hdr = 0;
+ modi->code_length = 0;
+ modi->catches = BEAM_CATCHES_NIL;
+ modi->nif = NULL;
+ modi->num_breakpoints = 0;
+ modi->num_traced_exports = 0;
+#ifdef HIPE
+ modi->first_hipe_ref = NULL;
+ modi->first_hipe_sdesc = NULL;
+ modi->hipe_code_start = NULL;
+# ifdef DEBUG
+ modi->hipe_code_size = 0;
+# endif
+#endif
+}
static Module* module_alloc(Module* tmpl)
{
@@ -74,18 +92,16 @@ static Module* module_alloc(Module* tmpl)
erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module));
obj->module = tmpl->module;
- obj->curr.code_hdr = 0;
- obj->old.code_hdr = 0;
- obj->curr.code_length = 0;
- obj->old.code_length = 0;
obj->slot.index = -1;
- obj->curr.nif = NULL;
- obj->old.nif = NULL;
- obj->curr.num_breakpoints = 0;
- obj->old.num_breakpoints = 0;
- obj->curr.num_traced_exports = 0;
- obj->old.num_traced_exports = 0;
+ erts_module_instance_init(&obj->curr);
+ erts_module_instance_init(&obj->old);
obj->on_load = 0;
+#ifdef HIPE
+ obj->first_hipe_mfa = NULL;
+ obj->new_hipe_refs = NULL;
+ obj->new_hipe_sdesc = NULL;
+#endif
+ DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc");
return obj;
}
@@ -139,19 +155,14 @@ erts_get_module(Eterm mod, ErtsCodeIndex code_ix)
}
}
-Module*
-erts_put_module(Eterm mod)
+
+static Module* put_module(Eterm mod, IndexTable* mod_tab)
{
Module e;
- IndexTable* mod_tab;
int oldsz, newsz;
Module* res;
ASSERT(is_atom(mod));
- ERTS_SMP_LC_ASSERT(erts_initialized == 0
- || erts_has_code_write_permission());
-
- mod_tab = &module_tables[erts_staging_code_ix()];
e.module = atom_val(mod);
oldsz = index_table_sz(mod_tab);
res = (Module*) index_put_entry(mod_tab, (void*) &e);
@@ -160,6 +171,24 @@ erts_put_module(Eterm mod)
return res;
}
+Module*
+erts_put_module(Eterm mod)
+{
+ ERTS_SMP_LC_ASSERT(erts_initialized == 0
+ || erts_has_code_write_permission());
+
+ return put_module(mod, &module_tables[erts_staging_code_ix()]);
+}
+
+Module*
+erts_put_active_module(Eterm mod)
+{
+ ASSERT(is_atom(mod));
+ //SVERK Why not? ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+ return put_module(mod, &module_tables[erts_active_code_ix()]);
+}
+
Module *module_code(int i, ErtsCodeIndex code_ix)
{
return (Module*) erts_index_lookup(&module_tables[code_ix], i);
@@ -186,6 +215,11 @@ static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod)
dst_mod->curr = src_mod->curr;
dst_mod->old = src_mod->old;
dst_mod->on_load = src_mod->on_load;
+#ifdef HIPE
+ dst_mod->first_hipe_mfa = src_mod->first_hipe_mfa;
+ dst_mod->new_hipe_refs = src_mod->new_hipe_refs;
+ dst_mod->new_hipe_sdesc = src_mod->new_hipe_sdesc;
+#endif
}
void module_start_staging(void)
@@ -217,6 +251,7 @@ void module_start_staging(void)
src_mod = (Module*) erts_index_lookup(src, i);
dst_mod = (Module*) index_put_entry(dst, src_mod);
ASSERT(dst_mod != src_mod);
+
copy_module(dst_mod, src_mod);
}
newsz = index_table_sz(dst);
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 1c1afc8461..694583597b 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -30,6 +30,14 @@ struct erl_module_instance {
struct erl_module_nif* nif;
int num_breakpoints;
int num_traced_exports;
+#ifdef HIPE
+ struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */
+ struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */
+ void* hipe_code_start;
+# ifdef DEBUG
+ UWord hipe_code_size;
+# endif
+#endif
};
typedef struct erl_module {
@@ -40,10 +48,17 @@ typedef struct erl_module {
struct erl_module_instance curr;
struct erl_module_instance old; /* protected by "old_code" rwlock */
struct erl_module_instance* on_load;
+#ifdef HIPE
+ struct hipe_mfa_info* first_hipe_mfa;
+ struct hipe_ref* new_hipe_refs;
+ struct hipe_sdesc* new_hipe_sdesc;
+#endif
} Module;
+void erts_module_instance_init(struct erl_module_instance* modi);
Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix);
Module* erts_put_module(Eterm mod);
+Module* erts_put_active_module(Eterm mod); /* only while blocked */
void init_module_table(void);
void module_start_staging(void);
diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c
index 62739d2a78..df53f4db30 100644
--- a/erts/emulator/hipe/hipe_amd64.c
+++ b/erts/emulator/hipe/hipe_amd64.c
@@ -130,6 +130,13 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
return alloc_code(nrbytes);
}
+void hipe_free_code(void* code)
+{
+ ALLOC_CODE_STATS(--nr_allocs);
+ /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/
+ erts_free(ERTS_ALC_T_HIPE_EXEC, code);
+}
+
/* Make stub for native code calling exported beam function.
*/
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
@@ -234,6 +241,14 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
return code;
}
+void hipe_free_native_stub(void* stub)
+{
+ ALLOC_CODE_STATS(++nr_allocs);
+ /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/
+
+ erts_free(ERTS_ALC_T_HIPE_EXEC, stub);
+}
+
void hipe_arch_print_pcb(struct hipe_process_state *p)
{
#define U(n,x) \
diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h
index 6f959815bb..df38a80069 100644
--- a/erts/emulator/hipe/hipe_arch.h
+++ b/erts/emulator/hipe/hipe_arch.h
@@ -31,22 +31,30 @@ extern int hipe_patch_insn(void *address, Uint value, Eterm type);
extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline);
extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p);
-extern void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity);
+extern void hipe_free_code(void*);
+extern void *hipe_make_native_stub(void *exp, unsigned int beamArity);
+extern void hipe_free_native_stub(void*);
+
#if defined(__sparc__)
#include "hipe_sparc.h"
+#include "hipe_sparc_asm.h"
#endif
#if defined(__i386__)
#include "hipe_x86.h"
+#include "hipe_x86_asm.h"
#endif
#if defined(__x86_64__)
#include "hipe_amd64.h"
+#include "hipe_amd64_asm.h"
#endif
#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)
#include "hipe_ppc.h"
+#include "hipe_ppc_asm.h"
#endif
#if defined(__arm__)
#include "hipe_arm.h"
+#include "hipe_arm_asm.h"
#endif
#if !defined(AEXTERN)
diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c
index f8ef468341..64e35b62d8 100644
--- a/erts/emulator/hipe/hipe_arm.c
+++ b/erts/emulator/hipe/hipe_arm.c
@@ -198,6 +198,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
return address;
}
+void hipe_free_code(void* code, unsigned int bytes)
+{
+ /*SVERK: Leaking code memory */
+}
+
static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu)
{
unsigned int *address;
@@ -229,6 +234,11 @@ static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu)
return address;
}
+void hipe_free_native_stub(void* stub)
+{
+ /*SVERK: Leaking code stub */
+}
+
/*
* ARMv5's support for 32-bit immediates is effectively non-existent.
* Hence, every 32-bit immediate is stored in memory and loaded via
diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c
index 95c2d73971..8be6dc9aa9 100644
--- a/erts/emulator/hipe/hipe_bif0.c
+++ b/erts/emulator/hipe/hipe_bif0.c
@@ -54,6 +54,7 @@
#define BeamOpCode(Op) ((Uint)BeamOp(Op))
+
int term_to_Sint32(Eterm term, Sint *sp)
{
Sint val;
@@ -644,30 +645,21 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3)
pc = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari);
if (pc) {
- hipe_mfa_save_orig_beam_op(mfa.mod, mfa.fun, mfa.ari, pc);
-#if HIPE
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mfa.mod, mfa.fun, mfa.ari);
- printf(": planting call trap to %p at BEAM pc %p\r\n", address, pc);
-#endif
+ DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", pc, address);
hipe_set_call_trap(pc, address, is_closure);
BIF_RET(am_true);
-#endif
}
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mfa.mod, mfa.fun, mfa.ari);
- printf(": no BEAM pc found\r\n");
-#endif
+ DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "failed set call trap to %p, no beam code found", address);
BIF_RET(am_false);
}
BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
{
struct hipe_sdesc *sdesc;
+ Module* modp;
+ int do_commit;
- sdesc = hipe_decode_sdesc(BIF_ARG_1);
+ sdesc = hipe_decode_sdesc(BIF_ARG_1, &do_commit);
if (!sdesc) {
fprintf(stderr, "%s: bad sdesc!\r\n", __FUNCTION__);
BIF_ERROR(BIF_P, BADARG);
@@ -676,6 +668,21 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1)
fprintf(stderr, "%s: duplicate entry!\r\n", __FUNCTION__);
BIF_ERROR(BIF_P, BADARG);
}
+
+ /*
+ * Link into list of sdesc's in same module instance
+ */
+ modp = erts_put_active_module(make_atom(sdesc->m_aix));
+ ASSERT(modp);
+ if (do_commit) { /* Direct "hipe-patching" of early loaded module */
+ sdesc->next_in_modi = modp->curr.first_hipe_sdesc;
+ modp->curr.first_hipe_sdesc = sdesc;
+ }
+ else { /* Normal module loading/upgrade */
+ sdesc->next_in_modi = modp->new_hipe_sdesc;
+ modp->new_hipe_sdesc = sdesc;
+ }
+
BIF_RET(NIL);
}
@@ -970,7 +977,7 @@ BIF_RETTYPE hipe_bifs_get_fe_2(BIF_ALIST_2)
atom_buf[0] = '\0';
strncat(atom_buf, (char*)atom_tab(i)->name, atom_tab(i)->len);
- printf("no fun entry for %s %ld:%ld\n", atom_buf, uniq, index);
+ printf("no fun entry for %s %ld:%ld\n", atom_buf, (unsigned long)uniq, (unsigned long)index);
BIF_ERROR(BIF_P, BADARG);
}
BIF_RET(address_to_term((void *)fe, BIF_P));
@@ -997,11 +1004,15 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2)
BIF_RET(am_true);
}
+struct hipe_ref_head {
+ struct hipe_ref_head* next;
+ struct hipe_ref_head* prev;
+};
+
/*
- * MFA info hash table:
+ * An exported function called from or implemented by native code
* - maps MFA to native code entry point
- * - the MFAs it calls (refers_to)
- * - the references to it (referred_from)
+ * - all references to it (callers)
* - maps MFA to most recent trampoline [if powerpc or arm]
*/
struct hipe_mfa_info {
@@ -1011,16 +1022,19 @@ struct hipe_mfa_info {
} bucket;
Eterm m; /* atom */
Eterm f; /* atom */
- unsigned int a;
+ unsigned int a : sizeof(int)*8 - 1;
+ unsigned int is_stub : 1; /* if beam or not (yet) loaded */
void *remote_address;
- void *local_address;
- Eterm *beam_code;
- Uint orig_beam_op;
- struct hipe_mfa_info_list *refers_to;
- struct hipe_ref *referred_from;
+ void *new_address;
+ struct hipe_ref_head callers; /* sentinel in list of hipe_ref's */
+ struct hipe_mfa_info* next_in_mod;
#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
void *trampoline;
#endif
+#ifdef DEBUG
+ Export* dbg_export;
+#endif
+
};
static struct {
@@ -1038,6 +1052,25 @@ static struct {
erts_smp_rwmtx_t lock;
} hipe_mfa_info_table;
+
+/*
+ * An external native call site M:F(...)
+ * to be patched when the callee changes.
+ */
+struct hipe_ref {
+ struct hipe_ref_head head; /* list of refs to same calleee */
+ void *address;
+ void *trampoline;
+ unsigned int flags;
+ struct hipe_ref* next_from_modi; /* list of refs from same module instance */
+#if defined(DEBUG)
+ struct hipe_mfa_info* callee;
+ Eterm caller_m, caller_f, caller_a;
+#endif
+};
+#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */
+
+
static inline void hipe_mfa_info_table_init_lock(void)
{
erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock");
@@ -1108,15 +1141,18 @@ static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigne
res->m = m;
res->f = f;
res->a = arity;
+ res->is_stub = 0;
res->remote_address = NULL;
- res->local_address = NULL;
- res->beam_code = NULL;
- res->orig_beam_op = 0;
- res->refers_to = NULL;
- res->referred_from = NULL;
+ res->new_address = NULL;
+ res->callers.next = &res->callers;
+ res->callers.prev = &res->callers;
+ res->next_in_mod = NULL;
#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__)
res->trampoline = NULL;
#endif
+#ifdef DEBUG
+ res->dbg_export = NULL;
+#endif
return res;
}
@@ -1154,22 +1190,13 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter
return NULL;
}
-#if 0 /* XXX: unused */
-void *hipe_mfa_find_na(Eterm m, Eterm f, unsigned int arity)
-{
- const struct hipe_mfa_info *p;
-
- p = hipe_mfa_info_table_get(m, f, arity);
- return p ? p->address : NULL;
-}
-#endif
-
static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned int arity)
{
unsigned long h;
unsigned int i;
struct hipe_mfa_info *p;
unsigned int size;
+ Module* modp;
h = HIPE_MFA_HASH(m, f, arity);
i = h & hipe_mfa_info_table.mask;
@@ -1189,23 +1216,47 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f,
size = 1 << hipe_mfa_info_table.log2size;
if (hipe_mfa_info_table.used > (4*size/5)) /* rehash at 80% */
hipe_mfa_info_table_grow();
+
+ modp = erts_put_active_module(m);
+ ASSERT(modp);
+ p->next_in_mod = modp->first_hipe_mfa;
+ modp->first_hipe_mfa = p;
+
+ DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p);
+
return p;
}
-static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address, int is_exported)
+static void remove_mfa_info(struct hipe_mfa_info* rm)
+{
+ unsigned int i;
+ struct hipe_mfa_info *p;
+ struct hipe_mfa_info **prevp;
+
+ i = rm->bucket.hvalue & hipe_mfa_info_table.mask;
+ prevp = &hipe_mfa_info_table.bucket[i];
+ for (;;) {
+ p = *prevp;
+ ASSERT(p);
+ if (p == rm) {
+ *prevp = p->bucket.next;
+ ASSERT(hipe_mfa_info_table.used > 0);
+ hipe_mfa_info_table.used--;
+ return;
+ }
+ prevp = &p->bucket.next;
+ }
+}
+
+static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address)
{
struct hipe_mfa_info *p;
hipe_mfa_info_table_rwlock();
p = hipe_mfa_info_table_put_rwlocked(m, f, arity);
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(m, f, arity);
- printf(": changing address from %p to %p\r\n", p->local_address, address);
-#endif
- p->local_address = address;
- if (is_exported)
- p->remote_address = address;
+ DBG_TRACE_MFA(m,f,arity,"set native address in hipe_mfa_info at %p", p);
+ p->new_address = address;
+
hipe_mfa_info_table_rwunlock();
}
@@ -1237,168 +1288,85 @@ BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3)
{
struct hipe_mfa mfa;
void *address;
- int is_exported;
- if (!term_to_mfa(BIF_ARG_1, &mfa))
- BIF_ERROR(BIF_P, BADARG);
- address = term_to_address(BIF_ARG_2);
- if (!address)
- BIF_ERROR(BIF_P, BADARG);
- if (BIF_ARG_3 == am_true)
- is_exported = 1;
- else if (BIF_ARG_3 == am_false)
- is_exported = 0;
- else
+ switch (BIF_ARG_3) {
+ case am_true: /* is_exported */
+ if (!term_to_mfa(BIF_ARG_1, &mfa))
+ BIF_ERROR(BIF_P, BADARG);
+ address = term_to_address(BIF_ARG_2);
+ if (!address)
+ BIF_ERROR(BIF_P, BADARG);
+ hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address);
+ break;
+ case am_false:
+ break; /* ignore local functions */
+ default:
BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address, is_exported);
- BIF_RET(NIL);
-}
-
-BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1)
-{
- Eterm lst;
- struct hipe_mfa mfa;
- struct hipe_mfa_info *p;
-
- hipe_mfa_info_table_rwlock();
- lst = BIF_ARG_1;
- while (is_list(lst)) {
- if (!term_to_mfa(CAR(list_val(lst)), &mfa))
- break;
- lst = CDR(list_val(lst));
- p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
- if (p) {
- p->remote_address = NULL;
- p->local_address = NULL;
- if (p->beam_code) {
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mfa.mod, mfa.fun, mfa.ari);
- printf(": removing call trap from BEAM pc %p (new op %#lx)\r\n",
- p->beam_code, p->orig_beam_op);
-#endif
- p->beam_code[0] = p->orig_beam_op;
- p->beam_code = NULL;
- p->orig_beam_op = 0;
- } else {
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mfa.mod, mfa.fun, mfa.ari);
- printf(": no call trap to remove\r\n");
-#endif
- }
- }
}
- hipe_mfa_info_table_rwunlock();
- if (is_not_nil(lst))
- BIF_ERROR(BIF_P, BADARG);
BIF_RET(NIL);
}
-void hipe_mfa_save_orig_beam_op(Eterm mod, Eterm fun, unsigned int ari, Eterm *pc)
+
+/* Ask if we need to block all threads
+ * while loading/deleting (beam) code for this module?
+ */
+int hipe_need_blocking(Module* modp)
{
- Uint orig_beam_op;
struct hipe_mfa_info *p;
- orig_beam_op = pc[0];
- if (orig_beam_op != BeamOpCode(op_hipe_trap_call_closure) &&
- orig_beam_op != BeamOpCode(op_hipe_trap_call)) {
- hipe_mfa_info_table_rwlock();
- p = hipe_mfa_info_table_put_rwlocked(mod, fun, ari);
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mod, fun, ari);
- printf(": saving orig op %#lx from BEAM pc %p\r\n", orig_beam_op, pc);
-#endif
- p->beam_code = pc;
- p->orig_beam_op = orig_beam_op;
- hipe_mfa_info_table_rwunlock();
- } else {
-#ifdef DEBUG_LINKER
- printf("%s: ", __FUNCTION__);
- print_mfa(mod, fun, ari);
- printf(": orig op %#lx already saved\r\n", orig_beam_op);
-#endif
+ /* Need to block if we have at least one native caller to this module
+ * or native code to make unaccessible.
+ */
+ hipe_mfa_info_table_rlock();
+ for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) {
+ ASSERT(!p->new_address);
+ if (p->callers.next != &p->callers || !p->is_stub) {
+ break;
+ }
}
+ hipe_mfa_info_table_runlock();
+ return (p != NULL);
}
-static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote)
-{
- Export *export_entry;
- void *StubAddress;
-
- ASSERT(is_remote);
-
- export_entry = erts_export_get_or_make_stub(m, f, arity);
- StubAddress = hipe_make_native_stub(export_entry, arity);
- if (!StubAddress)
- erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
- return StubAddress;
-}
-
-static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info **pp)
+static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a)
{
struct hipe_mfa_info *p;
- void *address;
p = hipe_mfa_info_table_get_locked(m, f, a);
- if (p) {
- /* find address, predicting for a runtime apply call */
- address = p->remote_address;
- if (!is_remote)
- address = p->local_address;
- if (address)
- return address;
-
- /* bummer, install stub, checking if one already existed */
- address = p->remote_address;
- if (address)
- return address;
- }
- /* Caller must take the slow path with the write lock held, but allow
- it to avoid some work if it already holds the write lock. */
- if (pp)
- *pp = p;
- return NULL;
-}
-
-static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info *p)
-{
- void *address;
-
- if (!p)
- p = hipe_mfa_info_table_put_rwlocked(m, f, a);
- address = hipe_make_stub(m, f, a, is_remote);
- /* XXX: how to tell if a BEAM MFA is exported or not? */
- p->remote_address = address;
- return address;
+ return p ? p->remote_address : NULL;
}
-static void *hipe_get_na_nofail_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote)
+static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a)
{
- struct hipe_mfa_info *p;
- void *address;
+ struct hipe_mfa_info *p = hipe_mfa_info_table_put_rwlocked(m, f, a);
- address = hipe_get_na_try_locked(m, f, a, is_remote, &p);
- if (address)
- return address;
+ if (!p->remote_address) {
+ Export* export_entry = erts_export_get_or_make_stub(m, f, a);
+ void* stubAddress = hipe_make_native_stub(export_entry, a);
+ if (!stubAddress)
+ erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
- address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, p);
- return address;
+ p->remote_address = stubAddress;
+ p->is_stub = 1;
+#ifdef DEBUG
+ p->dbg_export = export_entry;
+#endif
+ }
+ return p->remote_address;
}
-static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a, int is_remote)
+static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a)
{
void *address;
hipe_mfa_info_table_rlock();
- address = hipe_get_na_try_locked(m, f, a, is_remote, NULL);
+ address = hipe_get_na_try_locked(m, f, a);
hipe_mfa_info_table_runlock();
if (address)
return address;
hipe_mfa_info_table_rwlock();
- address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, NULL);
+ address = hipe_get_na_slow_rwlocked(m, f, a);
hipe_mfa_info_table_rwunlock();
return address;
}
@@ -1408,7 +1376,7 @@ void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a)
{
if (is_not_atom(m) || is_not_atom(f) || a > 255)
return NULL;
- return hipe_get_na_nofail(m, f, a, 1);
+ return hipe_get_na_nofail(m, f, a);
}
/* primop, but called like a BIF for error handling purposes */
@@ -1420,25 +1388,19 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3)
if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2))
BIF_ERROR(BIF_P, BADARG);
arity = unsigned_val(BIF_ARG_3); /* no error check */
- address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity, 1);
+ address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity);
BIF_RET((Eterm)address); /* semi-Ok */
}
-BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2)
+BIF_RETTYPE hipe_bifs_find_na_or_make_stub_1(BIF_ALIST_1)
{
struct hipe_mfa mfa;
void *address;
- int is_remote;
if (!term_to_mfa(BIF_ARG_1, &mfa))
BIF_ERROR(BIF_P, BADARG);
- if (BIF_ARG_2 == am_true)
- is_remote = 1;
- else if (BIF_ARG_2 == am_false)
- is_remote = 0;
- else
- BIF_ERROR(BIF_P, BADARG);
- address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari, is_remote);
+
+ address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari);
BIF_RET(address_to_term(address, BIF_P));
}
@@ -1460,7 +1422,7 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2)
f = ep->code[1];
} else
goto badfun;
- address = hipe_get_na_nofail(m, f, BIF_ARG_2, 1);
+ address = hipe_get_na_nofail(m, f, BIF_ARG_2);
BIF_RET((Eterm)address);
badfun:
@@ -1471,60 +1433,19 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2)
int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a)
{
- struct hipe_mfa_info *mfa;
- long mfa_offset, ra_offset;
- struct hipe_mfa_info **bucket;
- unsigned int i, nrbuckets;
+ const struct hipe_sdesc* sdesc = hipe_find_sdesc((unsigned long)ra);
- /* Note about locking: the table is only updated from the
- loader, which runs with the rest of the system suspended. */
- /* XXX: alas not true; see comment at hipe_mfa_info_table.lock */
- hipe_mfa_info_table_rlock();
- bucket = hipe_mfa_info_table.bucket;
- nrbuckets = 1 << hipe_mfa_info_table.log2size;
- mfa = NULL;
- mfa_offset = LONG_MAX;
- for (i = 0; i < nrbuckets; ++i) {
- struct hipe_mfa_info *b = bucket[i];
- while (b != NULL) {
- ra_offset = (char*)ra - (char*)b->local_address;
- if (ra_offset > 0 && ra_offset < mfa_offset) {
- mfa_offset = ra_offset;
- mfa = b;
- }
- b = b->bucket.next;
- }
- }
- if (mfa) {
- *m = mfa->m;
- *f = mfa->f;
- *a = mfa->a;
- }
- hipe_mfa_info_table_runlock();
- return mfa ? 1 : 0;
-}
+ if (!sdesc || sdesc->m_aix == atom_val(am_Empty))
+ return 0;
-/*
- * Patch Reference Handling.
- */
-struct hipe_mfa_info_list {
- struct hipe_mfa_info *mfa;
- struct hipe_mfa_info_list *next;
-};
+ *m = make_atom(sdesc->m_aix);
+ *f = make_atom(sdesc->f_aix);
+ *a = sdesc->a;
+ return 1;
+}
-struct hipe_ref {
- struct hipe_mfa_info *caller_mfa;
- void *address;
- void *trampoline;
- unsigned int flags;
- struct hipe_ref *next;
-};
-#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */
-#define REF_FLAG_IS_REMOTE 2 /* bit 1: 0 == local, 1 == remote */
-#define REF_FLAG_PENDING_REDIRECT 4 /* bit 2: 1 == pending redirect */
-#define REF_FLAG_PENDING_REMOVE 8 /* bit 3: 1 == pending remove */
-/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,'remote'|'local'})
+/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,DoCommit})
*/
BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
{
@@ -1535,9 +1456,9 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
void *trampoline;
unsigned int flags;
struct hipe_mfa_info *callee_mfa;
- struct hipe_mfa_info *caller_mfa;
- struct hipe_mfa_info_list *refers_to;
struct hipe_ref *ref;
+ Module* modp;
+ int do_commit;
if (!term_to_mfa(BIF_ARG_1, &callee))
goto badarg;
@@ -1569,62 +1490,90 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2)
goto badarg;
}
switch (tuple[5]) {
- case am_local:
- break;
- case am_remote:
- flags |= REF_FLAG_IS_REMOTE;
- break;
- default:
- goto badarg;
+ case am_true: do_commit = 1; break;
+ case am_false: do_commit = 0; break;
+ default: goto badarg;
}
hipe_mfa_info_table_rwlock();
callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari);
- caller_mfa = hipe_mfa_info_table_put_rwlocked(caller.mod, caller.fun, caller.ari);
-
- refers_to = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*refers_to));
- refers_to->mfa = callee_mfa;
- refers_to->next = caller_mfa->refers_to;
- caller_mfa->refers_to = refers_to;
- ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*ref));
- ref->caller_mfa = caller_mfa;
+ ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref));
ref->address = address;
ref->trampoline = trampoline;
ref->flags = flags;
- ref->next = callee_mfa->referred_from;
- callee_mfa->referred_from = ref;
+
+ /*
+ * Link into list of refs to same callee
+ */
+ ASSERT(callee_mfa->callers.next->prev == &callee_mfa->callers);
+ ASSERT(callee_mfa->callers.prev->next == &callee_mfa->callers);
+ ref->head.next = callee_mfa->callers.next;
+ ref->head.prev = &callee_mfa->callers;
+ ref->head.next->prev = &ref->head;
+ ref->head.prev->next = &ref->head;
+
+ /*
+ * Link into list of refs from same module instance
+ */
+ modp = erts_put_active_module(caller.mod);
+ ASSERT(modp);
+ if (do_commit) { /* Direct "hipe-patching" of early loaded module */
+ ref->next_from_modi = modp->curr.first_hipe_ref;
+ modp->curr.first_hipe_ref = ref;
+ }
+ else { /* Normal module loading/upgrade */
+ ref->next_from_modi = modp->new_hipe_refs;
+ modp->new_hipe_refs = ref;
+ }
+
+#if defined(DEBUG)
+ ref->callee = callee_mfa;
+ ref->caller_m = caller.mod;
+ ref->caller_f = caller.fun;
+ ref->caller_a = caller.ari;
+#endif
hipe_mfa_info_table_rwunlock();
+ DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p) do_commit=%d",
+ ref, callee.mod, callee.fun, callee.ari, ref->address, do_commit);
+ DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p) do_commit=%d",
+ ref, caller.mod, caller.fun, caller.ari, ref->address, do_commit);
BIF_RET(NIL);
badarg:
BIF_ERROR(BIF_P, BADARG);
}
-/* Given a CalleeMFA, mark each ref to it as pending-redirect.
- * This ensures that remove_refs_from() won't remove them: any
- * removal is instead done at the end of redirect_referred_from().
- */
-BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */
+
+static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me)
{
- struct hipe_mfa mfa;
- const struct hipe_mfa_info *p;
- struct hipe_ref *ref;
+ Module* modp = erts_get_module(unlink_me->m, erts_active_code_ix());
+ struct hipe_mfa_info** prevp = &modp->first_hipe_mfa;
+ struct hipe_mfa_info* p;
- if (!term_to_mfa(BIF_ARG_1, &mfa))
- BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_info_table_rwlock();
- p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
- if (p)
- for (ref = p->referred_from; ref != NULL; ref = ref->next)
- ref->flags |= REF_FLAG_PENDING_REDIRECT;
- hipe_mfa_info_table_rwunlock();
- BIF_RET(NIL);
+ ASSERT(modp);
+ for (;;) {
+ p = *prevp;
+ ASSERT(p && p->m == unlink_me->m);
+ if (p == unlink_me) {
+ *prevp = p->next_in_mod;
+ break;
+ }
+ prevp = &p->next_in_mod;
+ }
+}
+
+static void purge_mfa(struct hipe_mfa_info* p)
+{
+ ASSERT(p->is_stub);
+ remove_mfa_info(p);
+ hipe_free_native_stub(p->remote_address);
+ erts_free(ERTS_ALC_T_HIPE, p);
}
/* Called by init:restart after unloading all hipe compiled modules
- * to work around bug causing execution of deallocated beam code.
- * Can be removed when delete/purge of native modules works better.
+ * to work around old bug that caused execution of deallocated beam code.
+ * Can be removed now when delete/purge of native modules works better.
* Test: Do init:restart in debug compiled vm with hipe compiled kernel.
*/
static void hipe_purge_all_refs(void)
@@ -1634,126 +1583,205 @@ static void hipe_purge_all_refs(void)
hipe_mfa_info_table_rwlock();
+ ASSERT(hipe_mfa_info_table.used == 0);
bucket = hipe_mfa_info_table.bucket;
nrbuckets = 1 << hipe_mfa_info_table.log2size;
for (i = 0; i < nrbuckets; ++i) {
+ ASSERT(bucket[i] == NULL);
while (bucket[i] != NULL) {
+ Module* modp;
struct hipe_mfa_info* mfa = bucket[i];
bucket[i] = mfa->bucket.next;
- while (mfa->refers_to) {
- struct hipe_mfa_info_list *to = mfa->refers_to;
- mfa->refers_to = to->next;
- erts_free(ERTS_ALC_T_HIPE, to);
- }
- while (mfa->referred_from) {
- struct hipe_ref* from = mfa->referred_from;
- mfa->referred_from = from->next;
- erts_free(ERTS_ALC_T_HIPE, from);
- }
+ ASSERT(mfa->callers.next == &mfa->callers);
+
+ modp = erts_get_module(mfa->m, erts_active_code_ix());
+ if (modp) {
+ /* unsafe write to active module */
+ modp->first_hipe_mfa = NULL;
+ }
erts_free(ERTS_ALC_T_HIPE, mfa);
}
}
+ hipe_mfa_info_table.used = 0;
hipe_mfa_info_table_rwunlock();
}
BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1)
{
- struct hipe_mfa mfa;
- struct hipe_mfa_info *caller_mfa, *callee_mfa;
- struct hipe_mfa_info_list *refers_to, *tmp_refers_to;
- struct hipe_ref **prev, *ref;
-
if (BIF_ARG_1 == am_all) {
hipe_purge_all_refs();
BIF_RET(am_ok);
}
- if (!term_to_mfa(BIF_ARG_1, &mfa))
- BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_info_table_rwlock();
- caller_mfa = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
- if (caller_mfa) {
- refers_to = caller_mfa->refers_to;
- while (refers_to) {
- callee_mfa = refers_to->mfa;
- prev = &callee_mfa->referred_from;
- ref = *prev;
- while (ref) {
- if (ref->caller_mfa == caller_mfa) {
- if (ref->flags & REF_FLAG_PENDING_REDIRECT) {
- ref->flags |= REF_FLAG_PENDING_REMOVE;
- prev = &ref->next;
- ref = ref->next;
- } else {
- struct hipe_ref *tmp = ref;
- ref = ref->next;
- *prev = ref;
- erts_free(ERTS_ALC_T_HIPE, tmp);
- }
- } else {
- prev = &ref->next;
- ref = ref->next;
- }
- }
- tmp_refers_to = refers_to;
- refers_to = refers_to->next;
- erts_free(ERTS_ALC_T_HIPE, tmp_refers_to);
- }
- caller_mfa->refers_to = NULL;
+ ASSERT(!"hipe_bifs_remove_refs_from_1() called");
+ BIF_ERROR(BIF_P, BADARG);
+}
+
+int hipe_purge_need_blocking(Module* modp)
+{
+ /* SVERK: Verify if this is really necessary */
+ return (modp->old.first_hipe_ref ||
+ modp->old.first_hipe_sdesc ||
+ (!modp->curr.code_hdr && modp->first_hipe_mfa));
+}
+
+void hipe_purge_module(Module* modp)
+{
+ struct hipe_ref* ref;
+ struct hipe_sdesc* sdesc;
+
+ ASSERT(modp);
+
+ DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module");
+
+ /*
+ * Remove all hipe_ref's (external calls) from the old module instance
+ */
+ ref = modp->old.first_hipe_ref;
+
+ while (ref) {
+ struct hipe_ref* free_ref = ref;
+
+ ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+ DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref,
+ ref->callee->m, ref->callee->f, ref->callee->a);
+ DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref,
+ ref->caller_m, ref->caller_f, ref->caller_a);
+ ASSERT(ref->caller_m == make_atom(modp->module));
+
+ /*
+ * Unlink from other refs to same callee
+ */
+ ASSERT(ref->head.next->prev == &ref->head);
+ ASSERT(ref->head.prev->next == &ref->head);
+ ASSERT(ref->head.next != &ref->head);
+ ASSERT(ref->head.prev != &ref->head);
+ ref->head.next->prev = ref->head.prev;
+ ref->head.prev->next = ref->head.next;
+
+ /*
+ * Was this the last ref to that callee?
+ */
+ if (ref->head.next == ref->head.prev) {
+ struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers);
+ if (p->is_stub) {
+ unlink_mfa_from_mod(p);
+ purge_mfa(p);
+ }
+ }
+
+ ref = ref->next_from_modi;
+ erts_free(ERTS_ALC_T_HIPE, free_ref);
+ }
+ modp->old.first_hipe_ref = NULL;
+
+ /*
+ * Remove all hipe_sdesc's for the old module instance
+ */
+ sdesc = modp->old.first_hipe_sdesc;
+
+ while (sdesc) {
+ struct hipe_sdesc* free_sdesc = sdesc;
+
+ ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+ DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue);
+ ASSERT(sdesc->m_aix == modp->module);
+
+ sdesc = sdesc->next_in_modi;
+ hipe_destruct_sdesc(free_sdesc);
+ }
+ modp->old.first_hipe_sdesc = NULL;
+
+ /*
+ * Remove unreferred hipe_mfa_info's
+ */
+ if (modp->curr.code_hdr == NULL) {
+ struct hipe_mfa_info** prevp = &modp->first_hipe_mfa;
+ struct hipe_mfa_info* p = *prevp;
+ ERTS_SMP_LC_ASSERT(!p || erts_smp_thr_progress_is_blocking());
+ for (; p; p = *prevp) {
+ if (p->callers.next == &p->callers) {
+ *prevp = p->next_in_mod;
+ purge_mfa(p);
+ }
+ else
+ prevp = &p->next_in_mod;
+ }
+ }
+ if (modp->old.hipe_code_start) {
+#ifdef DEBUG
+ sys_memset(modp->old.hipe_code_start, 0xfe, modp->old.hipe_code_size);
+#endif
+ hipe_free_code(modp->old.hipe_code_start);
+ modp->old.hipe_code_start = NULL;
}
- hipe_mfa_info_table_rwunlock();
- BIF_RET(am_ok);
}
-/* redirect_referred_from(CalleeMFA)
- * Redirect all pending-redirect refs in CalleeMFA's referred_from.
- * Then remove any pending-redirect && pending-remove refs from CalleeMFA's referred_from.
+/*
+ * Redirect all existing native calls to this module
*/
-BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1)
+void hipe_redirect_to_module(Module* modp)
{
- struct hipe_mfa mfa;
struct hipe_mfa_info *p;
- struct hipe_ref **prev, *ref;
- int is_remote, res;
- void *new_address;
+ struct hipe_ref_head* refh;
+
+ ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking());
+
+ for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) {
+ if (p->new_address) {
+ if (p->is_stub) {
+ hipe_free_native_stub(p->remote_address);
+ p->is_stub = 0;
+ }
+ DBG_TRACE_MFA(p->m, p->f, p->a, "Commit new_address %p", p->new_address);
+ p->remote_address = p->new_address;
+ p->new_address = NULL;
+#ifdef DEBUG
+ p->dbg_export = NULL;
+#endif
+ }
+ else if (!p->is_stub) {
+ Export* exp = erts_export_get_or_make_stub(p->m, p->f, p->a);
+ p->remote_address = hipe_make_native_stub(exp, p->a);
+ DBG_TRACE_MFA(p->m, p->f, p->a, "Commit stub %p", p->remote_address);
+ if (!p->remote_address)
+ erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n");
+ p->is_stub = 1;
+#ifdef DEBUG
+ p->dbg_export = exp;
+#endif
+ }
+ else {
+ DBG_TRACE_MFA(p->m, p->f, p->a, "Commit no-op, already stub");
+ ASSERT(p->remote_address && p->dbg_export);
+ }
- if (!term_to_mfa(BIF_ARG_1, &mfa))
- BIF_ERROR(BIF_P, BADARG);
- hipe_mfa_info_table_rwlock();
- p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari);
- if (p) {
- prev = &p->referred_from;
- ref = *prev;
- while (ref) {
- if (ref->flags & REF_FLAG_PENDING_REDIRECT) {
- is_remote = ref->flags & REF_FLAG_IS_REMOTE;
- new_address = hipe_get_na_nofail_rwlocked(p->m, p->f, p->a, is_remote);
- if (ref->flags & REF_FLAG_IS_LOAD_MFA)
- res = hipe_patch_insn(ref->address, (Uint)new_address, am_load_mfa);
- else
- res = hipe_patch_call(ref->address, new_address, ref->trampoline);
- if (res)
- fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__);
- ref->flags &= ~REF_FLAG_PENDING_REDIRECT;
- if (ref->flags & REF_FLAG_PENDING_REMOVE) {
- struct hipe_ref *tmp = ref;
- ref = ref->next;
- *prev = ref;
- erts_free(ERTS_ALC_T_HIPE, tmp);
- } else {
- prev = &ref->next;
- ref = ref->next;
- }
- } else {
- prev = &ref->next;
- ref = ref->next;
- }
+ DBG_TRACE_MFA(p->m,p->f,p->a,"START REDIRECT towards hipe_mfa_info at %p", p);
+ for (refh = p->callers.next; refh != &p->callers; refh = refh->next) {
+ struct hipe_ref* ref = (struct hipe_ref*) refh;
+ int res;
+
+ DBG_TRACE_MFA(p->m,p->f,p->a, " REDIRECT ref at %p FROM %T:%T/%u (%p -> %p)",
+ ref, ref->caller_m, ref->caller_f, ref->caller_a,
+ ref->address, p->remote_address);
+
+ DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a,
+ " REDIRECT ref at %p TO %T:%T/%u (%p -> %p)",
+ ref, p->m,p->f,p->a, ref->address, p->remote_address);
+
+ if (ref->flags & REF_FLAG_IS_LOAD_MFA)
+ res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa);
+ else
+ res = hipe_patch_call(ref->address, p->remote_address, ref->trampoline);
+ if (res)
+ fprintf(stderr, "%s: patch failed", __FUNCTION__);
}
+ DBG_TRACE_MFA(p->m,p->f,p->a,"DONE REDIRECT towards hipe_mfa_info at %p", p);
}
- hipe_mfa_info_table_rwunlock();
- BIF_RET(NIL);
}
BIF_RETTYPE hipe_bifs_check_crc_1(BIF_ALIST_1)
diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h
index c9a8216368..1914405e2d 100644
--- a/erts/emulator/hipe/hipe_bif0.h
+++ b/erts/emulator/hipe/hipe_bif0.h
@@ -42,7 +42,10 @@ extern void hipe_primop_set_trampoline(Eterm name, void *trampoline);
#endif
/* needed in beam_load.c */
-void hipe_mfa_save_orig_beam_op(Eterm m, Eterm f, unsigned int a, Eterm *pc);
+int hipe_need_blocking(Module*);
+int hipe_purge_need_blocking(Module*);
+void hipe_purge_module(Module*);
+void hipe_redirect_to_module(Module* modp);
/* these are also needed in hipe_amd64.c */
extern void *term_to_address(Eterm);
diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab
index 99237aae05..eae84383df 100644
--- a/erts/emulator/hipe/hipe_bif0.tab
+++ b/erts/emulator/hipe/hipe_bif0.tab
@@ -54,7 +54,7 @@ bif hipe_bifs:set_native_address/3
#bif hipe_bifs:address_to_fun/1
bif hipe_bifs:set_funinfo_native_address/3
-bif hipe_bifs:invalidate_funinfo_native_addresses/1
+#bif hipe_bifs:invalidate_funinfo_native_addresses/1
bif hipe_bifs:update_code_size/3
bif hipe_bifs:code_size/1
@@ -72,7 +72,7 @@ bif hipe_bifs:term_to_word/1
bif hipe_bifs:get_fe/2
bif hipe_bifs:set_native_address_in_fe/2
-bif hipe_bifs:find_na_or_make_stub/2
+bif hipe_bifs:find_na_or_make_stub/1
bif hipe_bifs:check_crc/1
bif hipe_bifs:system_crc/0
@@ -84,9 +84,7 @@ bif hipe_bifs:patch_insn/3
bif hipe_bifs:patch_call/3
bif hipe_bifs:add_ref/2
-bif hipe_bifs:mark_referred_from/1
bif hipe_bifs:remove_refs_from/1
-bif hipe_bifs:redirect_referred_from/1
# atoms used by add_ref/2
atom call
diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c
index 9b2048c457..a1a6e2ad02 100644
--- a/erts/emulator/hipe/hipe_ppc.c
+++ b/erts/emulator/hipe/hipe_ppc.c
@@ -214,6 +214,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
return address;
}
+void hipe_free_code(void* code, unsigned int bytes)
+{
+ /*SVERK: Leaking code memory */
+}
+
static unsigned int *alloc_stub(Uint nrwords)
{
unsigned int *address;
@@ -241,6 +246,11 @@ static unsigned int *alloc_stub(Uint nrwords)
return address;
}
+void hipe_free_native_stub(void* stub)
+{
+ /*SVERK: Leaking code stubs */
+}
+
static void patch_imm16(Uint32 *address, unsigned int imm16)
{
unsigned int insn = *address;
diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h
index 09804e3016..1369b392fe 100644
--- a/erts/emulator/hipe/hipe_risc_glue.h
+++ b/erts/emulator/hipe/hipe_risc_glue.h
@@ -66,15 +66,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y)
static __inline__ void hipe_arch_glue_init(void)
{
- static struct hipe_sdesc_with_exnra nbif_return_sdesc = {
- .exnra = (unsigned long)&nbif_fail,
- .sdesc = {
- .bucket = { .hvalue = (unsigned long)&nbif_return },
- .fsize = 0,
- .has_exnra = 1,
- .arity = 0
- },
- };
+ static struct hipe_sdesc_with_exnra nbif_return_sdesc;
+
+ nbif_return_sdesc.exnra = (unsigned long)nbif_fail;
+ nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return;
+ nbif_return_sdesc.sdesc.fsize = 0;
+ nbif_return_sdesc.sdesc.has_exnra = 1;
+ nbif_return_sdesc.sdesc.stk_nargs = 0;
+ nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty);
+ nbif_return_sdesc.sdesc.f_aix = atom_val(am_return);
+ nbif_return_sdesc.sdesc.a = 0;
+
hipe_init_sdesc_table(&nbif_return_sdesc.sdesc);
}
diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c
index 17bef0718c..3d082668a6 100644
--- a/erts/emulator/hipe/hipe_stack.c
+++ b/erts/emulator/hipe/hipe_stack.c
@@ -102,6 +102,28 @@ struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc)
return sdesc;
}
+void hipe_destruct_sdesc(struct hipe_sdesc *sdesc)
+{
+ unsigned int i;
+ struct hipe_sdesc** prevp;
+ void* free_me;
+
+ i = (sdesc->bucket.hvalue >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask;
+ prevp = &hipe_sdesc_table.bucket[i];
+
+ for (; *prevp != sdesc; prevp = &(*prevp)->bucket.next)
+ ASSERT(*prevp);
+
+ *prevp = sdesc->bucket.next;
+ hipe_sdesc_table.used -= 1;
+
+ if (sdesc->has_exnra)
+ free_me = ErtsContainerStruct(sdesc, struct hipe_sdesc_with_exnra, sdesc);
+ else
+ free_me = sdesc;
+ erts_free(ERTS_ALC_T_HIPE, free_me);
+}
+
void hipe_init_sdesc_table(struct hipe_sdesc *sdesc)
{
unsigned int log2size, size;
@@ -121,31 +143,46 @@ void hipe_init_sdesc_table(struct hipe_sdesc *sdesc)
* representation. If different representations are needed in
* the future, this code has to be made target dependent.
*/
-struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
+struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp)
{
Uint ra, exnra;
Eterm *live;
- Uint fsize, arity, nlive, i, nslots, off;
+ Uint fsize, nargs, stk_nargs, nlive, i, nslots, off;
Uint livebitswords, sdescbytes;
void *p;
struct hipe_sdesc *sdesc;
-
- if (is_not_tuple(arg) ||
- (tuple_val(arg))[0] != make_arityval(6) ||
- term_to_Uint((tuple_val(arg))[1], &ra) == 0 ||
- term_to_Uint((tuple_val(arg))[2], &exnra) == 0 ||
- is_not_small((tuple_val(arg))[3]) ||
- (fsize = unsigned_val((tuple_val(arg))[3])) > 65535 ||
- is_not_small((tuple_val(arg))[4]) ||
- (arity = unsigned_val((tuple_val(arg))[4])) > 255 ||
- is_not_tuple((tuple_val(arg))[5]))
+ Eterm* mfa_tpl;
+ Eterm* tp;
+
+ if (is_not_tuple(arg))
+ return 0;
+
+ tp = tuple_val(arg);
+ if (tp[0] != make_arityval(7) ||
+ term_to_Uint(tp[1], &ra) == 0 ||
+ term_to_Uint(tp[2], &exnra) == 0 ||
+ is_not_small(tp[3]) ||
+ (fsize = unsigned_val(tp[3])) > 65535 ||
+ is_not_small(tp[4]) ||
+ (stk_nargs = unsigned_val(tp[4])) > 255 ||
+ is_not_tuple(tp[5]) ||
+ is_not_tuple(tp[6]) ||
+ (mfa_tpl = tuple_val(tp[6]))[0] != make_arityval(3) ||
+ is_not_atom(mfa_tpl[1]) ||
+ is_not_atom(mfa_tpl[2]) ||
+ is_not_small(mfa_tpl[3]) ||
+ (nargs = unsigned_val(mfa_tpl[3])) > 255)
return 0;
+
+ if (stk_nargs > nargs)
+ return 0;
+
/* Get tuple with live slots */
- live = tuple_val((tuple_val(arg))[5]) + 1;
+ live = tuple_val(tp[5]) + 1;
/* Get number of live slots */
nlive = arityval(live[-1]);
- /* Calculate size of frame = locals + ra + arguments */
- nslots = fsize + 1 + arity;
+ /* Calculate size of frame = locals + ra + stack arguments */
+ nslots = fsize + 1 + stk_nargs;
/* Check that only valid slots are given. */
for (i = 0; i < nlive; ++i) {
if (is_not_small(live[i]) ||
@@ -154,8 +191,14 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
return 0;
}
+ switch(tp[7]) {
+ case am_true: *do_commitp = 1; break;
+ case am_false: *do_commitp = 0; break;
+ default: return 0;
+ }
+
/* Calculate number of words for the live bitmap. */
- livebitswords = (fsize + arity + 1 + 31) / 32;
+ livebitswords = (fsize + stk_nargs + 1 + 31) / 32;
/* Calculate number of bytes needed for the stack descriptor. */
sdescbytes =
(exnra
@@ -172,12 +215,17 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
} else
sdesc = p;
+ sdesc->m_aix = atom_val(mfa_tpl[1]);
+ sdesc->f_aix = atom_val(mfa_tpl[2]);
+ sdesc->a = nargs;
+
+
/* Initialise head of sdesc. */
sdesc->bucket.next = 0;
sdesc->bucket.hvalue = ra;
sdesc->fsize = fsize;
sdesc->has_exnra = (exnra ? 1 : 0);
- sdesc->arity = arity;
+ sdesc->stk_nargs = stk_nargs;
/* Clear all live-bits */
for (i = 0; i < livebitswords; ++i)
sdesc->livebits[i] = 0;
@@ -186,13 +234,5 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg)
off = unsigned_val(live[i]);
sdesc->livebits[off / 32] |= (1 << (off & 31));
}
-#ifdef DEBUG
- {
- Eterm mfa_tpl = tuple_val(arg)[6];
- sdesc->dbg_M = tuple_val(mfa_tpl)[1];
- sdesc->dbg_F = tuple_val(mfa_tpl)[2];
- sdesc->dbg_A = tuple_val(mfa_tpl)[3];
- }
-#endif
return sdesc;
}
diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h
index 81f2e53dbc..1863b0db8c 100644
--- a/erts/emulator/hipe/hipe_stack.h
+++ b/erts/emulator/hipe/hipe_stack.h
@@ -37,12 +37,12 @@ struct hipe_sdesc {
} bucket;
unsigned int fsize : 23; /* frame size */
unsigned int has_exnra : 1; /* exn handler presence flag */
- unsigned int arity : 8;
-#ifdef DEBUG
- Eterm dbg_M, dbg_F;
- unsigned dbg_A;
-#endif
- unsigned int livebits[1]; /* size depends on arch & data in summary field */
+ unsigned int stk_nargs : 8; /* arguments on stack */
+ Uint32 m_aix;
+ Uint32 f_aix;
+ Uint32 a;
+ struct hipe_sdesc* next_in_modi;
+ Uint32 livebits[1]; /* size depends on arch & data in summary field */
};
struct hipe_sdesc_with_exnra {
@@ -55,9 +55,10 @@ static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc)
return sdesc->fsize;
}
+/* Nr of arguments pushed on stack */
static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc)
{
- return sdesc->arity;
+ return sdesc->stk_nargs;
}
static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc)
@@ -79,8 +80,9 @@ struct hipe_sdesc_table {
extern struct hipe_sdesc_table hipe_sdesc_table;
extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*);
+extern void hipe_destruct_sdesc(struct hipe_sdesc*);
extern void hipe_init_sdesc_table(struct hipe_sdesc*);
-extern struct hipe_sdesc *hipe_decode_sdesc(Eterm);
+extern struct hipe_sdesc *hipe_decode_sdesc(Eterm, int* do_commitp);
#if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96)
#define __builtin_expect(x, expected_value) (x)
diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c
index 5f6c8c200e..8a3ba8bc7f 100644
--- a/erts/emulator/hipe/hipe_x86.c
+++ b/erts/emulator/hipe/hipe_x86.c
@@ -184,6 +184,15 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *
return alloc_code(nrbytes);
}
+void hipe_free_code(void* code, unsigned int bytes)
+{
+ /* SVERK: Leak !!!
+ ALLOC_CODE_STATS(--nr_allocs);
+ ALLOC_CODE_STATS(total_alloc -= bytes);
+
+ erts_free(ERTS_ALC_T_HIPE_EXEC, code);*/
+}
+
void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
{
/*
@@ -264,6 +273,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity)
return code;
}
+void hipe_free_native_stub(void* stub)
+{
+ /* SVERK: leak leak drip drop */
+}
+
void hipe_arch_print_pcb(struct hipe_process_state *p)
{
#define U(n,x) \
diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h
index 7802076f70..e92e6ea718 100644
--- a/erts/emulator/hipe/hipe_x86_gc.h
+++ b/erts/emulator/hipe/hipe_x86_gc.h
@@ -65,18 +65,14 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state)
state->sdesc0 = sdesc;
return sdesc;
#else
- unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS;
- if ((int)nstkarity < 0)
- nstkarity = 0;
state->sdesc0[0].fsize = 0;
state->sdesc0[0].has_exnra = 0;
- state->sdesc0[0].arity = nstkarity;
+ state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 :
+ p->hipe.narity - NR_ARG_REGS);
state->sdesc0[0].livebits[0] = 0;
-# ifdef DEBUG
- state->sdesc0[0].dbg_M = 0;
- state->sdesc0[0].dbg_F = am_undefined;
- state->sdesc0[0].dbg_A = 0;
-# endif
+ state->sdesc0[0].m_aix = 0;
+ state->sdesc0[0].f_aix = atom_val(am_undefined);
+ state->sdesc0[0].a = 0;
/* XXX: this appears to prevent a gcc-4.1.1 bug on x86 */
__asm__ __volatile__("" : : "m"(*state) : "memory");
return &state->sdesc0[0];
diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h
index 0e9fcd61f1..de2b061706 100644
--- a/erts/emulator/hipe/hipe_x86_glue.h
+++ b/erts/emulator/hipe/hipe_x86_glue.h
@@ -58,18 +58,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y)
static __inline__ void hipe_arch_glue_init(void)
{
- static struct hipe_sdesc_with_exnra nbif_return_sdesc = {
- .exnra = (unsigned long)nbif_fail,
- .sdesc = {
- .bucket = { .hvalue = (unsigned long)nbif_return },
- .fsize = 0,
- .has_exnra = 1,
- .arity = 0,
- #ifdef DEBUG
- .dbg_F = am_return,
- #endif
- },
- };
+ static struct hipe_sdesc_with_exnra nbif_return_sdesc;
+
+ nbif_return_sdesc.exnra = (unsigned long)nbif_fail;
+ nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return;
+ nbif_return_sdesc.sdesc.fsize = 0;
+ nbif_return_sdesc.sdesc.has_exnra = 1;
+ nbif_return_sdesc.sdesc.stk_nargs = 0;
+ nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty);
+ nbif_return_sdesc.sdesc.f_aix = atom_val(am_return);
+ nbif_return_sdesc.sdesc.a = 0;
+
hipe_init_sdesc_table(&nbif_return_sdesc.sdesc);
}
diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c
index 9426b166e2..31582b3a2e 100644
--- a/erts/emulator/hipe/hipe_x86_stack.c
+++ b/erts/emulator/hipe/hipe_x86_stack.c
@@ -60,7 +60,6 @@ void hipe_print_nstack(Process *p)
unsigned int mask;
unsigned int sdesc_size;
unsigned int i;
- unsigned int nstkarity;
static const char dashes[2*sizeof(long)+5] = {
[0 ... 2*sizeof(long)+3] = '-'
};
@@ -68,12 +67,10 @@ void hipe_print_nstack(Process *p)
nsp = p->hipe.nsp;
nsp_end = p->hipe.nstend;
- nstkarity = p->hipe.narity - NR_ARG_REGS;
- if ((int)nstkarity < 0)
- nstkarity = 0;
sdesc0.fsize = 0;
sdesc0.has_exnra = 0;
- sdesc0.arity = nstkarity;
+ sdesc0.stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 :
+ p->hipe.narity - NR_ARG_REGS);
sdesc0.livebits[0] = ~1;
sdesc = &sdesc0;
diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl
index 8427bb134d..b76ba6427f 100644
--- a/erts/emulator/test/code_SUITE.erl
+++ b/erts/emulator/test/code_SUITE.erl
@@ -445,62 +445,64 @@ module_md5_ok(Code) ->
make_stub(Config) when is_list(Config) ->
catch erlang:purge_module(my_code_test),
MD5 = erlang:md5(<<>>),
+ Arg3 = {[], [], MD5, 0, 0},
Data = proplists:get_value(data_dir, Config),
File = filename:join(Data, "my_code_test"),
{ok,my_code_test,Code} = compile:file(File, [binary]),
- my_code_test = code:make_stub_module(my_code_test, Code, {[],[],MD5}),
+ my_code_test = code:make_stub_module(my_code_test, Code, Arg3),
true = erlang:delete_module(my_code_test),
true = erlang:purge_module(my_code_test),
my_code_test = code:make_stub_module(my_code_test,
make_unaligned_sub_binary(Code),
- {[],[],MD5}),
+ Arg3),
true = erlang:delete_module(my_code_test),
true = erlang:purge_module(my_code_test),
my_code_test = code:make_stub_module(my_code_test, zlib:gzip(Code),
- {[],[],MD5}),
+ Arg3),
true = erlang:delete_module(my_code_test),
true = erlang:purge_module(my_code_test),
%% Should fail.
{'EXIT',{badarg,_}} =
- (catch code:make_stub_module(my_code_test, <<"bad">>, {[],[],MD5})),
+ (catch code:make_stub_module(my_code_test, <<"bad">>, Arg3)),
{'EXIT',{badarg,_}} =
(catch code:make_stub_module(my_code_test,
bit_sized_binary(Code),
- {[],[],MD5})),
+ Arg3)),
{'EXIT',{badarg,_}} =
(catch code:make_stub_module(my_code_test_with_wrong_name,
- Code, {[],[],MD5})),
+ Code, Arg3)),
ok.
make_stub_many_funs(Config) when is_list(Config) ->
catch erlang:purge_module(many_funs),
MD5 = erlang:md5(<<>>),
+ Arg3 = {[], [], MD5, 0, 0},
Data = proplists:get_value(data_dir, Config),
File = filename:join(Data, "many_funs"),
{ok,many_funs,Code} = compile:file(File, [binary]),
- many_funs = code:make_stub_module(many_funs, Code, {[],[],MD5}),
+ many_funs = code:make_stub_module(many_funs, Code, Arg3),
true = erlang:delete_module(many_funs),
true = erlang:purge_module(many_funs),
many_funs = code:make_stub_module(many_funs,
make_unaligned_sub_binary(Code),
- {[],[],MD5}),
+ Arg3),
true = erlang:delete_module(many_funs),
true = erlang:purge_module(many_funs),
%% Should fail.
{'EXIT',{badarg,_}} =
- (catch code:make_stub_module(many_funs, <<"bad">>, {[],[],MD5})),
+ (catch code:make_stub_module(many_funs, <<"bad">>, Arg3)),
{'EXIT',{badarg,_}} =
(catch code:make_stub_module(many_funs,
bit_sized_binary(Code),
- {[],[],MD5})),
+ Arg3)),
ok.
constant_pools(Config) when is_list(Config) ->
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-03 14:19:33 +0000