diff options
30 files changed, 1005 insertions, 734 deletions
diff --git a/bootstrap/lib/kernel/ebin/code_server.beam b/bootstrap/lib/kernel/ebin/code_server.beam Binary files differindex 0763c4c93b..e369cc2f36 100644 --- a/bootstrap/lib/kernel/ebin/code_server.beam +++ b/bootstrap/lib/kernel/ebin/code_server.beam diff --git a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam Binary files differindex 2e639db331..f560389756 100644 --- a/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam +++ b/bootstrap/lib/kernel/ebin/hipe_unified_loader.beam diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 153fa205ed..237513095a 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -36,6 +36,12 @@ #include "erl_nif.h" #include "erl_bits.h" #include "erl_thr_progress.h" +#ifdef HIPE +# include "hipe_bif0.h" +# define IF_HIPE(X) (X) +#else +# define IF_HIPE(X) (0) +#endif #ifdef HIPE # include "hipe_stack.h" @@ -169,6 +175,11 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) if (res == BIF_ARG_1) { erts_end_staging_code_ix(); erts_commit_staging_code_ix(); +#ifdef HIPE + if (!modp) + modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); + hipe_redirect_to_module(modp); +#endif } else { erts_abort_staging_code_ix(); @@ -241,7 +252,7 @@ struct m { Uint exception; }; -static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int); +static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int, int); #ifdef ERTS_SMP static void smp_code_ix_commiter(void*); @@ -377,8 +388,9 @@ finish_loading_1(BIF_ALIST_1) for (i = 0; i < n; i++) { if (p[i].modp->curr.num_breakpoints > 0 || p[i].modp->curr.num_traced_exports > 0 || - erts_is_default_trace_enabled()) { - /* tracing involved, fallback with thread blocking */ + erts_is_default_trace_enabled() || + IF_HIPE(hipe_need_blocking(p[i].modp))) { + /* tracing or hipe need thread blocking */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; @@ -436,32 +448,36 @@ finish_loading_1(BIF_ALIST_1) } done: - return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n); + return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n, 1); } static Eterm staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, - struct m* loaded, int nloaded) + struct m* mods, int nmods, int free_mods) { #ifdef ERTS_SMP if (is_blocking || !commit) #endif { if (commit) { + int i; erts_end_staging_code_ix(); erts_commit_staging_code_ix(); - if (loaded) { - int i; - for (i=0; i < nloaded; i++) { - set_default_trace_pattern(loaded[i].module); + + for (i=0; i < nmods; i++) { + if (mods[i].modp->curr.code_hdr) { + set_default_trace_pattern(mods[i].module); } + #ifdef HIPE + hipe_redirect_to_module(mods[i].modp); + #endif } } else { erts_abort_staging_code_ix(); } - if (loaded) { - erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + if (free_mods) { + erts_free(ERTS_ALC_T_LOADER_TMP, mods); } if (is_blocking) { erts_smp_thr_progress_unblock(); @@ -474,8 +490,8 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, else { ASSERT(is_value(res)); - if (loaded) { - erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + if (free_mods) { + erts_free(ERTS_ALC_T_LOADER_TMP, mods); } erts_end_staging_code_ix(); /* @@ -653,8 +669,9 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } else { if (modp->curr.num_breakpoints > 0 || - modp->curr.num_traced_exports > 0) { - /* we have tracing, retry single threaded */ + modp->curr.num_traced_exports > 0 || + IF_HIPE(hipe_need_blocking(modp))) { + /* tracing or hipe need to go single threaded */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; @@ -668,7 +685,14 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) success = 1; } } - return staging_epilogue(BIF_P, success, res, is_blocking, NULL, 0); + { + struct m mod; + Eterm retval; + mod.module = BIF_ARG_1; + mod.modp = modp; + retval = staging_epilogue(BIF_P, success, res, is_blocking, &mod, 1, 0); + return retval; + } } BIF_RETTYPE module_loaded_1(BIF_ALIST_1) @@ -809,6 +833,9 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) } modp->curr.code_hdr->on_load_function_ptr = NULL; set_default_trace_pattern(BIF_ARG_1); + #ifdef HIPE + hipe_redirect_to_module(modp); + #endif } else if (BIF_ARG_2 == am_false) { int i; @@ -1619,15 +1646,18 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) /* * Unload any NIF library */ - if (modp->old.nif != NULL) { + if (modp->old.nif != NULL + || IF_HIPE(hipe_purge_need_blocking(modp))) { /* ToDo: Do unload nif without blocking */ erts_rwunlock_old_code(code_ix); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; erts_rwlock_old_code(code_ix); - erts_unload_nif(modp->old.nif); - modp->old.nif = NULL; + if (modp->old.nif) { + erts_unload_nif(modp->old.nif); + modp->old.nif = NULL; + } } /* @@ -1646,7 +1676,9 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) modp->old.code_length = 0; modp->old.catches = BEAM_CATCHES_NIL; erts_remove_from_ranges(code); - +#ifdef HIPE + hipe_purge_module(modp); +#endif ERTS_BIF_PREP_RET(ret, am_true); } @@ -1719,6 +1751,8 @@ delete_code(Module* modp) (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); + DBG_TRACE_MFA(ep->code[0],ep->code[1],ep->code[2], + "export trace cleared, code_ix=%d", code_ix); erts_clear_export_break(modp, ep->code+3); } else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler @@ -1727,17 +1761,16 @@ delete_code(Module* modp) ep->addressv[code_ix] = ep->code+3; ep->code[3] = (BeamInstr) em_call_error_handler; ep->code[4] = 0; + DBG_TRACE_MFA(ep->code[0],ep->code[1],ep->code[2], + "export invalidation, code_ix=%d", code_ix); } } ASSERT(modp->curr.num_breakpoints == 0); ASSERT(modp->curr.num_traced_exports == 0); + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "delete_code old.first_hipe_ref=%p", modp->curr.first_hipe_ref); modp->old = modp->curr; - modp->curr.code_hdr = NULL; - modp->curr.code_length = 0; - modp->curr.catches = BEAM_CATCHES_NIL; - modp->curr.nif = NULL; - + erts_module_instance_init(&modp->curr); } @@ -1751,9 +1784,11 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) * if not, delete old code; error if old code already exists. */ - if (modp->curr.code_hdr && modp->old.code_hdr) { - return am_not_purged; - } else if (!modp->old.code_hdr) { /* Make the current version old. */ + if (modp->curr.code_hdr) { + if (modp->old.code_hdr) { + return am_not_purged; + } + /* Make the current version old. */ delete_code(modp); } return NIL; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 0ba06058a5..ccb0f786ec 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -6105,6 +6105,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) Uint sz; int i; + DBG_TRACE_MFA(fi[0], fi[1], fi[2], "call_error_handler"); /* * Search for the error_handler module. */ diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index d69b18e22f..9b206f9a23 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -482,7 +482,8 @@ static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code, Uint size); + BeamCodeHeader* code, Uint size, + void* hipe_code_start, UWord hipe_code_size); static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -842,9 +843,7 @@ erts_finish_loading(Binary* magic, Process* c_p, erts_alloc(ERTS_ALC_T_PREPARED_CODE, sizeof(struct erl_module_instance)); inst_p = mod_tab_p->on_load; - inst_p->nif = 0; - inst_p->num_breakpoints = 0; - inst_p->num_traced_exports = 0; + erts_module_instance_init(inst_p); } inst_p->code_hdr = stp->hdr; @@ -1094,7 +1093,8 @@ loader_state_dtor(Binary* magic) static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code_hdr, Uint size) + BeamCodeHeader* code_hdr, Uint size, + void* hipe_code_start, UWord hipe_code_size) { Module* modp; Eterm retval; @@ -1117,6 +1117,17 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modp->curr.code_hdr = code_hdr; modp->curr.code_length = size; modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ +#if defined(HIPE) + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code new_hipe_refs = %p", modp->new_hipe_refs); + modp->curr.first_hipe_ref = modp->new_hipe_refs; + modp->curr.first_hipe_sdesc = modp->new_hipe_sdesc; + modp->curr.hipe_code_start = hipe_code_start; + modp->new_hipe_refs = NULL; + modp->new_hipe_sdesc = NULL; +# ifdef DEBUG + modp->curr.hipe_code_size = hipe_code_size; +# endif +#endif /* * Update ranges (used for finding a function from a PC value). @@ -4773,9 +4784,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) } ep = erts_export_put(stp->module, stp->export[i].function, stp->export[i].arity); - if (!on_load) { - ep->addressv[erts_staging_code_ix()] = address; - } else { + if (on_load) { /* * on_load: Don't make any of the exported functions * callable yet. Keep any function in the current @@ -4783,6 +4792,8 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) */ ep->code[4] = (BeamInstr) address; } + else + ep->addressv[erts_staging_code_ix()] = address; } /* @@ -5989,17 +6000,12 @@ code_module_md5_1(BIF_ALIST_1) static BeamInstr* make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode) { + DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", &fp[5]); fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); fp[1] = native; fp[2] = mod; fp[3] = func; fp[4] = arity; -#ifdef HIPE - if (native) { - fp[5] = BeamOpCode(op_move_return_n); - hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5); - } -#endif fp[5] = OpCode; return fp + WORDS_PER_FUNCTION; } @@ -6087,6 +6093,8 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) if (stp->export[i].function == function && stp->export[i].arity == arity) { Export* ep = erts_export_put(mod, function, arity); ep->addressv[erts_staging_code_ix()] = fp+5; + DBG_TRACE_MFA(mod,function,arity,"set beam stub at %p in export at %p (code_ix=%d)", + fp+5, ep, erts_staging_code_ix()); return; } } @@ -6282,6 +6290,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) byte* temp_alloc = NULL; byte* bytes; Uint size; + UWord hipe_code_start = NULL, hipe_code_size = 0; /* * Must initialize stp->lambdas here because the error handling code @@ -6297,7 +6306,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) goto error; } tp = tuple_val(Info); - if (tp[0] != make_arityval(3)) { + if (tp[0] != make_arityval(5)) { goto error; } Funcs = tp[1]; @@ -6314,6 +6323,15 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) } size = binary_size(Beam); +#ifdef HIPE + if (!term_to_Uint(tp[4], &hipe_code_start)) + goto error; +# ifdef DEBUG + if (!term_to_Uint(tp[5], &hipe_code_size)) + goto error; +# endif +#endif + /* * Scan the Beam binary and read the interesting sections. */ @@ -6476,7 +6494,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) */ rval = stub_insert_new_code(p, 0, p->group_leader, Mod, - code_hdr, code_size); + code_hdr, code_size, + (void*)hipe_code_start, hipe_code_size); if (rval != NIL) { goto error; } @@ -6517,3 +6536,46 @@ static int safe_mul(UWord a, UWord b, UWord* resp) } } +#ifdef ENABLE_DBG_TRACE_MFA + +#define MFA_MAX 10 +Eterm dbg_trace_m[MFA_MAX]; +Eterm dbg_trace_f[MFA_MAX]; +Uint dbg_trace_a[MFA_MAX]; +unsigned int dbg_trace_ix = 0; + +void dbg_set_traced_mfa(const char* m, const char* f, Uint a) +{ + unsigned i = dbg_trace_ix++; + ASSERT(i < MFA_MAX); + dbg_trace_m[i] = am_atom_put(m, strlen(m)); + dbg_trace_f[i] = am_atom_put(f, strlen(f)); + dbg_trace_a[i] = a; +} + +int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a) +{ + unsigned int i; + for (i = 0; i < dbg_trace_ix; ++i) { + if (m == dbg_trace_m[i] && + (!f || (f == dbg_trace_f[i] && a == dbg_trace_a[i]))) { + + return i+1; + } + } + return 0; +} + +void dbg_vtrace_mfa(unsigned ix, const char* format, ...) +{ + va_list arglist; + va_start(arglist, format); + ASSERT(--ix < MFA_MAX); + erts_fprintf(stderr, "MFA TRACE %T:%T/%u: ", + dbg_trace_m[ix], dbg_trace_f[ix], (int)dbg_trace_a[ix]); + + erts_vfprintf(stderr, format, arglist); + va_end(arglist); +} + +#endif /* ENABLE_DBG_TRACE_MFA */ diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 6be4031822..5f0f34fcdd 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -151,4 +151,24 @@ struct BeamCodeLineTab_ { #define LOC_FILE(Loc) ((Loc) >> 24) #define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) +#ifdef DEBUG +# define ENABLE_DBG_TRACE_MFA +#endif + +#ifdef ENABLE_DBG_TRACE_MFA + +void dbg_set_traced_mfa(const char* m, const char* f, Uint a); +int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a); +void dbg_vtrace_mfa(unsigned ix, const char* format, ...); +#define DBG_TRACE_MFA(M,F,A,FMT, ...) do {\ + unsigned ix;\ + if ((ix=dbg_is_traced_mfa(M,F,A))) \ + dbg_vtrace_mfa(ix, FMT"\n", ##__VA_ARGS__);\ + }while(0) + +#else +# define dbg_set_traced_mfa(M,F,A) +# define DBG_TRACE_MFA(M,F,A,FMT, ...) +#endif /* ENABLE_DBG_TRACE_MFA */ + #endif /* _BEAM_LOAD_H */ diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 2a19211987..9da7fae1dc 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -145,6 +145,9 @@ export_alloc(struct export_entry* tmpl_e) blob->entryv[ix].ep = &blob->exp; } ix = 0; + + DBG_TRACE_MFA(obj->code[0], obj->code[1], obj->code[2], + "export allocation at %p", obj); } else { /* Existing entry in another table, use free entry in blob */ blob = entry_to_blob(tmpl_e); @@ -163,9 +166,14 @@ export_free(struct export_entry* obj) obj->slot.index = -1; for (i=0; i < ERTS_NUM_CODE_IX; i++) { if (blob->entryv[i].slot.index >= 0) { + DBG_TRACE_MFA(blob->exp.code[0], blob->exp.code[1], blob->exp.code[2], + "export entry slot %u freed for %p", + (obj - blob->entryv), &blob->exp); return; } } + DBG_TRACE_MFA(blob->exp.code[0], blob->exp.code[1], blob->exp.code[2], + "export blob deallocation at %p", &blob->exp); erts_free(ERTS_ALC_T_EXPORT, blob); erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob)); } diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index d4f6e17c56..93ce8fcf9f 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -26,6 +26,7 @@ #include "erl_vm.h" #include "global.h" #include "module.h" +#include "beam_catches.h" #ifdef DEBUG # define IF_DEBUG(x) x @@ -67,6 +68,23 @@ static int module_cmp(Module* tmpl, Module* obj) return tmpl->module != obj->module; } +void erts_module_instance_init(struct erl_module_instance* modi) +{ + modi->code_hdr = 0; + modi->code_length = 0; + modi->catches = BEAM_CATCHES_NIL; + modi->nif = NULL; + modi->num_breakpoints = 0; + modi->num_traced_exports = 0; +#ifdef HIPE + modi->first_hipe_ref = NULL; + modi->first_hipe_sdesc = NULL; + modi->hipe_code_start = NULL; +# ifdef DEBUG + modi->hipe_code_size = 0; +# endif +#endif +} static Module* module_alloc(Module* tmpl) { @@ -74,18 +92,16 @@ static Module* module_alloc(Module* tmpl) erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module)); obj->module = tmpl->module; - obj->curr.code_hdr = 0; - obj->old.code_hdr = 0; - obj->curr.code_length = 0; - obj->old.code_length = 0; obj->slot.index = -1; - obj->curr.nif = NULL; - obj->old.nif = NULL; - obj->curr.num_breakpoints = 0; - obj->old.num_breakpoints = 0; - obj->curr.num_traced_exports = 0; - obj->old.num_traced_exports = 0; + erts_module_instance_init(&obj->curr); + erts_module_instance_init(&obj->old); obj->on_load = 0; +#ifdef HIPE + obj->first_hipe_mfa = NULL; + obj->new_hipe_refs = NULL; + obj->new_hipe_sdesc = NULL; +#endif + DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc"); return obj; } @@ -139,19 +155,14 @@ erts_get_module(Eterm mod, ErtsCodeIndex code_ix) } } -Module* -erts_put_module(Eterm mod) + +static Module* put_module(Eterm mod, IndexTable* mod_tab) { Module e; - IndexTable* mod_tab; int oldsz, newsz; Module* res; ASSERT(is_atom(mod)); - ERTS_SMP_LC_ASSERT(erts_initialized == 0 - || erts_has_code_write_permission()); - - mod_tab = &module_tables[erts_staging_code_ix()]; e.module = atom_val(mod); oldsz = index_table_sz(mod_tab); res = (Module*) index_put_entry(mod_tab, (void*) &e); @@ -160,6 +171,24 @@ erts_put_module(Eterm mod) return res; } +Module* +erts_put_module(Eterm mod) +{ + ERTS_SMP_LC_ASSERT(erts_initialized == 0 + || erts_has_code_write_permission()); + + return put_module(mod, &module_tables[erts_staging_code_ix()]); +} + +Module* +erts_put_active_module(Eterm mod) +{ + ASSERT(is_atom(mod)); + //SVERK Why not? ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + return put_module(mod, &module_tables[erts_active_code_ix()]); +} + Module *module_code(int i, ErtsCodeIndex code_ix) { return (Module*) erts_index_lookup(&module_tables[code_ix], i); @@ -186,6 +215,11 @@ static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod) dst_mod->curr = src_mod->curr; dst_mod->old = src_mod->old; dst_mod->on_load = src_mod->on_load; +#ifdef HIPE + dst_mod->first_hipe_mfa = src_mod->first_hipe_mfa; + dst_mod->new_hipe_refs = src_mod->new_hipe_refs; + dst_mod->new_hipe_sdesc = src_mod->new_hipe_sdesc; +#endif } void module_start_staging(void) @@ -217,6 +251,7 @@ void module_start_staging(void) src_mod = (Module*) erts_index_lookup(src, i); dst_mod = (Module*) index_put_entry(dst, src_mod); ASSERT(dst_mod != src_mod); + copy_module(dst_mod, src_mod); } newsz = index_table_sz(dst); diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 1c1afc8461..694583597b 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -30,6 +30,14 @@ struct erl_module_instance { struct erl_module_nif* nif; int num_breakpoints; int num_traced_exports; +#ifdef HIPE + struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */ + struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */ + void* hipe_code_start; +# ifdef DEBUG + UWord hipe_code_size; +# endif +#endif }; typedef struct erl_module { @@ -40,10 +48,17 @@ typedef struct erl_module { struct erl_module_instance curr; struct erl_module_instance old; /* protected by "old_code" rwlock */ struct erl_module_instance* on_load; +#ifdef HIPE + struct hipe_mfa_info* first_hipe_mfa; + struct hipe_ref* new_hipe_refs; + struct hipe_sdesc* new_hipe_sdesc; +#endif } Module; +void erts_module_instance_init(struct erl_module_instance* modi); Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix); Module* erts_put_module(Eterm mod); +Module* erts_put_active_module(Eterm mod); /* only while blocked */ void init_module_table(void); void module_start_staging(void); diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index 62739d2a78..df53f4db30 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -130,6 +130,13 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code) +{ + ALLOC_CODE_STATS(--nr_allocs); + /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + /* Make stub for native code calling exported beam function. */ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) @@ -234,6 +241,14 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + ALLOC_CODE_STATS(++nr_allocs); + /*ALLOC_CODE_STATS(total_alloc += alloc_bytes);*/ + + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h index 6f959815bb..df38a80069 100644 --- a/erts/emulator/hipe/hipe_arch.h +++ b/erts/emulator/hipe/hipe_arch.h @@ -31,22 +31,30 @@ extern int hipe_patch_insn(void *address, Uint value, Eterm type); extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline); extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p); -extern void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity); +extern void hipe_free_code(void*); +extern void *hipe_make_native_stub(void *exp, unsigned int beamArity); +extern void hipe_free_native_stub(void*); + #if defined(__sparc__) #include "hipe_sparc.h" +#include "hipe_sparc_asm.h" #endif #if defined(__i386__) #include "hipe_x86.h" +#include "hipe_x86_asm.h" #endif #if defined(__x86_64__) #include "hipe_amd64.h" +#include "hipe_amd64_asm.h" #endif #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) #include "hipe_ppc.h" +#include "hipe_ppc_asm.h" #endif #if defined(__arm__) #include "hipe_arm.h" +#include "hipe_arm_asm.h" #endif #if !defined(AEXTERN) diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c index f8ef468341..64e35b62d8 100644 --- a/erts/emulator/hipe/hipe_arm.c +++ b/erts/emulator/hipe/hipe_arm.c @@ -198,6 +198,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return address; } +void hipe_free_code(void* code, unsigned int bytes) +{ + /*SVERK: Leaking code memory */ +} + static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu) { unsigned int *address; @@ -229,6 +234,11 @@ static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu) return address; } +void hipe_free_native_stub(void* stub) +{ + /*SVERK: Leaking code stub */ +} + /* * ARMv5's support for 32-bit immediates is effectively non-existent. * Hence, every 32-bit immediate is stored in memory and loaded via diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 95c2d73971..8be6dc9aa9 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -54,6 +54,7 @@ #define BeamOpCode(Op) ((Uint)BeamOp(Op)) + int term_to_Sint32(Eterm term, Sint *sp) { Sint val; @@ -644,30 +645,21 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) pc = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari); if (pc) { - hipe_mfa_save_orig_beam_op(mfa.mod, mfa.fun, mfa.ari, pc); -#if HIPE -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": planting call trap to %p at BEAM pc %p\r\n", address, pc); -#endif + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", pc, address); hipe_set_call_trap(pc, address, is_closure); BIF_RET(am_true); -#endif } -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": no BEAM pc found\r\n"); -#endif + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "failed set call trap to %p, no beam code found", address); BIF_RET(am_false); } BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) { struct hipe_sdesc *sdesc; + Module* modp; + int do_commit; - sdesc = hipe_decode_sdesc(BIF_ARG_1); + sdesc = hipe_decode_sdesc(BIF_ARG_1, &do_commit); if (!sdesc) { fprintf(stderr, "%s: bad sdesc!\r\n", __FUNCTION__); BIF_ERROR(BIF_P, BADARG); @@ -676,6 +668,21 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) fprintf(stderr, "%s: duplicate entry!\r\n", __FUNCTION__); BIF_ERROR(BIF_P, BADARG); } + + /* + * Link into list of sdesc's in same module instance + */ + modp = erts_put_active_module(make_atom(sdesc->m_aix)); + ASSERT(modp); + if (do_commit) { /* Direct "hipe-patching" of early loaded module */ + sdesc->next_in_modi = modp->curr.first_hipe_sdesc; + modp->curr.first_hipe_sdesc = sdesc; + } + else { /* Normal module loading/upgrade */ + sdesc->next_in_modi = modp->new_hipe_sdesc; + modp->new_hipe_sdesc = sdesc; + } + BIF_RET(NIL); } @@ -970,7 +977,7 @@ BIF_RETTYPE hipe_bifs_get_fe_2(BIF_ALIST_2) atom_buf[0] = '\0'; strncat(atom_buf, (char*)atom_tab(i)->name, atom_tab(i)->len); - printf("no fun entry for %s %ld:%ld\n", atom_buf, uniq, index); + printf("no fun entry for %s %ld:%ld\n", atom_buf, (unsigned long)uniq, (unsigned long)index); BIF_ERROR(BIF_P, BADARG); } BIF_RET(address_to_term((void *)fe, BIF_P)); @@ -997,11 +1004,15 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2) BIF_RET(am_true); } +struct hipe_ref_head { + struct hipe_ref_head* next; + struct hipe_ref_head* prev; +}; + /* - * MFA info hash table: + * An exported function called from or implemented by native code * - maps MFA to native code entry point - * - the MFAs it calls (refers_to) - * - the references to it (referred_from) + * - all references to it (callers) * - maps MFA to most recent trampoline [if powerpc or arm] */ struct hipe_mfa_info { @@ -1011,16 +1022,19 @@ struct hipe_mfa_info { } bucket; Eterm m; /* atom */ Eterm f; /* atom */ - unsigned int a; + unsigned int a : sizeof(int)*8 - 1; + unsigned int is_stub : 1; /* if beam or not (yet) loaded */ void *remote_address; - void *local_address; - Eterm *beam_code; - Uint orig_beam_op; - struct hipe_mfa_info_list *refers_to; - struct hipe_ref *referred_from; + void *new_address; + struct hipe_ref_head callers; /* sentinel in list of hipe_ref's */ + struct hipe_mfa_info* next_in_mod; #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) void *trampoline; #endif +#ifdef DEBUG + Export* dbg_export; +#endif + }; static struct { @@ -1038,6 +1052,25 @@ static struct { erts_smp_rwmtx_t lock; } hipe_mfa_info_table; + +/* + * An external native call site M:F(...) + * to be patched when the callee changes. + */ +struct hipe_ref { + struct hipe_ref_head head; /* list of refs to same calleee */ + void *address; + void *trampoline; + unsigned int flags; + struct hipe_ref* next_from_modi; /* list of refs from same module instance */ +#if defined(DEBUG) + struct hipe_mfa_info* callee; + Eterm caller_m, caller_f, caller_a; +#endif +}; +#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ + + static inline void hipe_mfa_info_table_init_lock(void) { erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock"); @@ -1108,15 +1141,18 @@ static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigne res->m = m; res->f = f; res->a = arity; + res->is_stub = 0; res->remote_address = NULL; - res->local_address = NULL; - res->beam_code = NULL; - res->orig_beam_op = 0; - res->refers_to = NULL; - res->referred_from = NULL; + res->new_address = NULL; + res->callers.next = &res->callers; + res->callers.prev = &res->callers; + res->next_in_mod = NULL; #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) res->trampoline = NULL; #endif +#ifdef DEBUG + res->dbg_export = NULL; +#endif return res; } @@ -1154,22 +1190,13 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter return NULL; } -#if 0 /* XXX: unused */ -void *hipe_mfa_find_na(Eterm m, Eterm f, unsigned int arity) -{ - const struct hipe_mfa_info *p; - - p = hipe_mfa_info_table_get(m, f, arity); - return p ? p->address : NULL; -} -#endif - static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned int arity) { unsigned long h; unsigned int i; struct hipe_mfa_info *p; unsigned int size; + Module* modp; h = HIPE_MFA_HASH(m, f, arity); i = h & hipe_mfa_info_table.mask; @@ -1189,23 +1216,47 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, size = 1 << hipe_mfa_info_table.log2size; if (hipe_mfa_info_table.used > (4*size/5)) /* rehash at 80% */ hipe_mfa_info_table_grow(); + + modp = erts_put_active_module(m); + ASSERT(modp); + p->next_in_mod = modp->first_hipe_mfa; + modp->first_hipe_mfa = p; + + DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p); + return p; } -static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address, int is_exported) +static void remove_mfa_info(struct hipe_mfa_info* rm) +{ + unsigned int i; + struct hipe_mfa_info *p; + struct hipe_mfa_info **prevp; + + i = rm->bucket.hvalue & hipe_mfa_info_table.mask; + prevp = &hipe_mfa_info_table.bucket[i]; + for (;;) { + p = *prevp; + ASSERT(p); + if (p == rm) { + *prevp = p->bucket.next; + ASSERT(hipe_mfa_info_table.used > 0); + hipe_mfa_info_table.used--; + return; + } + prevp = &p->bucket.next; + } +} + +static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address) { struct hipe_mfa_info *p; hipe_mfa_info_table_rwlock(); p = hipe_mfa_info_table_put_rwlocked(m, f, arity); -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(m, f, arity); - printf(": changing address from %p to %p\r\n", p->local_address, address); -#endif - p->local_address = address; - if (is_exported) - p->remote_address = address; + DBG_TRACE_MFA(m,f,arity,"set native address in hipe_mfa_info at %p", p); + p->new_address = address; + hipe_mfa_info_table_rwunlock(); } @@ -1237,168 +1288,85 @@ BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) { struct hipe_mfa mfa; void *address; - int is_exported; - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - address = term_to_address(BIF_ARG_2); - if (!address) - BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_3 == am_true) - is_exported = 1; - else if (BIF_ARG_3 == am_false) - is_exported = 0; - else + switch (BIF_ARG_3) { + case am_true: /* is_exported */ + if (!term_to_mfa(BIF_ARG_1, &mfa)) + BIF_ERROR(BIF_P, BADARG); + address = term_to_address(BIF_ARG_2); + if (!address) + BIF_ERROR(BIF_P, BADARG); + hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address); + break; + case am_false: + break; /* ignore local functions */ + default: BIF_ERROR(BIF_P, BADARG); - hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address, is_exported); - BIF_RET(NIL); -} - -BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1) -{ - Eterm lst; - struct hipe_mfa mfa; - struct hipe_mfa_info *p; - - hipe_mfa_info_table_rwlock(); - lst = BIF_ARG_1; - while (is_list(lst)) { - if (!term_to_mfa(CAR(list_val(lst)), &mfa)) - break; - lst = CDR(list_val(lst)); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) { - p->remote_address = NULL; - p->local_address = NULL; - if (p->beam_code) { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": removing call trap from BEAM pc %p (new op %#lx)\r\n", - p->beam_code, p->orig_beam_op); -#endif - p->beam_code[0] = p->orig_beam_op; - p->beam_code = NULL; - p->orig_beam_op = 0; - } else { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": no call trap to remove\r\n"); -#endif - } - } } - hipe_mfa_info_table_rwunlock(); - if (is_not_nil(lst)) - BIF_ERROR(BIF_P, BADARG); BIF_RET(NIL); } -void hipe_mfa_save_orig_beam_op(Eterm mod, Eterm fun, unsigned int ari, Eterm *pc) + +/* Ask if we need to block all threads + * while loading/deleting (beam) code for this module? + */ +int hipe_need_blocking(Module* modp) { - Uint orig_beam_op; struct hipe_mfa_info *p; - orig_beam_op = pc[0]; - if (orig_beam_op != BeamOpCode(op_hipe_trap_call_closure) && - orig_beam_op != BeamOpCode(op_hipe_trap_call)) { - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_put_rwlocked(mod, fun, ari); -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mod, fun, ari); - printf(": saving orig op %#lx from BEAM pc %p\r\n", orig_beam_op, pc); -#endif - p->beam_code = pc; - p->orig_beam_op = orig_beam_op; - hipe_mfa_info_table_rwunlock(); - } else { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mod, fun, ari); - printf(": orig op %#lx already saved\r\n", orig_beam_op); -#endif + /* Need to block if we have at least one native caller to this module + * or native code to make unaccessible. + */ + hipe_mfa_info_table_rlock(); + for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) { + ASSERT(!p->new_address); + if (p->callers.next != &p->callers || !p->is_stub) { + break; + } } + hipe_mfa_info_table_runlock(); + return (p != NULL); } -static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote) -{ - Export *export_entry; - void *StubAddress; - - ASSERT(is_remote); - - export_entry = erts_export_get_or_make_stub(m, f, arity); - StubAddress = hipe_make_native_stub(export_entry, arity); - if (!StubAddress) - erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); - return StubAddress; -} - -static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info **pp) +static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a) { struct hipe_mfa_info *p; - void *address; p = hipe_mfa_info_table_get_locked(m, f, a); - if (p) { - /* find address, predicting for a runtime apply call */ - address = p->remote_address; - if (!is_remote) - address = p->local_address; - if (address) - return address; - - /* bummer, install stub, checking if one already existed */ - address = p->remote_address; - if (address) - return address; - } - /* Caller must take the slow path with the write lock held, but allow - it to avoid some work if it already holds the write lock. */ - if (pp) - *pp = p; - return NULL; -} - -static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info *p) -{ - void *address; - - if (!p) - p = hipe_mfa_info_table_put_rwlocked(m, f, a); - address = hipe_make_stub(m, f, a, is_remote); - /* XXX: how to tell if a BEAM MFA is exported or not? */ - p->remote_address = address; - return address; + return p ? p->remote_address : NULL; } -static void *hipe_get_na_nofail_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote) +static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a) { - struct hipe_mfa_info *p; - void *address; + struct hipe_mfa_info *p = hipe_mfa_info_table_put_rwlocked(m, f, a); - address = hipe_get_na_try_locked(m, f, a, is_remote, &p); - if (address) - return address; + if (!p->remote_address) { + Export* export_entry = erts_export_get_or_make_stub(m, f, a); + void* stubAddress = hipe_make_native_stub(export_entry, a); + if (!stubAddress) + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); - address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, p); - return address; + p->remote_address = stubAddress; + p->is_stub = 1; +#ifdef DEBUG + p->dbg_export = export_entry; +#endif + } + return p->remote_address; } -static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a, int is_remote) +static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a) { void *address; hipe_mfa_info_table_rlock(); - address = hipe_get_na_try_locked(m, f, a, is_remote, NULL); + address = hipe_get_na_try_locked(m, f, a); hipe_mfa_info_table_runlock(); if (address) return address; hipe_mfa_info_table_rwlock(); - address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, NULL); + address = hipe_get_na_slow_rwlocked(m, f, a); hipe_mfa_info_table_rwunlock(); return address; } @@ -1408,7 +1376,7 @@ void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a) { if (is_not_atom(m) || is_not_atom(f) || a > 255) return NULL; - return hipe_get_na_nofail(m, f, a, 1); + return hipe_get_na_nofail(m, f, a); } /* primop, but called like a BIF for error handling purposes */ @@ -1420,25 +1388,19 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3) if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2)) BIF_ERROR(BIF_P, BADARG); arity = unsigned_val(BIF_ARG_3); /* no error check */ - address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity, 1); + address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity); BIF_RET((Eterm)address); /* semi-Ok */ } -BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_find_na_or_make_stub_1(BIF_ALIST_1) { struct hipe_mfa mfa; void *address; - int is_remote; if (!term_to_mfa(BIF_ARG_1, &mfa)) BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_2 == am_true) - is_remote = 1; - else if (BIF_ARG_2 == am_false) - is_remote = 0; - else - BIF_ERROR(BIF_P, BADARG); - address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari, is_remote); + + address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari); BIF_RET(address_to_term(address, BIF_P)); } @@ -1460,7 +1422,7 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) f = ep->code[1]; } else goto badfun; - address = hipe_get_na_nofail(m, f, BIF_ARG_2, 1); + address = hipe_get_na_nofail(m, f, BIF_ARG_2); BIF_RET((Eterm)address); badfun: @@ -1471,60 +1433,19 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a) { - struct hipe_mfa_info *mfa; - long mfa_offset, ra_offset; - struct hipe_mfa_info **bucket; - unsigned int i, nrbuckets; + const struct hipe_sdesc* sdesc = hipe_find_sdesc((unsigned long)ra); - /* Note about locking: the table is only updated from the - loader, which runs with the rest of the system suspended. */ - /* XXX: alas not true; see comment at hipe_mfa_info_table.lock */ - hipe_mfa_info_table_rlock(); - bucket = hipe_mfa_info_table.bucket; - nrbuckets = 1 << hipe_mfa_info_table.log2size; - mfa = NULL; - mfa_offset = LONG_MAX; - for (i = 0; i < nrbuckets; ++i) { - struct hipe_mfa_info *b = bucket[i]; - while (b != NULL) { - ra_offset = (char*)ra - (char*)b->local_address; - if (ra_offset > 0 && ra_offset < mfa_offset) { - mfa_offset = ra_offset; - mfa = b; - } - b = b->bucket.next; - } - } - if (mfa) { - *m = mfa->m; - *f = mfa->f; - *a = mfa->a; - } - hipe_mfa_info_table_runlock(); - return mfa ? 1 : 0; -} + if (!sdesc || sdesc->m_aix == atom_val(am_Empty)) + return 0; -/* - * Patch Reference Handling. - */ -struct hipe_mfa_info_list { - struct hipe_mfa_info *mfa; - struct hipe_mfa_info_list *next; -}; + *m = make_atom(sdesc->m_aix); + *f = make_atom(sdesc->f_aix); + *a = sdesc->a; + return 1; +} -struct hipe_ref { - struct hipe_mfa_info *caller_mfa; - void *address; - void *trampoline; - unsigned int flags; - struct hipe_ref *next; -}; -#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ -#define REF_FLAG_IS_REMOTE 2 /* bit 1: 0 == local, 1 == remote */ -#define REF_FLAG_PENDING_REDIRECT 4 /* bit 2: 1 == pending redirect */ -#define REF_FLAG_PENDING_REMOVE 8 /* bit 3: 1 == pending remove */ -/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,'remote'|'local'}) +/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,DoCommit}) */ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) { @@ -1535,9 +1456,9 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) void *trampoline; unsigned int flags; struct hipe_mfa_info *callee_mfa; - struct hipe_mfa_info *caller_mfa; - struct hipe_mfa_info_list *refers_to; struct hipe_ref *ref; + Module* modp; + int do_commit; if (!term_to_mfa(BIF_ARG_1, &callee)) goto badarg; @@ -1569,62 +1490,90 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) goto badarg; } switch (tuple[5]) { - case am_local: - break; - case am_remote: - flags |= REF_FLAG_IS_REMOTE; - break; - default: - goto badarg; + case am_true: do_commit = 1; break; + case am_false: do_commit = 0; break; + default: goto badarg; } hipe_mfa_info_table_rwlock(); callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari); - caller_mfa = hipe_mfa_info_table_put_rwlocked(caller.mod, caller.fun, caller.ari); - - refers_to = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*refers_to)); - refers_to->mfa = callee_mfa; - refers_to->next = caller_mfa->refers_to; - caller_mfa->refers_to = refers_to; - ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*ref)); - ref->caller_mfa = caller_mfa; + ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref)); ref->address = address; ref->trampoline = trampoline; ref->flags = flags; - ref->next = callee_mfa->referred_from; - callee_mfa->referred_from = ref; + + /* + * Link into list of refs to same callee + */ + ASSERT(callee_mfa->callers.next->prev == &callee_mfa->callers); + ASSERT(callee_mfa->callers.prev->next == &callee_mfa->callers); + ref->head.next = callee_mfa->callers.next; + ref->head.prev = &callee_mfa->callers; + ref->head.next->prev = &ref->head; + ref->head.prev->next = &ref->head; + + /* + * Link into list of refs from same module instance + */ + modp = erts_put_active_module(caller.mod); + ASSERT(modp); + if (do_commit) { /* Direct "hipe-patching" of early loaded module */ + ref->next_from_modi = modp->curr.first_hipe_ref; + modp->curr.first_hipe_ref = ref; + } + else { /* Normal module loading/upgrade */ + ref->next_from_modi = modp->new_hipe_refs; + modp->new_hipe_refs = ref; + } + +#if defined(DEBUG) + ref->callee = callee_mfa; + ref->caller_m = caller.mod; + ref->caller_f = caller.fun; + ref->caller_a = caller.ari; +#endif hipe_mfa_info_table_rwunlock(); + DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p) do_commit=%d", + ref, callee.mod, callee.fun, callee.ari, ref->address, do_commit); + DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p) do_commit=%d", + ref, caller.mod, caller.fun, caller.ari, ref->address, do_commit); BIF_RET(NIL); badarg: BIF_ERROR(BIF_P, BADARG); } -/* Given a CalleeMFA, mark each ref to it as pending-redirect. - * This ensures that remove_refs_from() won't remove them: any - * removal is instead done at the end of redirect_referred_from(). - */ -BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */ + +static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me) { - struct hipe_mfa mfa; - const struct hipe_mfa_info *p; - struct hipe_ref *ref; + Module* modp = erts_get_module(unlink_me->m, erts_active_code_ix()); + struct hipe_mfa_info** prevp = &modp->first_hipe_mfa; + struct hipe_mfa_info* p; - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) - for (ref = p->referred_from; ref != NULL; ref = ref->next) - ref->flags |= REF_FLAG_PENDING_REDIRECT; - hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); + ASSERT(modp); + for (;;) { + p = *prevp; + ASSERT(p && p->m == unlink_me->m); + if (p == unlink_me) { + *prevp = p->next_in_mod; + break; + } + prevp = &p->next_in_mod; + } +} + +static void purge_mfa(struct hipe_mfa_info* p) +{ + ASSERT(p->is_stub); + remove_mfa_info(p); + hipe_free_native_stub(p->remote_address); + erts_free(ERTS_ALC_T_HIPE, p); } /* Called by init:restart after unloading all hipe compiled modules - * to work around bug causing execution of deallocated beam code. - * Can be removed when delete/purge of native modules works better. + * to work around old bug that caused execution of deallocated beam code. + * Can be removed now when delete/purge of native modules works better. * Test: Do init:restart in debug compiled vm with hipe compiled kernel. */ static void hipe_purge_all_refs(void) @@ -1634,126 +1583,205 @@ static void hipe_purge_all_refs(void) hipe_mfa_info_table_rwlock(); + ASSERT(hipe_mfa_info_table.used == 0); bucket = hipe_mfa_info_table.bucket; nrbuckets = 1 << hipe_mfa_info_table.log2size; for (i = 0; i < nrbuckets; ++i) { + ASSERT(bucket[i] == NULL); while (bucket[i] != NULL) { + Module* modp; struct hipe_mfa_info* mfa = bucket[i]; bucket[i] = mfa->bucket.next; - while (mfa->refers_to) { - struct hipe_mfa_info_list *to = mfa->refers_to; - mfa->refers_to = to->next; - erts_free(ERTS_ALC_T_HIPE, to); - } - while (mfa->referred_from) { - struct hipe_ref* from = mfa->referred_from; - mfa->referred_from = from->next; - erts_free(ERTS_ALC_T_HIPE, from); - } + ASSERT(mfa->callers.next == &mfa->callers); + + modp = erts_get_module(mfa->m, erts_active_code_ix()); + if (modp) { + /* unsafe write to active module */ + modp->first_hipe_mfa = NULL; + } erts_free(ERTS_ALC_T_HIPE, mfa); } } + hipe_mfa_info_table.used = 0; hipe_mfa_info_table_rwunlock(); } BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) { - struct hipe_mfa mfa; - struct hipe_mfa_info *caller_mfa, *callee_mfa; - struct hipe_mfa_info_list *refers_to, *tmp_refers_to; - struct hipe_ref **prev, *ref; - if (BIF_ARG_1 == am_all) { hipe_purge_all_refs(); BIF_RET(am_ok); } - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - caller_mfa = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (caller_mfa) { - refers_to = caller_mfa->refers_to; - while (refers_to) { - callee_mfa = refers_to->mfa; - prev = &callee_mfa->referred_from; - ref = *prev; - while (ref) { - if (ref->caller_mfa == caller_mfa) { - if (ref->flags & REF_FLAG_PENDING_REDIRECT) { - ref->flags |= REF_FLAG_PENDING_REMOVE; - prev = &ref->next; - ref = ref->next; - } else { - struct hipe_ref *tmp = ref; - ref = ref->next; - *prev = ref; - erts_free(ERTS_ALC_T_HIPE, tmp); - } - } else { - prev = &ref->next; - ref = ref->next; - } - } - tmp_refers_to = refers_to; - refers_to = refers_to->next; - erts_free(ERTS_ALC_T_HIPE, tmp_refers_to); - } - caller_mfa->refers_to = NULL; + ASSERT(!"hipe_bifs_remove_refs_from_1() called"); + BIF_ERROR(BIF_P, BADARG); +} + +int hipe_purge_need_blocking(Module* modp) +{ + /* SVERK: Verify if this is really necessary */ + return (modp->old.first_hipe_ref || + modp->old.first_hipe_sdesc || + (!modp->curr.code_hdr && modp->first_hipe_mfa)); +} + +void hipe_purge_module(Module* modp) +{ + struct hipe_ref* ref; + struct hipe_sdesc* sdesc; + + ASSERT(modp); + + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module"); + + /* + * Remove all hipe_ref's (external calls) from the old module instance + */ + ref = modp->old.first_hipe_ref; + + while (ref) { + struct hipe_ref* free_ref = ref; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, + ref->callee->m, ref->callee->f, ref->callee->a); + DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, + ref->caller_m, ref->caller_f, ref->caller_a); + ASSERT(ref->caller_m == make_atom(modp->module)); + + /* + * Unlink from other refs to same callee + */ + ASSERT(ref->head.next->prev == &ref->head); + ASSERT(ref->head.prev->next == &ref->head); + ASSERT(ref->head.next != &ref->head); + ASSERT(ref->head.prev != &ref->head); + ref->head.next->prev = ref->head.prev; + ref->head.prev->next = ref->head.next; + + /* + * Was this the last ref to that callee? + */ + if (ref->head.next == ref->head.prev) { + struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); + if (p->is_stub) { + unlink_mfa_from_mod(p); + purge_mfa(p); + } + } + + ref = ref->next_from_modi; + erts_free(ERTS_ALC_T_HIPE, free_ref); + } + modp->old.first_hipe_ref = NULL; + + /* + * Remove all hipe_sdesc's for the old module instance + */ + sdesc = modp->old.first_hipe_sdesc; + + while (sdesc) { + struct hipe_sdesc* free_sdesc = sdesc; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); + ASSERT(sdesc->m_aix == modp->module); + + sdesc = sdesc->next_in_modi; + hipe_destruct_sdesc(free_sdesc); + } + modp->old.first_hipe_sdesc = NULL; + + /* + * Remove unreferred hipe_mfa_info's + */ + if (modp->curr.code_hdr == NULL) { + struct hipe_mfa_info** prevp = &modp->first_hipe_mfa; + struct hipe_mfa_info* p = *prevp; + ERTS_SMP_LC_ASSERT(!p || erts_smp_thr_progress_is_blocking()); + for (; p; p = *prevp) { + if (p->callers.next == &p->callers) { + *prevp = p->next_in_mod; + purge_mfa(p); + } + else + prevp = &p->next_in_mod; + } + } + if (modp->old.hipe_code_start) { +#ifdef DEBUG + sys_memset(modp->old.hipe_code_start, 0xfe, modp->old.hipe_code_size); +#endif + hipe_free_code(modp->old.hipe_code_start); + modp->old.hipe_code_start = NULL; } - hipe_mfa_info_table_rwunlock(); - BIF_RET(am_ok); } -/* redirect_referred_from(CalleeMFA) - * Redirect all pending-redirect refs in CalleeMFA's referred_from. - * Then remove any pending-redirect && pending-remove refs from CalleeMFA's referred_from. +/* + * Redirect all existing native calls to this module */ -BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1) +void hipe_redirect_to_module(Module* modp) { - struct hipe_mfa mfa; struct hipe_mfa_info *p; - struct hipe_ref **prev, *ref; - int is_remote, res; - void *new_address; + struct hipe_ref_head* refh; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + for (p = modp->first_hipe_mfa; p; p = p->next_in_mod) { + if (p->new_address) { + if (p->is_stub) { + hipe_free_native_stub(p->remote_address); + p->is_stub = 0; + } + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit new_address %p", p->new_address); + p->remote_address = p->new_address; + p->new_address = NULL; +#ifdef DEBUG + p->dbg_export = NULL; +#endif + } + else if (!p->is_stub) { + Export* exp = erts_export_get_or_make_stub(p->m, p->f, p->a); + p->remote_address = hipe_make_native_stub(exp, p->a); + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit stub %p", p->remote_address); + if (!p->remote_address) + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); + p->is_stub = 1; +#ifdef DEBUG + p->dbg_export = exp; +#endif + } + else { + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit no-op, already stub"); + ASSERT(p->remote_address && p->dbg_export); + } - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) { - prev = &p->referred_from; - ref = *prev; - while (ref) { - if (ref->flags & REF_FLAG_PENDING_REDIRECT) { - is_remote = ref->flags & REF_FLAG_IS_REMOTE; - new_address = hipe_get_na_nofail_rwlocked(p->m, p->f, p->a, is_remote); - if (ref->flags & REF_FLAG_IS_LOAD_MFA) - res = hipe_patch_insn(ref->address, (Uint)new_address, am_load_mfa); - else - res = hipe_patch_call(ref->address, new_address, ref->trampoline); - if (res) - fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__); - ref->flags &= ~REF_FLAG_PENDING_REDIRECT; - if (ref->flags & REF_FLAG_PENDING_REMOVE) { - struct hipe_ref *tmp = ref; - ref = ref->next; - *prev = ref; - erts_free(ERTS_ALC_T_HIPE, tmp); - } else { - prev = &ref->next; - ref = ref->next; - } - } else { - prev = &ref->next; - ref = ref->next; - } + DBG_TRACE_MFA(p->m,p->f,p->a,"START REDIRECT towards hipe_mfa_info at %p", p); + for (refh = p->callers.next; refh != &p->callers; refh = refh->next) { + struct hipe_ref* ref = (struct hipe_ref*) refh; + int res; + + DBG_TRACE_MFA(p->m,p->f,p->a, " REDIRECT ref at %p FROM %T:%T/%u (%p -> %p)", + ref, ref->caller_m, ref->caller_f, ref->caller_a, + ref->address, p->remote_address); + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, + " REDIRECT ref at %p TO %T:%T/%u (%p -> %p)", + ref, p->m,p->f,p->a, ref->address, p->remote_address); + + if (ref->flags & REF_FLAG_IS_LOAD_MFA) + res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa); + else + res = hipe_patch_call(ref->address, p->remote_address, ref->trampoline); + if (res) + fprintf(stderr, "%s: patch failed", __FUNCTION__); } + DBG_TRACE_MFA(p->m,p->f,p->a,"DONE REDIRECT towards hipe_mfa_info at %p", p); } - hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); } BIF_RETTYPE hipe_bifs_check_crc_1(BIF_ALIST_1) diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index c9a8216368..1914405e2d 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -42,7 +42,10 @@ extern void hipe_primop_set_trampoline(Eterm name, void *trampoline); #endif /* needed in beam_load.c */ -void hipe_mfa_save_orig_beam_op(Eterm m, Eterm f, unsigned int a, Eterm *pc); +int hipe_need_blocking(Module*); +int hipe_purge_need_blocking(Module*); +void hipe_purge_module(Module*); +void hipe_redirect_to_module(Module* modp); /* these are also needed in hipe_amd64.c */ extern void *term_to_address(Eterm); diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index 99237aae05..eae84383df 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -54,7 +54,7 @@ bif hipe_bifs:set_native_address/3 #bif hipe_bifs:address_to_fun/1 bif hipe_bifs:set_funinfo_native_address/3 -bif hipe_bifs:invalidate_funinfo_native_addresses/1 +#bif hipe_bifs:invalidate_funinfo_native_addresses/1 bif hipe_bifs:update_code_size/3 bif hipe_bifs:code_size/1 @@ -72,7 +72,7 @@ bif hipe_bifs:term_to_word/1 bif hipe_bifs:get_fe/2 bif hipe_bifs:set_native_address_in_fe/2 -bif hipe_bifs:find_na_or_make_stub/2 +bif hipe_bifs:find_na_or_make_stub/1 bif hipe_bifs:check_crc/1 bif hipe_bifs:system_crc/0 @@ -84,9 +84,7 @@ bif hipe_bifs:patch_insn/3 bif hipe_bifs:patch_call/3 bif hipe_bifs:add_ref/2 -bif hipe_bifs:mark_referred_from/1 bif hipe_bifs:remove_refs_from/1 -bif hipe_bifs:redirect_referred_from/1 # atoms used by add_ref/2 atom call diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c index 9b2048c457..a1a6e2ad02 100644 --- a/erts/emulator/hipe/hipe_ppc.c +++ b/erts/emulator/hipe/hipe_ppc.c @@ -214,6 +214,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return address; } +void hipe_free_code(void* code, unsigned int bytes) +{ + /*SVERK: Leaking code memory */ +} + static unsigned int *alloc_stub(Uint nrwords) { unsigned int *address; @@ -241,6 +246,11 @@ static unsigned int *alloc_stub(Uint nrwords) return address; } +void hipe_free_native_stub(void* stub) +{ + /*SVERK: Leaking code stubs */ +} + static void patch_imm16(Uint32 *address, unsigned int imm16) { unsigned int insn = *address; diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h index 09804e3016..1369b392fe 100644 --- a/erts/emulator/hipe/hipe_risc_glue.h +++ b/erts/emulator/hipe/hipe_risc_glue.h @@ -66,15 +66,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct hipe_sdesc_with_exnra nbif_return_sdesc = { - .exnra = (unsigned long)&nbif_fail, - .sdesc = { - .bucket = { .hvalue = (unsigned long)&nbif_return }, - .fsize = 0, - .has_exnra = 1, - .arity = 0 - }, - }; + static struct hipe_sdesc_with_exnra nbif_return_sdesc; + + nbif_return_sdesc.exnra = (unsigned long)nbif_fail; + nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return; + nbif_return_sdesc.sdesc.fsize = 0; + nbif_return_sdesc.sdesc.has_exnra = 1; + nbif_return_sdesc.sdesc.stk_nargs = 0; + nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty); + nbif_return_sdesc.sdesc.f_aix = atom_val(am_return); + nbif_return_sdesc.sdesc.a = 0; + hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); } diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c index 17bef0718c..3d082668a6 100644 --- a/erts/emulator/hipe/hipe_stack.c +++ b/erts/emulator/hipe/hipe_stack.c @@ -102,6 +102,28 @@ struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc) return sdesc; } +void hipe_destruct_sdesc(struct hipe_sdesc *sdesc) +{ + unsigned int i; + struct hipe_sdesc** prevp; + void* free_me; + + i = (sdesc->bucket.hvalue >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask; + prevp = &hipe_sdesc_table.bucket[i]; + + for (; *prevp != sdesc; prevp = &(*prevp)->bucket.next) + ASSERT(*prevp); + + *prevp = sdesc->bucket.next; + hipe_sdesc_table.used -= 1; + + if (sdesc->has_exnra) + free_me = ErtsContainerStruct(sdesc, struct hipe_sdesc_with_exnra, sdesc); + else + free_me = sdesc; + erts_free(ERTS_ALC_T_HIPE, free_me); +} + void hipe_init_sdesc_table(struct hipe_sdesc *sdesc) { unsigned int log2size, size; @@ -121,31 +143,46 @@ void hipe_init_sdesc_table(struct hipe_sdesc *sdesc) * representation. If different representations are needed in * the future, this code has to be made target dependent. */ -struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) +struct hipe_sdesc *hipe_decode_sdesc(Eterm arg, int* do_commitp) { Uint ra, exnra; Eterm *live; - Uint fsize, arity, nlive, i, nslots, off; + Uint fsize, nargs, stk_nargs, nlive, i, nslots, off; Uint livebitswords, sdescbytes; void *p; struct hipe_sdesc *sdesc; - - if (is_not_tuple(arg) || - (tuple_val(arg))[0] != make_arityval(6) || - term_to_Uint((tuple_val(arg))[1], &ra) == 0 || - term_to_Uint((tuple_val(arg))[2], &exnra) == 0 || - is_not_small((tuple_val(arg))[3]) || - (fsize = unsigned_val((tuple_val(arg))[3])) > 65535 || - is_not_small((tuple_val(arg))[4]) || - (arity = unsigned_val((tuple_val(arg))[4])) > 255 || - is_not_tuple((tuple_val(arg))[5])) + Eterm* mfa_tpl; + Eterm* tp; + + if (is_not_tuple(arg)) + return 0; + + tp = tuple_val(arg); + if (tp[0] != make_arityval(7) || + term_to_Uint(tp[1], &ra) == 0 || + term_to_Uint(tp[2], &exnra) == 0 || + is_not_small(tp[3]) || + (fsize = unsigned_val(tp[3])) > 65535 || + is_not_small(tp[4]) || + (stk_nargs = unsigned_val(tp[4])) > 255 || + is_not_tuple(tp[5]) || + is_not_tuple(tp[6]) || + (mfa_tpl = tuple_val(tp[6]))[0] != make_arityval(3) || + is_not_atom(mfa_tpl[1]) || + is_not_atom(mfa_tpl[2]) || + is_not_small(mfa_tpl[3]) || + (nargs = unsigned_val(mfa_tpl[3])) > 255) return 0; + + if (stk_nargs > nargs) + return 0; + /* Get tuple with live slots */ - live = tuple_val((tuple_val(arg))[5]) + 1; + live = tuple_val(tp[5]) + 1; /* Get number of live slots */ nlive = arityval(live[-1]); - /* Calculate size of frame = locals + ra + arguments */ - nslots = fsize + 1 + arity; + /* Calculate size of frame = locals + ra + stack arguments */ + nslots = fsize + 1 + stk_nargs; /* Check that only valid slots are given. */ for (i = 0; i < nlive; ++i) { if (is_not_small(live[i]) || @@ -154,8 +191,14 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) return 0; } + switch(tp[7]) { + case am_true: *do_commitp = 1; break; + case am_false: *do_commitp = 0; break; + default: return 0; + } + /* Calculate number of words for the live bitmap. */ - livebitswords = (fsize + arity + 1 + 31) / 32; + livebitswords = (fsize + stk_nargs + 1 + 31) / 32; /* Calculate number of bytes needed for the stack descriptor. */ sdescbytes = (exnra @@ -172,12 +215,17 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) } else sdesc = p; + sdesc->m_aix = atom_val(mfa_tpl[1]); + sdesc->f_aix = atom_val(mfa_tpl[2]); + sdesc->a = nargs; + + /* Initialise head of sdesc. */ sdesc->bucket.next = 0; sdesc->bucket.hvalue = ra; sdesc->fsize = fsize; sdesc->has_exnra = (exnra ? 1 : 0); - sdesc->arity = arity; + sdesc->stk_nargs = stk_nargs; /* Clear all live-bits */ for (i = 0; i < livebitswords; ++i) sdesc->livebits[i] = 0; @@ -186,13 +234,5 @@ struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) off = unsigned_val(live[i]); sdesc->livebits[off / 32] |= (1 << (off & 31)); } -#ifdef DEBUG - { - Eterm mfa_tpl = tuple_val(arg)[6]; - sdesc->dbg_M = tuple_val(mfa_tpl)[1]; - sdesc->dbg_F = tuple_val(mfa_tpl)[2]; - sdesc->dbg_A = tuple_val(mfa_tpl)[3]; - } -#endif return sdesc; } diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index 81f2e53dbc..1863b0db8c 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -37,12 +37,12 @@ struct hipe_sdesc { } bucket; unsigned int fsize : 23; /* frame size */ unsigned int has_exnra : 1; /* exn handler presence flag */ - unsigned int arity : 8; -#ifdef DEBUG - Eterm dbg_M, dbg_F; - unsigned dbg_A; -#endif - unsigned int livebits[1]; /* size depends on arch & data in summary field */ + unsigned int stk_nargs : 8; /* arguments on stack */ + Uint32 m_aix; + Uint32 f_aix; + Uint32 a; + struct hipe_sdesc* next_in_modi; + Uint32 livebits[1]; /* size depends on arch & data in summary field */ }; struct hipe_sdesc_with_exnra { @@ -55,9 +55,10 @@ static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc) return sdesc->fsize; } +/* Nr of arguments pushed on stack */ static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc) { - return sdesc->arity; + return sdesc->stk_nargs; } static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc) @@ -79,8 +80,9 @@ struct hipe_sdesc_table { extern struct hipe_sdesc_table hipe_sdesc_table; extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*); +extern void hipe_destruct_sdesc(struct hipe_sdesc*); extern void hipe_init_sdesc_table(struct hipe_sdesc*); -extern struct hipe_sdesc *hipe_decode_sdesc(Eterm); +extern struct hipe_sdesc *hipe_decode_sdesc(Eterm, int* do_commitp); #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) #define __builtin_expect(x, expected_value) (x) diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c index 5f6c8c200e..8a3ba8bc7f 100644 --- a/erts/emulator/hipe/hipe_x86.c +++ b/erts/emulator/hipe/hipe_x86.c @@ -184,6 +184,15 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int bytes) +{ + /* SVERK: Leak !!! + ALLOC_CODE_STATS(--nr_allocs); + ALLOC_CODE_STATS(total_alloc -= bytes); + + erts_free(ERTS_ALC_T_HIPE_EXEC, code);*/ +} + void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { /* @@ -264,6 +273,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + /* SVERK: leak leak drip drop */ +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h index 7802076f70..e92e6ea718 100644 --- a/erts/emulator/hipe/hipe_x86_gc.h +++ b/erts/emulator/hipe/hipe_x86_gc.h @@ -65,18 +65,14 @@ nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) state->sdesc0 = sdesc; return sdesc; #else - unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS; - if ((int)nstkarity < 0) - nstkarity = 0; state->sdesc0[0].fsize = 0; state->sdesc0[0].has_exnra = 0; - state->sdesc0[0].arity = nstkarity; + state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : + p->hipe.narity - NR_ARG_REGS); state->sdesc0[0].livebits[0] = 0; -# ifdef DEBUG - state->sdesc0[0].dbg_M = 0; - state->sdesc0[0].dbg_F = am_undefined; - state->sdesc0[0].dbg_A = 0; -# endif + state->sdesc0[0].m_aix = 0; + state->sdesc0[0].f_aix = atom_val(am_undefined); + state->sdesc0[0].a = 0; /* XXX: this appears to prevent a gcc-4.1.1 bug on x86 */ __asm__ __volatile__("" : : "m"(*state) : "memory"); return &state->sdesc0[0]; diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h index 0e9fcd61f1..de2b061706 100644 --- a/erts/emulator/hipe/hipe_x86_glue.h +++ b/erts/emulator/hipe/hipe_x86_glue.h @@ -58,18 +58,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct hipe_sdesc_with_exnra nbif_return_sdesc = { - .exnra = (unsigned long)nbif_fail, - .sdesc = { - .bucket = { .hvalue = (unsigned long)nbif_return }, - .fsize = 0, - .has_exnra = 1, - .arity = 0, - #ifdef DEBUG - .dbg_F = am_return, - #endif - }, - }; + static struct hipe_sdesc_with_exnra nbif_return_sdesc; + + nbif_return_sdesc.exnra = (unsigned long)nbif_fail; + nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return; + nbif_return_sdesc.sdesc.fsize = 0; + nbif_return_sdesc.sdesc.has_exnra = 1; + nbif_return_sdesc.sdesc.stk_nargs = 0; + nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty); + nbif_return_sdesc.sdesc.f_aix = atom_val(am_return); + nbif_return_sdesc.sdesc.a = 0; + hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); } diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c index 9426b166e2..31582b3a2e 100644 --- a/erts/emulator/hipe/hipe_x86_stack.c +++ b/erts/emulator/hipe/hipe_x86_stack.c @@ -60,7 +60,6 @@ void hipe_print_nstack(Process *p) unsigned int mask; unsigned int sdesc_size; unsigned int i; - unsigned int nstkarity; static const char dashes[2*sizeof(long)+5] = { [0 ... 2*sizeof(long)+3] = '-' }; @@ -68,12 +67,10 @@ void hipe_print_nstack(Process *p) nsp = p->hipe.nsp; nsp_end = p->hipe.nstend; - nstkarity = p->hipe.narity - NR_ARG_REGS; - if ((int)nstkarity < 0) - nstkarity = 0; sdesc0.fsize = 0; sdesc0.has_exnra = 0; - sdesc0.arity = nstkarity; + sdesc0.stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : + p->hipe.narity - NR_ARG_REGS); sdesc0.livebits[0] = ~1; sdesc = &sdesc0; diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 8427bb134d..b76ba6427f 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -445,62 +445,64 @@ module_md5_ok(Code) -> make_stub(Config) when is_list(Config) -> catch erlang:purge_module(my_code_test), MD5 = erlang:md5(<<>>), + Arg3 = {[], [], MD5, 0, 0}, Data = proplists:get_value(data_dir, Config), File = filename:join(Data, "my_code_test"), {ok,my_code_test,Code} = compile:file(File, [binary]), - my_code_test = code:make_stub_module(my_code_test, Code, {[],[],MD5}), + my_code_test = code:make_stub_module(my_code_test, Code, Arg3), true = erlang:delete_module(my_code_test), true = erlang:purge_module(my_code_test), my_code_test = code:make_stub_module(my_code_test, make_unaligned_sub_binary(Code), - {[],[],MD5}), + Arg3), true = erlang:delete_module(my_code_test), true = erlang:purge_module(my_code_test), my_code_test = code:make_stub_module(my_code_test, zlib:gzip(Code), - {[],[],MD5}), + Arg3), true = erlang:delete_module(my_code_test), true = erlang:purge_module(my_code_test), %% Should fail. {'EXIT',{badarg,_}} = - (catch code:make_stub_module(my_code_test, <<"bad">>, {[],[],MD5})), + (catch code:make_stub_module(my_code_test, <<"bad">>, Arg3)), {'EXIT',{badarg,_}} = (catch code:make_stub_module(my_code_test, bit_sized_binary(Code), - {[],[],MD5})), + Arg3)), {'EXIT',{badarg,_}} = (catch code:make_stub_module(my_code_test_with_wrong_name, - Code, {[],[],MD5})), + Code, Arg3)), ok. make_stub_many_funs(Config) when is_list(Config) -> catch erlang:purge_module(many_funs), MD5 = erlang:md5(<<>>), + Arg3 = {[], [], MD5, 0, 0}, Data = proplists:get_value(data_dir, Config), File = filename:join(Data, "many_funs"), {ok,many_funs,Code} = compile:file(File, [binary]), - many_funs = code:make_stub_module(many_funs, Code, {[],[],MD5}), + many_funs = code:make_stub_module(many_funs, Code, Arg3), true = erlang:delete_module(many_funs), true = erlang:purge_module(many_funs), many_funs = code:make_stub_module(many_funs, make_unaligned_sub_binary(Code), - {[],[],MD5}), + Arg3), true = erlang:delete_module(many_funs), true = erlang:purge_module(many_funs), %% Should fail. {'EXIT',{badarg,_}} = - (catch code:make_stub_module(many_funs, <<"bad">>, {[],[],MD5})), + (catch code:make_stub_module(many_funs, <<"bad">>, Arg3)), {'EXIT',{badarg,_}} = (catch code:make_stub_module(many_funs, bit_sized_binary(Code), - {[],[],MD5})), + Arg3)), ok. constant_pools(Config) when is_list(Config) -> diff --git a/lib/hipe/cerl/erl_bif_types.erl b/lib/hipe/cerl/erl_bif_types.erl index c9cc1cfe25..0e43af4f60 100644 --- a/lib/hipe/cerl/erl_bif_types.erl +++ b/lib/hipe/cerl/erl_bif_types.erl @@ -1060,8 +1060,8 @@ type(hipe_bifs, enter_code, 2, Xs, Opaques) -> t_sup(t_nil(), t_binary())]) end, Opaques); type(hipe_bifs, enter_sdesc, 1, Xs, Opaques) -> strict(hipe_bifs, enter_sdesc, 1, Xs, fun (_) -> t_nil() end, Opaques); -type(hipe_bifs, find_na_or_make_stub, 2, Xs, Opaques) -> - strict(hipe_bifs, find_na_or_make_stub, 2, Xs, +type(hipe_bifs, find_na_or_make_stub, 1, Xs, Opaques) -> + strict(hipe_bifs, find_na_or_make_stub, 1, Xs, fun (_) -> t_integer() end, Opaques); % address type(hipe_bifs, fun_to_address, 1, Xs, Opaques) -> strict(hipe_bifs, fun_to_address, 1, Xs, @@ -1071,12 +1071,6 @@ type(hipe_bifs, get_fe, 2, Xs, Opaques) -> type(hipe_bifs, get_rts_param, 1, Xs, Opaques) -> strict(hipe_bifs, get_rts_param, 1, Xs, fun (_) -> t_sup(t_integer(), t_nil()) end, Opaques); -type(hipe_bifs, invalidate_funinfo_native_addresses, 1, Xs, Opaques) -> - strict(hipe_bifs, invalidate_funinfo_native_addresses, 1, Xs, - fun (_) -> t_nil() end, Opaques); -type(hipe_bifs, mark_referred_from, 1, Xs, Opaques) -> - strict(hipe_bifs, mark_referred_from, 1, Xs, - fun (_) -> t_nil() end, Opaques); type(hipe_bifs, merge_term, 1, Xs, Opaques) -> strict(hipe_bifs, merge_term, 1, Xs, fun ([X]) -> X end, Opaques); type(hipe_bifs, nstack_used_size, 0, _, _Opaques) -> @@ -1088,9 +1082,6 @@ type(hipe_bifs, patch_insn, 3, Xs, Opaques) -> type(hipe_bifs, primop_address, 1, Xs, Opaques) -> strict(hipe_bifs, primop_address, 1, Xs, fun (_) -> t_sup(t_integer(), t_atom('false')) end, Opaques); -type(hipe_bifs, redirect_referred_from, 1, Xs, Opaques) -> - strict(hipe_bifs, redirect_referred_from, 1, Xs, - fun (_) -> t_nil() end, Opaques); type(hipe_bifs, ref, 1, Xs, Opaques) -> strict(hipe_bifs, ref, 1, Xs, fun (_) -> t_immarray() end, Opaques); type(hipe_bifs, ref_get, 1, Xs, Opaques) -> @@ -2469,8 +2460,7 @@ arg_types(hipe_bifs, add_ref, 2) -> [t_mfa(), t_tuple([t_mfa(), t_integer(), t_sup(t_atom('call'), t_atom('load_mfa')), - t_trampoline(), - t_sup(t_atom('remote'), t_atom('local'))])]; + t_trampoline()])]; arg_types(hipe_bifs, alloc_data, 2) -> [t_integer(), t_integer()]; arg_types(hipe_bifs, array, 2) -> @@ -2511,18 +2501,14 @@ arg_types(hipe_bifs, enter_code, 2) -> [t_binary(), t_sup(t_nil(), t_tuple())]; arg_types(hipe_bifs, enter_sdesc, 1) -> [t_tuple([t_integer(), t_integer(), t_integer(), t_integer(), t_integer(), t_mfa()])]; -arg_types(hipe_bifs, find_na_or_make_stub, 2) -> - [t_mfa(), t_boolean()]; +arg_types(hipe_bifs, find_na_or_make_stub, 1) -> + [t_mfa()]; arg_types(hipe_bifs, fun_to_address, 1) -> [t_mfa()]; arg_types(hipe_bifs, get_fe, 2) -> [t_atom(), t_tuple([t_integer(), t_integer(), t_integer()])]; arg_types(hipe_bifs, get_rts_param, 1) -> [t_fixnum()]; -arg_types(hipe_bifs, invalidate_funinfo_native_addresses, 1) -> - [t_list(t_mfa())]; -arg_types(hipe_bifs, mark_referred_from, 1) -> - [t_mfa()]; arg_types(hipe_bifs, merge_term, 1) -> [t_any()]; arg_types(hipe_bifs, nstack_used_size, 0) -> @@ -2533,8 +2519,6 @@ arg_types(hipe_bifs, patch_insn, 3) -> [t_integer(), t_integer(), t_insn_type()]; arg_types(hipe_bifs, primop_address, 1) -> [t_atom()]; -arg_types(hipe_bifs, redirect_referred_from, 1) -> - [t_mfa()]; arg_types(hipe_bifs, ref, 1) -> [t_immediate()]; arg_types(hipe_bifs, ref_get, 1) -> diff --git a/lib/kernel/src/code_server.erl b/lib/kernel/src/code_server.erl index 835b7eb588..90ecb22074 100644 --- a/lib/kernel/src/code_server.erl +++ b/lib/kernel/src/code_server.erl @@ -1130,7 +1130,7 @@ try_load_module_2(File, Mod, Bin, From, Architecture, #state{moddb=Db}=St) -> case catch hipe_unified_loader:load_native_code(Mod, Bin, Architecture) of {module,Mod} = Module -> - ets:insert(Db, [{{native,Mod},true},{Mod,File}]), + ets:insert(Db, {Mod,File}), {reply,Module,St}; no_native -> try_load_module_3(File, Mod, Bin, From, Architecture, St); @@ -1139,10 +1139,9 @@ try_load_module_2(File, Mod, Bin, From, Architecture, {reply,ok,St} end. -try_load_module_3(File, Mod, Bin, From, Architecture, St0) -> +try_load_module_3(File, Mod, Bin, From, _Architecture, St0) -> Action = fun({module,_}=Module, #state{moddb=Db}=S) -> ets:insert(Db, {Mod,File}), - post_beam_load([Mod], Architecture, S), {reply,Module,S}; ({error,on_load_failure}=Error, S) -> {reply,Error,S}; @@ -1153,24 +1152,14 @@ try_load_module_3(File, Mod, Bin, From, Architecture, St0) -> Res = erlang:load_module(Mod, Bin), handle_on_load(Res, Action, Mod, From, St0). -hipe_result_to_status(Result, #state{moddb=Db}) -> +hipe_result_to_status(Result, #state{}) -> case Result of - {module,Mod} -> - ets:insert(Db, [{{native,Mod},true}]), + {module,_} -> Result; _ -> {error,Result} end. -post_beam_load(_, undefined, _) -> - %% HiPE is disabled. - ok; -post_beam_load(Mods0, _Architecture, #state{moddb=Db}) -> - %% post_beam_load/2 can potentially be very expensive because it - %% blocks multi-scheduling. Therefore, we only want to call - %% it with modules that are known to have native code loaded. - Mods = [M || M <- Mods0, ets:member(Db, {native,M})], - hipe_unified_loader:post_beam_load(Mods). int_list([H|T]) when is_integer(H) -> int_list(T); int_list([_|_]) -> false; @@ -1313,15 +1302,12 @@ abort_if_sticky(L, Db) -> [_|_] -> {error,Sticky} end. -do_finish_loading(Prepared, #state{moddb=Db}=St) -> +do_finish_loading(Prepared, #state{moddb=Db}) -> MagicBins = [B || {_,{B,_}} <- Prepared], case erlang:finish_loading(MagicBins) of ok -> MFs = [{M,F} || {M,{_,F}} <- Prepared], true = ets:insert(Db, MFs), - Ms = [M || {M,_} <- MFs], - Architecture = erlang:system_info(hipe_architecture), - post_beam_load(Ms, Architecture, St), ok; {Reason,Ms} -> {error,[{M,Reason} || M <- Ms]} diff --git a/lib/kernel/src/hipe_unified_loader.erl b/lib/kernel/src/hipe_unified_loader.erl index 7b1e6c2591..75a5995695 100644 --- a/lib/kernel/src/hipe_unified_loader.erl +++ b/lib/kernel/src/hipe_unified_loader.erl @@ -41,10 +41,11 @@ % I think the real solution would be to let BIF erlang:load_module/2 redirect all % hipe calls to the module and thereby remove post_beam_load. +% SVERK: Can we remove -compile(no_native) now when post_beam_load is gone? + -export([chunk_name/1, %% Only the code and code_server modules may call the entries below! load_native_code/3, - post_beam_load/1, load_module/4, load/3]). @@ -101,15 +102,13 @@ word_size(Architecture) -> load_native_code(_Mod, _Bin, undefined) -> no_native; load_native_code(Mod, Bin, Architecture) when is_atom(Mod), is_binary(Bin) -> - %% patch_to_emu(Mod), case code:get_chunk(Bin, chunk_name(Architecture)) of undefined -> no_native; NativeCode when is_binary(NativeCode) -> erlang:system_flag(multi_scheduling, block_normal), try - OldReferencesToPatch = patch_to_emu_step1(Mod), - case load_module(Mod, NativeCode, Bin, OldReferencesToPatch, - Architecture) of + put(hipe_patch_closures, false), + case load_common(Mod, NativeCode, Bin, Architecture) of bad_crc -> no_native; Result -> Result end @@ -120,22 +119,6 @@ load_native_code(Mod, Bin, Architecture) when is_atom(Mod), is_binary(Bin) -> %%======================================================================== --spec post_beam_load([module()]) -> 'ok'. - -post_beam_load([])-> - ok; -post_beam_load([_|_]=Mods) -> - erlang:system_flag(multi_scheduling, block_normal), - try - _ = [patch_to_emu(Mod) || Mod <- Mods], - ok - after - erlang:system_flag(multi_scheduling, unblock_normal) - end, - ok. - -%%======================================================================== - version_check(Version, Mod) when is_atom(Mod) -> Ver = ?VERSION_STRING(), case Version < Ver of @@ -153,19 +136,12 @@ version_check(Version, Mod) when is_atom(Mod) -> load_module(Mod, Bin, Beam, Architecture) -> erlang:system_flag(multi_scheduling, block_normal), try - load_module_nosmp(Mod, Bin, Beam, Architecture) + put(hipe_patch_closures, false), + load_common(Mod, Bin, Beam, Architecture) after erlang:system_flag(multi_scheduling, unblock_normal) end. -load_module_nosmp(Mod, Bin, Beam, Architecture) -> - load_module(Mod, Bin, Beam, [], Architecture). - -load_module(Mod, Bin, Beam, OldReferencesToPatch, Architecture) -> - ?debug_msg("************ Loading Module ~w ************\n",[Mod]), - %% Loading a whole module, let the BEAM loader patch closures. - put(hipe_patch_closures, false), - load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture). %%======================================================================== @@ -175,20 +151,17 @@ load_module(Mod, Bin, Beam, OldReferencesToPatch, Architecture) -> load(Mod, Bin, Architecture) -> erlang:system_flag(multi_scheduling, block_normal), try - load_nosmp(Mod, Bin, Architecture) + ?debug_msg("********* Loading funs in module ~w *********\n",[Mod]), + %% Loading just some functions in a module; patch closures separately. + put(hipe_patch_closures, true), + load_common(Mod, Bin, [], Architecture) after erlang:system_flag(multi_scheduling, unblock_normal) end. -load_nosmp(Mod, Bin, Architecture) -> - ?debug_msg("********* Loading funs in module ~w *********\n",[Mod]), - %% Loading just some functions in a module; patch closures separately. - put(hipe_patch_closures, true), - load_common(Mod, Bin, [], [], Architecture). - %%------------------------------------------------------------------------ -load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) -> +load_common(Mod, Bin, Beam, Architecture) -> %% Unpack the binary. [{Version, CheckSum}, ConstAlign, ConstSize, ConstMap, LabelMap, ExportMap, @@ -228,15 +201,14 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) -> Architecture), %% Patch references to code labels in data seg. ok = patch_consts(LabelMap, ConstAddr, CodeAddress, WriteWord), + %% Find out which functions are being loaded (and where). - %% Note: FunDefs are sorted descending. - {MFAs,FunDefs} = exports(ExportMap, CodeAddress), - %% Remove references to old versions of the module. - ReferencesToPatch = get_refs_from(MFAs, []), - %% io:format("References to patch: ~w~n", [ReferencesToPatch]), - ok = remove_refs_from(MFAs), + %% Note: FunDefs are sorted descending address order. + FunDefs = exports(ExportMap, CodeAddress), + %% Patch all dynamic references in the code. %% Function calls, Atoms, Constants, System calls + ok = patch(Refs, CodeAddress, ConstMap2, FunDefs, TrampolineMap), %% Tell the system where the loaded funs are. @@ -250,7 +222,7 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) -> lists:foreach(fun({FE, DestAddress}) -> hipe_bifs:set_native_address_in_fe(FE, DestAddress) end, erase(closures_to_patch)), - export_funs(FunDefs), + set_beam_call_traps(FunDefs), ok; BeamBinary when is_binary(BeamBinary) -> %% Find all closures in the code. @@ -259,15 +231,10 @@ load_common(Mod, Bin, Beam, OldReferencesToPatch, Architecture) -> AddressesOfClosuresToPatch = calculate_addresses(ClosurePatches, CodeAddress, FunDefs), export_funs(FunDefs), - export_funs(Mod, MD5, BeamBinary, - FunDefs, AddressesOfClosuresToPatch) + make_beam_stub(Mod, MD5, BeamBinary, FunDefs, AddressesOfClosuresToPatch, + CodeAddress, byte_size(CodeBinary)) end, - %% Redirect references to the old module to the new module's BEAM stub. - patch_to_emu_step2(OldReferencesToPatch), - %% Patch referring functions to call the new function - %% The call to export_funs/1 above updated the native addresses - %% for the targets, so passing 'FunDefs' is not needed. - redirect(ReferencesToPatch), + %% Final clean up. _ = erase(hipe_patch_closures), _ = erase(hipe_assert_code_area), @@ -371,21 +338,21 @@ trampoline_map_lookup(Primop, Map) -> is_exported :: boolean()}). exports(ExportMap, BaseAddress) -> - exports(ExportMap, BaseAddress, [], []). + exports(ExportMap, BaseAddress, []). -exports([Offset,M,F,A,IsClosure,IsExported|Rest], BaseAddress, MFAs, FunDefs) -> +exports([Offset,M,F,A,IsClosure,IsExported|Rest], BaseAddress, FunDefs) -> case IsExported andalso erlang:is_builtin(M, F, A) of true -> - exports(Rest, BaseAddress, MFAs, FunDefs); + exports(Rest, BaseAddress, FunDefs); _false -> MFA = {M,F,A}, Address = BaseAddress + Offset, FunDef = #fundef{address=Address, mfa=MFA, is_closure=IsClosure, is_exported=IsExported}, - exports(Rest, BaseAddress, [MFA|MFAs], [FunDef|FunDefs]) + exports(Rest, BaseAddress, [FunDef|FunDefs]) end; -exports([], _, MFAs, FunDefs) -> - {MFAs, FunDefs}. +exports([], _, FunDefs) -> + FunDefs. mod({M,_F,_A}) -> M. @@ -424,9 +391,9 @@ find_closure_refs([], Refs) -> %%------------------------------------------------------------------------ -export_funs([FunDef | FunDefs]) -> +set_beam_call_traps([FunDef | FunDefs]) -> #fundef{address=Address, mfa=MFA, is_closure=IsClosure, - is_exported=IsExported} = FunDef, + is_exported=_IsExported} = FunDef, ?IF_DEBUG({M,F,A} = MFA, no_debug), ?IF_DEBUG( case IsClosure of @@ -437,15 +404,32 @@ export_funs([FunDef | FunDefs]) -> ?debug_msg("LINKING: ~w:~w/~w to closure (0x~.16b)\n", [M,F,A, Address]) end, no_debug), - hipe_bifs:set_funinfo_native_address(MFA, Address, IsExported), hipe_bifs:set_native_address(MFA, Address, IsClosure), + set_beam_call_traps(FunDefs); +set_beam_call_traps([]) -> + ok. + +export_funs([FunDef | FunDefs]) -> + #fundef{address=Address, mfa=MFA, is_closure=_IsClosure, + is_exported=IsExported} = FunDef, + ?IF_DEBUG({M,F,A} = MFA, no_debug), + ?IF_DEBUG( + case _IsClosure of + false -> + ?debug_msg("LINKING: ~w:~w/~w to (0x~.16b)\n", + [M,F,A, Address]); + true -> + ?debug_msg("LINKING: ~w:~w/~w to closure (0x~.16b)\n", + [M,F,A, Address]) + end, no_debug), + hipe_bifs:set_funinfo_native_address(MFA, Address, IsExported), export_funs(FunDefs); export_funs([]) -> ok. -export_funs(Mod, MD5, Beam, FunDefs, ClosuresToPatch) -> +make_beam_stub(Mod, MD5, Beam, FunDefs, ClosuresToPatch, CodeAddress, CodeSize) -> Fs = [{F,A,Address} || #fundef{address=Address, mfa={_M,F,A}} <- FunDefs], - Mod = code:make_stub_module(Mod, Beam, {Fs,ClosuresToPatch,MD5}), + Mod = code:make_stub_module(Mod, Beam, {Fs,ClosuresToPatch,MD5,CodeAddress,CodeSize}), ok. %%======================================================================== @@ -515,8 +499,7 @@ patch_mfa_call_list([Offset|Offsets], BaseAddress, DestMFA, DestAddress, FunDefs patch_mfa_call_list([], _, _, _, _, _, _) -> ok. patch_call_insn(CallAddress, DestAddress, Trampoline) -> - %% This assertion is false when we're called from redirect/2. - %% ?ASSERT(assert_local_patch(CallAddress)), + ?ASSERT(assert_local_patch(CallAddress)), hipe_bifs:patch_call(CallAddress, DestAddress, Trampoline). %% ____________________________________________________________________ @@ -561,15 +544,15 @@ patch_atom(Address, Atom) -> patch_instr(Address, hipe_bifs:atom_to_word(Atom), atom). patch_sdesc(?STACK_DESC(SymExnRA, FSize, Arity, Live), - Address, {_ConstMap2,CodeAddress}, _FunDefs) -> + Address, {_ConstMap2,CodeAddress}, FunDefs) -> ExnRA = case SymExnRA of [] -> 0; % No catch LabelOffset -> CodeAddress + LabelOffset end, ?ASSERT(assert_local_patch(Address)), - DBG_MFA = ?IF_DEBUG(address_to_mfa_lth(Address, _FunDefs), {undefined,undefined,0}), - hipe_bifs:enter_sdesc({Address, ExnRA, FSize, Arity, Live, DBG_MFA}). + MFA = address_to_mfa_lth(Address, FunDefs), + hipe_bifs:enter_sdesc({Address, ExnRA, FSize, Arity, Live, MFA, get(hipe_patch_closures)}). %%---------------------------------------------------------------- @@ -617,6 +600,7 @@ patch_closure(DestMFA, Uniq, Index, Address, FunDefs) -> %% RemoteOrLocal ::= 'remote' | 'local' %% patch_load_mfa(CodeAddress, DestMFA, FunDefs, RemoteOrLocal) -> + ?ASSERT(assert_local_patch(CodeAddress)), DestAddress = case bif_address(DestMFA) of false -> @@ -626,7 +610,6 @@ patch_load_mfa(CodeAddress, DestMFA, FunDefs, RemoteOrLocal) -> BifAddress when is_integer(BifAddress) -> BifAddress end, - ?ASSERT(assert_local_patch(CodeAddress)), patch_instr(CodeAddress, DestAddress, 'load_mfa'). %%---------------------------------------------------------------- @@ -781,23 +764,17 @@ find_const(ConstNo, []) -> %% RemoteOrLocal ::= 'remote' | 'local'. %% -%% -%% -record(ref, {caller_mfa, address, ref_type, trampoline, remote_or_local}). -%% - add_ref(CalleeMFA, Address, FunDefs, RefType, Trampoline, RemoteOrLocal) -> CallerMFA = address_to_mfa_lth(Address, FunDefs), - %% just a sanity assertion below - true = case RemoteOrLocal of - local -> - {M1,_,_} = CalleeMFA, - {M2,_,_} = CallerMFA, - M1 =:= M2; - remote -> - true - end, - %% io:format("Adding ref ~w\n",[{CallerMFA, CalleeMFA, Address, RefType}]), - hipe_bifs:add_ref(CalleeMFA, {CallerMFA,Address,RefType,Trampoline,RemoteOrLocal}). + case RemoteOrLocal of + local -> + %% just a sanity assertion + {M,_,_} = CalleeMFA, + {M,_,_} = CallerMFA; + remote -> + hipe_bifs:add_ref(CalleeMFA, {CallerMFA,Address,RefType,Trampoline,get(hipe_patch_closures)}) + end, + ok. % For FunDefs sorted from low to high addresses address_to_mfa_lth(Address, FunDefs) -> @@ -825,81 +802,6 @@ address_to_mfa_lth(_Address, [], Prev) -> %% ?error_msg("Local adddress not found ~w\n",[Address]), %% exit({?MODULE, local_address_not_found}). -%%---------------------------------------------------------------- -%% Change callers of the given module to instead trap to BEAM. -%% load_native_code/3 calls this just before loading native code. -%% -patch_to_emu(Mod) -> - patch_to_emu_step2(patch_to_emu_step1(Mod)). - -%% Step 1 must occur before the loading of native code updates -%% references information or creates a new BEAM stub module. -patch_to_emu_step1(Mod) -> - case is_loaded(Mod) of - true -> - %% Get exported functions - MFAs = [{Mod,Fun,Arity} || {Fun,Arity} <- Mod:module_info(exports)], - %% get_refs_from/2 only finds references from compiled static - %% call sites to the module, but some native address entries - %% were added as the result of dynamic apply calls. We must - %% purge them too, but we have no explicit record of them. - %% Therefore invalidate all native addresses for the module. - hipe_bifs:invalidate_funinfo_native_addresses(MFAs), - %% Find all call sites that call these MFAs. As a side-effect, - %% create native stubs for any MFAs that are referred. - ReferencesToPatch = get_refs_from(MFAs, []), - ok = remove_refs_from(MFAs), - ReferencesToPatch; - false -> - %% The first time we load the module, no redirection needs to be done. - [] - end. - -%% Step 2 must occur after the new BEAM stub module is created. -patch_to_emu_step2(ReferencesToPatch) -> - redirect(ReferencesToPatch). - --spec is_loaded(Module::atom()) -> boolean(). -%% @doc Checks whether a module is loaded or not. -is_loaded(M) when is_atom(M) -> - try hipe_bifs:fun_to_address({M,module_info,0}) of - I when is_integer(I) -> true - catch _:_ -> false - end. - -%%-------------------------------------------------------------------- -%% Given a list of MFAs, tag them with their referred_from references. -%% The resulting {MFA,Refs} list is later passed to redirect/1, once -%% the MFAs have been bound to (possibly new) native-code addresses. -%% -get_refs_from(MFAs, []) -> - mark_referred_from(MFAs), - MFAs. - -mark_referred_from(MFAs) -> - lists:foreach(fun(MFA) -> hipe_bifs:mark_referred_from(MFA) end, MFAs). - -%%-------------------------------------------------------------------- -%% Given a list of MFAs with referred_from references, update their -%% callers to refer to their new native-code addresses. -%% -%% The {MFA,Refs} list must come from get_refs_from/2. -%% -redirect(MFAs) -> - lists:foreach(fun(MFA) -> hipe_bifs:redirect_referred_from(MFA) end, MFAs). - -%%-------------------------------------------------------------------- -%% Given a list of MFAs, remove all referred_from references having -%% any of them as CallerMFA. -%% -%% This is the only place using refers_to. Whenever a reference is -%% added from CallerMFA to CalleeMFA, CallerMFA is added to CalleeMFA's -%% referred_from list, and CalleeMFA is added to CallerMFA's refers_to -%% list. The refers_to list is used here to find the CalleeMFAs whose -%% referred_from lists should be updated. -%% -remove_refs_from(MFAs) -> - lists:foreach(fun(MFA) -> hipe_bifs:remove_refs_from(MFA) end, MFAs). %%-------------------------------------------------------------------- @@ -913,12 +815,13 @@ get_native_address(MFA, FunDefs, RemoteOrLocal) -> case mfa_to_address(MFA, FunDefs, RemoteOrLocal) of Adr when is_integer(Adr) -> Adr; false -> - IsRemote = case RemoteOrLocal of - remote -> true; - local -> false - end, - hipe_bifs:find_na_or_make_stub(MFA, IsRemote) + remote -> + hipe_bifs:find_na_or_make_stub(MFA); + local -> + ?error_msg("Local function ~p not found\n",[MFA]), + exit({function_not_found,MFA}) + end end. mfa_to_address(MFA, [#fundef{address=Adr, mfa=MFA, diff --git a/lib/kernel/test/code_SUITE.erl b/lib/kernel/test/code_SUITE.erl index c5167efa56..1be4c364d5 100644 --- a/lib/kernel/test/code_SUITE.erl +++ b/lib/kernel/test/code_SUITE.erl @@ -483,23 +483,25 @@ load_binary(Config) when is_list(Config) -> code:delete(code_b_test), ok. + upgrade(Config) -> DataDir = proplists:get_value(data_dir, Config), - %%T = [beam, hipe], - T = [beam], - - [upgrade_do(DataDir, Client, U1, U2, O1, O2) - || Client<-T, U1<-T, U2<-T, O1<-T, O2<-T], + T = [beam, hipe], + %%T = [beam], + %%T = [hipe], + [upgrade_do(DataDir, Client, T) || Client <- T], ok. -upgrade_do(DataDir, Client, U1, U2, O1, O2) -> +upgrade_do(DataDir, Client, T) -> compile_load(upgrade_client, DataDir, undefined, Client), - upgrade_client:run(DataDir, U1, U2, O1, O2), + [upgrade_client:run(DataDir, U1, U2, O1, O2) + || U1<-T, U2<-T, O1<-T, O2<-T], ok. compile_load(Mod, Dir, Ver, CodeType) -> + erlang:display({"{{{{{{{{{{{{{{{{Loading",Mod,Ver,CodeType}), Version = case Ver of undefined -> io:format("Compiling '~p' as ~p\n", [Mod, CodeType]), @@ -516,9 +518,15 @@ compile_load(Mod, Dir, Ver, CodeType) -> CompOpts = [binary, report] ++ Target ++ Version, Src = filename:join(Dir, atom_to_list(Mod) ++ ".erl"), + T1 = erlang:now(), {ok,Mod,Code} = compile:file(Src, CompOpts), + T2 = erlang:now(), ObjFile = filename:basename(Src,".erl") ++ ".beam", {module,Mod} = code:load_binary(Mod, ObjFile, Code), + T3 = erlang:now(), + io:format("Compile time ~p ms, Load time ~p ms\n", + [timer:now_diff(T2,T1) div 1000, timer:now_diff(T3,T2) div 1000]), + erlang:display({"}}}}}}}}}}}}}}}Loaded",Mod,Ver,CodeType}), ok. dir_req(Config) when is_list(Config) -> @@ -810,8 +818,6 @@ check_funs({'$M_EXPR','$F_EXPR',_}, {code_server,start_link,1}]) -> 0; check_funs({'$M_EXPR','$F_EXPR',_}, [{erlang,spawn_link,1},{code_server,start_link,1}]) -> 0; -check_funs({'$M_EXPR',module_info,1}, - [{hipe_unified_loader,patch_to_emu_step1,1} | _]) -> 0; check_funs({'$M_EXPR','$F_EXPR',2}, [{hipe_unified_loader,write_words,3} | _]) -> 0; check_funs({'$M_EXPR','$F_EXPR',2}, @@ -823,11 +829,7 @@ check_funs({'$M_EXPR','$F_EXPR',2}, {hipe_unified_loader,sort_and_write,5} | _]) -> 0; check_funs({'$M_EXPR','$F_EXPR',1}, [{lists,foreach,2}, - {hipe_unified_loader,patch_consts,3} | _]) -> 0; -check_funs({'$M_EXPR','$F_EXPR',1}, - [{lists,foreach,2}, - {hipe_unified_loader,mark_referred_from,1}, - {hipe_unified_loader,get_refs_from,2}| _]) -> 0; + {hipe_unified_loader,patch_consts,4} | _]) -> 0; check_funs({'$M_EXPR',warning_msg,2}, [{code_server,finish_on_load_report,2} | _]) -> 0; check_funs({'$M_EXPR','$F_EXPR',1}, diff --git a/lib/kernel/test/code_SUITE_data/upgrade_client.erl b/lib/kernel/test/code_SUITE_data/upgrade_client.erl index bb655e01d3..7ca0df7f5c 100644 --- a/lib/kernel/test/code_SUITE_data/upgrade_client.erl +++ b/lib/kernel/test/code_SUITE_data/upgrade_client.erl @@ -9,6 +9,8 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) -> %% Load version 1 of upgradee code_SUITE:compile_load(upgradee, Dir, 1, Upgradee1), + Tracer = start_tracing(), + ?line 1 = upgradee:exp1(), ?line 1 = upgradee:exp1exp2(), ?line 1 = upgradee:exp1loc2(), @@ -56,6 +58,16 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) -> ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc1), ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2), + Env1 = "Env1", + put(loc1_fun, upgradee:get_local_fun(Env1)), + erlang:display(sverk_break), + ?line {1,Env1} = (get(loc1_fun))(), + + put(exp1exp2_fun, upgradee:get_exp1exp2_fun()), + ?line 1 = (get(exp1exp2_fun))(), + + ?line 13 = check_tracing(Tracer), + %% %% Load version 1 of other %% @@ -78,6 +90,8 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) -> ?line {'EXIT',{undef,_}} = proxy_call(P, other, exp2), ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2), + ?line 5 = check_tracing(Tracer), + %% %% Load version 2 of upgradee %% @@ -130,6 +144,15 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) -> ?line {'EXIT',{undef,_}} = proxy_call(P, other, exp2), ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2), + ?line {1,Env1} = (get(loc1_fun))(), + Env2 = "Env2", + put(loc2_fun, upgradee:get_local_fun(Env2)), + ?line {2,Env2} = (get(loc2_fun))(), + + ?line 2 = (get(exp1exp2_fun))(), + + ?line 10 = check_tracing(Tracer), + %% %% Load version 2 of other %% @@ -182,17 +205,26 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) -> ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc1loc2), ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2), + ?line {1,Env1} = (get(loc1_fun))(), + ?line {2,Env2} = (get(loc2_fun))(), + ?line 2 = (get(exp1exp2_fun))(), + + ?line 10 = check_tracing(Tracer), %% %% Upgrade proxy to version 2 %% P ! upgrade_order, - %% - io:format("Delete version 2 of 'upgradee'\n",[]), + io:format("Purge version 1 of 'upgradee'\n",[]), %% + put(loc1_fun,undefined), code:purge(upgradee), + + %% + io:format("Delete version 2 of 'upgradee'\n",[]), + %% code:delete(upgradee), ?line {'EXIT',{undef,_}} = (catch upgradee:exp2()), @@ -239,17 +271,24 @@ run(Dir, Upgradee1, Upgradee2, Other1, Other2) -> ?line {'EXIT',{undef,_}} = proxy_call(P, other, exp1loc2), ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc1loc2), ?line {'EXIT',{undef,_}} = proxy_call(P, other, loc2), + + ?line {'EXIT',{undef,_}} = (catch (get(exp1exp2_fun))()), + ?line 14 = check_tracing(Tracer), + unlink(P), exit(P, die_please), io:format("Purge 'upgradee'\n",[]), + put(loc2_fun,undefined), code:purge(upgradee), io:format("Delete and purge 'other'\n",[]), code:purge(other), code:delete(other), code:purge(other), + + stop_tracing(Tracer), ok. proxy_call(Pid, CallType, Func) -> @@ -257,3 +296,48 @@ proxy_call(Pid, CallType, Func) -> receive {Pid, call_result, Func, Ret} -> Ret end. + + +start_tracing() -> + Self = self(), + {Tracer,_} = spawn_opt(fun() -> tracer_loop(Self) end, [link,monitor]), + ?line 1 = erlang:trace_pattern({error_handler,undefined_function,3}, + true, [global]), + ?line 1 = erlang:trace(Self, true, [call,{tracer,Tracer}]), + Tracer. + + +tracer_loop(Receiver) -> + receive + die_please -> + ok; + {do_trace_delivered, Tracee} -> + _ = erlang:trace_delivered(Tracee), + tracer_loop(Receiver); + + Msg -> + Receiver ! Msg, + tracer_loop(Receiver) + end. + +check_tracing(Tracer) -> + Tracer ! {do_trace_delivered, self()}, + check_tracing_loop(0). + +check_tracing_loop(N) -> + Self = self(), + receive + {trace, _Pid, call, {_M, _F, _Args}} = Msg -> + io:format("Trace: ~p\n",[Msg]), + check_tracing_loop(N+1); + {trace_delivered, Self, _} -> + N + end. + + +stop_tracing(Tracer) -> + erlang:trace(self(), false, [call]), + Tracer ! die_please, + receive + {'DOWN', _, process, Tracer, _} -> ok + end. diff --git a/lib/kernel/test/code_SUITE_data/upgradee.erl b/lib/kernel/test/code_SUITE_data/upgradee.erl index 62b1d95e30..8ca660c19c 100644 --- a/lib/kernel/test/code_SUITE_data/upgradee.erl +++ b/lib/kernel/test/code_SUITE_data/upgradee.erl @@ -8,6 +8,9 @@ -export([exp1/0]). % only exported in v1 -export([exp1loc2/0]). % exported in v1, local in v2 -export([exp1exp2/0]). % exported in v1 and v2 +-export([get_local_fun/1]). +-export([get_exp1exp2_fun/0]). +-export([exp1exp2_fun/0]). exp1() -> ?VERSION. loc1() -> ?VERSION. @@ -20,6 +23,9 @@ loc1() -> ?VERSION. -export([exp2/0]). -export([loc1exp2/0]). -export([exp1exp2/0]). +-export([get_local_fun/1]). +-export([get_exp1exp2_fun/0]). +-export([exp1exp2_fun/0]). exp2() -> ?VERSION. loc2() -> ?VERSION. @@ -31,6 +37,12 @@ exp1loc2() -> ?VERSION. loc1exp2() -> ?VERSION. loc1loc2() -> ?VERSION. +get_local_fun(Env) -> fun() -> {?VERSION,Env} end. +get_exp1exp2_fun() -> fun ?MODULE:exp1exp2_fun/0. + +exp1exp2_fun() -> + ?VERSION. + dispatch_loop() -> receive upgrade_order -> |