diff options
Diffstat (limited to 'erts/emulator')
42 files changed, 1659 insertions, 1363 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 21bcbbab27..fa4de5805e 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -884,7 +884,9 @@ HIPE_OBJS= \ $(OBJDIR)/hipe_bif2.o \ $(OBJDIR)/hipe_debug.o \ $(OBJDIR)/hipe_gc.o \ + $(OBJDIR)/hipe_load.o \ $(OBJDIR)/hipe_mode_switch.o \ + $(OBJDIR)/hipe_module.o \ $(OBJDIR)/hipe_native_bif.o \ $(OBJDIR)/hipe_stack.o $(HIPE_ARCH_OBJS) ifdef HIPE_ENABLED diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index bce57fb7c1..099abad4e1 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -36,6 +36,12 @@ #include "erl_nif.h" #include "erl_bits.h" #include "erl_thr_progress.h" +#ifdef HIPE +# include "hipe_bif0.h" +# define IF_HIPE(X) (X) +#else +# define IF_HIPE(X) (0) +#endif #ifdef HIPE # include "hipe_stack.h" @@ -143,8 +149,17 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) { +#if !defined(HIPE) + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else Module* modp; - Eterm res; + Eterm res, mod; + + if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1) || + is_not_atom(mod = erts_module_for_prepared_code + (((ProcBin*)binary_val(BIF_ARG_1))->val))) { + BIF_ERROR(BIF_P, BADARG); + } if (!erts_try_seize_code_write_permission(BIF_P)) { ERTS_BIF_YIELD3(bif_export[BIF_code_make_stub_module_3], @@ -154,7 +169,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); - modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); + modp = erts_get_module(mod, erts_active_code_ix()); if (modp && modp->curr.num_breakpoints > 0) { ASSERT(modp->curr.code_hdr != NULL); @@ -166,9 +181,12 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); - if (res == BIF_ARG_1) { + if (res == mod) { erts_end_staging_code_ix(); erts_commit_staging_code_ix(); + if (!modp) + modp = erts_get_module(mod, erts_active_code_ix()); + hipe_redirect_to_module(modp); } else { erts_abort_staging_code_ix(); @@ -177,6 +195,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); return res; +#endif } BIF_RETTYPE @@ -241,7 +260,7 @@ struct m { Uint exception; }; -static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int); +static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int, int); #ifdef ERTS_SMP static void smp_code_ix_commiter(void*); @@ -377,8 +396,9 @@ finish_loading_1(BIF_ALIST_1) for (i = 0; i < n; i++) { if (p[i].modp->curr.num_breakpoints > 0 || p[i].modp->curr.num_traced_exports > 0 || - erts_is_default_trace_enabled()) { - /* tracing involved, fallback with thread blocking */ + erts_is_default_trace_enabled() || + IF_HIPE(hipe_need_blocking(p[i].modp))) { + /* tracing or hipe need thread blocking */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; @@ -436,32 +456,36 @@ finish_loading_1(BIF_ALIST_1) } done: - return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n); + return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n, 1); } static Eterm staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, - struct m* loaded, int nloaded) + struct m* mods, int nmods, int free_mods) { #ifdef ERTS_SMP if (is_blocking || !commit) #endif { if (commit) { + int i; erts_end_staging_code_ix(); erts_commit_staging_code_ix(); - if (loaded) { - int i; - for (i=0; i < nloaded; i++) { - set_default_trace_pattern(loaded[i].module); + + for (i=0; i < nmods; i++) { + if (mods[i].modp->curr.code_hdr) { + set_default_trace_pattern(mods[i].module); } + #ifdef HIPE + hipe_redirect_to_module(mods[i].modp); + #endif } } else { erts_abort_staging_code_ix(); } - if (loaded) { - erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + if (free_mods) { + erts_free(ERTS_ALC_T_LOADER_TMP, mods); } if (is_blocking) { erts_smp_thr_progress_unblock(); @@ -474,8 +498,8 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, else { ASSERT(is_value(res)); - if (loaded) { - erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + if (free_mods) { + erts_free(ERTS_ALC_T_LOADER_TMP, mods); } erts_end_staging_code_ix(); /* @@ -653,8 +677,9 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } else { if (modp->curr.num_breakpoints > 0 || - modp->curr.num_traced_exports > 0) { - /* we have tracing, retry single threaded */ + modp->curr.num_traced_exports > 0 || + IF_HIPE(hipe_need_blocking(modp))) { + /* tracing or hipe need to go single threaded */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; @@ -668,7 +693,14 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) success = 1; } } - return staging_epilogue(BIF_P, success, res, is_blocking, NULL, 0); + { + struct m mod; + Eterm retval; + mod.module = BIF_ARG_1; + mod.modp = modp; + retval = staging_epilogue(BIF_P, success, res, is_blocking, &mod, 1, 0); + return retval; + } } BIF_RETTYPE module_loaded_1(BIF_ALIST_1) @@ -809,6 +841,9 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) } modp->curr.code_hdr->on_load_function_ptr = NULL; set_default_trace_pattern(BIF_ARG_1); + #ifdef HIPE + hipe_redirect_to_module(modp); + #endif } else if (BIF_ARG_2 == am_false) { int i; @@ -1037,6 +1072,10 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) char* mod_start; Uint mod_size; Eterm* sp; +#ifdef HIPE + void *nat_start = NULL; + Uint nat_size = 0; +#endif *redsp += 1; @@ -1066,6 +1105,20 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) } } +#ifdef HIPE + /* + * Check all continuation pointers stored on the native stack if the module + * has native code. + */ + if (modp->old.hipe_code) { + nat_start = modp->old.hipe_code->text_segment; + nat_size = modp->old.hipe_code->text_segment_size; + if (nat_size && nstack_any_cps_in_segment(rp, nat_start, nat_size)) { + return am_true; + } + } +#endif + /* * Check all continuation pointers stored in stackdump * and clear exception stackdump if there is a pointer @@ -1082,8 +1135,16 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) rp->ftrace = NIL; } else { int i; + char *area_start = mod_start; + Uint area_size = mod_size; +#ifdef HIPE + if (rp->freason & EXF_NATIVE) { + area_start = nat_start; + area_size = nat_size; + } +#endif for (i = 0; i < s->depth; i++) { - if (ErtsInArea(s->trace[i], mod_start, mod_size)) { + if (ErtsInArea(s->trace[i], area_start, area_size)) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; @@ -1619,15 +1680,18 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) /* * Unload any NIF library */ - if (modp->old.nif != NULL) { + if (modp->old.nif != NULL + || IF_HIPE(hipe_purge_need_blocking(modp))) { /* ToDo: Do unload nif without blocking */ erts_rwunlock_old_code(code_ix); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; erts_rwlock_old_code(code_ix); - erts_unload_nif(modp->old.nif); - modp->old.nif = NULL; + if (modp->old.nif) { + erts_unload_nif(modp->old.nif); + modp->old.nif = NULL; + } } /* @@ -1646,7 +1710,9 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) modp->old.code_length = 0; modp->old.catches = BEAM_CATCHES_NIL; erts_remove_from_ranges(code); - +#ifdef HIPE + hipe_purge_module(modp, is_blocking); +#endif ERTS_BIF_PREP_RET(ret, am_true); } @@ -1719,6 +1785,8 @@ delete_code(Module* modp) (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); + DBG_TRACE_MFA_P(&ep->info.mfa, + "export trace cleared, code_ix=%d", code_ix); erts_clear_export_break(modp, &ep->info); } else ASSERT(ep->beam[0] == (BeamInstr) em_call_error_handler @@ -1727,17 +1795,15 @@ delete_code(Module* modp) ep->addressv[code_ix] = ep->beam; ep->beam[0] = (BeamInstr) em_call_error_handler; ep->beam[1] = 0; + DBG_TRACE_MFA_P(&ep->info.mfa, + "export invalidation, code_ix=%d", code_ix); } } ASSERT(modp->curr.num_breakpoints == 0); ASSERT(modp->curr.num_traced_exports == 0); modp->old = modp->curr; - modp->curr.code_hdr = NULL; - modp->curr.code_length = 0; - modp->curr.catches = BEAM_CATCHES_NIL; - modp->curr.nif = NULL; - + erts_module_instance_init(&modp->curr); } @@ -1751,9 +1817,11 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) * if not, delete old code; error if old code already exists. */ - if (modp->curr.code_hdr && modp->old.code_hdr) { - return am_not_purged; - } else if (!modp->old.code_hdr) { /* Make the current version old. */ + if (modp->curr.code_hdr) { + if (modp->old.code_hdr) { + return am_not_purged; + } + /* Make the current version old. */ delete_code(modp); } return NIL; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 3b02df8bac..84e2d42af6 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -6121,6 +6121,7 @@ call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) Uint sz; int i; + DBG_TRACE_MFA_P(mfa, "call_error_handler"); /* * Search for the error_handler module. */ @@ -6676,9 +6677,6 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) funp->fe = fe; funp->num_free = num_free; funp->creator = p->common.id; -#ifdef HIPE - funp->native_address = fe->native_address; -#endif funp->arity = (int)fe->address[-1] - num_free; for (i = 0; i < num_free; i++) { *hp++ = reg[i]; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index ee40c308ce..36cf64e08d 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -45,6 +45,7 @@ #include "hipe_bif0.h" #include "hipe_mode_switch.h" #include "hipe_arch.h" +#include "hipe_load.h" #endif ErlDrvBinary* erts_gzinflate_buffer(char*, int); @@ -481,9 +482,12 @@ static void free_loader_state(Binary* magic); static ErlHeapFragment* new_literal_fragment(Uint size); static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); +#ifdef HIPE static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code, Uint size); + BeamCodeHeader* code_hdr, Uint size, + HipeModule *hipe_code); +#endif static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -538,8 +542,6 @@ static Eterm compilation_info_for_module(Process* p, BeamCodeHeader*); static Eterm md5_of_module(Process* p, BeamCodeHeader*); static Eterm has_native(BeamCodeHeader*); static Eterm native_addresses(Process* p, BeamCodeHeader*); -int patch_funentries(Eterm Patchlist); -int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); static int must_swap_floats; @@ -843,9 +845,7 @@ erts_finish_loading(Binary* magic, Process* c_p, erts_alloc(ERTS_ALC_T_PREPARED_CODE, sizeof(struct erl_module_instance)); inst_p = mod_tab_p->on_load; - inst_p->nif = 0; - inst_p->num_breakpoints = 0; - inst_p->num_traced_exports = 0; + erts_module_instance_init(inst_p); } inst_p->code_hdr = stp->hdr; @@ -944,6 +944,13 @@ erts_module_for_prepared_code(Binary* magic) LoaderState* stp; if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) { +#ifdef HIPE + HipeLoaderState *hipe_stp; + if ((hipe_stp = hipe_get_loader_state(magic)) + && hipe_stp->text_segment != 0) { + return hipe_stp->module; + } +#endif return NIL; } stp = ERTS_MAGIC_BIN_DATA(magic); @@ -1092,10 +1099,12 @@ loader_state_dtor(Binary* magic) ASSERT(stp->genop_blocks == 0); } +#ifdef HIPE static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code_hdr, Uint size) + BeamCodeHeader* code_hdr, Uint size, + HipeModule *hipe_code) { Module* modp; Eterm retval; @@ -1118,6 +1127,9 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modp->curr.code_hdr = code_hdr; modp->curr.code_length = size; modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code " + "first_hipe_ref = %p", hipe_code->first_hipe_ref); + modp->curr.hipe_code = hipe_code; /* * Update ranges (used for finding a function from a PC value). @@ -1126,6 +1138,7 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, erts_update_ranges((BeamInstr*)modp->curr.code_hdr, size); return NIL; } +#endif static int init_iff_file(LoaderState* stp, byte* code, Uint size) @@ -4780,9 +4793,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) } ep = erts_export_put(stp->module, stp->export[i].function, stp->export[i].arity); - if (!on_load) { - ep->addressv[erts_staging_code_ix()] = address; - } else { + if (on_load) { /* * on_load: Don't make any of the exported functions * callable yet. Keep any function in the current @@ -4790,6 +4801,8 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) */ ep->beam[1] = (BeamInstr) address; } + else + ep->addressv[erts_staging_code_ix()] = address; } /* @@ -5991,23 +6004,19 @@ code_module_md5_1(BIF_ALIST_1) return res; } +#ifdef HIPE #define WORDS_PER_FUNCTION (sizeof(ErtsCodeInfo) / sizeof(UWord) + 1) static BeamInstr* make_stub(ErtsCodeInfo* info, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode) { + DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", erts_codeinfo_to_code(info)); ASSERT(WORDS_PER_FUNCTION == 6); info->op = (BeamInstr) BeamOp(op_i_func_info_IaaI); info->native = native; info->mfa.module = mod; info->mfa.function = func; info->mfa.arity = arity; -#ifdef HIPE - if (native) { - erts_codeinfo_to_code(info)[0] = BeamOpCode(op_move_return_n); - hipe_mfa_save_orig_beam_op(mod, func, arity, erts_codeinfo_to_code(info)); - } -#endif erts_codeinfo_to_code(info)[0] = OpCode; return erts_codeinfo_to_code(info)+1; } @@ -6072,9 +6081,7 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci) { unsigned int i; unsigned int n = stp->num_exps; -#ifdef HIPE Lambda* lp; -#endif if (is_bif(ci->mfa.module, ci->mfa.function, ci->mfa.arity)) { ci->native = 0; @@ -6095,6 +6102,8 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci) ci->mfa.function, ci->mfa.arity); ep->addressv[erts_staging_code_ix()] = erts_codeinfo_to_code(ci); + DBG_TRACE_MFA_P(&ci->mfa,"set beam stub at %p in export at %p (code_ix=%d)", + erts_codeinfo_to_code(ci), ep, erts_staging_code_ix()); return; } } @@ -6104,7 +6113,6 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci) * Search the lambda table to find out which. */ -#ifdef HIPE n = stp->num_lambdas; for (i = 0, lp = stp->lambdas; i < n; i++, lp++) { ErlFunEntry* fe = stp->lambdas[i].fe; @@ -6113,7 +6121,6 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci) fe->address = erts_codeinfo_to_code(ci); } } -#endif return; } @@ -6122,10 +6129,9 @@ stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci) [{Adr, Patchtyppe} | Addresses] and the address of a fun_entry. */ -int +static int patch(Eterm Addresses, Uint fe) { -#ifdef HIPE Eterm* listp; Eterm tuple; Eterm* tp; @@ -6161,15 +6167,13 @@ patch(Eterm Addresses, Uint fe) } -#endif return 1; } -int +static int patch_funentries(Eterm Patchlist) { -#ifdef HIPE while (!is_nil(Patchlist)) { Eterm Info; Eterm MFA; @@ -6258,23 +6262,23 @@ patch_funentries(Eterm Patchlist) return 0; } -#endif return 1; /* Signal that all went well */ } - /* * Do a dummy load of a module. No threaded code will be loaded. * Used for loading native code. * Will also patch all references to fun_entries to point to * the new fun_entries created. */ - Eterm -erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) +erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) { Binary* magic; + Binary* hipe_magic; LoaderState* stp; + HipeLoaderState* hipe_stp; + HipeModule *hipe_code; BeamInstr Funcs; BeamInstr Patchlist; Eterm MD5Bin; @@ -6297,8 +6301,12 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) */ magic = erts_alloc_loader_state(); stp = ERTS_MAGIC_BIN_DATA(magic); + hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); - if (is_not_atom(Mod)) { + if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) || + !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val, + hipe_stp = hipe_get_loader_state(hipe_magic)) || + hipe_stp->module == NIL || hipe_stp->text_segment == 0) { goto error; } if (is_not_tuple(Info)) { @@ -6326,7 +6334,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Scan the Beam binary and read the interesting sections. */ - stp->module = Mod; + stp->module = hipe_stp->module; stp->group_leader = p->group_leader; stp->num_functions = n; if (!init_iff_file(stp, bytes, size)) { @@ -6435,12 +6443,8 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * as the body until we know what kind of trap we should put there. */ code_hdr->functions[i] = (ErtsCodeInfo*)fp; -#ifdef HIPE op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */ -#else - op = (Eterm) BeamOpCode(op_move_return_n); -#endif - fp = make_stub((ErtsCodeInfo*)fp, Mod, func, arity, + fp = make_stub((ErtsCodeInfo*)fp, hipe_stp->module, func, arity, (Uint)native_address, op); } @@ -6481,11 +6485,20 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) } /* + * Initialise HiPE module + */ + hipe_code->text_segment = hipe_stp->text_segment; + hipe_code->text_segment_size = hipe_stp->text_segment_size; + hipe_code->data_segment = hipe_stp->data_segment; + hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs; + hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc; + + /* * Insert the module in the module table. */ - rval = stub_insert_new_code(p, 0, p->group_leader, Mod, - code_hdr, code_size); + rval = stub_insert_new_code(p, 0, p->group_leader, hipe_stp->module, + code_hdr, code_size, hipe_code); if (rval != NIL) { goto error; } @@ -6501,18 +6514,69 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) } if (patch_funentries(Patchlist)) { + Eterm mod = hipe_stp->module; + /* Prevent code from being freed */ + hipe_stp->text_segment = 0; + hipe_stp->data_segment = 0; + hipe_stp->new_hipe_refs = NULL; + hipe_stp->new_hipe_sdesc = NULL; + erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); - return Mod; + hipe_free_loader_state(hipe_stp); + + return mod; } error: + erts_free(ERTS_ALC_T_HIPE, hipe_code); erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); BIF_ERROR(p, BADARG); } +int erts_commit_hipe_patch_load(Eterm hipe_magic_bin) +{ + Binary* hipe_magic; + HipeLoaderState* hipe_stp; + HipeModule *hipe_code; + Module* modp; + + if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) || + !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val, + hipe_stp = hipe_get_loader_state(hipe_magic)) || + hipe_stp->module == NIL || hipe_stp->text_segment == 0) { + return 0; + } + + modp = erts_get_module(hipe_stp->module, erts_active_code_ix()); + if (!modp) + return 0; + + /* + * Initialise HiPE module + */ + hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); + hipe_code->text_segment = hipe_stp->text_segment; + hipe_code->text_segment_size = hipe_stp->text_segment_size; + hipe_code->data_segment = hipe_stp->data_segment; + hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs; + hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc; + + modp->curr.hipe_code = hipe_code; + + /* Prevent code from being freed */ + hipe_stp->text_segment = 0; + hipe_stp->data_segment = 0; + hipe_stp->new_hipe_refs = NULL; + hipe_stp->new_hipe_sdesc = NULL; + + return 1; +} + #undef WORDS_PER_FUNCTION +#endif /* HIPE */ + static int safe_mul(UWord a, UWord b, UWord* resp) { @@ -6526,3 +6590,46 @@ static int safe_mul(UWord a, UWord b, UWord* resp) } } +#ifdef ENABLE_DBG_TRACE_MFA + +#define MFA_MAX 10 +Eterm dbg_trace_m[MFA_MAX]; +Eterm dbg_trace_f[MFA_MAX]; +Uint dbg_trace_a[MFA_MAX]; +unsigned int dbg_trace_ix = 0; + +void dbg_set_traced_mfa(const char* m, const char* f, Uint a) +{ + unsigned i = dbg_trace_ix++; + ASSERT(i < MFA_MAX); + dbg_trace_m[i] = am_atom_put(m, strlen(m)); + dbg_trace_f[i] = am_atom_put(f, strlen(f)); + dbg_trace_a[i] = a; +} + +int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a) +{ + unsigned int i; + for (i = 0; i < dbg_trace_ix; ++i) { + if (m == dbg_trace_m[i] && + (!f || (f == dbg_trace_f[i] && a == dbg_trace_a[i]))) { + + return i+1; + } + } + return 0; +} + +void dbg_vtrace_mfa(unsigned ix, const char* format, ...) +{ + va_list arglist; + va_start(arglist, format); + ASSERT(--ix < MFA_MAX); + erts_fprintf(stderr, "MFA TRACE %T:%T/%u: ", + dbg_trace_m[ix], dbg_trace_f[ix], (int)dbg_trace_a[ix]); + + erts_vfprintf(stderr, format, arglist); + va_end(arglist); +} + +#endif /* ENABLE_DBG_TRACE_MFA */ diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index f67dfd7a81..6a3110d0f5 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -143,4 +143,34 @@ struct BeamCodeLineTab_ { #define LOC_FILE(Loc) ((Loc) >> 24) #define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) + +/* + * MFA event debug "tracing" usage: + * + * #define ENABLE_DBG_TRACE_MFA + * call dbg_set_traced_mfa("mymod","myfunc",arity) + * for the function(s) to trace, in some init function. + * + * Run and get stderr printouts when interesting things happen to your MFA. + */ +#ifdef ENABLE_DBG_TRACE_MFA + +void dbg_set_traced_mfa(const char* m, const char* f, Uint a); +int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a); +void dbg_vtrace_mfa(unsigned ix, const char* format, ...); +#define DBG_TRACE_MFA(M,F,A,FMT, ...) do {\ + unsigned ix;\ + if ((ix=dbg_is_traced_mfa(M,F,A))) \ + dbg_vtrace_mfa(ix, FMT"\n", ##__VA_ARGS__);\ + }while(0) + +#define DBG_TRACE_MFA_P(MFA, FMT, ...) \ + DBG_TRACE_MFA((MFA)->module, (MFA)->function, (MFA)->arity, FMT, ##__VA_ARGS__) + +#else +# define dbg_set_traced_mfa(M,F,A) +# define DBG_TRACE_MFA(M,F,A,FMT, ...) +# define DBG_TRACE_MFA_P(MFA,FMT, ...) +#endif /* ENABLE_DBG_TRACE_MFA */ + #endif /* _BEAM_LOAD_H */ diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 3c2c9def3b..56aa78a08a 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -373,10 +373,16 @@ set_default_exec_alloc_opts(struct au_init *ip) ip->init.util.rmbcmt = 0; ip->init.util.acul = 0; +# ifdef ERTS_HAVE_EXEC_MMAPPER ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc; ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc; ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc; ip->init.util.mseg_mmapper = &erts_exec_mmapper; +# else + ip->init.util.mseg_alloc = &erts_alcu_exec_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_exec_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_exec_mseg_dealloc; +# endif } #endif /* ERTS_ALC_A_EXEC */ @@ -1571,7 +1577,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) break; case 'X': if (has_prefix("scs", argv[i]+3)) { -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER init->mseg.exec_mmap.scs = #endif get_mb_value(argv[i]+6, argv, &i); @@ -2852,7 +2858,7 @@ erts_allocator_info(int to, void *arg) erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n"); erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis); #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER erts_print(to, arg, "=allocator:erts_mmap.exec_mmap\n"); erts_mmap_info(&erts_exec_mmapper, &to, arg, NULL, NULL, &emis); #endif @@ -3010,7 +3016,7 @@ erts_allocator_options(void *proc) #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) terms[length++] = ERTS_MAKE_AM("literal_mmap"); #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER terms[length++] = ERTS_MAKE_AM("exec_mmap"); #endif features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; @@ -3102,7 +3108,7 @@ reply_alloc_info(void *vair) # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) struct erts_mmap_info_struct mmap_info_literal; # endif -# ifdef ERTS_ALC_A_EXEC +# ifdef ERTS_HAVE_EXEC_MMAPPER struct erts_mmap_info_struct mmap_info_exec; # endif #endif @@ -3232,7 +3238,7 @@ reply_alloc_info(void *vair) erts_bld_atom(hpp,szp,"literal_mmap"), ainfo); # endif -# ifdef ERTS_ALC_A_EXEC +# ifdef ERTS_HAVE_EXEC_MMAPPER ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list); ainfo = (air->only_sz ? NIL : @@ -4129,12 +4135,20 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *dptr; Uint size; + int free_pattern = n; ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE); - sys_memset((void *) dptr, n, size + FENCE_SZ); +#ifdef ERTS_ALC_A_EXEC +# if defined(__i386__) || defined(__x86_64__) + if (ERTS_ALC_T2A(ERTS_ALC_N2T(n)) == ERTS_ALC_A_EXEC) { + free_pattern = 0x0f; /* Illegal instruction */ + } +# endif +#endif + sys_memset((void *) dptr, free_pattern, size + FENCE_SZ); (*real_af->free)(n, real_af->extra, dptr); diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index dd8ca5fac1..163244237d 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -907,7 +907,9 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, #elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) -/* Used by literal allocator that has its own mmapper (super carrier) */ +/* For allocators that have their own mmapper (super carrier), + * like literal_alloc and exec_alloc on amd64 + */ void* erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { @@ -948,6 +950,50 @@ erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, } #endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ +#if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) + +/* + * For exec_alloc on non-amd64 that just need memory with PROT_EXEC + */ +void* +erts_alcu_exec_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +{ + void* res = erts_alcu_mseg_alloc(allctr, size_p, flags); + + if (res) { + int r = mprotect(res, *size_p, PROT_EXEC | PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + return res; +} + +void* +erts_alcu_exec_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) +{ + void *res; + + if (seg && old_size) { + int r = mprotect(seg, old_size, PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p); + if (res) { + int r = mprotect(res, *new_size_p, PROT_EXEC | PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + return res; +} + +void +erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) +{ + int r = mprotect(seg, size, PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + erts_alcu_mseg_dealloc(allctr, seg, size, flags); +} +#endif /* ERTS_ALC_A_EXEC && !ERTS_HAVE_EXEC_MMAPPER */ + #endif /* HAVE_ERTS_MSEG */ void* diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index f50f09907a..2958bdf8d1 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -210,6 +210,12 @@ void* erts_alcu_mmapper_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); void erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); # endif + +# if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) +void* erts_alcu_exec_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); +void* erts_alcu_exec_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); +void erts_alcu_exec_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); +# endif #endif /* HAVE_ERTS_MSEG */ void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign); diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 73c3e19c1c..6ba055d92c 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -57,9 +57,6 @@ typedef struct erl_fun_thing { Eterm thing_word; /* Subtag FUN_SUBTAG. */ ErlFunEntry* fe; /* Pointer to fun entry. */ struct erl_off_heap_header* next; -#ifdef HIPE - UWord* native_address; /* Native code for the fun. */ -#endif Uint arity; /* The arity of the fun. */ Uint num_free; /* Number of free variables (in env). */ /* -- The following may be compound Erlang terms ---------------------- */ diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 060fde074c..94568e79a9 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -146,6 +146,8 @@ export_alloc(struct export_entry* tmpl_e) blob->entryv[ix].ep = &blob->exp; } ix = 0; + + DBG_TRACE_MFA_P(&obj->info.mfa, "export allocation at %p", obj); } else { /* Existing entry in another table, use free entry in blob */ blob = entry_to_blob(tmpl_e); @@ -164,9 +166,12 @@ export_free(struct export_entry* obj) obj->slot.index = -1; for (i=0; i < ERTS_NUM_CODE_IX; i++) { if (blob->entryv[i].slot.index >= 0) { + DBG_TRACE_MFA_P(&blob->exp.info.mfa, "export entry slot %u freed for %p", + (obj - blob->entryv), &blob->exp); return; } } + DBG_TRACE_MFA_P(&blob->exp.info.mfa, "export blob deallocation at %p", &blob->exp); erts_free(ERTS_ALC_T_EXPORT, blob); erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob)); } diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index c297bb26b3..98df8a0726 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -3750,7 +3750,6 @@ dec_term_atom_common: if (funp->fe->native_address == NULL) { hipe_set_closure_stub(funp->fe, num_free); } - funp->native_address = funp->fe->native_address; #endif hp = factory->hp; @@ -3822,9 +3821,6 @@ dec_term_atom_common: funp->fe = erts_put_fun_entry(module, old_uniq, old_index); funp->arity = funp->fe->address[-1] - num_free; -#ifdef HIPE - funp->native_address = funp->fe->native_address; -#endif hp = factory->hp; /* Environment */ diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 7ee046ac12..537aaf3177 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1048,6 +1048,7 @@ void erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa); Eterm erts_module_info_0(Process* p, Eterm module); Eterm erts_module_info_1(Process* p, Eterm module, Eterm what); Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); +int erts_commit_hipe_patch_load(Eterm hipe_magic_bin); /* beam_ranges.c */ void erts_init_ranges(void); diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 4f36377450..705dd1d43f 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -26,6 +26,7 @@ #include "erl_vm.h" #include "global.h" #include "module.h" +#include "beam_catches.h" #ifdef DEBUG # define IF_DEBUG(x) x @@ -67,6 +68,18 @@ static int module_cmp(Module* tmpl, Module* obj) return tmpl->module != obj->module; } +void erts_module_instance_init(struct erl_module_instance* modi) +{ + modi->code_hdr = 0; + modi->code_length = 0; + modi->catches = BEAM_CATCHES_NIL; + modi->nif = NULL; + modi->num_breakpoints = 0; + modi->num_traced_exports = 0; +#ifdef HIPE + modi->hipe_code = NULL; +#endif +} static Module* module_alloc(Module* tmpl) { @@ -74,18 +87,11 @@ static Module* module_alloc(Module* tmpl) erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module)); obj->module = tmpl->module; - obj->curr.code_hdr = 0; - obj->old.code_hdr = 0; - obj->curr.code_length = 0; - obj->old.code_length = 0; obj->slot.index = -1; - obj->curr.nif = NULL; - obj->old.nif = NULL; - obj->curr.num_breakpoints = 0; - obj->old.num_breakpoints = 0; - obj->curr.num_traced_exports = 0; - obj->old.num_traced_exports = 0; + erts_module_instance_init(&obj->curr); + erts_module_instance_init(&obj->old); obj->on_load = 0; + DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc"); return obj; } @@ -119,6 +125,7 @@ void init_module_table(void) erts_smp_atomic_init_nob(&tot_module_bytes, 0); } + Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix) { @@ -139,19 +146,14 @@ erts_get_module(Eterm mod, ErtsCodeIndex code_ix) } } -Module* -erts_put_module(Eterm mod) + +static Module* put_module(Eterm mod, IndexTable* mod_tab) { Module e; - IndexTable* mod_tab; int oldsz, newsz; Module* res; ASSERT(is_atom(mod)); - ERTS_SMP_LC_ASSERT(erts_initialized == 0 - || erts_has_code_write_permission()); - - mod_tab = &module_tables[erts_staging_code_ix()]; e.module = atom_val(mod); oldsz = index_table_sz(mod_tab); res = (Module*) index_put_entry(mod_tab, (void*) &e); @@ -160,6 +162,15 @@ erts_put_module(Eterm mod) return res; } +Module* +erts_put_module(Eterm mod) +{ + ERTS_SMP_LC_ASSERT(erts_initialized == 0 + || erts_has_code_write_permission()); + + return put_module(mod, &module_tables[erts_staging_code_ix()]); +} + Module *module_code(int i, ErtsCodeIndex code_ix) { return (Module*) erts_index_lookup(&module_tables[code_ix], i); @@ -181,6 +192,13 @@ static ErtsCodeIndex dbg_load_code_ix = 0; static int entries_at_start_staging = 0; +static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod) +{ + dst_mod->curr = src_mod->curr; + dst_mod->old = src_mod->old; + dst_mod->on_load = src_mod->on_load; +} + void module_start_staging(void) { IndexTable* src = &module_tables[erts_active_code_ix()]; @@ -199,10 +217,7 @@ void module_start_staging(void) src_mod = (Module*) erts_index_lookup(src, i); dst_mod = (Module*) erts_index_lookup(dst, i); ASSERT(src_mod->module == dst_mod->module); - - dst_mod->curr = src_mod->curr; - dst_mod->old = src_mod->old; - dst_mod->on_load = src_mod->on_load; + copy_module(dst_mod, src_mod); } /* @@ -214,9 +229,7 @@ void module_start_staging(void) dst_mod = (Module*) index_put_entry(dst, src_mod); ASSERT(dst_mod != src_mod); - dst_mod->curr = src_mod->curr; - dst_mod->old = src_mod->old; - dst_mod->on_load = src_mod->on_load; + copy_module(dst_mod, src_mod); } newsz = index_table_sz(dst); erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 1c1afc8461..8f44e11f3d 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -23,6 +23,10 @@ #include "index.h" +#ifdef HIPE +#include "hipe_module.h" +#endif + struct erl_module_instance { BeamCodeHeader* code_hdr; int code_length; /* Length of loaded code in bytes. */ @@ -30,6 +34,9 @@ struct erl_module_instance { struct erl_module_nif* nif; int num_breakpoints; int num_traced_exports; +#ifdef HIPE + HipeModule *hipe_code; +#endif }; typedef struct erl_module { @@ -42,6 +49,7 @@ typedef struct erl_module { struct erl_module_instance* on_load; } Module; +void erts_module_instance_init(struct erl_module_instance* modi); Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix); Module* erts_put_module(Eterm mod); diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index 62739d2a78..e3cff4a4ba 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -73,8 +73,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Sint rel32; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + rel32 = (Sint)destAddress - (Sint)callAddress - 4; if ((Sint)(Sint32)rel32 != rel32) return -1; @@ -83,29 +83,6 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - /* * Memory allocator for executable code. * @@ -116,9 +93,6 @@ static void atexit_alloc_code_stats(void) */ static void *alloc_code(unsigned int alloc_bytes) { - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } @@ -130,6 +104,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int bytes) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + /* Make stub for native code calling exported beam function. */ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) @@ -234,6 +213,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h index 6f959815bb..059b8e7f29 100644 --- a/erts/emulator/hipe/hipe_arch.h +++ b/erts/emulator/hipe/hipe_arch.h @@ -30,23 +30,31 @@ extern void hipe_patch_load_fe(Uint *address, Uint value); extern int hipe_patch_insn(void *address, Uint value, Eterm type); extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline); -extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p); -extern void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity); +extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, struct process *p); +extern void hipe_free_code(void*, unsigned int); +extern void *hipe_make_native_stub(void *exp, unsigned int beamArity); +extern void hipe_free_native_stub(void*); + #if defined(__sparc__) #include "hipe_sparc.h" +#include "hipe_sparc_asm.h" #endif #if defined(__i386__) #include "hipe_x86.h" +#include "hipe_x86_asm.h" #endif #if defined(__x86_64__) #include "hipe_amd64.h" +#include "hipe_amd64_asm.h" #endif #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) #include "hipe_ppc.h" +#include "hipe_ppc_asm.h" #endif #if defined(__arm__) #include "hipe_arm.h" +#include "hipe_arm_asm.h" #endif #if !defined(AEXTERN) diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c index f8ef468341..b61939724c 100644 --- a/erts/emulator/hipe/hipe_arm.c +++ b/erts/emulator/hipe/hipe_arm.c @@ -25,7 +25,6 @@ #endif #include "global.h" #include "erl_binary.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -57,30 +56,6 @@ void hipe_flush_icache_word(void *address) hipe_flush_icache_range(address, 4); } -/* - * Management of 32MB code segments for regular code and trampolines. - */ - -#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */ - -static struct segment { - unsigned int *base; /* [base,base+32MB[ */ - unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */ - unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */ - /* On ARM we always allocate a trampoline at base+32MB-8 for - nbif_callemu, so tramp_pos <= base+32MB-8. */ -} curseg; - -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) - -static void *new_code_mapping(void) -{ - return mmap(0, SEGMENT_NRBYTES, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, - -1, 0); -} static int check_callees(Eterm callees) { @@ -107,126 +82,53 @@ static int check_callees(Eterm callees) return arity; } -static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec) -{ - unsigned int *base, *address, *tramp_pos, nrfreewords; - int trampnr; - Eterm mfa, m, f; - unsigned int a, *trampoline; - - m = NIL; f = NIL; a = 0; /* silence stupid compiler warning */ - tramp_pos = curseg.tramp_pos; - address = curseg.code_pos; - nrfreewords = tramp_pos - address; - if (nrwords > nrfreewords) - return NULL; - curseg.code_pos = address + nrwords; - nrfreewords -= nrwords; +#define TRAMPOLINE_WORDS 2 - base = curseg.base; - for (trampnr = 1; trampnr <= nrcallees; ++trampnr) { - mfa = tuple_val(callees)[trampnr]; - if (is_atom(mfa)) - trampoline = hipe_primop_get_trampoline(mfa); - else { - m = tuple_val(mfa)[1]; - f = tuple_val(mfa)[2]; - a = unsigned_val(tuple_val(mfa)[3]); - trampoline = hipe_mfa_get_trampoline(m, f, a); - } - if (!in_area(trampoline, base, SEGMENT_NRBYTES)) { - if (nrfreewords < 2) - return NULL; - nrfreewords -= 2; - tramp_pos = trampoline = tramp_pos - 2; - trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - trampoline[1] = 0; /* callee's address */ - hipe_flush_icache_range(trampoline, 2*sizeof(int)); - if (is_atom(mfa)) - hipe_primop_set_trampoline(mfa, trampoline); - else - hipe_mfa_set_trampoline(m, f, a, trampoline); - } - trampvec[trampnr-1] = trampoline; +static void generate_trampolines(Uint32* address, + int nrcallees, Eterm callees, + Uint32** trampvec) +{ + Uint32* trampoline = address; + int i; + + for (i = 0; i < nrcallees; ++i) { + trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ + trampoline[1] = 0; /* callee's address */ + trampvec[i] = trampoline; + trampoline += TRAMPOLINE_WORDS; } - curseg.tramp_pos = tramp_pos; - return address; + hipe_flush_icache_range(address, nrcallees*2*sizeof(Uint32)); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) { - Uint nrwords; + Uint code_words; int nrcallees; Eterm trampvecbin; - unsigned int **trampvec; - unsigned int *address; - unsigned int *base; - struct segment oldseg; + Uint32 **trampvec; + Uint32 *address; if (nrbytes & 0x3) return NULL; - nrwords = nrbytes >> 2; + code_words = nrbytes / sizeof(Uint32); nrcallees = check_callees(callees); if (nrcallees < 0) return NULL; - trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*)); - trampvec = (unsigned int**)binary_bytes(trampvecbin); + trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*)); + trampvec = (Uint32**)binary_bytes(trampvecbin); - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - curseg.tramp_pos -= 2; - curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - curseg.tramp_pos[1] = (unsigned int)&nbif_callemu; + address = erts_alloc(ERTS_ALC_T_HIPE_EXEC, + (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32)); - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } + generate_trampolines(address + code_words, nrcallees, callees, trampvec); *trampolines = trampvecbin; return address; } -static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu) +void hipe_free_code(void* code, unsigned int bytes) { - unsigned int *address; - unsigned int *base; - struct segment oldseg; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - curseg.tramp_pos -= 2; - curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - curseg.tramp_pos[1] = (unsigned int)&nbif_callemu; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } - *tramp_callemu = (unsigned int*)((char*)curseg.base + SEGMENT_NRBYTES) - 2; - return address; + erts_free(ERTS_ALC_T_HIPE_EXEC, code); } /* @@ -266,8 +168,8 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { unsigned int *code; - unsigned int *tramp_callemu; int callemu_offset; + int is_short_jmp; /* * Native code calls BEAM via a stub looking as follows: @@ -277,36 +179,57 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) * b nbif_callemu * .long callee_exp * + * or if nbif_callemu is too far away: + * + * mov r0, #beamArity + * ldr r8, [pc,#0] // callee_exp + * ldr pc, [pc,#0] // nbif_callemu + * .long callee_exp + * .long nbif_callemu + * * I'm using r0 and r8 since they aren't used for - * parameter passing in native code. The branch to - * nbif_callemu may need to go via a trampoline. - * (Trampolines are allowed to modify r12, but they don't.) + * parameter passing in native code. */ - code = alloc_stub(4, &tramp_callemu); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 5*sizeof(Uint32)); if (!code) return NULL; callemu_offset = ((int)&nbif_callemu - ((int)&code[2] + 8)) >> 2; - if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) { - callemu_offset = ((int)tramp_callemu - ((int)&code[2] + 8)) >> 2; - if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) - abort(); + is_short_jmp = (callemu_offset >= -0x00800000 && + callemu_offset <= 0x007FFFFF); +#ifdef DEBUG + if (is_short_jmp && (callemu_offset % 3)==0) { + is_short_jmp = 0; } +#endif /* mov r0, #beamArity */ code[0] = 0xE3A00000 | (beamArity & 0xFF); /* ldr r8, [pc,#0] // callee_exp */ code[1] = 0xE59F8000; - /* b nbif_callemu */ - code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF); + if (is_short_jmp) { + /* b nbif_callemu */ + code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF); + } + else { + /* ldr pc, [pc,#0] // nbif_callemu */ + code[2] = 0xE59FF000; + /* .long nbif_callemu */ + code[4] = (unsigned int)&nbif_callemu; + } /* .long callee_exp */ code[3] = (unsigned int)callee_exp; - hipe_flush_icache_range(code, 4*sizeof(int)); + hipe_flush_icache_range(code, 5*sizeof(Uint32)); return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + static void patch_b(Uint32 *address, Sint32 offset, Uint32 AA) { Uint32 oldI = *address; diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 9faa6e1649..453a452590 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -44,6 +44,7 @@ #include "hipe_mode_switch.h" #include "hipe_native_bif.h" #include "hipe_bif0.h" +#include "hipe_load.h" /* We need hipe_literals.h for HIPE_SYSTEM_CRC, but it redefines a few constants. #undef them here to avoid warnings. */ #undef F_TIMO @@ -54,6 +55,7 @@ #define BeamOpCode(Op) ((Uint)BeamOp(Op)) + int term_to_Sint32(Eterm term, Sint *sp) { Sint val; @@ -374,15 +376,31 @@ BIF_RETTYPE hipe_bifs_ref_set_2(BIF_ALIST_2) } /* + * BIFs for loading code. + */ + +static HipeLoaderState *get_loader_state(Eterm term) +{ + ProcBin *pb; + + if (!ERTS_TERM_IS_MAGIC_BINARY(term)) return NULL; + + pb = (ProcBin*) binary_val(term); + return hipe_get_loader_state(pb->val); +} + + +/* * Allocate memory and copy machine code to it. */ -BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_enter_code_3(BIF_ALIST_3) { Uint nrbytes; void *bytes; void *address; Eterm trampolines; Eterm *hp; + HipeLoaderState *stp; #ifndef DEBUG ERTS_DECLARE_DUMMY(Uint bitoffs); ERTS_DECLARE_DUMMY(Uint bitsize); @@ -391,7 +409,8 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) Uint bitsize; #endif - if (is_not_binary(BIF_ARG_1)) + if (is_not_binary(BIF_ARG_1) || + (!(stp = get_loader_state(BIF_ARG_3)))) BIF_ERROR(BIF_P, BADARG); nrbytes = binary_size(BIF_ARG_1); ERTS_GET_BINARY_BYTES(BIF_ARG_1, bytes, bitoffs, bitsize); @@ -406,11 +425,15 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) nrcallees = arityval(tuple_val(BIF_ARG_2)[0]); else nrcallees = 0; + // XXX: Is there any reason to not just BIF_ERROR, so that the runtime + // survives? erts_exit(ERTS_ERROR_EXIT, "%s: failed to allocate %lu bytes and %lu trampolines\r\n", __func__, (unsigned long)nrbytes, (unsigned long)nrcallees); } memcpy(address, bytes, nrbytes); hipe_flush_icache_range(address, nrbytes); + stp->text_segment = address; + stp->text_segment_size = nrbytes; hp = HAlloc(BIF_P, 3); hp[0] = make_arityval(2); hp[1] = address_to_term(address, BIF_P); @@ -423,25 +446,34 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) /* * Allocate memory for arbitrary non-Erlang data. */ -BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_alloc_data_3(BIF_ALIST_3) { - Uint align, nrbytes; - void *block; + Uint align; + HipeLoaderState *stp; if (is_not_small(BIF_ARG_1) || is_not_small(BIF_ARG_2) || + (!(stp = get_loader_state(BIF_ARG_3))) || (align = unsigned_val(BIF_ARG_1), !IS_POWER_OF_TWO(align))) BIF_ERROR(BIF_P, BADARG); - nrbytes = unsigned_val(BIF_ARG_2); - if (nrbytes == 0) + + if (stp->data_segment_size || stp->data_segment) + BIF_ERROR(BIF_P, BADARG); + + stp->data_segment_size = unsigned_val(BIF_ARG_2); + if (stp->data_segment_size == 0) BIF_RET(make_small(0)); - block = erts_alloc(ERTS_ALC_T_HIPE, nrbytes); - if ((unsigned long)block & (align-1)) { - fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte aligned\r\n", - __FUNCTION__, (unsigned long)nrbytes, block, (unsigned long)align); - erts_free(ERTS_ALC_T_HIPE, block); + stp->data_segment = erts_alloc(ERTS_ALC_T_HIPE, stp->data_segment_size); + if ((unsigned long)stp->data_segment & (align-1)) { + fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte " + "aligned\r\n", + __FUNCTION__, (unsigned long)stp->data_segment_size, + stp->data_segment, (unsigned long)align); + erts_free(ERTS_ALC_T_HIPE, stp->data_segment); + stp->data_segment = NULL; + stp->data_segment_size = 0; BIF_ERROR(BIF_P, EXC_NOTSUP); } - BIF_RET(address_to_term(block, BIF_P)); + BIF_RET(address_to_term(stp->data_segment, BIF_P)); } /* @@ -537,13 +569,13 @@ BIF_RETTYPE hipe_bifs_merge_term_1(BIF_ALIST_1) BIF_RET(val); } -struct mfa_t { +struct hipe_mfa { Eterm mod; Eterm fun; Uint ari; }; -static int term_to_mfa(Eterm term, struct mfa_t *mfa) +static int term_to_mfa(Eterm term, struct hipe_mfa *mfa) { Eterm mod, fun, a; Uint ari; @@ -601,7 +633,7 @@ static Uint *hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity) Uint *hipe_bifs_find_pc_from_mfa(Eterm term) { - struct mfa_t mfa; + struct hipe_mfa mfa; if (!term_to_mfa(term, &mfa)) return NULL; @@ -616,12 +648,20 @@ BIF_RETTYPE hipe_bifs_fun_to_address_1(BIF_ALIST_1) BIF_RET(address_to_term(pc, BIF_P)); } +BIF_RETTYPE hipe_bifs_commit_patch_load_1(BIF_ALIST_1) +{ + if (!erts_commit_hipe_patch_load(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + BIF_RET(am_ok); +} + BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) { Eterm *pc; void *address; int is_closure; - struct mfa_t mfa; + struct hipe_mfa mfa; switch (BIF_ARG_3) { case am_false: @@ -644,28 +684,22 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) pc = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari); if (pc) { - hipe_mfa_save_orig_beam_op(mfa.mod, mfa.fun, mfa.ari, pc); -#if HIPE -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": planting call trap to %p at BEAM pc %p\r\n", address, pc); -#endif + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", pc, address); hipe_set_call_trap(pc, address, is_closure); BIF_RET(am_true); -#endif } -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": no BEAM pc found\r\n"); -#endif + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "failed set call trap to %p, no beam code found", address); BIF_RET(am_false); } -BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) +BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2) { - struct sdesc *sdesc; + struct hipe_sdesc *sdesc; + HipeLoaderState* stp; + + stp = get_loader_state(BIF_ARG_2); + if (!stp) + BIF_ERROR(BIF_P, BADARG); sdesc = hipe_decode_sdesc(BIF_ARG_1); if (!sdesc) { @@ -676,6 +710,13 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) fprintf(stderr, "%s: duplicate entry!\r\n", __FUNCTION__); BIF_ERROR(BIF_P, BADARG); } + + /* + * Link into list of sdesc's in same module instance + */ + sdesc->next_in_modi = stp->new_hipe_sdesc; + stp->new_hipe_sdesc = sdesc; + BIF_RET(NIL); } @@ -778,9 +819,6 @@ BIF_RETTYPE hipe_bifs_bif_address_3(BIF_ALIST_3) struct primop { HashBucket bucket; /* bucket.hvalue == atom_val(name) */ const void *address; -#if defined(__arm__) - void *trampoline; -#endif }; static struct primop primops[] = { @@ -839,29 +877,6 @@ static struct primop *primop_table_get(Eterm name) return hash_get(&primop_table, &tmpl); } -#if defined(__arm__) -static struct primop *primop_table_put(Eterm name) -{ - struct primop tmpl; - - init_primop_table(); - tmpl.bucket.hvalue = atom_val(name); - return hash_put(&primop_table, &tmpl); -} - -void *hipe_primop_get_trampoline(Eterm name) -{ - struct primop *primop = primop_table_get(name); - return primop ? primop->trampoline : NULL; -} - -void hipe_primop_set_trampoline(Eterm name, void *trampoline) -{ - struct primop *primop = primop_table_put(name); - primop->trampoline = trampoline; -} -#endif - /* * hipe_bifs_primop_address(Atom) -> address or false */ @@ -970,7 +985,7 @@ BIF_RETTYPE hipe_bifs_get_fe_2(BIF_ALIST_2) atom_buf[0] = '\0'; strncat(atom_buf, (char*)atom_tab(i)->name, atom_tab(i)->len); - printf("no fun entry for %s %ld:%ld\n", atom_buf, uniq, index); + printf("no fun entry for %s %ld:%ld\n", atom_buf, (unsigned long)uniq, (unsigned long)index); BIF_ERROR(BIF_P, BADARG); } BIF_RET(address_to_term((void *)fe, BIF_P)); @@ -997,30 +1012,35 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2) BIF_RET(am_true); } +struct hipe_ref_head { + struct hipe_ref_head* next; + struct hipe_ref_head* prev; +}; + /* - * MFA info hash table: + * An exported function called from or implemented by native code * - maps MFA to native code entry point - * - the MFAs it calls (refers_to) - * - the references to it (referred_from) + * - all references to it (callers) * - maps MFA to most recent trampoline [if powerpc or arm] */ struct hipe_mfa_info { + HashBucket mod2mfa; struct { unsigned long hvalue; struct hipe_mfa_info *next; } bucket; Eterm m; /* atom */ Eterm f; /* atom */ - unsigned int a; + unsigned int a : sizeof(int)*8 - 1; + unsigned int is_stub : 1; /* if beam or not (yet) loaded */ void *remote_address; - void *local_address; - Eterm *beam_code; - Uint orig_beam_op; - struct hipe_mfa_info_list *refers_to; - struct ref *referred_from; -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) - void *trampoline; + void *new_address; + struct hipe_ref_head callers; /* sentinel in list of hipe_ref's */ + struct hipe_mfa_info* next_in_mod; +#ifdef DEBUG + Export* dbg_export; #endif + }; static struct { @@ -1038,6 +1058,82 @@ static struct { erts_smp_rwmtx_t lock; } hipe_mfa_info_table; +Hash mod2mfa_tab; /* map from module atom to list of hipe_mfa_info */ + +static HashValue mod2mfa_hash(struct hipe_mfa_info* mfa) +{ + return mfa->mod2mfa.hvalue; +} + +static int mod2mfa_cmp(HashBucket* tmpl, struct hipe_mfa_info* mfa) +{ + return tmpl->hvalue != mfa->mod2mfa.hvalue; +} + +static struct hipe_mfa_info* mod2mfa_alloc(struct hipe_mfa_info* tmpl) +{ + return tmpl; /* hash_put always use mfa itself at template */ +} + +static void mod2mfa_free(struct hipe_mfa_info* mfa) +{ +} + +static void mod2mfa_tab_init(void) +{ + HashFunctions f; + static int init_done = 0; + + if (init_done) + return; + init_done = 1; + + f.hash = (H_FUN) mod2mfa_hash; + f.cmp = (HCMP_FUN) mod2mfa_cmp; + f.alloc = (HALLOC_FUN) mod2mfa_alloc; + f.free = (HFREE_FUN) mod2mfa_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; + + hash_init(ERTS_ALC_T_HIPE, &mod2mfa_tab, "mod2mfa_tab", 50, f); +} + +static struct hipe_mfa_info* mod2mfa_get(Module* modp) +{ + HashBucket tmpl; + tmpl.hvalue = modp->module; + return hash_get(&mod2mfa_tab, &tmpl); +} + +static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa) +{ + mfa->mod2mfa.hvalue = atom_val(mfa->m); + return hash_put(&mod2mfa_tab, mfa); +} + + + +/* + * An external native call site M:F(...) + * to be patched when the callee changes. + */ +struct hipe_ref { + struct hipe_ref_head head; /* list of refs to same calleee */ + void *address; +#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) + void *trampoline; +#endif + unsigned int flags; + struct hipe_ref* next_from_modi; /* list of refs from same module instance */ +#if defined(DEBUG) + struct hipe_mfa_info* callee; + Eterm caller_m, caller_f, caller_a; +#endif +}; +#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ + + static inline void hipe_mfa_info_table_init_lock(void) { erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock"); @@ -1063,6 +1159,16 @@ static inline void hipe_mfa_info_table_rwunlock(void) erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock); } +static ERTS_INLINE +struct hipe_mfa_info* mod2mfa_get_safe(Module* modp) +{ + struct hipe_mfa_info* mfa; + hipe_mfa_info_table_rlock(); + mfa = mod2mfa_get(modp); + hipe_mfa_info_table_runlock(); + return mfa; +} + #define HIPE_MFA_HASH(M,F,A) (atom_val(M) ^ atom_val(F) ^ (A)) static struct hipe_mfa_info **hipe_mfa_info_table_alloc_bucket(unsigned int size) @@ -1108,14 +1214,14 @@ static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigne res->m = m; res->f = f; res->a = arity; + res->is_stub = 0; res->remote_address = NULL; - res->local_address = NULL; - res->beam_code = NULL; - res->orig_beam_op = 0; - res->refers_to = NULL; - res->referred_from = NULL; -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) - res->trampoline = NULL; + res->new_address = NULL; + res->callers.next = &res->callers; + res->callers.prev = &res->callers; + res->next_in_mod = NULL; +#ifdef DEBUG + res->dbg_export = NULL; #endif return res; @@ -1133,6 +1239,8 @@ void hipe_mfa_info_table_init(void) hipe_mfa_info_table.bucket = hipe_mfa_info_table_alloc_bucket(size); hipe_mfa_info_table_init_lock(); + + mod2mfa_tab_init(); } static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eterm f, unsigned int arity) @@ -1154,21 +1262,12 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter return NULL; } -#if 0 /* XXX: unused */ -void *hipe_mfa_find_na(Eterm m, Eterm f, unsigned int arity) -{ - const struct hipe_mfa_info *p; - - p = hipe_mfa_info_table_get(m, f, arity); - return p ? p->address : NULL; -} -#endif - static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned int arity) { unsigned long h; unsigned int i; struct hipe_mfa_info *p; + struct hipe_mfa_info *first_in_mod; unsigned int size; h = HIPE_MFA_HASH(m, f, arity); @@ -1189,216 +1288,137 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, size = 1 << hipe_mfa_info_table.log2size; if (hipe_mfa_info_table.used > (4*size/5)) /* rehash at 80% */ hipe_mfa_info_table_grow(); - return p; -} -static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address, int is_exported) -{ - struct hipe_mfa_info *p; + first_in_mod = mod2mfa_put(p); + if (p != first_in_mod) { + p->next_in_mod = first_in_mod->next_in_mod; + first_in_mod->next_in_mod = p; + } + else { + p->next_in_mod = NULL; + } - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_put_rwlocked(m, f, arity); -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(m, f, arity); - printf(": changing address from %p to %p\r\n", p->local_address, address); -#endif - p->local_address = address; - if (is_exported) - p->remote_address = address; - hipe_mfa_info_table_rwunlock(); + DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p); + + return p; } -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) -void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int arity) +static void remove_mfa_info(struct hipe_mfa_info* rm) { + unsigned int i; struct hipe_mfa_info *p; - void *trampoline; - - hipe_mfa_info_table_rlock(); - p = hipe_mfa_info_table_get_locked(m, f, arity); - trampoline = p ? p->trampoline : NULL; - hipe_mfa_info_table_runlock(); - return trampoline; + struct hipe_mfa_info **prevp; + + i = rm->bucket.hvalue & hipe_mfa_info_table.mask; + prevp = &hipe_mfa_info_table.bucket[i]; + for (;;) { + p = *prevp; + ASSERT(p); + if (p == rm) { + *prevp = p->bucket.next; + ASSERT(hipe_mfa_info_table.used > 0); + hipe_mfa_info_table.used--; + return; + } + prevp = &p->bucket.next; + } } -void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int arity, void *trampoline) +static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address) { struct hipe_mfa_info *p; hipe_mfa_info_table_rwlock(); p = hipe_mfa_info_table_put_rwlocked(m, f, arity); - p->trampoline = trampoline; + DBG_TRACE_MFA(m,f,arity,"set native address in hipe_mfa_info at %p", p); + p->new_address = address; + hipe_mfa_info_table_rwunlock(); } -#endif BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) { - struct mfa_t mfa; + struct hipe_mfa mfa; void *address; - int is_exported; - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - address = term_to_address(BIF_ARG_2); - if (!address) - BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_3 == am_true) - is_exported = 1; - else if (BIF_ARG_3 == am_false) - is_exported = 0; - else + switch (BIF_ARG_3) { + case am_true: /* is_exported */ + if (!term_to_mfa(BIF_ARG_1, &mfa)) + BIF_ERROR(BIF_P, BADARG); + address = term_to_address(BIF_ARG_2); + if (!address) + BIF_ERROR(BIF_P, BADARG); + hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address); + break; + case am_false: + break; /* ignore local functions */ + default: BIF_ERROR(BIF_P, BADARG); - hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address, is_exported); - BIF_RET(NIL); -} - -BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1) -{ - Eterm lst; - struct mfa_t mfa; - struct hipe_mfa_info *p; - - hipe_mfa_info_table_rwlock(); - lst = BIF_ARG_1; - while (is_list(lst)) { - if (!term_to_mfa(CAR(list_val(lst)), &mfa)) - break; - lst = CDR(list_val(lst)); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) { - p->remote_address = NULL; - p->local_address = NULL; - if (p->beam_code) { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": removing call trap from BEAM pc %p (new op %#lx)\r\n", - p->beam_code, p->orig_beam_op); -#endif - p->beam_code[0] = p->orig_beam_op; - p->beam_code = NULL; - p->orig_beam_op = 0; - } else { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": no call trap to remove\r\n"); -#endif - } - } } - hipe_mfa_info_table_rwunlock(); - if (is_not_nil(lst)) - BIF_ERROR(BIF_P, BADARG); BIF_RET(NIL); } -void hipe_mfa_save_orig_beam_op(Eterm mod, Eterm fun, unsigned int ari, Eterm *pc) + +/* Ask if we need to block all threads + * while loading/deleting code for this module? + */ +int hipe_need_blocking(Module* modp) { - Uint orig_beam_op; struct hipe_mfa_info *p; - orig_beam_op = pc[0]; - if (orig_beam_op != BeamOpCode(op_hipe_trap_call_closure) && - orig_beam_op != BeamOpCode(op_hipe_trap_call)) { - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_put_rwlocked(mod, fun, ari); -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mod, fun, ari); - printf(": saving orig op %#lx from BEAM pc %p\r\n", orig_beam_op, pc); -#endif - p->beam_code = pc; - p->orig_beam_op = orig_beam_op; - hipe_mfa_info_table_rwunlock(); - } else { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mod, fun, ari); - printf(": orig op %#lx already saved\r\n", orig_beam_op); -#endif + /* Need to block if we have at least one native caller to this module + * or native code to make unaccessible. + */ + hipe_mfa_info_table_rlock(); + for (p = mod2mfa_get(modp); p; p = p->next_in_mod) { + ASSERT(!p->new_address); + if (p->callers.next != &p->callers || !p->is_stub) { + break; + } } + hipe_mfa_info_table_runlock(); + return (p != NULL); } -static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote) -{ - Export *export_entry; - void *StubAddress; - - ASSERT(is_remote); - - export_entry = erts_export_get_or_make_stub(m, f, arity); - StubAddress = hipe_make_native_stub(export_entry, arity); - if (!StubAddress) - erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); - return StubAddress; -} - -static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info **pp) +static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a) { struct hipe_mfa_info *p; - void *address; p = hipe_mfa_info_table_get_locked(m, f, a); - if (p) { - /* find address, predicting for a runtime apply call */ - address = p->remote_address; - if (!is_remote) - address = p->local_address; - if (address) - return address; - - /* bummer, install stub, checking if one already existed */ - address = p->remote_address; - if (address) - return address; - } - /* Caller must take the slow path with the write lock held, but allow - it to avoid some work if it already holds the write lock. */ - if (pp) - *pp = p; - return NULL; + return p ? p->remote_address : NULL; } -static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info *p) +static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a) { - void *address; + struct hipe_mfa_info *p = hipe_mfa_info_table_put_rwlocked(m, f, a); - if (!p) - p = hipe_mfa_info_table_put_rwlocked(m, f, a); - address = hipe_make_stub(m, f, a, is_remote); - /* XXX: how to tell if a BEAM MFA is exported or not? */ - p->remote_address = address; - return address; -} + if (!p->remote_address) { + Export* export_entry = erts_export_get_or_make_stub(m, f, a); + void* stubAddress = hipe_make_native_stub(export_entry, a); + if (!stubAddress) + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); -static void *hipe_get_na_nofail_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote) -{ - struct hipe_mfa_info *p; - void *address; - - address = hipe_get_na_try_locked(m, f, a, is_remote, &p); - if (address) - return address; - - address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, p); - return address; + p->remote_address = stubAddress; + p->is_stub = 1; +#ifdef DEBUG + p->dbg_export = export_entry; +#endif + } + return p->remote_address; } -static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a, int is_remote) +static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a) { void *address; hipe_mfa_info_table_rlock(); - address = hipe_get_na_try_locked(m, f, a, is_remote, NULL); + address = hipe_get_na_try_locked(m, f, a); hipe_mfa_info_table_runlock(); if (address) return address; hipe_mfa_info_table_rwlock(); - address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, NULL); + address = hipe_get_na_slow_rwlocked(m, f, a); hipe_mfa_info_table_rwunlock(); return address; } @@ -1408,7 +1428,7 @@ void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a) { if (is_not_atom(m) || is_not_atom(f) || a > 255) return NULL; - return hipe_get_na_nofail(m, f, a, 1); + return hipe_get_na_nofail(m, f, a); } /* primop, but called like a BIF for error handling purposes */ @@ -1420,25 +1440,19 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3) if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2)) BIF_ERROR(BIF_P, BADARG); arity = unsigned_val(BIF_ARG_3); /* no error check */ - address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity, 1); + address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity); BIF_RET((Eterm)address); /* semi-Ok */ } -BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_find_na_or_make_stub_1(BIF_ALIST_1) { - struct mfa_t mfa; + struct hipe_mfa mfa; void *address; - int is_remote; if (!term_to_mfa(BIF_ARG_1, &mfa)) BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_2 == am_true) - is_remote = 1; - else if (BIF_ARG_2 == am_false) - is_remote = 0; - else - BIF_ERROR(BIF_P, BADARG); - address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari, is_remote); + + address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari); BIF_RET(address_to_term(address, BIF_P)); } @@ -1460,7 +1474,7 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) f = ep->info.mfa.function; } else goto badfun; - address = hipe_get_na_nofail(m, f, BIF_ARG_2, 1); + address = hipe_get_na_nofail(m, f, BIF_ARG_2); BIF_RET((Eterm)address); badfun: @@ -1471,73 +1485,31 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a) { - struct hipe_mfa_info *mfa; - long mfa_offset, ra_offset; - struct hipe_mfa_info **bucket; - unsigned int i, nrbuckets; + const struct hipe_sdesc* sdesc = hipe_find_sdesc((unsigned long)ra); - /* Note about locking: the table is only updated from the - loader, which runs with the rest of the system suspended. */ - /* XXX: alas not true; see comment at hipe_mfa_info_table.lock */ - hipe_mfa_info_table_rlock(); - bucket = hipe_mfa_info_table.bucket; - nrbuckets = 1 << hipe_mfa_info_table.log2size; - mfa = NULL; - mfa_offset = LONG_MAX; - for (i = 0; i < nrbuckets; ++i) { - struct hipe_mfa_info *b = bucket[i]; - while (b != NULL) { - ra_offset = (char*)ra - (char*)b->local_address; - if (ra_offset > 0 && ra_offset < mfa_offset) { - mfa_offset = ra_offset; - mfa = b; - } - b = b->bucket.next; - } - } - if (mfa) { - *m = mfa->m; - *f = mfa->f; - *a = mfa->a; - } - hipe_mfa_info_table_runlock(); - return mfa ? 1 : 0; -} + if (!sdesc || sdesc->m_aix == atom_val(am_Empty)) + return 0; -/* - * Patch Reference Handling. - */ -struct hipe_mfa_info_list { - struct hipe_mfa_info *mfa; - struct hipe_mfa_info_list *next; -}; + *m = make_atom(sdesc->m_aix); + *f = make_atom(sdesc->f_aix); + *a = sdesc->a; + return 1; +} -struct ref { - struct hipe_mfa_info *caller_mfa; - void *address; - void *trampoline; - unsigned int flags; - struct ref *next; -}; -#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ -#define REF_FLAG_IS_REMOTE 2 /* bit 1: 0 == local, 1 == remote */ -#define REF_FLAG_PENDING_REDIRECT 4 /* bit 2: 1 == pending redirect */ -#define REF_FLAG_PENDING_REMOVE 8 /* bit 3: 1 == pending remove */ -/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,'remote'|'local'}) +/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,LoaderState}) */ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) { - struct mfa_t callee; + struct hipe_mfa callee; Eterm *tuple; - struct mfa_t caller; + struct hipe_mfa caller; void *address; void *trampoline; unsigned int flags; struct hipe_mfa_info *callee_mfa; - struct hipe_mfa_info *caller_mfa; - struct hipe_mfa_info_list *refers_to; - struct ref *ref; + struct hipe_ref *ref; + HipeLoaderState* stp; if (!term_to_mfa(BIF_ARG_1, &callee)) goto badarg; @@ -1568,63 +1540,90 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) if (!trampoline) goto badarg; } - switch (tuple[5]) { - case am_local: - break; - case am_remote: - flags |= REF_FLAG_IS_REMOTE; - break; - default: - goto badarg; - } + stp = get_loader_state(tuple[5]); + if (!stp) + goto badarg; + hipe_mfa_info_table_rwlock(); callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari); - caller_mfa = hipe_mfa_info_table_put_rwlocked(caller.mod, caller.fun, caller.ari); - refers_to = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*refers_to)); - refers_to->mfa = callee_mfa; - refers_to->next = caller_mfa->refers_to; - caller_mfa->refers_to = refers_to; - - ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*ref)); - ref->caller_mfa = caller_mfa; + ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref)); ref->address = address; +#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) ref->trampoline = trampoline; +#endif ref->flags = flags; - ref->next = callee_mfa->referred_from; - callee_mfa->referred_from = ref; + + /* + * Link into list of refs to same callee + */ + ASSERT(callee_mfa->callers.next->prev == &callee_mfa->callers); + ASSERT(callee_mfa->callers.prev->next == &callee_mfa->callers); + ref->head.next = callee_mfa->callers.next; + ref->head.prev = &callee_mfa->callers; + ref->head.next->prev = &ref->head; + ref->head.prev->next = &ref->head; + + /* + * Link into list of refs from same module instance + */ + ref->next_from_modi = stp->new_hipe_refs; + stp->new_hipe_refs = ref; + +#if defined(DEBUG) + ref->callee = callee_mfa; + ref->caller_m = caller.mod; + ref->caller_f = caller.fun; + ref->caller_a = caller.ari; +#endif hipe_mfa_info_table_rwunlock(); + DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p)", + ref, callee.mod, callee.fun, callee.ari, ref->address); + DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p)", + ref, caller.mod, caller.fun, caller.ari, ref->address); BIF_RET(NIL); badarg: BIF_ERROR(BIF_P, BADARG); } -/* Given a CalleeMFA, mark each ref to it as pending-redirect. - * This ensures that remove_refs_from() won't remove them: any - * removal is instead done at the end of redirect_referred_from(). - */ -BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */ + +static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me) { - struct mfa_t mfa; - const struct hipe_mfa_info *p; - struct ref *ref; + struct hipe_mfa_info* p; - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) - for (ref = p->referred_from; ref != NULL; ref = ref->next) - ref->flags |= REF_FLAG_PENDING_REDIRECT; - hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); + p = hash_get(&mod2mfa_tab, unlink_me); + ASSERT(p); + if (p == unlink_me) { + hash_erase(&mod2mfa_tab, p); + if (p->next_in_mod) + mod2mfa_put(p->next_in_mod); + } + else { + struct hipe_mfa_info** prevp; + + do { + prevp = &p->next_in_mod; + p = *prevp; + ASSERT(p && p->m == unlink_me->m); + } while (p != unlink_me); + + *prevp = p->next_in_mod; + } +} + +static void purge_mfa(struct hipe_mfa_info* p) +{ + ASSERT(p->is_stub); + remove_mfa_info(p); + hipe_free_native_stub(p->remote_address); + erts_free(ERTS_ALC_T_HIPE, p); } /* Called by init:restart after unloading all hipe compiled modules - * to work around bug causing execution of deallocated beam code. - * Can be removed when delete/purge of native modules works better. + * to work around old bug that caused execution of deallocated beam code. + * Can be removed now when delete/purge of native modules works better. * Test: Do init:restart in debug compiled vm with hipe compiled kernel. */ static void hipe_purge_all_refs(void) @@ -1634,126 +1633,249 @@ static void hipe_purge_all_refs(void) hipe_mfa_info_table_rwlock(); + ASSERT(hipe_mfa_info_table.used == 0); bucket = hipe_mfa_info_table.bucket; nrbuckets = 1 << hipe_mfa_info_table.log2size; for (i = 0; i < nrbuckets; ++i) { + ASSERT(bucket[i] == NULL); while (bucket[i] != NULL) { struct hipe_mfa_info* mfa = bucket[i]; bucket[i] = mfa->bucket.next; - while (mfa->refers_to) { - struct hipe_mfa_info_list *to = mfa->refers_to; - mfa->refers_to = to->next; - erts_free(ERTS_ALC_T_HIPE, to); - } - while (mfa->referred_from) { - struct ref* from = mfa->referred_from; - mfa->referred_from = from->next; - erts_free(ERTS_ALC_T_HIPE, from); - } - erts_free(ERTS_ALC_T_HIPE, mfa); + hash_erase(&mod2mfa_tab, mfa); + erts_free(ERTS_ALC_T_HIPE, mfa); } } + hipe_mfa_info_table.used = 0; hipe_mfa_info_table_rwunlock(); } BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) { - struct mfa_t mfa; - struct hipe_mfa_info *caller_mfa, *callee_mfa; - struct hipe_mfa_info_list *refers_to, *tmp_refers_to; - struct ref **prev, *ref; - if (BIF_ARG_1 == am_all) { hipe_purge_all_refs(); BIF_RET(am_ok); } - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - caller_mfa = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (caller_mfa) { - refers_to = caller_mfa->refers_to; - while (refers_to) { - callee_mfa = refers_to->mfa; - prev = &callee_mfa->referred_from; - ref = *prev; - while (ref) { - if (ref->caller_mfa == caller_mfa) { - if (ref->flags & REF_FLAG_PENDING_REDIRECT) { - ref->flags |= REF_FLAG_PENDING_REMOVE; - prev = &ref->next; - ref = ref->next; - } else { - struct ref *tmp = ref; - ref = ref->next; - *prev = ref; - erts_free(ERTS_ALC_T_HIPE, tmp); - } - } else { - prev = &ref->next; - ref = ref->next; - } - } - tmp_refers_to = refers_to; - refers_to = refers_to->next; - erts_free(ERTS_ALC_T_HIPE, tmp_refers_to); - } - caller_mfa->refers_to = NULL; + ASSERT(!"hipe_bifs_remove_refs_from_1() called"); + BIF_ERROR(BIF_P, BADARG); +} + +int hipe_purge_need_blocking(Module* modp) +{ + /* SVERK: Verify if this is really necessary */ + if (modp->old.hipe_code) { + if (modp->old.hipe_code->first_hipe_ref || + modp->old.hipe_code->first_hipe_sdesc) + return 1; + } + if (!modp->curr.code_hdr) { + return mod2mfa_get_safe(modp) != NULL; + } + return 0; +} + +void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module, + int is_blocking) +{ + struct hipe_ref* ref = first_ref; + + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + + while (ref) { + struct hipe_ref* free_ref = ref; + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, + ref->callee->m, ref->callee->f, ref->callee->a); + DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, + ref->caller_m, ref->caller_f, ref->caller_a); + ASSERT(ref->caller_m == caller_module); + + /* + * Unlink from other refs to same callee + */ + ASSERT(ref->head.next->prev == &ref->head); + ASSERT(ref->head.prev->next == &ref->head); + ASSERT(ref->head.next != &ref->head); + ASSERT(ref->head.prev != &ref->head); + ref->head.next->prev = ref->head.prev; + ref->head.prev->next = ref->head.next; + + /* + * Was this the last ref to that callee? + */ + if (ref->head.next == ref->head.prev) { + struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); + if (p->is_stub) { + if (!is_blocking) + hipe_mfa_info_table_rwlock(); + unlink_mfa_from_mod(p); + purge_mfa(p); + if (!is_blocking) + hipe_mfa_info_table_rwunlock(); + } + } + + ref = ref->next_from_modi; + erts_free(ERTS_ALC_T_HIPE, free_ref); + } +} + +void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module, + int is_blocking) +{ + struct hipe_sdesc* sdesc = first_sdesc; + + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + + ERTS_SMP_LC_ASSERT(is_blocking); /*XXX Fix safe sdesc destruction */ + + while (sdesc) { + struct hipe_sdesc* free_sdesc = sdesc; + + DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); + ASSERT(make_atom(sdesc->m_aix) == module); + + sdesc = sdesc->next_in_modi; + hipe_destruct_sdesc(free_sdesc); + } +} + + +void hipe_purge_module(Module* modp, int is_blocking) +{ + ASSERT(modp); + + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module"); + + if (modp->old.hipe_code) { + /* + * Remove all hipe_ref's (external calls) from the old module instance + */ + if (modp->old.hipe_code->first_hipe_ref) { + ERTS_SMP_LC_ASSERT(is_blocking); + + hipe_purge_refs(modp->old.hipe_code->first_hipe_ref, + make_atom(modp->module), is_blocking); + modp->old.hipe_code->first_hipe_ref = NULL; + } + + /* + * Remove all hipe_sdesc's for the old module instance + */ + if (modp->old.hipe_code->first_hipe_sdesc) { + ERTS_SMP_LC_ASSERT(is_blocking); + + hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc, + make_atom(modp->module), is_blocking); + modp->old.hipe_code->first_hipe_sdesc = NULL; + } + + hipe_free_module(modp->old.hipe_code); + modp->old.hipe_code = NULL; + } + + + /* + * Remove unreferred hipe_mfa_info's + * when all module instances are removed (like in init:restart) + */ + if (is_blocking && modp->curr.code_hdr == NULL) { + struct hipe_mfa_info* was_first = mod2mfa_get(modp); + struct hipe_mfa_info* is_first = was_first; + struct hipe_mfa_info** prevp = &is_first; + struct hipe_mfa_info *p; + + if (was_first) { + for (p = was_first ; p; p = *prevp) { + if (p->callers.next == &p->callers) { + *prevp = p->next_in_mod; + if (p != was_first) + purge_mfa(p); + } + else + prevp = &p->next_in_mod; + } + if (was_first != is_first) { + hash_erase(&mod2mfa_tab, was_first); + purge_mfa(was_first); + if (is_first) + mod2mfa_put(is_first); + } + } } - hipe_mfa_info_table_rwunlock(); - BIF_RET(am_ok); } -/* redirect_referred_from(CalleeMFA) - * Redirect all pending-redirect refs in CalleeMFA's referred_from. - * Then remove any pending-redirect && pending-remove refs from CalleeMFA's referred_from. +/* + * Redirect all existing native calls to this module */ -BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1) +void hipe_redirect_to_module(Module* modp) { - struct mfa_t mfa; struct hipe_mfa_info *p; - struct ref **prev, *ref; - int is_remote, res; - void *new_address; + struct hipe_ref_head* refh; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + for (p = mod2mfa_get(modp); p; p = p->next_in_mod) { + if (p->new_address) { + if (p->is_stub) { + hipe_free_native_stub(p->remote_address); + p->is_stub = 0; + } + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit new_address %p", p->new_address); + p->remote_address = p->new_address; + p->new_address = NULL; +#ifdef DEBUG + p->dbg_export = NULL; +#endif + } + else if (!p->is_stub) { + Export* exp = erts_export_get_or_make_stub(p->m, p->f, p->a); + p->remote_address = hipe_make_native_stub(exp, p->a); + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit stub %p", p->remote_address); + if (!p->remote_address) + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); + p->is_stub = 1; +#ifdef DEBUG + p->dbg_export = exp; +#endif + } + else { + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit no-op, already stub"); + ASSERT(p->remote_address && p->dbg_export); + } - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) { - prev = &p->referred_from; - ref = *prev; - while (ref) { - if (ref->flags & REF_FLAG_PENDING_REDIRECT) { - is_remote = ref->flags & REF_FLAG_IS_REMOTE; - new_address = hipe_get_na_nofail_rwlocked(p->m, p->f, p->a, is_remote); - if (ref->flags & REF_FLAG_IS_LOAD_MFA) - res = hipe_patch_insn(ref->address, (Uint)new_address, am_load_mfa); - else - res = hipe_patch_call(ref->address, new_address, ref->trampoline); - if (res) - fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__); - ref->flags &= ~REF_FLAG_PENDING_REDIRECT; - if (ref->flags & REF_FLAG_PENDING_REMOVE) { - struct ref *tmp = ref; - ref = ref->next; - *prev = ref; - erts_free(ERTS_ALC_T_HIPE, tmp); - } else { - prev = &ref->next; - ref = ref->next; - } - } else { - prev = &ref->next; - ref = ref->next; - } + DBG_TRACE_MFA(p->m,p->f,p->a,"START REDIRECT towards hipe_mfa_info at %p", p); + for (refh = p->callers.next; refh != &p->callers; refh = refh->next) { + struct hipe_ref* ref = (struct hipe_ref*) refh; + int res; + + DBG_TRACE_MFA(p->m,p->f,p->a, " REDIRECT ref at %p FROM %T:%T/%u (%p -> %p)", + ref, ref->caller_m, ref->caller_f, ref->caller_a, + ref->address, p->remote_address); + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, + " REDIRECT ref at %p TO %T:%T/%u (%p -> %p)", + ref, p->m,p->f,p->a, ref->address, p->remote_address); + + if (ref->flags & REF_FLAG_IS_LOAD_MFA) + res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa); + else { +#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) + void* trampoline = ref->trampoline; +#else + void* trampoline = NULL; +#endif + res = hipe_patch_call(ref->address, p->remote_address, trampoline); + } + if (res) + fprintf(stderr, "%s: patch failed", __FUNCTION__); } + DBG_TRACE_MFA(p->m,p->f,p->a,"DONE REDIRECT towards hipe_mfa_info at %p", p); } - hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); } BIF_RETTYPE hipe_bifs_check_crc_1(BIF_ALIST_1) @@ -1807,89 +1929,6 @@ void hipe_patch_address(Uint *address, Eterm patchtype, Uint value) } } -struct modinfo { - HashBucket bucket; /* bucket.hvalue == atom_val(the module name) */ - unsigned int code_size; -}; - -static Hash modinfo_table; - -static HashValue modinfo_hash(void *tmpl) -{ - Eterm mod = (Eterm)tmpl; - return atom_val(mod); -} - -static int modinfo_cmp(void *tmpl, void *bucket) -{ - /* bucket->hvalue == modinfo_hash(tmpl), so just return 0 (match) */ - return 0; -} - -static void *modinfo_alloc(void *tmpl) -{ - struct modinfo *p; - - p = (struct modinfo*)erts_alloc(ERTS_ALC_T_HIPE, sizeof(*p)); - p->code_size = 0; - return &p->bucket; -} - -static void init_modinfo_table(void) -{ - HashFunctions f; - static int init_done = 0; - - if (init_done) - return; - init_done = 1; - f.hash = (H_FUN) modinfo_hash; - f.cmp = (HCMP_FUN) modinfo_cmp; - f.alloc = (HALLOC_FUN) modinfo_alloc; - f.free = (HFREE_FUN) NULL; - f.meta_alloc = (HMALLOC_FUN) erts_alloc; - f.meta_free = (HMFREE_FUN) erts_free; - f.meta_print = (HMPRINT_FUN) erts_print; - hash_init(ERTS_ALC_T_HIPE, &modinfo_table, "modinfo_table", 11, f); -} - -BIF_RETTYPE hipe_bifs_update_code_size_3(BIF_ALIST_3) -{ - struct modinfo *p; - Sint code_size; - - init_modinfo_table(); - - if (is_not_atom(BIF_ARG_1) || - is_not_small(BIF_ARG_3) || - (code_size = signed_val(BIF_ARG_3)) < 0) - BIF_ERROR(BIF_P, BADARG); - - p = (struct modinfo*)hash_put(&modinfo_table, (void*)BIF_ARG_1); - - if (is_nil(BIF_ARG_2)) /* some MFAs, not whole module */ - p->code_size += code_size; - else /* whole module */ - p->code_size = code_size; - BIF_RET(NIL); -} - -BIF_RETTYPE hipe_bifs_code_size_1(BIF_ALIST_1) -{ - struct modinfo *p; - unsigned int code_size; - - init_modinfo_table(); - - if (is_not_atom(BIF_ARG_1)) - BIF_ERROR(BIF_P, BADARG); - - p = (struct modinfo*)hash_get(&modinfo_table, (void*)BIF_ARG_1); - - code_size = p ? p->code_size : 0; - BIF_RET(make_small(code_size)); -} - BIF_RETTYPE hipe_bifs_patch_insn_3(BIF_ALIST_3) { Uint *address, value; @@ -1925,3 +1964,23 @@ BIF_RETTYPE hipe_bifs_patch_call_3(BIF_ALIST_3) BIF_ERROR(BIF_P, BADARG); BIF_RET(NIL); } + +BIF_RETTYPE hipe_bifs_alloc_loader_state_1(BIF_ALIST_1) +{ + Binary *magic; + Eterm *hp; + Eterm res; + + if (is_not_atom(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + magic = hipe_alloc_loader_state(BIF_ARG_1); + + if (!magic) + BIF_ERROR(BIF_P, BADARG); + + hp = HAlloc(BIF_P, PROC_BIN_SIZE); + res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic); + erts_refc_dec(&magic->refc, 1); + BIF_RET(res); +} diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index c9a8216368..4a59bacc6e 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -32,17 +32,14 @@ extern void hipe_mfa_info_table_init(void); extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a); extern BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3); extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a); -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) -extern void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int a); -extern void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int a, void *trampoline); -#endif -#if defined(__arm__) -extern void *hipe_primop_get_trampoline(Eterm name); -extern void hipe_primop_set_trampoline(Eterm name, void *trampoline); -#endif /* needed in beam_load.c */ -void hipe_mfa_save_orig_beam_op(Eterm m, Eterm f, unsigned int a, Eterm *pc); +int hipe_need_blocking(Module*); +int hipe_purge_need_blocking(Module*); +void hipe_purge_refs(struct hipe_ref*, Eterm, int is_blocking); +void hipe_purge_sdescs(struct hipe_sdesc*, Eterm, int is_blocking); +void hipe_purge_module(Module*, int is_blocking); +void hipe_redirect_to_module(Module* modp); /* these are also needed in hipe_amd64.c */ extern void *term_to_address(Eterm); diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index 99237aae05..264ea2c34a 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -44,22 +44,20 @@ bif hipe_bifs:ref/1 bif hipe_bifs:ref_get/1 bif hipe_bifs:ref_set/2 -bif hipe_bifs:enter_code/2 -bif hipe_bifs:alloc_data/2 +bif hipe_bifs:enter_code/3 +bif hipe_bifs:alloc_data/3 bif hipe_bifs:constants_size/0 bif hipe_bifs:merge_term/1 bif hipe_bifs:fun_to_address/1 +bif hipe_bifs:commit_patch_load/1 bif hipe_bifs:set_native_address/3 #bif hipe_bifs:address_to_fun/1 bif hipe_bifs:set_funinfo_native_address/3 -bif hipe_bifs:invalidate_funinfo_native_addresses/1 +#bif hipe_bifs:invalidate_funinfo_native_addresses/1 -bif hipe_bifs:update_code_size/3 -bif hipe_bifs:code_size/1 - -bif hipe_bifs:enter_sdesc/1 +bif hipe_bifs:enter_sdesc/2 bif hipe_bifs:bif_address/3 bif hipe_bifs:primop_address/1 @@ -72,7 +70,7 @@ bif hipe_bifs:term_to_word/1 bif hipe_bifs:get_fe/2 bif hipe_bifs:set_native_address_in_fe/2 -bif hipe_bifs:find_na_or_make_stub/2 +bif hipe_bifs:find_na_or_make_stub/1 bif hipe_bifs:check_crc/1 bif hipe_bifs:system_crc/0 @@ -84,9 +82,9 @@ bif hipe_bifs:patch_insn/3 bif hipe_bifs:patch_call/3 bif hipe_bifs:add_ref/2 -bif hipe_bifs:mark_referred_from/1 bif hipe_bifs:remove_refs_from/1 -bif hipe_bifs:redirect_referred_from/1 + +bif hipe_bifs:alloc_loader_state/1 # atoms used by add_ref/2 atom call diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index 68c65dea27..e6ce7ce628 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -38,7 +38,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int sdesc_size; unsigned long ra; unsigned int i; @@ -123,7 +123,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int sdesc_size; unsigned long ra; unsigned int i; @@ -244,7 +244,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area, /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; /* arch-specific nstack walk state */ struct nstack_walk_state walk_state; @@ -311,7 +311,7 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size) { Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; /* arch-specific nstack walk state */ struct nstack_walk_state walk_state; @@ -356,3 +356,36 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size) } return 0; } + +int +nstack_any_cps_in_segment(Process *p, char* seg_start, Uint seg_size) +{ + Eterm *nsp; + Eterm *nsp_end; + const struct hipe_sdesc *sdesc; + /* arch-specific nstack walk state */ + struct nstack_walk_state walk_state; + + if (!p->hipe.nstack || !nstack_walk_init_check(p)) + return 0; + ASSERT(p->hipe.nsp && p->hipe.nstend); + nsp = nstack_walk_nsp_begin(p); + nsp_end = nstack_walk_nsp_end(p); + sdesc = nstack_walk_init_sdesc_ignore_trap(p, &walk_state); + + /* Check the topmost frame */ + if (ErtsInArea(sdesc->bucket.hvalue, seg_start, seg_size)) + return 1; + + while (!nstack_walk_nsp_reached_end(nsp, nsp_end)) { + unsigned sdesc_size = nstack_walk_frame_size(sdesc); + unsigned long ra = nstack_walk_frame_ra(nsp, sdesc); + if (ra == (unsigned long)nbif_stack_trap_ra) + ra = (unsigned long)p->hipe.ngra; + if (ErtsInArea(ra, seg_start, seg_size)) + return 1; + sdesc = hipe_find_sdesc(ra); + nsp = nstack_walk_next_frame(nsp, sdesc_size); + } + return 0; +} diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c new file mode 100644 index 0000000000..2998ed87a2 --- /dev/null +++ b/erts/emulator/hipe/hipe_load.c @@ -0,0 +1,105 @@ +/* + * %CopyrightBegin% + + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_load.c + * + * HiPE atomic code loader + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "sys.h" +#include "global.h" +#include "erl_binary.h" +#include "hipe_load.h" +#include "hipe_bif0.h" + +void hipe_free_loader_state(HipeLoaderState *stp) +{ + if (stp->module == NIL) return; + + // TODO: Needs to be freed separately. We'd like have a unified executable + // code allocator, so postpone this for now. + /* if (stp->text_segment) */ + /* erts_free(ERTS_ALC_T_HIPE, stp->text_segment); */ + stp->text_segment = NULL; + stp->text_segment_size = 0; + + if (stp->data_segment) + erts_free(ERTS_ALC_T_HIPE, stp->data_segment); + stp->data_segment = NULL; + stp->data_segment_size = 0; + + if (stp->new_hipe_refs) { + hipe_purge_refs(stp->new_hipe_refs, stp->module, 0); + stp->new_hipe_refs = NULL; + } + if (stp->new_hipe_sdesc) { + hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module, 0); + stp->new_hipe_sdesc = NULL; + } + + stp->module = NIL; +} + +static void +hipe_loader_state_dtor(Binary* magic) +{ + HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(magic) == hipe_loader_state_dtor); + + hipe_free_loader_state(stp); +} + +Binary *hipe_alloc_loader_state(Eterm module) +{ + HipeLoaderState *stp; + Binary *magic; + + if (is_not_atom(module)) return NULL; + + magic = erts_create_magic_binary(sizeof(HipeLoaderState), + hipe_loader_state_dtor); + erts_refc_inc(&magic->refc, 1); + stp = ERTS_MAGIC_BIN_DATA(magic); + + stp->module = module; + stp->text_segment = NULL; + stp->text_segment_size = 0; + stp->data_segment = NULL; + stp->data_segment_size = 0; + + stp->new_hipe_refs = NULL; + stp->new_hipe_sdesc = NULL; + + return magic; +} + +HipeLoaderState * +hipe_get_loader_state(Binary *magic) +{ + if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor) + return NULL; + + return (HipeLoaderState*) ERTS_MAGIC_BIN_DATA(magic); +} diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h new file mode 100644 index 0000000000..40c8a8aa2a --- /dev/null +++ b/erts/emulator/hipe/hipe_load.h @@ -0,0 +1,48 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_load.h + * + * HiPE atomic code loader + */ +#ifndef HIPE_LOAD_H +#define HIPE_LOAD_H + +#include "global.h" + +typedef struct hipe_loader_state { + Eterm module; /* Module name, atom */ + + void *text_segment; + Uint text_segment_size; + + void *data_segment; + Uint data_segment_size; + + struct hipe_ref* new_hipe_refs; + struct hipe_sdesc* new_hipe_sdesc; + +} HipeLoaderState; + +extern Binary *hipe_alloc_loader_state(Eterm module); +extern void hipe_free_loader_state(HipeLoaderState*); +extern HipeLoaderState *hipe_get_loader_state(Binary *binary); + +#endif /* HIPE_LOAD_H */ diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index b9d4226705..4573980e1e 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -435,9 +435,6 @@ static const struct rts_param rts_params[] = { presence or absence of struct erl_fun_thing's "next" field. */ { 5, "EFT_CREATOR", 1, offsetof(struct erl_fun_thing, creator) }, { 6, "EFT_FE", 1, offsetof(struct erl_fun_thing, fe) }, -#ifdef HIPE - { 7, "EFT_NATIVE_ADDRESS", 1, offsetof(struct erl_fun_thing, native_address) }, -#endif { 8, "EFT_ARITY", 1, offsetof(struct erl_fun_thing, arity) }, { 9, "EFT_NUM_FREE", 1, offsetof(struct erl_fun_thing, num_free) }, { 10, "EFT_ENV", 1, offsetof(struct erl_fun_thing, env[0]) }, diff --git a/erts/emulator/hipe/hipe_module.c b/erts/emulator/hipe/hipe_module.c new file mode 100644 index 0000000000..469f077dd2 --- /dev/null +++ b/erts/emulator/hipe/hipe_module.c @@ -0,0 +1,35 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "sys.h" +#include "hipe_arch.h" +#include "hipe_module.h" + +void hipe_free_module(HipeModule *mod) +{ + hipe_free_code(mod->text_segment, mod->text_segment_size); + if (mod->data_segment) /* Some modules lack data segments */ + erts_free(ERTS_ALC_T_HIPE, mod->data_segment); + + erts_free(ERTS_ALC_T_HIPE, mod); +} diff --git a/erts/emulator/hipe/hipe_module.h b/erts/emulator/hipe/hipe_module.h new file mode 100644 index 0000000000..b489f567cb --- /dev/null +++ b/erts/emulator/hipe/hipe_module.h @@ -0,0 +1,45 @@ +/* + * %CopyrightBegin% + + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_module.h + * + * + */ +#ifndef HIPE_MODULE_H +#define HIPE_MODULE_H + +/* Forward-declare type to resolve circular dependency with module.h */ +typedef struct hipe_module HipeModule; + +#include "global.h" + +struct hipe_module { + void *text_segment; + Uint text_segment_size; + void *data_segment; + + struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */ + struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */ +}; + +extern void hipe_free_module(HipeModule *mod); + +#endif /* HIPE_MODULE_H */ diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c index 9b2048c457..4413748936 100644 --- a/erts/emulator/hipe/hipe_ppc.c +++ b/erts/emulator/hipe/hipe_ppc.c @@ -25,7 +25,6 @@ #endif #include "global.h" #include "erl_binary.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -68,34 +67,6 @@ void hipe_flush_icache_range(void *address, unsigned int nbytes) asm volatile("sync\n\tisync"); } -/* - * Management of 32MB code segments for regular code and trampolines. - */ - -#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */ - -static struct segment { - unsigned int *base; /* [base,base+32MB[ */ - unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */ - unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */ -} curseg; - -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) - -/* Darwin breakage */ -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif - -static void *new_code_mapping(void) -{ - return mmap(0, SEGMENT_NRBYTES, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, - -1, 0); -} - static int check_callees(Eterm callees) { Eterm *tuple; @@ -119,126 +90,71 @@ static int check_callees(Eterm callees) return arity; } -static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec) + +static void generate_trampolines(Uint32* address, + int nrcallees, Eterm callees, + Uint32** trampvec) { - unsigned int *base, *address, *tramp_pos, nrfreewords; - int trampnr; + Uint32* trampoline = address; + int i; - tramp_pos = curseg.tramp_pos; - address = curseg.code_pos; - nrfreewords = tramp_pos - address; - if (nrwords > nrfreewords) - return NULL; - curseg.code_pos = address + nrwords; - nrfreewords -= nrwords; - - base = curseg.base; - for (trampnr = 1; trampnr <= nrcallees; ++trampnr) { - Eterm mfa = tuple_val(callees)[trampnr]; - Eterm m = tuple_val(mfa)[1]; - Eterm f = tuple_val(mfa)[2]; - unsigned int a = unsigned_val(tuple_val(mfa)[3]); - unsigned int *trampoline = hipe_mfa_get_trampoline(m, f, a); - if (!in_area(trampoline, base, SEGMENT_NRBYTES)) { + for (i = 0; i < nrcallees; ++i) { #if defined(__powerpc64__) - if (nrfreewords < 7) - return NULL; - nrfreewords -= 7; - tramp_pos = trampoline = tramp_pos - 7; - trampoline[0] = 0x3D600000; /* addis r11,r0,0 */ - trampoline[1] = 0x616B0000; /* ori r11,r11,0 */ - trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */ - trampoline[3] = 0x656B0000; /* oris r11,r11,0 */ - trampoline[4] = 0x616B0000; /* ori r11,r11,0 */ - trampoline[5] = 0x7D6903A6; /* mtctr r11 */ - trampoline[6] = 0x4E800420; /* bctr */ - hipe_flush_icache_range(trampoline, 7*sizeof(int)); +# define TRAMPOLINE_WORDS 7 + trampoline[0] = 0x3D600000; /* addis r11,r0,0 */ + trampoline[1] = 0x616B0000; /* ori r11,r11,0 */ + trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */ + trampoline[3] = 0x656B0000; /* oris r11,r11,0 */ + trampoline[4] = 0x616B0000; /* ori r11,r11,0 */ + trampoline[5] = 0x7D6903A6; /* mtctr r11 */ + trampoline[6] = 0x4E800420; /* bctr */ #else - if (nrfreewords < 4) - return NULL; - nrfreewords -= 4; - tramp_pos = trampoline = tramp_pos - 4; - trampoline[0] = 0x39600000; /* addi r11,r0,0 */ - trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */ - trampoline[2] = 0x7D6903A6; /* mtctr r11 */ - trampoline[3] = 0x4E800420; /* bctr */ - hipe_flush_icache_range(trampoline, 4*sizeof(int)); +# define TRAMPOLINE_WORDS 4 + trampoline[0] = 0x39600000; /* addi r11,r0,0 */ + trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */ + trampoline[2] = 0x7D6903A6; /* mtctr r11 */ + trampoline[3] = 0x4E800420; /* bctr */ #endif - hipe_mfa_set_trampoline(m, f, a, trampoline); - } - trampvec[trampnr-1] = trampoline; + trampvec[i] = trampoline; + trampoline += TRAMPOLINE_WORDS; } - curseg.tramp_pos = tramp_pos; - return address; + hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_WORDS*sizeof(Uint32)); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) { - Uint nrwords; + Uint code_words; int nrcallees; Eterm trampvecbin; - unsigned int **trampvec; - unsigned int *address; - unsigned int *base; - struct segment oldseg; + Uint32 **trampvec; + Uint32 *address; if (nrbytes & 0x3) return NULL; - nrwords = nrbytes >> 2; + code_words = nrbytes / sizeof(Uint32); nrcallees = check_callees(callees); if (nrcallees < 0) return NULL; - trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*)); - trampvec = (unsigned int**)binary_bytes(trampvecbin); - - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } + trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*)); + trampvec = (Uint32**)binary_bytes(trampvecbin); + + address = erts_alloc(ERTS_ALC_T_HIPE_EXEC, + (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32)); + + generate_trampolines(address + code_words, nrcallees, callees, trampvec); *trampolines = trampvecbin; return address; } -static unsigned int *alloc_stub(Uint nrwords) +void hipe_free_code(void* code, unsigned int bytes) { - unsigned int *address; - unsigned int *base; - struct segment oldseg; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } - return address; + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); } static void patch_imm16(Uint32 *address, unsigned int imm16) @@ -288,12 +204,12 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { - unsigned int *code; + Uint32 *code; if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL) abort(); - code = alloc_stub(7); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 7*sizeof(Uint32)); if (!code) return NULL; @@ -312,7 +228,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) /* ba nbif_callemu */ code[6] = 0x48000002 | (unsigned long)&nbif_callemu; - hipe_flush_icache_range(code, 7*sizeof(int)); + hipe_flush_icache_range(code, 7*sizeof(Uint32)); return code; } @@ -360,7 +276,7 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { - unsigned int *code; + Uint32 *code; /* * Native code calls BEAM via a stub looking as follows: @@ -383,7 +299,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL) abort(); - code = alloc_stub(4); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 4*sizeof(Uint32)); if (!code) return NULL; @@ -396,7 +312,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) /* ba nbif_callemu */ code[3] = 0x48000002 | (unsigned long)&nbif_callemu; - hipe_flush_icache_range(code, 4*sizeof(int)); + hipe_flush_icache_range(code, 4*sizeof(Uint32)); return code; } diff --git a/erts/emulator/hipe/hipe_risc_gc.h b/erts/emulator/hipe/hipe_risc_gc.h index 09568c140e..f019434f67 100644 --- a/erts/emulator/hipe/hipe_risc_gc.h +++ b/erts/emulator/hipe/hipe_risc_gc.h @@ -27,7 +27,7 @@ /* arch wrapper includes hipe_${arch}_asm.h to define NR_ARG_REGS */ struct nstack_walk_state { - const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ + const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ }; static inline int nstack_walk_init_check(const Process *p) @@ -43,20 +43,20 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p) return p->hipe.nsp + nstkarity; } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) { - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra); state->sdesc0 = sdesc; return sdesc; } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc_ignore_trap(const Process *p, struct nstack_walk_state *state) { unsigned long ra = (unsigned long)p->hipe.nra; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; if (ra == (unsigned long)&nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; sdesc = hipe_find_sdesc(ra); @@ -64,7 +64,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p, return sdesc; } -static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0) +static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0) { Eterm *nsp = p->hipe.nsp; p->hipe.nsp = nstack_walk_nsp_begin(p); @@ -103,7 +103,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp return nsp >= nsp_end; } -static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc) +static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc) { return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc); } @@ -114,7 +114,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i) } static inline unsigned long -nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc) +nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc) { return nsp[sdesc_fsize(sdesc)]; } diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h index 0284265307..1369b392fe 100644 --- a/erts/emulator/hipe/hipe_risc_glue.h +++ b/erts/emulator/hipe/hipe_risc_glue.h @@ -66,13 +66,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct sdesc_with_exnra nbif_return_sdesc = { - .exnra = (unsigned long)&nbif_fail, - .sdesc = { - .bucket = { .hvalue = (unsigned long)&nbif_return }, - .summary = (1<<8), - }, - }; + static struct hipe_sdesc_with_exnra nbif_return_sdesc; + + nbif_return_sdesc.exnra = (unsigned long)nbif_fail; + nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return; + nbif_return_sdesc.sdesc.fsize = 0; + nbif_return_sdesc.sdesc.has_exnra = 1; + nbif_return_sdesc.sdesc.stk_nargs = 0; + nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty); + nbif_return_sdesc.sdesc.f_aix = atom_val(am_return); + nbif_return_sdesc.sdesc.a = 0; + hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); } diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c index dc98c96b8f..4001bedeb6 100644 --- a/erts/emulator/hipe/hipe_risc_stack.c +++ b/erts/emulator/hipe/hipe_risc_stack.c @@ -56,8 +56,8 @@ void hipe_print_nstack(Process *p) { Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc1; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc1; + const struct hipe_sdesc *sdesc; unsigned long ra; unsigned long exnra; unsigned int mask; @@ -175,7 +175,7 @@ void hipe_print_nstack(Process *p) #define MINSTACK 128 #define NSKIPFRAMES 4 -void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) +void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc) { Eterm *nsp; Eterm *nsp_end; @@ -216,7 +216,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) void (*hipe_handle_stack_trap(Process *p))(void) { void (*ngra)(void) = p->hipe.ngra; - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); hipe_update_stack_trap(p, sdesc); return ngra; } @@ -237,7 +237,7 @@ void hipe_find_handler(Process *p) unsigned long ra; unsigned long exnra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; nsp = p->hipe.nsp; nsp_end = p->hipe.nstend; @@ -277,7 +277,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) Eterm *nsp_end; unsigned long ra, prev_ra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; int i; if (depth < 1) @@ -292,7 +292,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) ra = (unsigned long)p->hipe.nra; prev_ra = 0; i = 0; - for (;;) { + while (nsp < nsp_end) { if (ra == (unsigned long)nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; if (ra != prev_ra) { @@ -302,8 +302,6 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) break; prev_ra = ra; } - if (nsp >= nsp_end) - break; sdesc = hipe_find_sdesc(ra); nsp += arity + sdesc_fsize(sdesc); arity = sdesc_arity(sdesc); diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c index 23020f34ee..876b20bb15 100644 --- a/erts/emulator/hipe/hipe_sparc.c +++ b/erts/emulator/hipe/hipe_sparc.c @@ -24,7 +24,6 @@ #include "config.h" #endif #include "global.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -88,8 +87,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Uint32 relDest, newI; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + relDest = (Uint32)((Sint32)destAddress - (Sint32)callAddress); newI = (1 << 30) | (relDest >> 2); *(Uint32*)callAddress = newI; @@ -97,105 +96,9 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -/* - * Memory allocator for executable code. - * - * This is required on x86 because some combinations - * of Linux kernels and CPU generations default to - * non-executable memory mappings, causing ordinary - * malloc() memory to be non-executable. - */ -static unsigned int code_bytes; -static char *code_next; - -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - -static int morecore(unsigned int alloc_bytes) -{ - unsigned int map_bytes; - char *map_hint, *map_start; - - /* Page-align the amount to allocate. */ - map_bytes = (alloc_bytes + 4095) & ~4095; - - /* Round up small allocations. */ - if (map_bytes < 1024*1024) - map_bytes = 1024*1024; - else - ALLOC_CODE_STATS(++nr_large); - - /* Create a new memory mapping, ensuring it is executable - and in the low 2GB of the address space. Also attempt - to make it adjacent to the previous mapping. */ - map_hint = code_next + code_bytes; - if ((unsigned long)map_hint & 4095) - abort(); - map_start = mmap(map_hint, map_bytes, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS -#ifdef __x86_64__ - |MAP_32BIT -#endif - , - -1, 0); - if (map_start == MAP_FAILED) - return -1; - - ALLOC_CODE_STATS(total_mapped += map_bytes); - - /* Merge adjacent mappings, so the trailing portion of the previous - mapping isn't lost. In practice this is quite successful. */ - if (map_start == map_hint) { - ALLOC_CODE_STATS(++nr_joins); - code_bytes += map_bytes; - } else { - ALLOC_CODE_STATS(++nr_splits); - ALLOC_CODE_STATS(total_lost += code_bytes); - code_next = map_start; - code_bytes = map_bytes; - } - - ALLOC_CODE_STATS(atexit_alloc_code_stats()); - - return 0; -} - static void *alloc_code(unsigned int alloc_bytes) { - void *res; - - /* Align function entries. */ - alloc_bytes = (alloc_bytes + 3) & ~3; - - if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0) - return NULL; - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - res = code_next; - code_next += alloc_bytes; - code_bytes -= alloc_bytes; - return res; + return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) @@ -206,6 +109,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int nrbytes) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { unsigned int *code; @@ -235,6 +143,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c index e2e6eb74b1..b80e44bc37 100644 --- a/erts/emulator/hipe/hipe_stack.c +++ b/erts/emulator/hipe/hipe_stack.c @@ -43,10 +43,10 @@ */ struct hipe_sdesc_table hipe_sdesc_table; -static struct sdesc **alloc_bucket(unsigned int size) +static struct hipe_sdesc **alloc_bucket(unsigned int size) { - unsigned long nbytes = size * sizeof(struct sdesc*); - struct sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes); + unsigned long nbytes = size * sizeof(struct hipe_sdesc*); + struct hipe_sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes); sys_memzero(bucket, nbytes); return bucket; } @@ -54,7 +54,7 @@ static struct sdesc **alloc_bucket(unsigned int size) static void hipe_grow_sdesc_table(void) { unsigned int old_size, new_size, new_mask; - struct sdesc **old_bucket, **new_bucket; + struct hipe_sdesc **old_bucket, **new_bucket; unsigned int i; old_size = 1 << hipe_sdesc_table.log2size; @@ -66,9 +66,9 @@ static void hipe_grow_sdesc_table(void) new_bucket = alloc_bucket(new_size); hipe_sdesc_table.bucket = new_bucket; for (i = 0; i < old_size; ++i) { - struct sdesc *b = old_bucket[i]; + struct hipe_sdesc *b = old_bucket[i]; while (b != NULL) { - struct sdesc *next = b->bucket.next; + struct hipe_sdesc *next = b->bucket.next; unsigned int j = (b->bucket.hvalue >> HIPE_RA_LSR_COUNT) & new_mask; b->bucket.next = new_bucket[j]; new_bucket[j] = b; @@ -78,11 +78,11 @@ static void hipe_grow_sdesc_table(void) erts_free(ERTS_ALC_T_HIPE, old_bucket); } -struct sdesc *hipe_put_sdesc(struct sdesc *sdesc) +struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc) { unsigned long ra; unsigned int i; - struct sdesc *chain; + struct hipe_sdesc *chain; unsigned int size; ra = sdesc->bucket.hvalue; @@ -102,7 +102,29 @@ struct sdesc *hipe_put_sdesc(struct sdesc *sdesc) return sdesc; } -void hipe_init_sdesc_table(struct sdesc *sdesc) +void hipe_destruct_sdesc(struct hipe_sdesc *sdesc) +{ + unsigned int i; + struct hipe_sdesc** prevp; + void* free_me; + + i = (sdesc->bucket.hvalue >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask; + prevp = &hipe_sdesc_table.bucket[i]; + + for (; *prevp != sdesc; prevp = &(*prevp)->bucket.next) + ASSERT(*prevp); + + *prevp = sdesc->bucket.next; + hipe_sdesc_table.used -= 1; + + if (sdesc->has_exnra) + free_me = ErtsContainerStruct(sdesc, struct hipe_sdesc_with_exnra, sdesc); + else + free_me = sdesc; + erts_free(ERTS_ALC_T_HIPE, free_me); +} + +void hipe_init_sdesc_table(struct hipe_sdesc *sdesc) { unsigned int log2size, size; @@ -121,31 +143,46 @@ void hipe_init_sdesc_table(struct sdesc *sdesc) * representation. If different representations are needed in * the future, this code has to be made target dependent. */ -struct sdesc *hipe_decode_sdesc(Eterm arg) +struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) { Uint ra, exnra; Eterm *live; - Uint fsize, arity, nlive, i, nslots, off; + Uint fsize, nargs, stk_nargs, nlive, i, nslots, off; Uint livebitswords, sdescbytes; void *p; - struct sdesc *sdesc; - - if (is_not_tuple(arg) || - (tuple_val(arg))[0] != make_arityval(6) || - term_to_Uint((tuple_val(arg))[1], &ra) == 0 || - term_to_Uint((tuple_val(arg))[2], &exnra) == 0 || - is_not_small((tuple_val(arg))[3]) || - (fsize = unsigned_val((tuple_val(arg))[3])) > 65535 || - is_not_small((tuple_val(arg))[4]) || - (arity = unsigned_val((tuple_val(arg))[4])) > 255 || - is_not_tuple((tuple_val(arg))[5])) + struct hipe_sdesc *sdesc; + Eterm* mfa_tpl; + Eterm* tp; + + if (is_not_tuple(arg)) + return 0; + + tp = tuple_val(arg); + if (tp[0] != make_arityval(6) || + term_to_Uint(tp[1], &ra) == 0 || + term_to_Uint(tp[2], &exnra) == 0 || + is_not_small(tp[3]) || + (fsize = unsigned_val(tp[3])) > 65535 || + is_not_small(tp[4]) || + (stk_nargs = unsigned_val(tp[4])) > 255 || + is_not_tuple(tp[5]) || + is_not_tuple(tp[6]) || + (mfa_tpl = tuple_val(tp[6]))[0] != make_arityval(3) || + is_not_atom(mfa_tpl[1]) || + is_not_atom(mfa_tpl[2]) || + is_not_small(mfa_tpl[3]) || + (nargs = unsigned_val(mfa_tpl[3])) > 255) return 0; + + if (stk_nargs > nargs) + return 0; + /* Get tuple with live slots */ - live = tuple_val((tuple_val(arg))[5]) + 1; + live = tuple_val(tp[5]) + 1; /* Get number of live slots */ nlive = arityval(live[-1]); - /* Calculate size of frame = locals + ra + arguments */ - nslots = fsize + 1 + arity; + /* Calculate size of frame = locals + ra + stack arguments */ + nslots = fsize + 1 + stk_nargs; /* Check that only valid slots are given. */ for (i = 0; i < nlive; ++i) { if (is_not_small(live[i]) || @@ -155,27 +192,34 @@ struct sdesc *hipe_decode_sdesc(Eterm arg) } /* Calculate number of words for the live bitmap. */ - livebitswords = (fsize + arity + 1 + 31) / 32; + livebitswords = (fsize + stk_nargs + 1 + 31) / 32; /* Calculate number of bytes needed for the stack descriptor. */ sdescbytes = (exnra - ? offsetof(struct sdesc_with_exnra, sdesc.livebits) - : offsetof(struct sdesc, livebits)) + ? offsetof(struct hipe_sdesc_with_exnra, sdesc.livebits) + : offsetof(struct hipe_sdesc, livebits)) + livebitswords * sizeof(int); p = erts_alloc(ERTS_ALC_T_HIPE, sdescbytes); /* If we have an exception handler use the special sdesc_with_exnra structure. */ if (exnra) { - struct sdesc_with_exnra *sdesc_we = p; + struct hipe_sdesc_with_exnra *sdesc_we = p; sdesc_we->exnra = exnra; sdesc = &(sdesc_we->sdesc); } else sdesc = p; + sdesc->m_aix = atom_val(mfa_tpl[1]); + sdesc->f_aix = atom_val(mfa_tpl[2]); + sdesc->a = nargs; + + /* Initialise head of sdesc. */ sdesc->bucket.next = 0; sdesc->bucket.hvalue = ra; - sdesc->summary = (fsize << 9) | (exnra ? (1<<8) : 0) | arity; + sdesc->fsize = fsize; + sdesc->has_exnra = (exnra ? 1 : 0); + sdesc->stk_nargs = stk_nargs; /* Clear all live-bits */ for (i = 0; i < livebitswords; ++i) sdesc->livebits[i] = 0; @@ -184,13 +228,5 @@ struct sdesc *hipe_decode_sdesc(Eterm arg) off = unsigned_val(live[i]); sdesc->livebits[off / 32] |= (1 << (off & 31)); } -#ifdef DEBUG - { - Eterm mfa_tpl = tuple_val(arg)[6]; - sdesc->dbg_M = tuple_val(mfa_tpl)[1]; - sdesc->dbg_F = tuple_val(mfa_tpl)[2]; - sdesc->dbg_A = tuple_val(mfa_tpl)[3]; - } -#endif return sdesc; } diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index afa0ed4256..7e30358767 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -30,40 +30,43 @@ #include <stddef.h> /* offsetof() */ -struct sdesc { +struct hipe_sdesc { struct { unsigned long hvalue; /* return address */ - struct sdesc *next; /* hash collision chain */ + struct hipe_sdesc *next; /* hash collision chain */ } bucket; - unsigned int summary; /* frame size, exn handler presence flag, arity */ -#ifdef DEBUG - Eterm dbg_M, dbg_F; - unsigned dbg_A; -#endif - unsigned int livebits[1]; /* size depends on arch & data in summary field */ + unsigned int fsize : 23; /* frame size */ + unsigned int has_exnra : 1; /* exn handler presence flag */ + unsigned int stk_nargs : 8; /* arguments on stack */ + Uint32 m_aix; + Uint32 f_aix; + Uint32 a; + struct hipe_sdesc* next_in_modi; + Uint32 livebits[1]; /* size depends on arch & data in summary field */ }; -struct sdesc_with_exnra { +struct hipe_sdesc_with_exnra { unsigned long exnra; - struct sdesc sdesc; + struct hipe_sdesc sdesc; }; -static __inline__ unsigned int sdesc_fsize(const struct sdesc *sdesc) +static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc) { - return sdesc->summary >> 9; + return sdesc->fsize; } -static __inline__ unsigned int sdesc_arity(const struct sdesc *sdesc) +/* Nr of arguments pushed on stack */ +static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc) { - return sdesc->summary & 0xFF; + return sdesc->stk_nargs; } -static __inline__ unsigned long sdesc_exnra(const struct sdesc *sdesc) +static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc) { - if ((sdesc->summary & (1<<8))) { + if (sdesc->has_exnra) { const char *tmp; - tmp = (const char*)sdesc - offsetof(struct sdesc_with_exnra, sdesc); - return ((const struct sdesc_with_exnra*)tmp)->exnra; + tmp = (const char*)sdesc - offsetof(struct hipe_sdesc_with_exnra, sdesc); + return ((const struct hipe_sdesc_with_exnra*)tmp)->exnra; } return 0; } @@ -72,13 +75,14 @@ struct hipe_sdesc_table { unsigned int log2size; unsigned int mask; /* INV: mask == (1 << log2size)-1 */ unsigned int used; - struct sdesc **bucket; + struct hipe_sdesc **bucket; }; extern struct hipe_sdesc_table hipe_sdesc_table; -extern struct sdesc *hipe_put_sdesc(struct sdesc*); -extern void hipe_init_sdesc_table(struct sdesc*); -extern struct sdesc *hipe_decode_sdesc(Eterm); +extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*); +extern void hipe_destruct_sdesc(struct hipe_sdesc*); +extern void hipe_init_sdesc_table(struct hipe_sdesc*); +extern struct hipe_sdesc *hipe_decode_sdesc(Eterm); #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) #define __builtin_expect(x, expected_value) (x) @@ -86,10 +90,10 @@ extern struct sdesc *hipe_decode_sdesc(Eterm); #define likely(x) __builtin_expect((x),1) #define unlikely(x) __builtin_expect((x),0) -static __inline__ const struct sdesc *hipe_find_sdesc(unsigned long ra) +static __inline__ const struct hipe_sdesc *hipe_find_sdesc(unsigned long ra) { unsigned int i = (ra >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask; - const struct sdesc *sdesc = hipe_sdesc_table.bucket[i]; + const struct hipe_sdesc *sdesc = hipe_sdesc_table.bucket[i]; if (likely(sdesc->bucket.hvalue == ra)) return sdesc; do { @@ -103,7 +107,7 @@ AEXTERN(void,nbif_stack_trap_ra,(void)); extern void hipe_print_nstack(Process*); extern void hipe_find_handler(Process*); extern void (*hipe_handle_stack_trap(Process*))(void); -extern void hipe_update_stack_trap(Process*, const struct sdesc*); +extern void hipe_update_stack_trap(Process*, const struct hipe_sdesc*); extern int hipe_fill_stacktrace(Process*, int, Eterm**); #if 0 && defined(HIPE_NSTACK_GROWS_UP) @@ -134,5 +138,7 @@ extern void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop extern Eterm *sweep_literals_nstack(Process *p, Eterm *n_htop, char *area, Uint area_size); extern int nstack_any_heap_ref_ptrs(Process *, char* mod_start, Uint mod_size); +extern int nstack_any_cps_in_segment(Process *, char* seg_start, Uint seg_size); + #endif /* HIPE_STACK_H */ diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c index 5f6c8c200e..c7e24673ac 100644 --- a/erts/emulator/hipe/hipe_x86.c +++ b/erts/emulator/hipe/hipe_x86.c @@ -24,7 +24,6 @@ #include "config.h" #endif #include "global.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -62,118 +61,17 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Uint rel32; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + rel32 = (Uint)destAddress - (Uint)callAddress - 4; *(Uint32*)callAddress = rel32; hipe_flush_icache_word(callAddress); return 0; } -/* - * Memory allocator for executable code. - * - * This is required on x86 because some combinations - * of Linux kernels and CPU generations default to - * non-executable memory mappings, causing ordinary - * malloc() memory to be non-executable. - */ -static unsigned int code_bytes; -static char *code_next; - -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - -/* FreeBSD 6.1 and Darwin breakage */ -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif - -static int morecore(unsigned int alloc_bytes) -{ - unsigned int map_bytes; - char *map_hint, *map_start; - - /* Page-align the amount to allocate. */ - map_bytes = (alloc_bytes + 4095) & ~4095; - - /* Round up small allocations. */ - if (map_bytes < 1024*1024) - map_bytes = 1024*1024; - else - ALLOC_CODE_STATS(++nr_large); - - /* Create a new memory mapping, ensuring it is executable - and in the low 2GB of the address space. Also attempt - to make it adjacent to the previous mapping. */ - map_hint = code_next + code_bytes; - if ((unsigned long)map_hint & 4095) - abort(); - map_start = mmap(map_hint, map_bytes, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS -#ifdef __x86_64__ - |MAP_32BIT -#endif - , - -1, 0); - if (map_start == MAP_FAILED) - return -1; - - ALLOC_CODE_STATS(total_mapped += map_bytes); - - /* Merge adjacent mappings, so the trailing portion of the previous - mapping isn't lost. In practice this is quite successful. */ - if (map_start == map_hint) { - ALLOC_CODE_STATS(++nr_joins); - code_bytes += map_bytes; - } else { - ALLOC_CODE_STATS(++nr_splits); - ALLOC_CODE_STATS(total_lost += code_bytes); - code_next = map_start; - code_bytes = map_bytes; - } - - ALLOC_CODE_STATS(atexit_alloc_code_stats()); - - return 0; -} - static void *alloc_code(unsigned int alloc_bytes) { - void *res; - - /* Align function entries. */ - alloc_bytes = (alloc_bytes + 3) & ~3; - - if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0) - return NULL; - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - res = code_next; - code_next += alloc_bytes; - code_bytes -= alloc_bytes; - return res; + return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) @@ -184,6 +82,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int bytes) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { /* @@ -264,6 +167,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h index 00fe03d8f9..a703e24b8c 100644 --- a/erts/emulator/hipe/hipe_x86_gc.h +++ b/erts/emulator/hipe/hipe_x86_gc.h @@ -30,9 +30,9 @@ struct nstack_walk_state { #ifdef SKIP_YOUNGEST_FRAME - const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ + const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ #else - struct sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */ + struct hipe_sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */ #endif }; @@ -57,37 +57,36 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p) #endif } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) { #ifdef SKIP_YOUNGEST_FRAME - const struct sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]); + const struct hipe_sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]); state->sdesc0 = sdesc; return sdesc; #else - unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS; - if ((int)nstkarity < 0) - nstkarity = 0; - state->sdesc0[0].summary = (0 << 9) | (0 << 8) | nstkarity; + state->sdesc0[0].bucket.hvalue = 0; /* for nstack_any_cps_in_segment */ + state->sdesc0[0].fsize = 0; + state->sdesc0[0].has_exnra = 0; + state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : + p->hipe.narity - NR_ARG_REGS); state->sdesc0[0].livebits[0] = 0; -# ifdef DEBUG - state->sdesc0[0].dbg_M = 0; - state->sdesc0[0].dbg_F = am_undefined; - state->sdesc0[0].dbg_A = 0; -# endif + state->sdesc0[0].m_aix = 0; + state->sdesc0[0].f_aix = atom_val(am_undefined); + state->sdesc0[0].a = 0; /* XXX: this appears to prevent a gcc-4.1.1 bug on x86 */ __asm__ __volatile__("" : : "m"(*state) : "memory"); return &state->sdesc0[0]; #endif } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc_ignore_trap(const Process *p, struct nstack_walk_state *state) { #ifdef SKIP_YOUNGEST_FRAME unsigned long ra = p->hipe.nsp[0]; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; if (ra == (unsigned long)nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; sdesc = hipe_find_sdesc(ra); @@ -98,7 +97,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p, #endif } -static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0) +static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0) { #ifdef SKIP_YOUNGEST_FRAME Eterm *nsp = p->hipe.nsp; @@ -137,7 +136,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp return nsp >= nsp_end; } -static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc) +static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc) { return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc); } @@ -148,7 +147,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i) } static inline unsigned long -nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc) +nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc) { return nsp[sdesc_fsize(sdesc)]; } diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h index 818d7444e2..de2b061706 100644 --- a/erts/emulator/hipe/hipe_x86_glue.h +++ b/erts/emulator/hipe/hipe_x86_glue.h @@ -58,16 +58,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct sdesc_with_exnra nbif_return_sdesc = { - .exnra = (unsigned long)nbif_fail, - .sdesc = { - .bucket = { .hvalue = (unsigned long)nbif_return }, - .summary = (1<<8), - #ifdef DEBUG - .dbg_F = am_return, - #endif - }, - }; + static struct hipe_sdesc_with_exnra nbif_return_sdesc; + + nbif_return_sdesc.exnra = (unsigned long)nbif_fail; + nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return; + nbif_return_sdesc.sdesc.fsize = 0; + nbif_return_sdesc.sdesc.has_exnra = 1; + nbif_return_sdesc.sdesc.stk_nargs = 0; + nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty); + nbif_return_sdesc.sdesc.f_aix = atom_val(am_return); + nbif_return_sdesc.sdesc.a = 0; + hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); } diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c index f1559b1451..31582b3a2e 100644 --- a/erts/emulator/hipe/hipe_x86_stack.c +++ b/erts/emulator/hipe/hipe_x86_stack.c @@ -52,15 +52,14 @@ void hipe_print_nstack(Process *p) { Eterm *nsp; Eterm *nsp_end; - struct sdesc sdesc0; - const struct sdesc *sdesc1; - const struct sdesc *sdesc; + struct hipe_sdesc sdesc0; + const struct hipe_sdesc *sdesc1; + const struct hipe_sdesc *sdesc; unsigned long ra; unsigned long exnra; unsigned int mask; unsigned int sdesc_size; unsigned int i; - unsigned int nstkarity; static const char dashes[2*sizeof(long)+5] = { [0 ... 2*sizeof(long)+3] = '-' }; @@ -68,10 +67,10 @@ void hipe_print_nstack(Process *p) nsp = p->hipe.nsp; nsp_end = p->hipe.nstend; - nstkarity = p->hipe.narity - NR_ARG_REGS; - if ((int)nstkarity < 0) - nstkarity = 0; - sdesc0.summary = nstkarity; + sdesc0.fsize = 0; + sdesc0.has_exnra = 0; + sdesc0.stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : + p->hipe.narity - NR_ARG_REGS); sdesc0.livebits[0] = ~1; sdesc = &sdesc0; @@ -158,7 +157,7 @@ void hipe_print_nstack(Process *p) #define MINSTACK 128 #define NSKIPFRAMES 4 -void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) +void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc) { Eterm *nsp; Eterm *nsp_end; @@ -199,7 +198,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) void (*hipe_handle_stack_trap(Process *p))(void) { void (*ngra)(void) = p->hipe.ngra; - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); hipe_update_stack_trap(p, sdesc); return ngra; } @@ -220,7 +219,7 @@ void hipe_find_handler(Process *p) unsigned long ra; unsigned long exnra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int nstkarity; nsp = p->hipe.nsp; @@ -262,7 +261,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) Eterm *nsp_end; unsigned long ra, prev_ra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int nstkarity; int i; diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 7bbb406f29..3ab03971a5 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -21,6 +21,7 @@ # include "config.h" #endif +#define ERTS_WANT_MEM_MAPPERS #include "sys.h" #include "erl_process.h" #include "erl_smp.h" @@ -358,12 +359,11 @@ char* erts_literals_start; UWord erts_literals_size; #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER ErtsMemMapper erts_exec_mmapper; #endif - #define ERTS_MMAP_SIZE_SC_SA_INC(SZ) \ do { \ mm->size.supercarrier.used.total += (SZ); \ diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index fa51b663fa..92e9eb9e41 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -159,9 +159,10 @@ Eterm erts_mmap_info_options(ErtsMemMapper*, extern ErtsMemMapper erts_dflt_mmapper; # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) extern ErtsMemMapper erts_literal_mmapper; -# endif -# ifdef ERTS_ALC_A_EXEC +# ifdef ERTS_ALC_A_EXEC +# define ERTS_HAVE_EXEC_MMAPPER extern ErtsMemMapper erts_exec_mmapper; +# endif # endif #endif /* ERTS_WANT_MEM_MAPPERS */ diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index f3306a888c..e76082177e 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -1414,7 +1414,7 @@ erts_mseg_init(ErtsMsegInit_t *init) erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms"); -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER /* Initialize erts_exec_mapper *FIRST*, to increase probability * of getting low memory for HiPE AMD64's small code model. */ diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 8427bb134d..465ddfa7a8 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -23,8 +23,8 @@ versions/1,new_binary_types/1, call_purged_fun_code_gone/1, call_purged_fun_code_reload/1, call_purged_fun_code_there/1, multi_proc_purge/1, t_check_old_code/1, - external_fun/1,get_chunk/1,module_md5/1,make_stub/1, - make_stub_many_funs/1,constant_pools/1,constant_refc_binaries/1, + external_fun/1,get_chunk/1,module_md5/1, + constant_pools/1,constant_refc_binaries/1, false_dependency/1,coverage/1,fun_confusion/1, t_copy_literals/1, t_copy_literals_frags/1]). @@ -37,7 +37,7 @@ all() -> [versions, new_binary_types, call_purged_fun_code_gone, call_purged_fun_code_reload, call_purged_fun_code_there, multi_proc_purge, t_check_old_code, external_fun, get_chunk, - module_md5, make_stub, make_stub_many_funs, + module_md5, constant_pools, constant_refc_binaries, false_dependency, coverage, fun_confusion, t_copy_literals, t_copy_literals_frags]. @@ -442,67 +442,6 @@ module_md5_ok(Code) -> end. -make_stub(Config) when is_list(Config) -> - catch erlang:purge_module(my_code_test), - MD5 = erlang:md5(<<>>), - - Data = proplists:get_value(data_dir, Config), - File = filename:join(Data, "my_code_test"), - {ok,my_code_test,Code} = compile:file(File, [binary]), - - my_code_test = code:make_stub_module(my_code_test, Code, {[],[],MD5}), - true = erlang:delete_module(my_code_test), - true = erlang:purge_module(my_code_test), - - my_code_test = code:make_stub_module(my_code_test, - make_unaligned_sub_binary(Code), - {[],[],MD5}), - true = erlang:delete_module(my_code_test), - true = erlang:purge_module(my_code_test), - - my_code_test = code:make_stub_module(my_code_test, zlib:gzip(Code), - {[],[],MD5}), - true = erlang:delete_module(my_code_test), - true = erlang:purge_module(my_code_test), - - %% Should fail. - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(my_code_test, <<"bad">>, {[],[],MD5})), - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(my_code_test, - bit_sized_binary(Code), - {[],[],MD5})), - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(my_code_test_with_wrong_name, - Code, {[],[],MD5})), - ok. - -make_stub_many_funs(Config) when is_list(Config) -> - catch erlang:purge_module(many_funs), - MD5 = erlang:md5(<<>>), - - Data = proplists:get_value(data_dir, Config), - File = filename:join(Data, "many_funs"), - {ok,many_funs,Code} = compile:file(File, [binary]), - - many_funs = code:make_stub_module(many_funs, Code, {[],[],MD5}), - true = erlang:delete_module(many_funs), - true = erlang:purge_module(many_funs), - many_funs = code:make_stub_module(many_funs, - make_unaligned_sub_binary(Code), - {[],[],MD5}), - true = erlang:delete_module(many_funs), - true = erlang:purge_module(many_funs), - - %% Should fail. - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(many_funs, <<"bad">>, {[],[],MD5})), - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(many_funs, - bit_sized_binary(Code), - {[],[],MD5})), - ok. - constant_pools(Config) when is_list(Config) -> Data = proplists:get_value(data_dir, Config), File = filename:join(Data, "literals"), diff --git a/erts/emulator/test/hipe_SUITE.erl b/erts/emulator/test/hipe_SUITE.erl index a556b4ddc0..0b44dd7fb7 100644 --- a/erts/emulator/test/hipe_SUITE.erl +++ b/erts/emulator/test/hipe_SUITE.erl @@ -19,12 +19,17 @@ %% -module(hipe_SUITE). --export([all/0, t_copy_literals/1]). +-export([all/0 + ,t_copy_literals/1 + ,t_purge/1 + ]). all() -> case erlang:system_info(hipe_architecture) of undefined -> {skip, "HiPE is disabled"}; - _ -> [t_copy_literals] + _ -> [t_copy_literals + ,t_purge + ] end. t_copy_literals(doc) -> @@ -65,3 +70,51 @@ t_copy_literals(Config) when is_list(Config) -> true = erlang:delete_module(ref_cell), true = erlang:purge_module(ref_cell), ok. + +t_purge(doc) -> "Checks that native code is properly found and purged"; +t_purge(Config) when is_list(Config) -> + Data = proplists:get_value(data_dir, Config), + Priv = proplists:get_value(priv_dir, Config), + SrcFile = filename:join(Data, "ref_cell"), + BeamFile = filename:join(Priv, "ref_cell"), + {ok,ref_cell} = c:c(SrcFile, [{outdir,Priv},native]), + true = code:is_module_native(ref_cell), + + PA = ref_cell:start_link(), + + %% Unload, PA should still be running + true = erlang:delete_module(ref_cell), + %% Can't use ref_cel:call/2, it's in old code! + call(PA, {put_res_of, fun()-> hej end}), + hej = call(PA, get), + + %% Load same module again + code:load_abs(BeamFile), + true = code:is_module_native(ref_cell), + PB = ref_cell:start_link(), + + %% Purge old code, PA should be killed, PB should survive + unlink(PA), + ARef = monitor(process, PA), + true = erlang:purge_module(ref_cell), + receive {'DOWN', ARef, process, PA, killed} -> ok + after 1 -> ct:fail("PA was not killed") + end, + + %% Unload, PB should still be running + true = erlang:delete_module(ref_cell), + call(PB, {put_res_of, fun()-> svejs end}), + svejs = call(PB, get), + + unlink(PB), + BRef = monitor(process, PB), + true = erlang:purge_module(ref_cell), + receive {'DOWN', BRef, process, PB, killed} -> ok + after 1 -> ct:fail("PB was not killed") + end, + + ok. + +call(Pid, Call) -> + Pid ! {Call, self()}, + receive {Pid, Res} -> Res end. |