diff options
Diffstat (limited to 'erts/emulator/beam')
77 files changed, 1918 insertions, 1386 deletions
diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index a5e778e4aa..b47739059b 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -68,7 +68,7 @@ static Uint atom_space; /* Amount of atom text space used */ /* * Print info about atom tables */ -void atom_info(int to, void *to_arg) +void atom_info(fmtfn_t to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) @@ -470,7 +470,7 @@ init_atom_table(void) } void -dump_atoms(int to, void *to_arg) +dump_atoms(fmtfn_t to, void *to_arg) { int i = erts_atom_table.entries; @@ -483,3 +483,9 @@ dump_atoms(int to, void *to_arg) } } } + +Uint +erts_get_atom_limit(void) +{ + return erts_atom_table.limit; +}
\ No newline at end of file diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index ae60904785..f3b21e1687 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -136,8 +136,9 @@ Eterm erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc); int atom_erase(byte*, int); int atom_static_put(byte*, int); void init_atom_table(void); -void atom_info(int, void *); -void dump_atoms(int, void *); +void atom_info(fmtfn_t, void *); +void dump_atoms(fmtfn_t, void *); +Uint erts_get_atom_limit(void); int erts_atom_get(const char* name, int len, Eterm* ap, ErtsAtomEncoding enc); void erts_atom_get_text_space_sizes(Uint *reserved, Uint *used); #endif diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 153fa205ed..5c3565f498 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -36,6 +36,12 @@ #include "erl_nif.h" #include "erl_bits.h" #include "erl_thr_progress.h" +#ifdef HIPE +# include "hipe_bif0.h" +# define IF_HIPE(X) (X) +#else +# define IF_HIPE(X) (0) +#endif #ifdef HIPE # include "hipe_stack.h" @@ -143,8 +149,17 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) { +#if !defined(HIPE) + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else Module* modp; - Eterm res; + Eterm res, mod; + + if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1) || + is_not_atom(mod = erts_module_for_prepared_code + (((ProcBin*)binary_val(BIF_ARG_1))->val))) { + BIF_ERROR(BIF_P, BADARG); + } if (!erts_try_seize_code_write_permission(BIF_P)) { ERTS_BIF_YIELD3(bif_export[BIF_code_make_stub_module_3], @@ -154,7 +169,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); - modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); + modp = erts_get_module(mod, erts_active_code_ix()); if (modp && modp->curr.num_breakpoints > 0) { ASSERT(modp->curr.code_hdr != NULL); @@ -166,9 +181,12 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); - if (res == BIF_ARG_1) { + if (res == mod) { erts_end_staging_code_ix(); erts_commit_staging_code_ix(); + if (!modp) + modp = erts_get_module(mod, erts_active_code_ix()); + hipe_redirect_to_module(modp); } else { erts_abort_staging_code_ix(); @@ -177,6 +195,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); return res; +#endif } BIF_RETTYPE @@ -241,7 +260,7 @@ struct m { Uint exception; }; -static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int); +static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int, int); #ifdef ERTS_SMP static void smp_code_ix_commiter(void*); @@ -377,8 +396,9 @@ finish_loading_1(BIF_ALIST_1) for (i = 0; i < n; i++) { if (p[i].modp->curr.num_breakpoints > 0 || p[i].modp->curr.num_traced_exports > 0 || - erts_is_default_trace_enabled()) { - /* tracing involved, fallback with thread blocking */ + erts_is_default_trace_enabled() || + IF_HIPE(hipe_need_blocking(p[i].modp))) { + /* tracing or hipe need thread blocking */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; @@ -436,32 +456,36 @@ finish_loading_1(BIF_ALIST_1) } done: - return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n); + return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n, 1); } static Eterm staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, - struct m* loaded, int nloaded) + struct m* mods, int nmods, int free_mods) { #ifdef ERTS_SMP if (is_blocking || !commit) #endif { if (commit) { + int i; erts_end_staging_code_ix(); erts_commit_staging_code_ix(); - if (loaded) { - int i; - for (i=0; i < nloaded; i++) { - set_default_trace_pattern(loaded[i].module); + + for (i=0; i < nmods; i++) { + if (mods[i].modp->curr.code_hdr) { + set_default_trace_pattern(mods[i].module); } + #ifdef HIPE + hipe_redirect_to_module(mods[i].modp); + #endif } } else { erts_abort_staging_code_ix(); } - if (loaded) { - erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + if (free_mods) { + erts_free(ERTS_ALC_T_LOADER_TMP, mods); } if (is_blocking) { erts_smp_thr_progress_unblock(); @@ -474,8 +498,8 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, else { ASSERT(is_value(res)); - if (loaded) { - erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + if (free_mods) { + erts_free(ERTS_ALC_T_LOADER_TMP, mods); } erts_end_staging_code_ix(); /* @@ -653,8 +677,9 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } else { if (modp->curr.num_breakpoints > 0 || - modp->curr.num_traced_exports > 0) { - /* we have tracing, retry single threaded */ + modp->curr.num_traced_exports > 0 || + IF_HIPE(hipe_need_blocking(modp))) { + /* tracing or hipe need to go single threaded */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; @@ -668,7 +693,14 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) success = 1; } } - return staging_epilogue(BIF_P, success, res, is_blocking, NULL, 0); + { + struct m mod; + Eterm retval; + mod.module = BIF_ARG_1; + mod.modp = modp; + retval = staging_epilogue(BIF_P, success, res, is_blocking, &mod, 1, 0); + return retval; + } } BIF_RETTYPE module_loaded_1(BIF_ALIST_1) @@ -792,23 +824,26 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) */ for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i,code_ix); - if (ep == NULL || ep->code[0] != BIF_ARG_1) { + if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; } - if (ep->code[4] != 0) { - ep->addressv[code_ix] = (void *) ep->code[4]; - ep->code[4] = 0; + if (ep->beam[1] != 0) { + ep->addressv[code_ix] = (void *) ep->beam[1]; + ep->beam[1] = 0; } else { - if (ep->addressv[code_ix] == ep->code+3 && - ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep->addressv[code_ix] == ep->beam && + ep->beam[0] == (BeamInstr) em_apply_bif) { continue; } - ep->addressv[code_ix] = ep->code+3; - ep->code[3] = (BeamInstr) em_call_error_handler; + ep->addressv[code_ix] = ep->beam; + ep->beam[0] = (BeamInstr) em_call_error_handler; } } modp->curr.code_hdr->on_load_function_ptr = NULL; set_default_trace_pattern(BIF_ARG_1); + #ifdef HIPE + hipe_redirect_to_module(modp); + #endif } else if (BIF_ARG_2 == am_false) { int i; @@ -819,13 +854,13 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i,code_ix); - if (ep == NULL || ep->code[0] != BIF_ARG_1) { + if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; } - if (ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep->beam[0] == (BeamInstr) em_apply_bif) { continue; } - ep->code[4] = 0; + ep->beam[1] = 0; } } erts_smp_thr_progress_unblock(); @@ -849,9 +884,9 @@ set_default_trace_pattern(Eterm module) &trace_pattern_flags, &meta_tracer); if (trace_pattern_is_on) { - Eterm mfa[1]; - mfa[0] = module; - (void) erts_set_trace_pattern(0, mfa, 1, + ErtsCodeMFA mfa; + mfa.module = module; + (void) erts_set_trace_pattern(0, &mfa, 1, match_spec, meta_match_spec, 1, trace_pattern_flags, @@ -1037,6 +1072,10 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) char* mod_start; Uint mod_size; Eterm* sp; +#ifdef HIPE + void *nat_start = NULL; + Uint nat_size = 0; +#endif *redsp += 1; @@ -1054,6 +1093,11 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) || ErtsInArea(rp->cp, mod_start, mod_size)) { return am_true; } + + *redsp += 1; + + if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) + return am_true; *redsp += (STACK_START(rp) - rp->stop) / 32; @@ -1066,6 +1110,20 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) } } +#ifdef HIPE + /* + * Check all continuation pointers stored on the native stack if the module + * has native code. + */ + if (modp->old.hipe_code) { + nat_start = modp->old.hipe_code->text_segment; + nat_size = modp->old.hipe_code->text_segment_size; + if (nat_size && nstack_any_cps_in_segment(rp, nat_start, nat_size)) { + return am_true; + } + } +#endif + /* * Check all continuation pointers stored in stackdump * and clear exception stackdump if there is a pointer @@ -1082,8 +1140,16 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) rp->ftrace = NIL; } else { int i; + char *area_start = mod_start; + Uint area_size = mod_size; +#ifdef HIPE + if (rp->freason & EXF_NATIVE) { + area_start = nat_start; + area_size = nat_size; + } +#endif for (i = 0; i < s->depth; i++) { - if (ErtsInArea(s->trace[i], mod_start, mod_size)) { + if (ErtsInArea(s->trace[i], area_start, area_size)) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; @@ -1336,10 +1402,10 @@ erts_purge_state_add_fun(ErlFunEntry *fe) } Export * -erts_suspend_process_on_pending_purge_lambda(Process *c_p) +erts_suspend_process_on_pending_purge_lambda(Process *c_p, ErlFunEntry* fe) { erts_smp_mtx_lock(&purge_state.mtx); - if (is_value(purge_state.module)) { + if (purge_state.module == fe->module) { /* * The process c_p is about to call a fun in the code * that we are trying to purge. Suspend it and call @@ -1619,15 +1685,18 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) /* * Unload any NIF library */ - if (modp->old.nif != NULL) { + if (modp->old.nif != NULL + || IF_HIPE(hipe_purge_need_blocking(modp))) { /* ToDo: Do unload nif without blocking */ erts_rwunlock_old_code(code_ix); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; erts_rwlock_old_code(code_ix); - erts_unload_nif(modp->old.nif); - modp->old.nif = NULL; + if (modp->old.nif) { + erts_unload_nif(modp->old.nif); + modp->old.nif = NULL; + } } /* @@ -1646,7 +1715,9 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) modp->old.code_length = 0; modp->old.catches = BEAM_CATCHES_NIL; erts_remove_from_ranges(code); - +#ifdef HIPE + hipe_purge_module(modp, is_blocking); +#endif ERTS_BIF_PREP_RET(ret, am_true); } @@ -1710,34 +1781,34 @@ delete_code(Module* modp) for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i, code_ix); - if (ep != NULL && (ep->code[0] == module)) { - if (ep->addressv[code_ix] == ep->code+3) { - if (ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep != NULL && (ep->info.mfa.module == module)) { + if (ep->addressv[code_ix] == ep->beam) { + if (ep->beam[0] == (BeamInstr) em_apply_bif) { continue; } - else if (ep->code[3] == + else if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); - erts_clear_export_break(modp, ep->code+3); + DBG_TRACE_MFA_P(&ep->info.mfa, + "export trace cleared, code_ix=%d", code_ix); + erts_clear_export_break(modp, &ep->info); } - else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler + else ASSERT(ep->beam[0] == (BeamInstr) em_call_error_handler || !erts_initialized); } - ep->addressv[code_ix] = ep->code+3; - ep->code[3] = (BeamInstr) em_call_error_handler; - ep->code[4] = 0; + ep->addressv[code_ix] = ep->beam; + ep->beam[0] = (BeamInstr) em_call_error_handler; + ep->beam[1] = 0; + DBG_TRACE_MFA_P(&ep->info.mfa, + "export invalidation, code_ix=%d", code_ix); } } ASSERT(modp->curr.num_breakpoints == 0); ASSERT(modp->curr.num_traced_exports == 0); modp->old = modp->curr; - modp->curr.code_hdr = NULL; - modp->curr.code_length = 0; - modp->curr.catches = BEAM_CATCHES_NIL; - modp->curr.nif = NULL; - + erts_module_instance_init(&modp->curr); } @@ -1751,9 +1822,11 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) * if not, delete old code; error if old code already exists. */ - if (modp->curr.code_hdr && modp->old.code_hdr) { - return am_not_purged; - } else if (!modp->old.code_hdr) { /* Make the current version old. */ + if (modp->curr.code_hdr) { + if (modp->old.code_hdr) { + return am_not_purged; + } + /* Make the current version old. */ delete_code(modp); } return NIL; diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 920c8b1ed0..73158205b3 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -92,27 +92,27 @@ get_mtime(Process *c_p) /* ** Helpers */ -static ErtsTracer do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, +static ErtsTracer do_call_trace(Process* c_p, ErtsCodeInfo *info, Eterm* reg, int local, Binary* ms, ErtsTracer tracer); static void set_break(BpFunctions* f, Binary *match_spec, Uint break_flags, enum erts_break_op count_op, ErtsTracer tracer); -static void set_function_break(BeamInstr *pc, +static void set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, enum erts_break_op count_op, ErtsTracer tracer); static void clear_break(BpFunctions* f, Uint break_flags); -static int clear_function_break(BeamInstr *pc, Uint break_flags); +static int clear_function_break(ErtsCodeInfo *ci, Uint break_flags); -static BpDataTime* get_time_break(BeamInstr *pc); -static GenericBpData* check_break(BeamInstr *pc, Uint break_flags); +static BpDataTime* get_time_break(ErtsCodeInfo *ci); +static GenericBpData* check_break(ErtsCodeInfo *ci, Uint break_flags); -static void bp_meta_unref(BpMetaTracer* bmt); -static void bp_count_unref(BpCount* bcp); -static void bp_time_unref(BpDataTime* bdt); -static void consolidate_bp_data(Module* modp, BeamInstr* pc, int local); -static void uninstall_breakpoint(BeamInstr* pc); +static void bp_meta_unref(BpMetaTracer *bmt); +static void bp_count_unref(BpCount *bcp); +static void bp_time_unref(BpDataTime *bdt); +static void consolidate_bp_data(Module *modp, ErtsCodeInfo *ci, int local); +static void uninstall_breakpoint(ErtsCodeInfo *ci); /* bp_hash */ #define BP_TIME_ADD(pi0, pi1) \ @@ -139,7 +139,7 @@ erts_bp_init(void) { void -erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) +erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified) { ErtsCodeIndex code_ix = erts_active_code_ix(); Uint max_funcs = 0; @@ -164,41 +164,44 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) i = 0; for (current = 0; current < num_modules; current++) { BeamCodeHeader* code_hdr = module[current]->curr.code_hdr; - BeamInstr* code; + ErtsCodeInfo* ci; Uint num_functions = (Uint)(UWord) code_hdr->num_functions; Uint fi; if (specified > 0) { - if (mfa[0] != make_atom(module[current]->module)) { + if (mfa->module != make_atom(module[current]->module)) { /* Wrong module name */ continue; } } for (fi = 0; fi < num_functions; fi++) { - BeamInstr* pc; - int wi; - code = code_hdr->functions[fi]; - ASSERT(code[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - pc = code+5; - if (erts_is_native_break(pc)) { + ci = code_hdr->functions[fi]; + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + if (erts_is_native_break(ci)) { continue; } - if (is_nil(code[3])) { /* Ignore BIF stub */ + if (is_nil(ci->mfa.module)) { /* Ignore BIF stub */ continue; } - for (wi = 0; - wi < specified && (Eterm) code[2+wi] == mfa[wi]; - wi++) { - /* Empty loop body */ - } - if (wi == specified) { - /* Store match */ - f->matching[i].pc = pc; - f->matching[i].mod = module[current]; - i++; - } + switch (specified) { + case 3: + if (ci->mfa.arity != mfa->arity) + continue; + case 2: + if (ci->mfa.function != mfa->function) + continue; + case 1: + if (ci->mfa.module != mfa->module) + continue; + case 0: + break; + } + /* Store match */ + f->matching[i].ci = ci; + f->matching[i].mod = module[current]; + i++; } } f->matched = i; @@ -206,7 +209,7 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) } void -erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified) +erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified) { ErtsCodeIndex code_ix = erts_active_code_ix(); int i; @@ -218,27 +221,36 @@ erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified) for (i = 0; i < num_exps; i++) { Export* ep = export_list(i, code_ix); BeamInstr* pc; - int j; - for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) { - /* Empty loop body */ - } - if (j < specified) { - continue; - } - pc = ep->code+3; + switch (specified) { + case 3: + if (mfa->arity != ep->info.mfa.arity) + continue; + case 2: + if (mfa->function != ep->info.mfa.function) + continue; + case 1: + if (mfa->module != ep->info.mfa.module) + continue; + case 0: + break; + default: + ASSERT(0); + } + + pc = ep->beam; if (ep->addressv[code_ix] == pc) { if ((*pc == (BeamInstr) em_apply_bif || *pc == (BeamInstr) em_call_error_handler)) { continue; } ASSERT(*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); - } else if (erts_is_native_break(ep->addressv[code_ix])) { + } else if (erts_is_native_break(erts_code_to_codeinfo(ep->addressv[code_ix]))) { continue; } - f->matching[ne].pc = pc; - f->matching[ne].mod = erts_get_module(ep->code[0], code_ix); + f->matching[ne].ci = &ep->info; + f->matching[ne].mod = erts_get_module(ep->info.mfa.module, code_ix); ne++; } @@ -264,7 +276,7 @@ erts_consolidate_bp_data(BpFunctions* f, int local) ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); for (i = 0; i < n; i++) { - consolidate_bp_data(fs[i].mod, fs[i].pc, local); + consolidate_bp_data(fs[i].mod, fs[i].ci, local); } } @@ -276,14 +288,14 @@ erts_consolidate_bif_bp_data(void) ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); for (i = 0; i < BIF_SIZE; i++) { Export *ep = bif_export[i]; - consolidate_bp_data(0, ep->code+3, 0); + consolidate_bp_data(0, &ep->info, 0); } } static void -consolidate_bp_data(Module* modp, BeamInstr* pc, int local) +consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; GenericBpData* src; GenericBpData* dst; Uint flags; @@ -329,9 +341,10 @@ consolidate_bp_data(Module* modp, BeamInstr* pc, int local) } ASSERT(modp->curr.num_breakpoints >= 0); ASSERT(modp->curr.num_traced_exports >= 0); - ASSERT(*pc != (BeamInstr) BeamOp(op_i_generic_breakpoint)); + ASSERT(*erts_codeinfo_to_code(ci) != + (BeamInstr) BeamOp(op_i_generic_breakpoint)); } - pc[-4] = 0; + ci->native = 0; Free(g); return; } @@ -380,8 +393,9 @@ erts_install_breakpoints(BpFunctions* f) BeamInstr br = (BeamInstr) BeamOp(op_i_generic_breakpoint); for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - GenericBp* g = (GenericBp *) pc[-4]; + ErtsCodeInfo* ci = f->matching[i].ci; + BeamInstr *pc = erts_codeinfo_to_code(ci); + GenericBp* g = (GenericBp *) ci->native; if (*pc != br && g) { Module* modp = f->matching[i].mod; @@ -413,16 +427,16 @@ erts_uninstall_breakpoints(BpFunctions* f) Uint n = f->matched; for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - uninstall_breakpoint(pc); + uninstall_breakpoint(f->matching[i].ci); } } static void -uninstall_breakpoint(BeamInstr* pc) +uninstall_breakpoint(ErtsCodeInfo *ci) { + BeamInstr *pc = erts_codeinfo_to_code(ci); if (*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; if (g->data[erts_active_bp_ix()].flags == 0) { /* * The following write is not protected by any lock. We @@ -449,30 +463,30 @@ erts_set_mtrace_break(BpFunctions* f, Binary *match_spec, ErtsTracer tracer) } void -erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local) +erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; - set_function_break(pc, match_spec, flags, 0, erts_tracer_nil); + set_function_break(ci, match_spec, flags, 0, erts_tracer_nil); } void -erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, ErtsTracer tracer) +erts_set_mtrace_bif(ErtsCodeInfo *ci, Binary *match_spec, ErtsTracer tracer) { - set_function_break(pc, match_spec, ERTS_BPF_META_TRACE, 0, tracer); + set_function_break(ci, match_spec, ERTS_BPF_META_TRACE, 0, tracer); } void -erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op count_op) +erts_set_time_trace_bif(ErtsCodeInfo *ci, enum erts_break_op count_op) { - set_function_break(pc, NULL, + set_function_break(ci, NULL, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE, count_op, erts_tracer_nil); } void -erts_clear_time_trace_bif(BeamInstr *pc) { - clear_function_break(pc, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE); +erts_clear_time_trace_bif(ErtsCodeInfo *ci) { + clear_function_break(ci, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE); } void @@ -501,14 +515,14 @@ erts_clear_trace_break(BpFunctions* f) } void -erts_clear_call_trace_bif(BeamInstr *pc, int local) +erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; if (g) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; if (g->data[erts_staging_bp_ix()].flags & flags) { - clear_function_break(pc, flags); + clear_function_break(ci, flags); } } } @@ -520,9 +534,9 @@ erts_clear_mtrace_break(BpFunctions* f) } void -erts_clear_mtrace_bif(BeamInstr *pc) +erts_clear_mtrace_bif(ErtsCodeInfo *ci) { - clear_function_break(pc, ERTS_BPF_META_TRACE); + clear_function_break(ci, ERTS_BPF_META_TRACE); } void @@ -564,52 +578,48 @@ erts_clear_module_break(Module *modp) { } n = (Uint)(UWord) code_hdr->num_functions; for (i = 0; i < n; ++i) { - BeamInstr* pc; - - pc = code_hdr->functions[i] + 5; - if (erts_is_native_break(pc)) { + ErtsCodeInfo *ci = code_hdr->functions[i]; + if (erts_is_native_break(ci)) continue; - } - clear_function_break(pc, ERTS_BPF_ALL); + clear_function_break(ci, ERTS_BPF_ALL); } erts_commit_staged_bp(); for (i = 0; i < n; ++i) { - BeamInstr* pc; - - pc = code_hdr->functions[i] + 5; - if (erts_is_native_break(pc)) { + ErtsCodeInfo *ci = code_hdr->functions[i]; + if (erts_is_native_break(ci)) continue; - } - uninstall_breakpoint(pc); - consolidate_bp_data(modp, pc, 1); - ASSERT(pc[-4] == 0); + uninstall_breakpoint(ci); + consolidate_bp_data(modp, ci, 1); + ASSERT(ci->native == 0); } return n; } void -erts_clear_export_break(Module* modp, BeamInstr* pc) +erts_clear_export_break(Module* modp, ErtsCodeInfo *ci) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - clear_function_break(pc, ERTS_BPF_ALL); + clear_function_break(ci, ERTS_BPF_ALL); erts_commit_staged_bp(); - *pc = (BeamInstr) 0; - consolidate_bp_data(modp, pc, 0); - ASSERT(pc[-4] == 0); + *erts_codeinfo_to_code(ci) = (BeamInstr) 0; + consolidate_bp_data(modp, ci, 0); + ASSERT(ci->native == 0); } BeamInstr -erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) +erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) { GenericBp* g; GenericBpData* bp; Uint bp_flags; ErtsBpIndex ix = erts_active_bp_ix(); - g = (GenericBp *) I[-4]; + ASSERT(info->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + + g = (GenericBp *) info->native; bp = &g->data[ix]; bp_flags = bp->flags; ASSERT((bp_flags & ~ERTS_BPF_ALL) == 0); @@ -628,9 +638,9 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) if (bp_flags & ERTS_BPF_LOCAL_TRACE) { ASSERT((bp_flags & ERTS_BPF_GLOBAL_TRACE) == 0); - (void) do_call_trace(c_p, I, reg, 1, bp->local_ms, erts_tracer_true); + (void) do_call_trace(c_p, info, reg, 1, bp->local_ms, erts_tracer_true); } else if (bp_flags & ERTS_BPF_GLOBAL_TRACE) { - (void) do_call_trace(c_p, I, reg, 0, bp->local_ms, erts_tracer_true); + (void) do_call_trace(c_p, info, reg, 0, bp->local_ms, erts_tracer_true); } if (bp_flags & ERTS_BPF_META_TRACE) { @@ -638,7 +648,8 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) old_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer); - new_tracer = do_call_trace(c_p, I, reg, 1, bp->meta_ms, old_tracer); + new_tracer = do_call_trace(c_p, info, reg, 1, bp->meta_ms, old_tracer); + if (!ERTS_TRACER_COMPARE(new_tracer, old_tracer)) { if (old_tracer == erts_smp_atomic_cmpxchg_acqb( &bp->meta_tracer->tracer, @@ -657,7 +668,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) { Eterm w; - erts_trace_time_call(c_p, I, bp->time); + erts_trace_time_call(c_p, info, bp->time); w = (BeamInstr) *c_p->cp; if (! (w == (BeamInstr) BeamOp(op_i_return_time_trace) || w == (BeamInstr) BeamOp(op_return_trace) || @@ -665,7 +676,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) Eterm* E = c_p->stop; ASSERT(c_p->htop <= E && E <= c_p->hend); if (E - 2 < c_p->htop) { - (void) erts_garbage_collect(c_p, 2, reg, I[-1]); + (void) erts_garbage_collect(c_p, 2, reg, info->mfa.arity); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); } E = c_p->stop; @@ -673,7 +684,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) ASSERT(c_p->htop <= E && E <= c_p->hend); E -= 2; - E[0] = make_cp(I); + E[0] = make_cp(erts_codeinfo_to_code(info)); E[1] = make_cp(c_p->cp); /* original return address */ c_p->cp = beam_return_time_trace; c_p->stop = E; @@ -701,9 +712,9 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) Export* ep = bif_export[bif_index]; Uint32 flags = 0, flags_meta = 0; ErtsTracer meta_tracer = erts_tracer_nil; - int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif - * is actually in the - * export entry */ + int applying = (I == ep->beam); /* Yup, the apply code for a bif + * is actually in the + * export entry */ BeamInstr *cp = p->cp; GenericBp* g; GenericBpData* bp = NULL; @@ -711,7 +722,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - g = (GenericBp *) ep->fake_op_func_info_for_hipe[1]; + g = (GenericBp *) ep->info.native; if (g) { bp = &g->data[erts_active_bp_ix()]; bp_flags = bp->flags; @@ -727,7 +738,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) if (bp_flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE) && IS_TRACED_FL(p, F_TRACE_CALLS)) { int local = !!(bp_flags & ERTS_BPF_LOCAL_TRACE); - flags = erts_call_trace(p, ep->code, bp->local_ms, args, + flags = erts_call_trace(p, &ep->info, bp->local_ms, args, local, &ERTS_TRACER(p)); } if (bp_flags & ERTS_BPF_META_TRACE) { @@ -735,7 +746,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) meta_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer); old_tracer = meta_tracer; - flags_meta = erts_call_trace(p, ep->code, bp->meta_ms, args, + flags_meta = erts_call_trace(p, &ep->info, bp->meta_ms, args, 0, &meta_tracer); if (!ERTS_TRACER_COMPARE(old_tracer, meta_tracer)) { @@ -753,8 +764,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && IS_TRACED_FL(p, F_TRACE_CALLS)) { - BeamInstr *pc = (BeamInstr *)ep->code+3; - erts_trace_time_call(p, pc, bp->time); + erts_trace_time_call(p, &ep->info, bp->time); } /* Restore original continuation pointer (if changed). */ @@ -817,11 +827,11 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) class = exception_tag[GET_EXC_CLASS(reason)]; if (flags_meta & MATCH_SET_EXCEPTION_TRACE) { - erts_trace_exception(p, ep->code, class, value, + erts_trace_exception(p, &ep->info.mfa, class, value, &meta_tracer); } if (flags & MATCH_SET_EXCEPTION_TRACE) { - erts_trace_exception(p, ep->code, class, value, + erts_trace_exception(p, &ep->info.mfa, class, value, &ERTS_TRACER(p)); } if ((flags & MATCH_SET_RETURN_TO_TRACE) && p->catches > 0) { @@ -852,11 +862,11 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } } else { if (flags_meta & MATCH_SET_RX_TRACE) { - erts_trace_return(p, ep->code, result, &meta_tracer); + erts_trace_return(p, &ep->info.mfa, result, &meta_tracer); } /* MATCH_SET_RETURN_TO_TRACE cannot occur if(meta) */ if (flags & MATCH_SET_RX_TRACE) { - erts_trace_return(p, ep->code, result, &ERTS_TRACER(p)); + erts_trace_return(p, &ep->info.mfa, result, &ERTS_TRACER(p)); } if (flags & MATCH_SET_RETURN_TO_TRACE && IS_TRACED_FL(p, F_TRACE_RETURN_TO)) { @@ -875,7 +885,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } static ErtsTracer -do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, +do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg, int local, Binary* ms, ErtsTracer tracer) { Eterm* cpp; @@ -916,7 +926,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, ASSERT(is_CP(*cpp)); } ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - flags = erts_call_trace(c_p, I-3, ms, reg, local, &tracer); + flags = erts_call_trace(c_p, info, ms, reg, local, &tracer); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); if (cpp) { c_p->cp = cp_save; @@ -932,7 +942,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, if (need) { ASSERT(c_p->htop <= E && E <= c_p->hend); if (E - need < c_p->htop) { - (void) erts_garbage_collect(c_p, need, reg, I[-1]); + (void) erts_garbage_collect(c_p, need, reg, info->mfa.arity); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); E = c_p->stop; } @@ -948,12 +958,12 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, E -= 3; c_p->stop = E; ASSERT(c_p->htop <= E && E <= c_p->hend); - ASSERT(is_CP((Eterm) (UWord) (I - 3))); + ASSERT(is_CP((Eterm) (UWord) (&info->mfa.module))); ASSERT(IS_TRACER_VALID(tracer)); E[2] = make_cp(c_p->cp); E[1] = copy_object(tracer, c_p); - E[0] = make_cp(I - 3); /* We ARE at the beginning of an - instruction, + E[0] = make_cp(&info->mfa.module); + /* We ARE at the beginning of an instruction, the funcinfo is above i. */ c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE) ? beam_exception_trace : beam_return_trace; @@ -966,7 +976,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, } void -erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) +erts_trace_time_call(Process* c_p, ErtsCodeInfo *info, BpDataTime* bdt) { ErtsMonotonicTime time; process_breakpoint_time_t *pbt = NULL; @@ -995,14 +1005,14 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) pbt = Alloc(sizeof(process_breakpoint_time_t)); (void) ERTS_PROC_SET_CALL_TIME(c_p, pbt); } else { - ASSERT(pbt->pc); + ASSERT(pbt->ci); /* add time to previous code */ sitem.time = time - pbt->time; sitem.pid = c_p->common.id; sitem.count = 0; /* previous breakpoint */ - pbdt = get_time_break(pbt->pc); + pbdt = get_time_break(pbt->ci); /* if null then the breakpoint was removed */ if (pbdt) { @@ -1039,12 +1049,12 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) BP_TIME_ADD(item, &sitem); } - pbt->pc = I; + pbt->ci = info; pbt->time = time; } void -erts_trace_time_return(Process *p, BeamInstr *pc) +erts_trace_time_return(Process *p, ErtsCodeInfo *ci) { ErtsMonotonicTime time; process_breakpoint_time_t *pbt = NULL; @@ -1074,14 +1084,14 @@ erts_trace_time_return(Process *p, BeamInstr *pc) /* might have been removed due to * trace_pattern(false) */ - ASSERT(pbt->pc); + ASSERT(pbt->ci); sitem.time = time - pbt->time; sitem.pid = p->common.id; sitem.count = 0; /* previous breakpoint */ - pbdt = get_time_break(pbt->pc); + pbdt = get_time_break(pbt->ci); /* beware, the trace_pattern might have been removed */ if (pbdt) { @@ -1098,16 +1108,16 @@ erts_trace_time_return(Process *p, BeamInstr *pc) } } - pbt->pc = pc; + pbt->ci = ci; pbt->time = time; } } int -erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local) +erts_is_trace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, int local) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; - GenericBpData* bp = check_break(pc, flags); + GenericBpData* bp = check_break(ci, flags); if (bp) { if (match_spec_ret) { @@ -1119,10 +1129,10 @@ erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local) } int -erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, +erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, ErtsTracer *tracer_ret) { - GenericBpData* bp = check_break(pc, ERTS_BPF_META_TRACE); + GenericBpData* bp = check_break(ci, ERTS_BPF_META_TRACE); if (bp) { if (match_spec_ret) { @@ -1137,20 +1147,20 @@ erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, } int -erts_is_native_break(BeamInstr *pc) { +erts_is_native_break(ErtsCodeInfo *ci) { #ifdef HIPE - ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - return pc[0] == (BeamInstr) BeamOp(op_hipe_trap_call) - || pc[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure); + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + return erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call) + || erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure); #else return 0; #endif } int -erts_is_count_break(BeamInstr *pc, Uint *count_ret) +erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret) { - GenericBpData* bp = check_break(pc, ERTS_BPF_COUNT); + GenericBpData* bp = check_break(ci, ERTS_BPF_COUNT); if (bp) { if (count_ret) { @@ -1161,13 +1171,13 @@ erts_is_count_break(BeamInstr *pc, Uint *count_ret) return 0; } -int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { +int erts_is_time_break(Process *p, ErtsCodeInfo *ci, Eterm *retval) { Uint i, ix; bp_time_hash_t hash; Uint size; Eterm *hp, t; bp_data_time_item_t *item = NULL; - BpDataTime *bdt = get_time_break(pc); + BpDataTime *bdt = get_time_break(ci); if (bdt) { if (retval) { @@ -1221,26 +1231,25 @@ int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { } -BeamInstr * -erts_find_local_func(Eterm mfa[3]) { +ErtsCodeInfo * +erts_find_local_func(ErtsCodeMFA *mfa) { Module *modp; BeamCodeHeader* code_hdr; - BeamInstr* code_ptr; + ErtsCodeInfo* ci; Uint i,n; - if ((modp = erts_get_module(mfa[0], erts_active_code_ix())) == NULL) + if ((modp = erts_get_module(mfa->module, erts_active_code_ix())) == NULL) return NULL; if ((code_hdr = modp->curr.code_hdr) == NULL) return NULL; n = (BeamInstr) code_hdr->num_functions; for (i = 0; i < n; ++i) { - code_ptr = code_hdr->functions[i]; - ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]); - ASSERT(mfa[0] == ((Eterm) code_ptr[2]) || - is_nil((Eterm) code_ptr[2])); - if (mfa[1] == ((Eterm) code_ptr[3]) && - ((BeamInstr) mfa[2]) == code_ptr[4]) { - return code_ptr + 5; + ci = code_hdr->functions[i]; + ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == ci->op); + ASSERT(mfa->module == ci->mfa.module || is_nil(ci->mfa.module)); + if (mfa->function == ci->mfa.function && + mfa->arity == ci->mfa.arity) { + return ci; } } return NULL; @@ -1369,7 +1378,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { * the previous breakpoint. */ - pbdt = get_time_break(pbt->pc); + pbdt = get_time_break(pbt->ci); if (pbdt) { sitem.time = get_mtime(p) - pbt->time; sitem.pid = p->common.id; @@ -1417,14 +1426,14 @@ set_break(BpFunctions* f, Binary *match_spec, Uint break_flags, n = f->matched; for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - set_function_break(pc, match_spec, break_flags, + set_function_break(f->matching[i].ci, + match_spec, break_flags, count_op, tracer); } } static void -set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, +set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, enum erts_break_op count_op, ErtsTracer tracer) { GenericBp* g; @@ -1433,7 +1442,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, ErtsBpIndex ix = erts_staging_bp_ix(); ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); - g = (GenericBp *) pc[-4]; + g = (GenericBp *) ci->native; if (g == 0) { int i; if (count_op == ERTS_BREAK_RESTART || count_op == ERTS_BREAK_PAUSE) { @@ -1441,11 +1450,11 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, return; } g = Alloc(sizeof(GenericBp)); - g->orig_instr = *pc; + g->orig_instr = *erts_codeinfo_to_code(ci); for (i = 0; i < ERTS_NUM_BP_IX; i++) { g->data[i].flags = 0; } - pc[-4] = (BeamInstr) g; + ci->native = (BeamInstr) g; } bp = &g->data[ix]; @@ -1517,7 +1526,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0); bdt = Alloc(sizeof(BpDataTime)); erts_refc_init(&bdt->refc, 1); - bdt->n = erts_no_schedulers; + bdt->n = erts_no_total_schedulers; bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n)); for (i = 0; i < bdt->n; i++) { bp_hash_init(&(bdt->hash[i]), 32); @@ -1537,13 +1546,12 @@ clear_break(BpFunctions* f, Uint break_flags) n = f->matched; for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - clear_function_break(pc, break_flags); + clear_function_break(f->matching[i].ci, break_flags); } } static int -clear_function_break(BeamInstr *pc, Uint break_flags) +clear_function_break(ErtsCodeInfo *ci, Uint break_flags) { GenericBp* g; GenericBpData* bp; @@ -1552,7 +1560,7 @@ clear_function_break(BeamInstr *pc, Uint break_flags) ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); - if ((g = (GenericBp *) pc[-4]) == 0) { + if ((g = (GenericBp *) ci->native) == 0) { return 1; } @@ -1638,19 +1646,19 @@ bp_time_unref(BpDataTime* bdt) } static BpDataTime* -get_time_break(BeamInstr *pc) +get_time_break(ErtsCodeInfo *ci) { - GenericBpData* bp = check_break(pc, ERTS_BPF_TIME_TRACE); + GenericBpData* bp = check_break(ci, ERTS_BPF_TIME_TRACE); return bp ? bp->time : 0; } static GenericBpData* -check_break(BeamInstr *pc, Uint break_flags) +check_break(ErtsCodeInfo *ci, Uint break_flags) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; - ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - if (erts_is_native_break(pc)) { + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + if (erts_is_native_break(ci)) { return 0; } if (g) { diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 541af77211..224b46407d 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -46,7 +46,7 @@ typedef struct bp_data_time { /* Call time */ typedef struct { ErtsMonotonicTime time; - BeamInstr *pc; + ErtsCodeInfo *ci; } process_breakpoint_time_t; /* used within psd */ typedef struct { @@ -80,7 +80,7 @@ typedef struct generic_bp { #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2) #ifdef ERTS_SMP -#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->no - 1) +#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->thr_id - 1) #else #define bp_sched2ix_proc(p) (0) #endif @@ -95,7 +95,7 @@ enum erts_break_op{ typedef Uint32 ErtsBpIndex; typedef struct { - BeamInstr* pc; + ErtsCodeInfo *ci; Module* mod; } BpFunction; @@ -116,8 +116,8 @@ void erts_commit_staged_bp(void); ERTS_GLB_INLINE ErtsBpIndex erts_active_bp_ix(void); ERTS_GLB_INLINE ErtsBpIndex erts_staging_bp_ix(void); -void erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified); -void erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified); +void erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified); +void erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified); void erts_bp_free_matched_functions(BpFunctions* f); void erts_install_breakpoints(BpFunctions* f); @@ -128,15 +128,15 @@ void erts_consolidate_bif_bp_data(void); void erts_set_trace_break(BpFunctions *f, Binary *match_spec); void erts_clear_trace_break(BpFunctions *f); -void erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local); -void erts_clear_call_trace_bif(BeamInstr *pc, int local); +void erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local); +void erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local); void erts_set_mtrace_break(BpFunctions *f, Binary *match_spec, ErtsTracer tracer); void erts_clear_mtrace_break(BpFunctions *f); -void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, +void erts_set_mtrace_bif(ErtsCodeInfo *ci, Binary *match_spec, ErtsTracer tracer); -void erts_clear_mtrace_bif(BeamInstr *pc); +void erts_clear_mtrace_bif(ErtsCodeInfo *ci); void erts_set_debug_break(BpFunctions *f); void erts_clear_debug_break(BpFunctions *f); @@ -146,32 +146,32 @@ void erts_clear_count_break(BpFunctions *f); void erts_clear_all_breaks(BpFunctions* f); int erts_clear_module_break(Module *modp); -void erts_clear_export_break(Module *modp, BeamInstr* pc); +void erts_clear_export_break(Module *modp, ErtsCodeInfo* ci); -BeamInstr erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg); -BeamInstr erts_trace_break(Process *p, BeamInstr *pc, Eterm *args, +BeamInstr erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *ci, Eterm* reg); +BeamInstr erts_trace_break(Process *p, ErtsCodeInfo *ci, Eterm *args, Uint32 *ret_flags, ErtsTracer *tracer); -int erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local); -int erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, +int erts_is_trace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, int local); +int erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, ErtsTracer *tracer_ret); -int erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret, +int erts_is_mtrace_bif(ErtsCodeInfo *ci, Binary **match_spec_ret, ErtsTracer *tracer_ret); -int erts_is_native_break(BeamInstr *pc); -int erts_is_count_break(BeamInstr *pc, Uint *count_ret); -int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *call_time); +int erts_is_native_break(ErtsCodeInfo *ci); +int erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret); +int erts_is_time_break(Process *p, ErtsCodeInfo *ci, Eterm *call_time); -void erts_trace_time_call(Process* c_p, BeamInstr* pc, BpDataTime* bdt); -void erts_trace_time_return(Process* c_p, BeamInstr* pc); +void erts_trace_time_call(Process* c_p, ErtsCodeInfo *ci, BpDataTime* bdt); +void erts_trace_time_return(Process* c_p, ErtsCodeInfo *ci); void erts_schedule_time_break(Process *p, Uint out); void erts_set_time_break(BpFunctions *f, enum erts_break_op); void erts_clear_time_break(BpFunctions *f); -int erts_is_time_trace_bif(Process *p, BeamInstr *pc, Eterm *call_time); -void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op); -void erts_clear_time_trace_bif(BeamInstr *pc); +int erts_is_time_trace_bif(Process *p, ErtsCodeInfo *ci, Eterm *call_time); +void erts_set_time_trace_bif(ErtsCodeInfo *ci, enum erts_break_op); +void erts_clear_time_trace_bif(ErtsCodeInfo *ci); -BeamInstr *erts_find_local_func(Eterm mfa[3]); +ErtsCodeInfo *erts_find_local_func(ErtsCodeMFA *mfa); #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 8f47ed4d9d..e72d7f8de4 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -50,8 +50,8 @@ void dbg_bt(Process* p, Eterm* sp); void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg); -static int print_op(int to, void *to_arg, int op, int size, BeamInstr* addr); -static void print_bif_name(int to, void* to_arg, BifFunction bif); +static int print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr); +static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif); BIF_RETTYPE erts_debug_same_2(BIF_ALIST_2) @@ -116,7 +116,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) Eterm MFA = BIF_ARG_1; Eterm boolean = BIF_ARG_2; Eterm* tp; - Eterm mfa[3]; + ErtsCodeMFA mfa; int i; int specified = 0; Eterm res; @@ -132,23 +132,24 @@ erts_debug_breakpoint_2(BIF_ALIST_2) if (*tp != make_arityval(3)) { goto error; } - mfa[0] = tp[1]; - mfa[1] = tp[2]; - mfa[2] = tp[3]; - if (!is_atom(mfa[0]) || !is_atom(mfa[1]) || - (!is_small(mfa[2]) && mfa[2] != am_Underscore)) { + if (!is_atom(tp[1]) || !is_atom(tp[2]) || + (!is_small(tp[3]) && tp[3] != am_Underscore)) { goto error; } - for (i = 0; i < 3 && mfa[i] != am_Underscore; i++, specified++) { + for (i = 0; i < 3 && tp[i+1] != am_Underscore; i++, specified++) { /* Empty loop body */ } for (i = specified; i < 3; i++) { - if (mfa[i] != am_Underscore) { + if (tp[i+1] != am_Underscore) { goto error; } } - if (is_small(mfa[2])) { - mfa[2] = signed_val(mfa[2]); + + mfa.module = tp[1]; + mfa.function = tp[2]; + + if (is_small(tp[3])) { + mfa.arity = signed_val(tp[3]); } if (!erts_try_seize_code_write_permission(BIF_P)) { @@ -158,7 +159,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); - erts_bp_match_functions(&f, mfa, specified); + erts_bp_match_functions(&f, &mfa, specified); if (boolean == am_true) { erts_set_debug_break(&f); erts_install_breakpoints(&f); @@ -242,9 +243,9 @@ erts_debug_disassemble_1(BIF_ALIST_1) Eterm* tp; Eterm bin; Eterm mfa; - BeamInstr* funcinfo = NULL; /* Initialized to eliminate warning. */ + ErtsCodeMFA *cmfa = NULL; BeamCodeHeader* code_hdr; - BeamInstr* code_ptr = NULL; /* Initialized to eliminate warning. */ + BeamInstr *code_ptr; BeamInstr instr; BeamInstr uaddr; Uint hsz; @@ -252,7 +253,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) if (term_to_UWord(addr, &uaddr)) { code_ptr = (BeamInstr *) uaddr; - if ((funcinfo = find_function_from_pc(code_ptr)) == NULL) { + if ((cmfa = find_function_from_pc(code_ptr)) == NULL) { BIF_RET(am_false); } } else if (is_tuple(addr)) { @@ -283,24 +284,22 @@ erts_debug_disassemble_1(BIF_ALIST_1) * such as erts_debug:apply/4. Then search for it in the module. */ if ((ep = erts_find_function(mod, name, arity, code_ix)) != NULL) { - /* XXX: add "&& ep->address != ep->code+3" condition? + /* XXX: add "&& ep->address != ep->code" condition? * Consider a traced function. - * Its ep will have ep->address == ep->code+3. + * Its ep will have ep->address == ep->code. * erts_find_function() will return the non-NULL ep. * Below we'll try to derive a code_ptr from ep->address. * But this code_ptr will point to the start of the Export, * not the function's func_info instruction. BOOM !? */ - code_ptr = ((BeamInstr *) ep->addressv[code_ix]) - 5; - funcinfo = code_ptr+2; + cmfa = erts_code_to_codemfa(ep->addressv[code_ix]); } else if (modp == NULL || (code_hdr = modp->curr.code_hdr) == NULL) { BIF_RET(am_undef); } else { n = code_hdr->num_functions; for (i = 0; i < n; i++) { - code_ptr = code_hdr->functions[i]; - if (code_ptr[3] == name && code_ptr[4] == arity) { - funcinfo = code_ptr+2; + cmfa = &code_hdr->functions[i]->mfa; + if (cmfa->function == name && cmfa->arity == arity) { break; } } @@ -308,6 +307,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) BIF_RET(am_undef); } } + code_ptr = erts_codemfa_to_code(cmfa); } else { goto error; } @@ -333,9 +333,10 @@ erts_debug_disassemble_1(BIF_ALIST_1) (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr); hp = HAlloc(p, hsz); addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr); - ASSERT(is_atom(funcinfo[0]) || funcinfo[0] == NIL); - ASSERT(is_atom(funcinfo[1]) || funcinfo[1] == NIL); - mfa = TUPLE3(hp, (Eterm) funcinfo[0], (Eterm) funcinfo[1], make_small((Eterm) funcinfo[2])); + ASSERT(is_atom(cmfa->module) || is_nil(cmfa->module)); + ASSERT(is_atom(cmfa->function) || is_nil(cmfa->function)); + mfa = TUPLE3(hp, cmfa->module, cmfa->function, + make_small(cmfa->arity)); hp += 4; return TUPLE3(hp, addr, bin, mfa); } @@ -347,11 +348,12 @@ dbg_bt(Process* p, Eterm* sp) while (sp < stack) { if (is_CP(*sp)) { - BeamInstr* addr = find_function_from_pc(cp_val(*sp)); - if (addr) + ErtsCodeMFA* cmfa = find_function_from_pc(cp_val(*sp)); + if (cmfa) erts_fprintf(stderr, HEXF ": %T:%T/%bpu\n", - addr, (Eterm) addr[0], (Eterm) addr[1], addr[2]); + &cmfa->module, cmfa->module, + cmfa->function, cmfa->arity); } sp++; } @@ -360,17 +362,17 @@ dbg_bt(Process* p, Eterm* sp) void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg) { - BeamInstr* f = find_function_from_pc(addr); + ErtsCodeMFA* cmfa = find_function_from_pc(addr); - if (f == NULL) { + if (cmfa == NULL) { erts_fprintf(stderr, "???\n"); } else { int arity; int i; - addr = f; - arity = addr[2]; - erts_fprintf(stderr, HEXF ": %T:%T(", addr, (Eterm) addr[0], (Eterm) addr[1]); + arity = cmfa->arity; + erts_fprintf(stderr, HEXF ": %T:%T(", addr, + cmfa->module, cmfa->function); for (i = 0; i < arity; i++) erts_fprintf(stderr, i ? ", %T" : "%T", i ? reg[i] : x0); erts_fprintf(stderr, ")\n"); @@ -378,7 +380,7 @@ dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg) } static int -print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) +print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) { int i; BeamInstr tag; @@ -546,22 +548,24 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) break; case 'f': /* Destination label */ { - BeamInstr* f = find_function_from_pc((BeamInstr *)*ap); - if (f+3 != (BeamInstr *) *ap) { + ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap); + if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { erts_print(to, to_arg, "f(" HEXF ")", *ap); } else { - erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]); + erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, + cmfa->function, cmfa->arity); } ap++; } break; case 'p': /* Pointer (to label) */ { - BeamInstr* f = find_function_from_pc((BeamInstr *)*ap); - if (f+3 != (BeamInstr *) *ap) { + ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap); + if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { erts_print(to, to_arg, "p(" HEXF ")", *ap); } else { - erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]); + erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, + cmfa->function, cmfa->arity); } ap++; } @@ -574,7 +578,9 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) { Export* ex = (Export *) *ap; erts_print(to, to_arg, - "%T:%T/%bpu", (Eterm) ex->code[0], (Eterm) ex->code[1], ex->code[2]); + "%T:%T/%bpu", (Eterm) ex->info.mfa.module, + (Eterm) ex->info.mfa.function, + ex->info.mfa.arity); ap++; } break; @@ -741,7 +747,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) return size; } -static void print_bif_name(int to, void* to_arg, BifFunction bif) +static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif) { int i; diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 0ba06058a5..613bd5c14e 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -116,10 +116,10 @@ do { \ #define MAX(x, y) (((x) > (y)) ? (x) : (y)) #endif -#define GET_BIF_MODULE(p) ((Eterm) (((Export *) p)->code[0])) -#define GET_BIF_FUNCTION(p) ((Eterm) (((Export *) p)->code[1])) -#define GET_BIF_ARITY(p) ((Eterm) (((Export *) p)->code[2])) -#define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4])) +#define GET_BIF_MODULE(p) (p->info.mfa.module) +#define GET_BIF_FUNCTION(p) (p->info.mfa.function) +#define GET_BIF_ARITY(p) (p->info.mfa.arity) +#define GET_BIF_ADDRESS(p) ((BifFunction) (p->beam[1])) #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) @@ -1045,7 +1045,7 @@ void** beam_ops; static BifFunction translate_gc_bif(void* gcf) NOINLINE; static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) NOINLINE; -static BeamInstr* call_error_handler(Process* p, BeamInstr* ip, +static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) NOINLINE; static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE; static BeamInstr* apply(Process* p, Eterm module, Eterm function, @@ -1106,98 +1106,91 @@ init_emulator(void) #ifdef USE_VM_CALL_PROBES -#define DTRACE_LOCAL_CALL(p, m, f, a) \ +#define DTRACE_LOCAL_CALL(p, mfa) \ if (DTRACE_ENABLED(local_function_entry)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(local_function_entry, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(local_function_entry, process_name, mfa_buf, depth); \ } -#define DTRACE_GLOBAL_CALL(p, m, f, a) \ +#define DTRACE_GLOBAL_CALL(p, mfa) \ if (DTRACE_ENABLED(global_function_entry)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(global_function_entry, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(global_function_entry, process_name, mfa_buf, depth); \ } -#define DTRACE_RETURN(p, m, f, a) \ +#define DTRACE_RETURN(p, mfa) \ if (DTRACE_ENABLED(function_return)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(function_return, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(function_return, process_name, mfa_buf, depth); \ } -#define DTRACE_BIF_ENTRY(p, m, f, a) \ - if (DTRACE_ENABLED(bif_entry)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(bif_entry, process_name, mfa); \ +#define DTRACE_BIF_ENTRY(p, mfa) \ + if (DTRACE_ENABLED(bif_entry)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(bif_entry, process_name, mfa_buf); \ } -#define DTRACE_BIF_RETURN(p, m, f, a) \ - if (DTRACE_ENABLED(bif_return)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(bif_return, process_name, mfa); \ +#define DTRACE_BIF_RETURN(p, mfa) \ + if (DTRACE_ENABLED(bif_return)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(bif_return, process_name, mfa_buf); \ } -#define DTRACE_NIF_ENTRY(p, m, f, a) \ - if (DTRACE_ENABLED(nif_entry)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(nif_entry, process_name, mfa); \ +#define DTRACE_NIF_ENTRY(p, mfa) \ + if (DTRACE_ENABLED(nif_entry)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(nif_entry, process_name, mfa_buf); \ } -#define DTRACE_NIF_RETURN(p, m, f, a) \ - if (DTRACE_ENABLED(nif_return)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(nif_return, process_name, mfa); \ +#define DTRACE_NIF_RETURN(p, mfa) \ + if (DTRACE_ENABLED(nif_return)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(nif_return, process_name, mfa_buf); \ } #define DTRACE_GLOBAL_CALL_FROM_EXPORT(p,e) \ do { \ if (DTRACE_ENABLED(global_function_entry)) { \ BeamInstr* fp = (BeamInstr *) (((Export *) (e))->addressv[erts_active_code_ix()]); \ - DTRACE_GLOBAL_CALL((p), (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); \ + DTRACE_GLOBAL_CALL((p), erts_code_to_codemfa(fp)); \ } \ } while(0) #define DTRACE_RETURN_FROM_PC(p) \ do { \ - BeamInstr* fp; \ - if (DTRACE_ENABLED(function_return) && (fp = find_function_from_pc((p)->cp))) { \ - DTRACE_RETURN((p), (Eterm)fp[0], (Eterm)fp[1], (Uint)fp[2]); \ + ErtsCodeMFA* cmfa; \ + if (DTRACE_ENABLED(function_return) && (cmfa = find_function_from_pc((p)->cp))) { \ + DTRACE_RETURN((p), cmfa); \ } \ } while(0) #else /* USE_VM_PROBES */ -#define DTRACE_LOCAL_CALL(p, m, f, a) do {} while (0) -#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0) +#define DTRACE_LOCAL_CALL(p, mfa) do {} while (0) +#define DTRACE_GLOBAL_CALL(p, mfa) do {} while (0) #define DTRACE_GLOBAL_CALL_FROM_EXPORT(p, e) do {} while (0) -#define DTRACE_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_RETURN(p, mfa) do {} while (0) #define DTRACE_RETURN_FROM_PC(p) do {} while (0) -#define DTRACE_BIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_BIF_RETURN(p, m, f, a) do {} while (0) -#define DTRACE_NIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_BIF_ENTRY(p, mfa) do {} while (0) +#define DTRACE_BIF_RETURN(p, mfa) do {} while (0) +#define DTRACE_NIF_ENTRY(p, mfa) do {} while (0) +#define DTRACE_NIF_RETURN(p, mfa) do {} while (0) #endif /* USE_VM_PROBES */ #ifdef DEBUG @@ -1324,8 +1317,8 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) if (start_time != 0) { Sint64 diff = erts_timestamp_millis() - start_time; if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule) { - BeamInstr *inptr = find_function_from_pc(start_time_i); - BeamInstr *outptr = find_function_from_pc(c_p->i); + ErtsCodeMFA *inptr = find_function_from_pc(start_time_i); + ErtsCodeMFA *outptr = find_function_from_pc(c_p->i); monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff); } } @@ -1399,10 +1392,9 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) if (ERTS_PROC_IS_EXITING(c_p)) { strcpy(fun_buf, "<exiting>"); } else { - BeamInstr *fptr = find_function_from_pc(c_p->i); - if (fptr) { - dtrace_fun_decode(c_p, (Eterm)fptr[0], - (Eterm)fptr[1], (Uint)fptr[2], + ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i); + if (cmfa) { + dtrace_fun_decode(c_p, cmfa, NULL, fun_buf); } else { erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), @@ -1583,7 +1575,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) /* FALL THROUGH */ OpCase(i_call_only_f): { SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } @@ -1595,7 +1587,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) RESTORE_CP(E); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } @@ -1607,7 +1599,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) OpCase(i_call_f): { SET_CP(c_p, I+2); SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } @@ -2834,25 +2826,27 @@ do { \ Eterm result; BeamInstr *next; ErlHeapFragment *live_hf_end; + Export *export = (Export*)Arg(0); if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { /* If we have run out of reductions, we do a context switch before calling the bif */ - c_p->arity = ((Export *)Arg(0))->code[2]; - c_p->current = ((Export *)Arg(0))->code; + c_p->arity = GET_BIF_ARITY(export); + c_p->current = &export->info.mfa; goto context_switch3; } - ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(Arg(0)), GET_BIF_ADDRESS(Arg(0))); + ERTS_MSACC_SET_BIF_STATE_CACHED_X( + GET_BIF_MODULE(export), GET_BIF_ADDRESS(export)); - bf = GET_BIF_ADDRESS(Arg(0)); + bf = GET_BIF_ADDRESS(export); PRE_BIF_SWAPOUT(c_p); ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS - 1; if (FCALLS <= 0) { - save_calls(c_p, (Export *) Arg(0)); + save_calls(c_p, export); } PreFetch(1, next); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -2866,7 +2860,7 @@ do { \ ERTS_HOLE_CHECK(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); if (ERTS_IS_GC_DESIRED(c_p)) { - Uint arity = ((Export *)Arg(0))->code[2]; + Uint arity = GET_BIF_ARITY(export); result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity); E = c_p->stop; } @@ -3359,14 +3353,15 @@ do { \ * called from I[-3], I[-2], and I[-1] respectively. */ context_switch_fun: - c_p->arity = I[-1] + 1; + /* Add one for the environment of the fun */ + c_p->arity = erts_code_to_codemfa(I)->arity + 1; goto context_switch2; context_switch: - c_p->arity = I[-1]; + c_p->arity = erts_code_to_codemfa(I)->arity; - context_switch2: /* Entry for fun calls. */ - c_p->current = I-3; /* Pointer to Mod, Func, Arity */ + context_switch2: /* Entry for fun calls. */ + c_p->current = erts_code_to_codemfa(I); context_switch3: @@ -3507,7 +3502,8 @@ do { \ * code[4]: Not used */ HEAVY_SWAPOUT; - I = call_error_handler(c_p, I-3, reg, am_undefined_function); + I = call_error_handler(c_p, erts_code_to_codemfa(I), + reg, am_undefined_function); HEAVY_SWAPIN; if (I) { Goto(*I); @@ -3544,9 +3540,12 @@ do { \ * I[1]: Function pointer to NIF function * I[2]: Pointer to erl_module_nif * I[3]: Function pointer to dirty NIF + * + * This layout is determined by the NifExport struct */ BifFunction vbf; ErlHeapFragment *live_hf_end; + ErtsCodeMFA *codemfa; if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { /* If we have run out of reductions, we do a context @@ -3556,12 +3555,16 @@ do { \ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); - DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); - c_p->current = I-3; /* current and vbf set to please handle_error */ + codemfa = erts_code_to_codemfa(I); + + c_p->current = codemfa; /* current and vbf set to please handle_error */ + + DTRACE_NIF_ENTRY(c_p, codemfa); + SWAPOUT; c_p->fcalls = FCALLS - 1; PROCESS_MAIN_CHK_LOCKS(c_p); - bif_nif_arity = I[-1]; + bif_nif_arity = codemfa->arity; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -3588,19 +3591,19 @@ do { \ ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } - DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_NIF_RETURN(c_p, codemfa); goto apply_bif_or_nif_epilogue; OpCase(apply_bif): /* - * At this point, I points to the code[3] in the export entry for + * At this point, I points to the code[0] in the export entry for * the BIF: * - * code[0]: Module - * code[1]: Function - * code[2]: Arity - * code[3]: &&apply_bif - * code[4]: Function pointer to BIF function + * code[-3]: Module + * code[-2]: Function + * code[-1]: Arity + * code[0]: &&apply_bif + * code[1]: Function pointer to BIF function */ if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { @@ -3609,21 +3612,25 @@ do { \ goto context_switch; } - ERTS_MSACC_SET_BIF_STATE_CACHED_X((Eterm)I[-3], (BifFunction)Arg(0)); + codemfa = erts_code_to_codemfa(I); + + ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)Arg(0)); - c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ + + /* In case we apply process_info/1,2 or load_nif/1 */ + c_p->current = codemfa; c_p->i = I; /* In case we apply check_process_code/2. */ c_p->arity = 0; /* To allow garbage collection on ourselves * (check_process_code/2). */ - DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_BIF_ENTRY(c_p, codemfa); SWAPOUT; ERTS_DBG_CHK_REDS(c_p, FCALLS - 1); c_p->fcalls = FCALLS - 1; vbf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); - bif_nif_arity = I[-1]; + bif_nif_arity = codemfa->arity; ASSERT(bif_nif_arity <= 4); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -3645,7 +3652,7 @@ do { \ if (ERTS_MSACC_IS_ENABLED_CACHED_X()) ERTS_MSACC_UPDATE_CACHE_X(); ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - DTRACE_BIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_BIF_RETURN(c_p, codemfa); apply_bif_or_nif_epilogue: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); @@ -3712,8 +3719,9 @@ do { \ goto find_func_info; OpCase(i_func_info_IaaI): { + ErtsCodeInfo *ci = (ErtsCodeInfo*)I; c_p->freason = EXC_FUNCTION_CLAUSE; - c_p->current = I + 2; + c_p->current = &ci->mfa; goto handle_error; } @@ -4704,11 +4712,11 @@ do { \ */ OpCase(return_trace): { - BeamInstr* code = (BeamInstr *) (UWord) E[0]; + ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[0]); SWAPOUT; /* Needed for shared heap */ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - erts_trace_return(c_p, code, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); + erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); SWAPIN; c_p->cp = NULL; @@ -4719,9 +4727,8 @@ do { \ OpCase(i_generic_breakpoint): { BeamInstr real_I; - ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); HEAVY_SWAPOUT; - real_I = erts_generic_breakpoint(c_p, I, reg); + real_I = erts_generic_breakpoint(c_p, erts_code_to_codeinfo(I), reg); HEAVY_SWAPIN; ASSERT(VALID_INSTR(real_I)); Goto(real_I); @@ -4730,7 +4737,7 @@ do { \ OpCase(i_return_time_trace): { BeamInstr *pc = (BeamInstr *) (UWord) E[0]; SWAPOUT; - erts_trace_time_return(c_p, pc); + erts_trace_time_return(c_p, erts_code_to_codeinfo(pc)); SWAPIN; c_p->cp = NULL; SET_I((BeamInstr *) cp_val(E[1])); @@ -4915,16 +4922,18 @@ do { \ * I[ 0]: &&lb_hipe_trap_call * ... remainder of original BEAM code */ - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); + c_p->hipe.u.ncallee = (void(*)(void)) ci->native; ++hipe_trap_count; - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8)); + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8)); } OpCase(hipe_trap_call_closure): { - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); + c_p->hipe.u.ncallee = (void(*)(void)) ci->native; ++hipe_trap_count; - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8)); + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8)); } OpCase(hipe_trap_return): { HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN); @@ -4993,8 +5002,9 @@ do { \ * I[ 0]: &&lb_hipe_call_count * ... remainder of original BEAM code */ - struct hipe_call_count *hcc = (struct hipe_call_count*)I[-4]; - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + struct hipe_call_count *hcc = (struct hipe_call_count*)ci->native; + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); ASSERT(hcc != NULL); ASSERT(VALID_INSTR(hcc->opcode)); ++(hcc->count); @@ -5043,7 +5053,7 @@ do { \ } else { TestHeap(ERTS_SINT64_HEAP_SIZE(ts),0); r(0) = make_big(HTOP); -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) if (ts >= (((Uint64) 1) << 32)) { *HTOP = make_pos_bignum_header(2); BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff)); @@ -5063,7 +5073,7 @@ do { \ OpCase(i_debug_breakpoint): { HEAVY_SWAPOUT; - I = call_error_handler(c_p, I-3, reg, am_breakpoint); + I = call_error_handler(c_p, erts_code_to_codemfa(I), reg, am_breakpoint); HEAVY_SWAPIN; if (I) { Goto(*I); @@ -5136,10 +5146,10 @@ do { \ bif_table[i].name, bif_table[i].arity); bif_export[i] = ep; - ep->code[3] = (BeamInstr) OpCode(apply_bif); - ep->code[4] = (BeamInstr) bif_table[i].f; + ep->beam[0] = (BeamInstr) OpCode(apply_bif); + ep->beam[1] = (BeamInstr) bif_table[i].f; /* XXX: set func info for bifs */ - ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); + ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); } return; @@ -5219,8 +5229,8 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) goto do_dirty_schedule; context_switch: - c_p->arity = I[-1]; - c_p->current = I-3; /* Pointer to Mod, Func, Arity */ + c_p->current = erts_code_to_codemfa(I); /* Pointer to Mod, Func, Arity */ + c_p->arity = c_p->current->arity; { int reds_used; @@ -5323,7 +5333,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) I = c_p->i; - ASSERT(BeamOp(op_call_nif) == (BeamInstr *) *I); + ASSERT(em_call_nif == (BeamInstr *) *I); /* * Set fcalls even though we ignore it, so we don't @@ -5345,11 +5355,9 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) if (ERTS_PROC_IS_EXITING(c_p)) { strcpy(fun_buf, "<exiting>"); } else { - BeamInstr *fptr = find_function_from_pc(c_p->i); - if (fptr) { - dtrace_fun_decode(c_p, (Eterm)fptr[0], - (Eterm)fptr[1], (Uint)fptr[2], - NULL, fun_buf); + ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i); + if (cmfa) { + dtrace_fun_decode(c_p, cmfa, NULL, fun_buf); } else { erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), "<unknown/%p>", *I); @@ -5378,16 +5386,22 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) * I[1]: Function pointer to NIF function * I[2]: Pointer to erl_module_nif * I[3]: Function pointer to dirty NIF + * + * This layout is determined by the NifExport struct */ BifFunction vbf; + ErtsCodeMFA *codemfa; ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); - DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); - c_p->current = I-3; /* current and vbf set to please handle_error */ + codemfa = erts_code_to_codemfa(I); + + DTRACE_NIF_ENTRY(c_p, codemfa); + /* current and vbf set to please handle_error */ + c_p->current = codemfa; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); - arity = I[-1]; + arity = codemfa->arity; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -5422,7 +5436,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } - DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_NIF_RETURN(c_p, codemfa); ERTS_HOLE_CHECK(c_p); SWAPIN; I = c_p->i; @@ -5602,7 +5616,8 @@ next_catch(Process* c_p, Eterm *reg) { /* Can not follow cp here - code may be unloaded */ BeamInstr *cpp = c_p->cp; if (cpp == beam_exception_trace) { - erts_trace_exception(c_p, cp_val(ptr[0]), + ErtsCodeMFA *mfa = (ErtsCodeMFA*)cp_val(ptr[0]); + erts_trace_exception(c_p, mfa, reg[1], reg[2], ERTS_TRACER_FROM_ETERM(ptr+1)); /* Skip return_trace parameters */ @@ -5630,7 +5645,8 @@ next_catch(Process* c_p, Eterm *reg) { if (is_catch(*ptr) && active_catches) goto found_catch; } if (cp_val(*prev) == beam_exception_trace) { - erts_trace_exception(c_p, cp_val(ptr[0]), + ErtsCodeMFA *mfa = (ErtsCodeMFA*)cp_val(ptr[0]); + erts_trace_exception(c_p, mfa, reg[1], reg[2], ERTS_TRACER_FROM_ETERM(ptr+1)); } @@ -5836,7 +5852,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, for (i = 0; i < BIF_SIZE; i++) { if (bf == bif_table[i].f || bf == bif_table[i].traced) { Export *ep = bif_export[i]; - s->current = ep->code; + s->current = &ep->info.mfa; a = bif_table[i].arity; break; } @@ -5850,7 +5866,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, */ ASSERT(c_p->current); s->current = c_p->current; - a = s->current[2]; + a = s->current->arity; } /* Save first stack entry */ ASSERT(pc); @@ -5875,7 +5891,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, (GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) ) { int a; ASSERT(s->current); - a = s->current[2]; + a = s->current->arity; args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */ /* Save first stack entry */ ASSERT(c_p->cp); @@ -6046,7 +6062,7 @@ build_stacktrace(Process* c_p, Eterm exc) { erts_lookup_function_info(&fi, s->pc, 1); } else if (GET_EXC_INDEX(s->freason) == GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) { - erts_lookup_function_info(&fi, s->current, 1); + erts_lookup_function_info(&fi, erts_codemfa_to_code(s->current), 1); } else { erts_set_current_function(&fi, s->current); } @@ -6055,8 +6071,8 @@ build_stacktrace(Process* c_p, Eterm exc) { * If fi.current is still NULL, default to the initial function * (e.g. spawn_link(erlang, abs, [1])). */ - if (fi.current == NULL) { - erts_set_current_function(&fi, c_p->u.initial); + if (fi.mfa == NULL) { + erts_set_current_function(&fi, &c_p->u.initial); args = am_true; /* Just in case */ } else { args = get_args_from_exc(exc); @@ -6072,7 +6088,7 @@ build_stacktrace(Process* c_p, Eterm exc) { heap_size = fi.needed + 2; for (i = 0; i < depth; i++) { erts_lookup_function_info(stkp, s->trace[i], 1); - if (stkp->current) { + if (stkp->mfa) { heap_size += stkp->needed + 2; stkp++; } @@ -6096,7 +6112,7 @@ build_stacktrace(Process* c_p, Eterm exc) { } static BeamInstr* -call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) +call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) { Eterm* hp; Export* ep; @@ -6105,13 +6121,14 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) Uint sz; int i; + DBG_TRACE_MFA_P(mfa, "call_error_handler"); /* * Search for the error_handler module. */ ep = erts_find_function(erts_proc_get_error_handler(p), func, 3, erts_active_code_ix()); if (ep == NULL) { /* No error handler */ - p->current = fi; + p->current = mfa; p->freason = EXC_UNDEF; return 0; } @@ -6120,7 +6137,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) * Create a list with all arguments in the x registers. */ - arity = fi[2]; + arity = mfa->arity; sz = 2 * arity; if (HeapWordsLeft(p) < sz) { erts_garbage_collect(p, sz, reg, arity); @@ -6136,8 +6153,8 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) /* * Set up registers for call to error_handler:<func>/3. */ - reg[0] = fi[0]; - reg[1] = fi[1]; + reg[0] = mfa->module; + reg[1] = mfa->function; reg[2] = args; return ep->addressv[erts_active_code_ix()]; } @@ -6382,11 +6399,11 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_hibernate)) { + ErtsCodeMFA cmfa = { module, function, arity}; DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); - dtrace_fun_decode(c_p, module, function, arity, - process_name, mfa); - DTRACE2(process_hibernate, process_name, mfa); + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); + dtrace_fun_decode(c_p, &cmfa, process_name, mfa_buf); + DTRACE2(process_hibernate, process_name, mfa_buf); } #endif /* @@ -6433,7 +6450,7 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - c_p->current = bif_export[BIF_hibernate_3]->code; + c_p->current = &bif_export[BIF_hibernate_3]->info.mfa; c_p->flags |= F_HIBERNATE_SCHED; /* Needed also when woken! */ return 1; } @@ -6456,21 +6473,15 @@ call_fun(Process* p, /* Current process. */ if (is_fun_header(hdr)) { ErlFunThing* funp = (ErlFunThing *) fun_val(fun); - ErlFunEntry* fe; - BeamInstr* code_ptr; + ErlFunEntry* fe = funp->fe; + BeamInstr* code_ptr = fe->address; Eterm* var_ptr; - int actual_arity; - unsigned num_free; - - fe = funp->fe; - num_free = funp->num_free; - code_ptr = fe->address; - actual_arity = (int) code_ptr[-1]; + unsigned num_free = funp->num_free; + ErtsCodeMFA *mfa = erts_code_to_codemfa(code_ptr); + int actual_arity = mfa->arity; if (actual_arity == arity+num_free) { - DTRACE_LOCAL_CALL(p, (Eterm)code_ptr[-3], - (Eterm)code_ptr[-2], - code_ptr[-1]); + DTRACE_LOCAL_CALL(p, mfa); if (num_free == 0) { return code_ptr; } else { @@ -6536,7 +6547,7 @@ call_fun(Process* p, /* Current process. */ * and let it try again when the purge operation is * done (may succeed or not). */ - ep = erts_suspend_process_on_pending_purge_lambda(p); + ep = erts_suspend_process_on_pending_purge_lambda(p, fe); ASSERT(ep); } else { @@ -6575,10 +6586,10 @@ call_fun(Process* p, /* Current process. */ int actual_arity; ep = *((Export **) (export_val(fun) + 1)); - actual_arity = (int) ep->code[2]; + actual_arity = ep->info.mfa.arity; if (arity == actual_arity) { - DTRACE_GLOBAL_CALL(p, ep->code[0], ep->code[1], (Uint)ep->code[2]); + DTRACE_GLOBAL_CALL(p, &ep->info.mfa); return ep->addressv[erts_active_code_ix()]; } else { /* @@ -6666,9 +6677,6 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) funp->fe = fe; funp->num_free = num_free; funp->creator = p->common.id; -#ifdef HIPE - funp->native_address = fe->native_address; -#endif funp->arity = (int)fe->address[-1] - num_free; for (i = 0; i < num_free; i++) { *hp++ = reg[i]; @@ -7199,15 +7207,15 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) return 1; } - e.code[0] = Mod; - e.code[1] = Name; - e.code[2] = arity; + e.info.mfa.module = Mod; + e.info.mfa.function = Name; + e.info.mfa.arity = arity; if ((ep = export_get(&e)) == NULL) { return 0; } - return ep->addressv[erts_active_code_ix()] == ep->code+3 - && (ep->code[3] == (BeamInstr) em_apply_bif); + return ep->addressv[erts_active_code_ix()] == ep->beam + && (ep->beam[0] == (BeamInstr) em_apply_bif); } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index d69b18e22f..3cd395c2c1 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -39,11 +39,13 @@ #include "erl_binary.h" #include "erl_zlib.h" #include "erl_map.h" +#include "erl_process_dict.h" #ifdef HIPE #include "hipe_bif0.h" #include "hipe_mode_switch.h" #include "hipe_arch.h" +#include "hipe_load.h" #endif ErlDrvBinary* erts_gzinflate_buffer(char*, int); @@ -480,9 +482,12 @@ static void free_loader_state(Binary* magic); static ErlHeapFragment* new_literal_fragment(Uint size); static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); +#ifdef HIPE static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code, Uint size); + BeamCodeHeader* code_hdr, Uint size, + HipeModule *hipe_code); +#endif static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -537,8 +542,6 @@ static Eterm compilation_info_for_module(Process* p, BeamCodeHeader*); static Eterm md5_of_module(Process* p, BeamCodeHeader*); static Eterm has_native(BeamCodeHeader*); static Eterm native_addresses(Process* p, BeamCodeHeader*); -int patch_funentries(Eterm Patchlist); -int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); static int must_swap_floats; @@ -806,21 +809,21 @@ erts_finish_loading(Binary* magic, Process* c_p, for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i, code_ix); - if (ep == NULL || ep->code[0] != module) { + if (ep == NULL || ep->info.mfa.module != module) { continue; } - if (ep->addressv[code_ix] == ep->code+3) { - if (ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep->addressv[code_ix] == ep->beam) { + if (ep->beam[0] == (BeamInstr) em_apply_bif) { continue; - } else if (ep->code[3] == + } else if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(mod_tab_p->curr.num_traced_exports > 0); - erts_clear_export_break(mod_tab_p, ep->code+3); - ep->addressv[code_ix] = (BeamInstr *) ep->code[4]; - ep->code[4] = 0; + erts_clear_export_break(mod_tab_p, &ep->info); + ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; + ep->beam[1] = 0; } - ASSERT(ep->code[4] == 0); + ASSERT(ep->beam[1] == 0); } } ASSERT(mod_tab_p->curr.num_breakpoints == 0); @@ -842,9 +845,7 @@ erts_finish_loading(Binary* magic, Process* c_p, erts_alloc(ERTS_ALC_T_PREPARED_CODE, sizeof(struct erl_module_instance)); inst_p = mod_tab_p->on_load; - inst_p->nif = 0; - inst_p->num_breakpoints = 0; - inst_p->num_traced_exports = 0; + erts_module_instance_init(inst_p); } inst_p->code_hdr = stp->hdr; @@ -943,6 +944,13 @@ erts_module_for_prepared_code(Binary* magic) LoaderState* stp; if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) { +#ifdef HIPE + HipeLoaderState *hipe_stp; + if ((hipe_stp = hipe_get_loader_state(magic)) + && hipe_stp->text_segment != 0) { + return hipe_stp->module; + } +#endif return NIL; } stp = ERTS_MAGIC_BIN_DATA(magic); @@ -1091,10 +1099,12 @@ loader_state_dtor(Binary* magic) ASSERT(stp->genop_blocks == 0); } +#ifdef HIPE static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code_hdr, Uint size) + BeamCodeHeader* code_hdr, Uint size, + HipeModule *hipe_code) { Module* modp; Eterm retval; @@ -1117,6 +1127,9 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modp->curr.code_hdr = code_hdr; modp->curr.code_length = size; modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code " + "first_hipe_ref = %p", hipe_code->first_hipe_ref); + modp->curr.hipe_code = hipe_code; /* * Update ranges (used for finding a function from a PC value). @@ -1125,6 +1138,7 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, erts_update_ranges((BeamInstr*)modp->curr.code_hdr, size); return NIL; } +#endif static int init_iff_file(LoaderState* stp, byte* code, Uint size) @@ -1420,8 +1434,8 @@ load_import_table(LoaderState* stp) * the BIF function. */ if ((e = erts_active_export_entry(mod, func, arity)) != NULL) { - if (e->code[3] == (BeamInstr) em_apply_bif) { - stp->import[i].bf = (BifFunction) e->code[4]; + if (e->beam[0] == (BeamInstr) em_apply_bif) { + stp->import[i].bf = (BifFunction) e->beam[1]; if (func == am_load_nif && mod == am_erlang && arity == 2) { stp->may_load_nif = 1; } @@ -1514,7 +1528,7 @@ is_bif(Eterm mod, Eterm func, unsigned arity) if (e == NULL) { return 0; } - if (e->code[3] != (BeamInstr) em_apply_bif) { + if (e->beam[0] != (BeamInstr) em_apply_bif) { return 0; } if (mod == am_erlang && func == am_apply && arity == 3) { @@ -1893,7 +1907,7 @@ load_code(LoaderState* stp) * by both the nif functionality and line instructions. */ enum { - FUNC_INFO_SZ = 5 + FUNC_INFO_SZ = sizeof(ErtsCodeInfo) / sizeof(Eterm) }; code = stp->codev; @@ -2565,8 +2579,11 @@ load_code(LoaderState* stp) stp->function = code[ci-2]; stp->arity = code[ci-1]; + /* When this assert is triggered, it is normally a sign that + the size of the ops.tab i_func_info instruction is not + the same as FUNC_INFO_SZ */ ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ); - stp->hdr->functions[function_number] = (BeamInstr*) stp->labels[last_label].patches; + stp->hdr->functions[function_number] = (ErtsCodeInfo*) stp->labels[last_label].patches; offset = function_number; stp->labels[last_label].patches = offset; function_number++; @@ -4343,22 +4360,25 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src, static int hash_internal_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx) { + Eterm key_term; switch (Key.type) { case TAG_a: - *hx = atom_tab(atom_val(Key.val))->slot.bucket.hvalue; - return 1; + key_term = Key.val; + break; case TAG_i: - *hx = Key.val; - return 1; + key_term = make_small(Key.val); + break; case TAG_n: - *hx = make_internal_hash(NIL); - return 1; + key_term = NIL; + break; case TAG_q: - *hx = make_internal_hash(stp->literals[Key.val].term); - return 1; + key_term = stp->literals[Key.val].term; + break; default: return 0; } + *hx = erts_pd_make_hx(key_term); + return 1; } @@ -4513,7 +4533,7 @@ freeze_code(LoaderState* stp) * function table in the beginning of the file. */ - code_hdr->functions[stp->num_functions] = (codev + stp->ci - 1); + code_hdr->functions[stp->num_functions] = (ErtsCodeInfo*)(codev + stp->ci - 1); CHKBLK(ERTS_ALC_T_CODE,code_hdr); /* @@ -4773,16 +4793,16 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) } ep = erts_export_put(stp->module, stp->export[i].function, stp->export[i].arity); - if (!on_load) { - ep->addressv[erts_staging_code_ix()] = address; - } else { + if (on_load) { /* * on_load: Don't make any of the exported functions * callable yet. Keep any function in the current * code callable. */ - ep->code[4] = (BeamInstr) address; + ep->beam[1] = (BeamInstr) address; } + else + ep->addressv[erts_staging_code_ix()] = address; } /* @@ -4828,7 +4848,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) } fe->address = code_ptr; #ifdef HIPE - hipe_set_closure_stub(fe, stp->lambdas[i].num_free); + hipe_set_closure_stub(fe); #endif } } @@ -4959,7 +4979,7 @@ transform_engine(LoaderState* st) if (i >= st->num_imports || st->import[i].bf == NULL) goto restart; if (bif_number != -1 && - bif_export[bif_number]->code[4] != (BeamInstr) st->import[i].bf) { + bif_export[bif_number]->beam[1] != (BeamInstr) st->import[i].bf) { goto restart; } } @@ -5584,18 +5604,16 @@ functions_in_module(Process* p, /* Process whose heap to use. */ hp = HAlloc(p, need); hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { - BeamInstr* func_info = code_hdr->functions[i]; - Eterm name = (Eterm) func_info[3]; - int arity = (int) func_info[4]; + ErtsCodeInfo* ci = code_hdr->functions[i]; Eterm tuple; /* * If the function name is [], this entry is a stub for * a BIF that should be ignored. */ - ASSERT(is_atom(name) || is_nil(name)); - if (is_atom(name)) { - tuple = TUPLE2(hp, name, make_small(arity)); + ASSERT(is_atom(ci->mfa.function) || is_nil(ci->mfa.function)); + if (is_atom(ci->mfa.function)) { + tuple = TUPLE2(hp, ci->mfa.function, make_small(ci->mfa.arity)); hp += 3; result = CONS(hp, tuple, result); hp += 2; @@ -5683,17 +5701,17 @@ native_addresses(Process* p, BeamCodeHeader* code_hdr) hp = HAlloc(p, need); hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { - BeamInstr* func_info = code_hdr->functions[i]; - Eterm name = (Eterm) func_info[3]; - int arity = (int) func_info[4]; + ErtsCodeInfo *ci = code_hdr->functions[i]; Eterm tuple; - ASSERT(is_atom(name) || is_nil(name)); /* [] if BIF stub */ - if (func_info[1] != 0) { - Eterm addr; - ASSERT(is_atom(name)); - addr = erts_bld_uint(&hp, NULL, func_info[1]); - tuple = erts_bld_tuple(&hp, NULL, 3, name, make_small(arity), addr); + ASSERT(is_atom(ci->mfa.function) + || is_nil(ci->mfa.function)); /* [] if BIF stub */ + if (ci->native != 0) { + Eterm addr; + ASSERT(is_atom(ci->mfa.function)); + addr = erts_bld_uint(&hp, NULL, ci->native); + tuple = erts_bld_tuple(&hp, NULL, 3, ci->mfa.function, + make_small(ci->mfa.arity), addr); result = erts_bld_cons(&hp, NULL, tuple, result); } } @@ -5719,11 +5737,11 @@ exported_from_module(Process* p, /* Process whose heap to use. */ for (i = 0; i < export_list_size(code_ix); i++) { Export* ep = export_list(i,code_ix); - if (ep->code[0] == mod) { + if (ep->info.mfa.module == mod) { Eterm tuple; - if (ep->addressv[code_ix] == ep->code+3 && - ep->code[3] == (BeamInstr) em_call_error_handler) { + if (ep->addressv[code_ix] == ep->beam && + ep->beam[0] == (BeamInstr) em_call_error_handler) { /* There is a call to the function, but it does not exist. */ continue; } @@ -5733,7 +5751,8 @@ exported_from_module(Process* p, /* Process whose heap to use. */ hp = HAlloc(p, need); hend = hp + need; } - tuple = TUPLE2(hp, ep->code[1], make_small(ep->code[2])); + tuple = TUPLE2(hp, ep->info.mfa.function, + make_small(ep->info.mfa.arity)); hp += 3; result = CONS(hp, tuple, result); hp += 2; @@ -5807,7 +5826,6 @@ md5_of_module(Process* p, /* Process whose heap to use. */ Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) { - BeamInstr* current = fi->current; Eterm loc = NIL; if (fi->loc != LINE_INVALID_LOCATION) { @@ -5817,7 +5835,7 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) Eterm file_term = NIL; if (file == 0) { - Atom* ap = atom_tab(atom_val(fi->current[0])); + Atom* ap = atom_tab(atom_val(fi->mfa->module)); file_term = buf_to_intlist(&hp, ".erl", 4, NIL); file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, file_term); } else { @@ -5836,10 +5854,12 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) } if (is_list(args) || is_nil(args)) { - *mfa_p = TUPLE4(hp, current[0], current[1], args, loc); + *mfa_p = TUPLE4(hp, fi->mfa->module, fi->mfa->function, + args, loc); } else { - Eterm arity = make_small(current[2]); - *mfa_p = TUPLE4(hp, current[0], current[1], arity, loc); + Eterm arity = make_small(fi->mfa->arity); + *mfa_p = TUPLE4(hp, fi->mfa->module, fi->mfa->function, + arity, loc); } return hp + 5; } @@ -5850,9 +5870,9 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) * the function. */ void -erts_set_current_function(FunctionInfo* fi, BeamInstr* current) +erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa) { - fi->current = current; + fi->mfa = mfa; fi->needed = 5; fi->loc = LINE_INVALID_LOCATION; } @@ -5861,13 +5881,13 @@ erts_set_current_function(FunctionInfo* fi, BeamInstr* current) /* * Returns a pointer to {module, function, arity}, or NULL if not found. */ -BeamInstr* +ErtsCodeMFA* find_function_from_pc(BeamInstr* pc) { FunctionInfo fi; erts_lookup_function_info(&fi, pc, 0); - return fi.current; + return fi.mfa; } /* @@ -5984,24 +6004,21 @@ code_module_md5_1(BIF_ALIST_1) return res; } -#define WORDS_PER_FUNCTION 6 +#ifdef HIPE +#define WORDS_PER_FUNCTION (sizeof(ErtsCodeInfo) / sizeof(UWord) + 1) static BeamInstr* -make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode) +make_stub(ErtsCodeInfo* info, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode) { - fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); - fp[1] = native; - fp[2] = mod; - fp[3] = func; - fp[4] = arity; -#ifdef HIPE - if (native) { - fp[5] = BeamOpCode(op_move_return_n); - hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5); - } -#endif - fp[5] = OpCode; - return fp + WORDS_PER_FUNCTION; + DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", erts_codeinfo_to_code(info)); + ASSERT(WORDS_PER_FUNCTION == 6); + info->op = (BeamInstr) BeamOp(op_i_func_info_IaaI); + info->native = native; + info->mfa.module = mod; + info->mfa.function = func; + info->mfa.arity = arity; + erts_codeinfo_to_code(info)[0] = OpCode; + return erts_codeinfo_to_code(info)+1; } static byte* @@ -6060,22 +6077,17 @@ stub_read_export_table(LoaderState* stp) } static void -stub_final_touch(LoaderState* stp, BeamInstr* fp) +stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci) { unsigned int i; unsigned int n = stp->num_exps; - Eterm mod = fp[2]; - Eterm function = fp[3]; - int arity = fp[4]; -#ifdef HIPE Lambda* lp; -#endif - if (is_bif(mod, function, arity)) { - fp[1] = 0; - fp[2] = 0; - fp[3] = 0; - fp[4] = 0; + if (is_bif(ci->mfa.module, ci->mfa.function, ci->mfa.arity)) { + ci->native = 0; + ci->mfa.module = 0; + ci->mfa.function = 0; + ci->mfa.arity = 0; return; } @@ -6084,9 +6096,14 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) */ for (i = 0; i < n; i++) { - if (stp->export[i].function == function && stp->export[i].arity == arity) { - Export* ep = erts_export_put(mod, function, arity); - ep->addressv[erts_staging_code_ix()] = fp+5; + if (stp->export[i].function == ci->mfa.function && + stp->export[i].arity == ci->mfa.arity) { + Export* ep = erts_export_put(ci->mfa.module, + ci->mfa.function, + ci->mfa.arity); + ep->addressv[erts_staging_code_ix()] = erts_codeinfo_to_code(ci); + DBG_TRACE_MFA_P(&ci->mfa,"set beam stub at %p in export at %p (code_ix=%d)", + erts_codeinfo_to_code(ci), ep, erts_staging_code_ix()); return; } } @@ -6096,16 +6113,14 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) * Search the lambda table to find out which. */ -#ifdef HIPE n = stp->num_lambdas; for (i = 0, lp = stp->lambdas; i < n; i++, lp++) { ErlFunEntry* fe = stp->lambdas[i].fe; - if (lp->function == function && lp->arity == arity) { - fp[5] = (Eterm) BeamOpCode(op_hipe_trap_call_closure); - fe->address = &(fp[5]); + if (lp->function == ci->mfa.function && lp->arity == ci->mfa.arity) { + *erts_codeinfo_to_code(ci) = (Eterm) BeamOpCode(op_hipe_trap_call_closure); + fe->address = erts_codeinfo_to_code(ci); } } -#endif return; } @@ -6114,10 +6129,9 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) [{Adr, Patchtyppe} | Addresses] and the address of a fun_entry. */ -int +static int patch(Eterm Addresses, Uint fe) { -#ifdef HIPE Eterm* listp; Eterm tuple; Eterm* tp; @@ -6153,15 +6167,13 @@ patch(Eterm Addresses, Uint fe) } -#endif return 1; } -int +static int patch_funentries(Eterm Patchlist) { -#ifdef HIPE while (!is_nil(Patchlist)) { Eterm Info; Eterm MFA; @@ -6250,23 +6262,23 @@ patch_funentries(Eterm Patchlist) return 0; } -#endif return 1; /* Signal that all went well */ } - /* * Do a dummy load of a module. No threaded code will be loaded. * Used for loading native code. * Will also patch all references to fun_entries to point to * the new fun_entries created. */ - Eterm -erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) +erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) { Binary* magic; + Binary* hipe_magic; LoaderState* stp; + HipeLoaderState* hipe_stp; + HipeModule *hipe_code; BeamInstr Funcs; BeamInstr Patchlist; Eterm MD5Bin; @@ -6289,8 +6301,12 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) */ magic = erts_alloc_loader_state(); stp = ERTS_MAGIC_BIN_DATA(magic); + hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); - if (is_not_atom(Mod)) { + if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) || + !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val, + hipe_stp = hipe_get_loader_state(hipe_magic)) || + hipe_stp->module == NIL || hipe_stp->text_segment == 0) { goto error; } if (is_not_tuple(Info)) { @@ -6318,7 +6334,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Scan the Beam binary and read the interesting sections. */ - stp->module = Mod; + stp->module = hipe_stp->module; stp->group_leader = p->group_leader; stp->num_functions = n; if (!init_iff_file(stp, bytes, size)) { @@ -6426,20 +6442,17 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Set the pointer and make the stub. Put a return instruction * as the body until we know what kind of trap we should put there. */ - code_hdr->functions[i] = fp; -#ifdef HIPE + code_hdr->functions[i] = (ErtsCodeInfo*)fp; op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */ -#else - op = (Eterm) BeamOpCode(op_move_return_n); -#endif - fp = make_stub(fp, Mod, func, arity, (Uint)native_address, op); + fp = make_stub((ErtsCodeInfo*)fp, hipe_stp->module, func, arity, + (Uint)native_address, op); } /* * Insert the last pointer and the int_code_end instruction. */ - code_hdr->functions[i] = fp; + code_hdr->functions[i] = (ErtsCodeInfo*)fp; *fp++ = (BeamInstr) BeamOp(op_int_code_end); /* @@ -6472,11 +6485,20 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) } /* + * Initialise HiPE module + */ + hipe_code->text_segment = hipe_stp->text_segment; + hipe_code->text_segment_size = hipe_stp->text_segment_size; + hipe_code->data_segment = hipe_stp->data_segment; + hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs; + hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc; + + /* * Insert the module in the module table. */ - rval = stub_insert_new_code(p, 0, p->group_leader, Mod, - code_hdr, code_size); + rval = stub_insert_new_code(p, 0, p->group_leader, hipe_stp->module, + code_hdr, code_size, hipe_code); if (rval != NIL) { goto error; } @@ -6487,23 +6509,74 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) fp = code_base; for (i = 0; i < n; i++) { - stub_final_touch(stp, fp); + stub_final_touch(stp, (ErtsCodeInfo*)fp); fp += WORDS_PER_FUNCTION; } if (patch_funentries(Patchlist)) { + Eterm mod = hipe_stp->module; + /* Prevent code from being freed */ + hipe_stp->text_segment = 0; + hipe_stp->data_segment = 0; + hipe_stp->new_hipe_refs = NULL; + hipe_stp->new_hipe_sdesc = NULL; + erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); - return Mod; + hipe_free_loader_state(hipe_stp); + + return mod; } error: + erts_free(ERTS_ALC_T_HIPE, hipe_code); erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); BIF_ERROR(p, BADARG); } +int erts_commit_hipe_patch_load(Eterm hipe_magic_bin) +{ + Binary* hipe_magic; + HipeLoaderState* hipe_stp; + HipeModule *hipe_code; + Module* modp; + + if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) || + !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val, + hipe_stp = hipe_get_loader_state(hipe_magic)) || + hipe_stp->module == NIL || hipe_stp->text_segment == 0) { + return 0; + } + + modp = erts_get_module(hipe_stp->module, erts_active_code_ix()); + if (!modp) + return 0; + + /* + * Initialise HiPE module + */ + hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); + hipe_code->text_segment = hipe_stp->text_segment; + hipe_code->text_segment_size = hipe_stp->text_segment_size; + hipe_code->data_segment = hipe_stp->data_segment; + hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs; + hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc; + + modp->curr.hipe_code = hipe_code; + + /* Prevent code from being freed */ + hipe_stp->text_segment = 0; + hipe_stp->data_segment = 0; + hipe_stp->new_hipe_refs = NULL; + hipe_stp->new_hipe_sdesc = NULL; + + return 1; +} + #undef WORDS_PER_FUNCTION +#endif /* HIPE */ + static int safe_mul(UWord a, UWord b, UWord* resp) { @@ -6517,3 +6590,46 @@ static int safe_mul(UWord a, UWord b, UWord* resp) } } +#ifdef ENABLE_DBG_TRACE_MFA + +#define MFA_MAX 10 +Eterm dbg_trace_m[MFA_MAX]; +Eterm dbg_trace_f[MFA_MAX]; +Uint dbg_trace_a[MFA_MAX]; +unsigned int dbg_trace_ix = 0; + +void dbg_set_traced_mfa(const char* m, const char* f, Uint a) +{ + unsigned i = dbg_trace_ix++; + ASSERT(i < MFA_MAX); + dbg_trace_m[i] = am_atom_put(m, strlen(m)); + dbg_trace_f[i] = am_atom_put(f, strlen(f)); + dbg_trace_a[i] = a; +} + +int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a) +{ + unsigned int i; + for (i = 0; i < dbg_trace_ix; ++i) { + if (m == dbg_trace_m[i] && + (!f || (f == dbg_trace_f[i] && a == dbg_trace_a[i]))) { + + return i+1; + } + } + return 0; +} + +void dbg_vtrace_mfa(unsigned ix, const char* format, ...) +{ + va_list arglist; + va_start(arglist, format); + ASSERT(--ix < MFA_MAX); + erts_fprintf(stderr, "MFA TRACE %T:%T/%u: ", + dbg_trace_m[ix], dbg_trace_f[ix], (int)dbg_trace_a[ix]); + + erts_vfprintf(stderr, format, arglist); + va_end(arglist); +} + +#endif /* ENABLE_DBG_TRACE_MFA */ diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 6be4031822..d420a4346c 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -37,14 +37,6 @@ typedef struct gen_op_entry { extern const GenOpEntry gen_opc[]; -#ifdef NO_JUMP_TABLE -#define BeamOp(Op) (Op) -#else -extern void** beam_ops; -#define BeamOp(Op) beam_ops[(Op)] -#endif - - extern BeamInstr beam_debug_apply[]; extern BeamInstr* em_call_error_handler; extern BeamInstr* em_apply_bif; @@ -115,7 +107,7 @@ typedef struct beam_code_header { * The actual loaded code (for the first function) start just beyond * this table. */ - BeamInstr* functions[1]; + ErtsCodeInfo* functions[1]; }BeamCodeHeader; @@ -124,7 +116,8 @@ int erts_is_module_native(BeamCodeHeader* code); void erts_beam_bif_load_init(void); struct erl_fun_entry; void erts_purge_state_add_fun(struct erl_fun_entry *fe); -Export *erts_suspend_process_on_pending_purge_lambda(Process *c_p); +Export *erts_suspend_process_on_pending_purge_lambda(Process *c_p, + struct erl_fun_entry*); /* * Layout of the line table. @@ -151,4 +144,34 @@ struct BeamCodeLineTab_ { #define LOC_FILE(Loc) ((Loc) >> 24) #define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) + +/* + * MFA event debug "tracing" usage: + * + * #define ENABLE_DBG_TRACE_MFA + * call dbg_set_traced_mfa("mymod","myfunc",arity) + * for the function(s) to trace, in some init function. + * + * Run and get stderr printouts when interesting things happen to your MFA. + */ +#ifdef ENABLE_DBG_TRACE_MFA + +void dbg_set_traced_mfa(const char* m, const char* f, Uint a); +int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a); +void dbg_vtrace_mfa(unsigned ix, const char* format, ...); +#define DBG_TRACE_MFA(M,F,A,FMT, ...) do {\ + unsigned ix;\ + if ((ix=dbg_is_traced_mfa(M,F,A))) \ + dbg_vtrace_mfa(ix, FMT"\n", ##__VA_ARGS__);\ + }while(0) + +#define DBG_TRACE_MFA_P(MFA, FMT, ...) \ + DBG_TRACE_MFA((MFA)->module, (MFA)->function, (MFA)->arity, FMT, ##__VA_ARGS__) + +#else +# define dbg_set_traced_mfa(M,F,A) +# define DBG_TRACE_MFA(M,F,A,FMT, ...) +# define DBG_TRACE_MFA_P(MFA,FMT, ...) +#endif /* ENABLE_DBG_TRACE_MFA */ + #endif /* _BEAM_LOAD_H */ diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c index 55342a38c6..9b0335e83d 100644 --- a/erts/emulator/beam/beam_ranges.c +++ b/erts/emulator/beam/beam_ranges.c @@ -221,13 +221,13 @@ erts_ranges_sz(void) void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) { - BeamInstr** low; - BeamInstr** high; - BeamInstr** mid; + ErtsCodeInfo** low; + ErtsCodeInfo** high; + ErtsCodeInfo** mid; Range* rp; BeamCodeHeader* hdr; - fi->current = NULL; + fi->mfa = NULL; fi->needed = 5; fi->loc = LINE_INVALID_LOCATION; rp = find_range(pc); @@ -240,12 +240,12 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) high = low + hdr->num_functions; while (low < high) { mid = low + (high-low) / 2; - if (pc < mid[0]) { + if (pc < (BeamInstr*)(mid[0])) { high = mid; - } else if (pc < mid[1]) { - fi->current = mid[0]+2; + } else if (pc < (BeamInstr*)(mid[1])) { + fi->mfa = &mid[0]->mfa; if (full_info) { - BeamInstr** fp = hdr->functions; + ErtsCodeInfo** fp = hdr->functions; int idx = mid - fp; lookup_loc(fi, pc, hdr, idx); } @@ -316,7 +316,7 @@ lookup_loc(FunctionInfo* fi, const BeamInstr* pc, file = LOC_FILE(fi->loc); if (file == 0) { /* Special case: Module name with ".erl" appended */ - Atom* mod_atom = atom_tab(atom_val(fi->current[0])); + Atom* mod_atom = atom_tab(atom_val(fi->mfa->module)); fi->needed += 2*(mod_atom->len+4); } else { Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 84eab5f651..d886c2985e 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -4963,13 +4963,13 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, int i; sys_memset((void *) ep, 0, sizeof(Export)); for (i=0; i<ERTS_NUM_CODE_IX; i++) { - ep->addressv[i] = &ep->code[3]; + ep->addressv[i] = ep->beam; } - ep->code[0] = m; - ep->code[1] = f; - ep->code[2] = a; - ep->code[3] = (BeamInstr) em_apply_bif; - ep->code[4] = (BeamInstr) bif; + ep->info.mfa.module = m; + ep->info.mfa.function = f; + ep->info.mfa.arity = a; + ep->beam[0] = (BeamInstr) em_apply_bif; + ep->beam[1] = (BeamInstr) bif; } void erts_init_bif(void) diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 2203182a0d..0c85e19ef0 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -159,7 +159,7 @@ do { \ #define ERTS_BIF_ERROR_TRAPPED0(Proc, Reason, Bif) \ do { \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ return THE_NON_VALUE; \ } while (0) @@ -167,7 +167,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ return THE_NON_VALUE; \ } while (0) @@ -176,7 +176,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ return THE_NON_VALUE; \ @@ -186,7 +186,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ reg[2] = (Eterm) (A2); \ @@ -202,7 +202,7 @@ do { \ #define ERTS_BIF_PREP_ERROR_TRAPPED0(Ret, Proc, Reason, Bif) \ do { \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -210,7 +210,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -219,7 +219,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ (Ret) = THE_NON_VALUE; \ @@ -229,7 +229,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ reg[2] = (Eterm) (A2); \ diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 852b1135fe..32600f4338 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -667,6 +667,7 @@ gcbif erlang:floor/1 gcbif erlang:ceil/1 bif math:floor/1 bif math:ceil/1 +bif math:fmod/2 # # Obsolete diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 3c19e82b66..6e1e94b95b 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -44,23 +44,22 @@ static void process_killer(void); void do_break(void); void erl_crash_dump_v(char *file, int line, char* fmt, va_list args); -void erl_crash_dump(char* file, int line, char* fmt, ...); #ifdef DEBUG static void bin_check(void); #endif -static void print_garb_info(int to, void *to_arg, Process* p); +static void print_garb_info(fmtfn_t to, void *to_arg, Process* p); #ifdef OPPROF static void dump_frequencies(void); #endif -static void dump_attributes(int to, void *to_arg, byte* ptr, int size); +static void dump_attributes(fmtfn_t to, void *to_arg, byte* ptr, int size); extern char* erts_system_version[]; static void -port_info(int to, void *to_arg) +port_info(fmtfn_t to, void *to_arg) { int i, max = erts_ptab_max(&erts_port); for (i = 0; i < max; i++) { @@ -71,7 +70,7 @@ port_info(int to, void *to_arg) } void -process_info(int to, void *to_arg) +process_info(fmtfn_t to, void *to_arg) { int i, max = erts_ptab_max(&erts_proc); for (i = 0; i < max; i++) { @@ -148,14 +147,14 @@ process_killer(void) typedef struct { int is_first; - int to; + fmtfn_t to; void *to_arg; } PrintMonitorContext; static void doit_print_link(ErtsLink *lnk, void *vpcontext) { PrintMonitorContext *pcontext = vpcontext; - int to = pcontext->to; + fmtfn_t to = pcontext->to; void *to_arg = pcontext->to_arg; if (pcontext->is_first) { @@ -170,7 +169,7 @@ static void doit_print_link(ErtsLink *lnk, void *vpcontext) static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) { PrintMonitorContext *pcontext = vpcontext; - int to = pcontext->to; + fmtfn_t to = pcontext->to; void *to_arg = pcontext->to_arg; char *prefix = ", "; @@ -197,7 +196,7 @@ static void doit_print_monitor(ErtsMonitor *mon, void *vpcontext) /* Display info about an individual Erlang process */ void -print_process_info(int to, void *to_arg, Process *p) +print_process_info(fmtfn_t to, void *to_arg, Process *p) { time_t approx_started; int garbing = 0; @@ -231,9 +230,9 @@ print_process_info(int to, void *to_arg, Process *p) * Display the initial function name */ erts_print(to, to_arg, "Spawned as: %T:%T/%bpu\n", - p->u.initial[INITIAL_MOD], - p->u.initial[INITIAL_FUN], - p->u.initial[INITIAL_ARI]); + p->u.initial.module, + p->u.initial.function, + p->u.initial.arity); if (p->current != NULL) { if (running) { @@ -242,9 +241,9 @@ print_process_info(int to, void *to_arg, Process *p) erts_print(to, to_arg, "Current call: "); } erts_print(to, to_arg, "%T:%T/%bpu\n", - p->current[0], - p->current[1], - p->current[2]); + p->current->module, + p->current->function, + p->current->arity); } erts_print(to, to_arg, "Spawned by: %T\n", p->parent); @@ -291,16 +290,16 @@ print_process_info(int to, void *to_arg, Process *p) erts_print(to, to_arg, "timeout"); else erts_print(to, to_arg, "%T:%T/%bpu\n", - scb->ct[j]->code[0], - scb->ct[j]->code[1], - scb->ct[j]->code[2]); + scb->ct[j]->info.mfa.module, + scb->ct[j]->info.mfa.function, + scb->ct[j]->info.mfa.arity); } erts_print(to, to_arg, "\n"); } /* display the links only if there are any*/ if (ERTS_P_LINKS(p) || ERTS_P_MONITORS(p)) { - PrintMonitorContext context = {1,to}; + PrintMonitorContext context = {1, to, to_arg}; erts_print(to, to_arg,"Link list: ["); erts_doforall_links(ERTS_P_LINKS(p), &doit_print_link, &context); erts_doforall_monitors(ERTS_P_MONITORS(p), &doit_print_monitor, &context); @@ -348,7 +347,7 @@ print_process_info(int to, void *to_arg, Process *p) } static void -print_garb_info(int to, void *to_arg, Process* p) +print_garb_info(fmtfn_t to, void *to_arg, Process* p) { #ifdef ERTS_SMP /* ERTS_SMP: A scheduler is probably concurrently doing gc... */ @@ -365,7 +364,7 @@ print_garb_info(int to, void *to_arg, Process* p) } void -info(int to, void *to_arg) +info(fmtfn_t to, void *to_arg) { erts_memory(&to, to_arg, NULL, THE_NON_VALUE); atom_info(to, to_arg); @@ -381,7 +380,7 @@ info(int to, void *to_arg) } void -loaded(int to, void *to_arg) +loaded(fmtfn_t to, void *to_arg) { int i; int old = 0; @@ -478,7 +477,7 @@ loaded(int to, void *to_arg) static void -dump_attributes(int to, void *to_arg, byte* ptr, int size) +dump_attributes(fmtfn_t to, void *to_arg, byte* ptr, int size) { while (size-- > 0) { erts_print(to, to_arg, "%02X", *ptr++); @@ -662,6 +661,26 @@ bin_check(void) #endif +static Sint64 crash_dump_limit = ERTS_SINT64_MAX; +static Sint64 crash_dump_written = 0; + +static int crash_dump_limited_writer(void* vfdp, char* buf, size_t len) +{ + const char stop_msg[] = "\n=abort:CRASH DUMP SIZE LIMIT REACHED\n"; + + crash_dump_written += len; + if (crash_dump_written <= crash_dump_limit) { + return erts_write_fd(vfdp, buf, len); + } + + len -= (crash_dump_written - crash_dump_limit); + erts_write_fd(vfdp, buf, len); + erts_write_fd(vfdp, (char*)stop_msg, sizeof(stop_msg)-1); + + /* We assume that crash dump was called from erts_exit_vv() */ + erts_exit_epilogue(); +} + /* XXX THIS SHOULD BE IN SYSTEM !!!! */ void erl_crash_dump_v(char *file, int line, char* fmt, va_list args) @@ -679,6 +698,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) int secs; int env_erl_crash_dump_seconds_set = 1; int i; + fmtfn_t to = &erts_write_fd; + void* to_arg; if (ERTS_SOMEONE_IS_CRASH_DUMPING) return; @@ -759,6 +780,21 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) return; } + crash_dump_limit = ERTS_SINT64_MAX; + envsz = sizeof(env); + if (erts_sys_getenv__("ERL_CRASH_DUMP_BYTES", env, &envsz) == 0) { + Sint64 limit; + char* endptr; + errno = 0; + limit = ErtsStrToSint64(env, &endptr, 10); + if (errno == 0 && limit >= 0 && endptr != env && *endptr == 0) { + if (limit == 0) + return; + crash_dump_limit = limit; + to = &crash_dump_limited_writer; + } + } + if (erts_sys_getenv__("ERL_CRASH_DUMP",&dumpnamebuf[0],&dumpnamebufsize) != 0) dumpname = "erl_crash.dump"; else @@ -769,39 +805,40 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) fd = open(dumpname,O_WRONLY | O_CREAT | O_TRUNC,0640); if (fd < 0) return; /* Can't create the crash dump, skip it */ + to_arg = (void*)&fd; time(&now); - erts_fdprintf(fd, "=erl_crash_dump:0.3\n%s", ctime(&now)); + erts_cbprintf(to, to_arg, "=erl_crash_dump:0.3\n%s", ctime(&now)); if (file != NULL) - erts_fdprintf(fd, "The error occurred in file %s, line %d\n", file, line); + erts_cbprintf(to, to_arg, "The error occurred in file %s, line %d\n", file, line); if (fmt != NULL && *fmt != '\0') { - erts_fdprintf(fd, "Slogan: "); - erts_vfdprintf(fd, fmt, args); + erts_cbprintf(to, to_arg, "Slogan: "); + erts_vcbprintf(to, to_arg, fmt, args); } - erts_fdprintf(fd, "System version: "); - erts_print_system_version(fd, NULL, NULL); + erts_cbprintf(to, to_arg, "System version: "); + erts_print_system_version(to, to_arg, NULL); #if ERTS_SAVED_COMPILE_TIME - erts_fdprintf(fd, "%s\n", "Compiled: " ERLANG_COMPILE_DATE); + erts_cbprintf(to, to_arg, "%s\n", "Compiled: " ERLANG_COMPILE_DATE); #endif - erts_fdprintf(fd, "Taints: "); - erts_print_nif_taints(fd, NULL); - erts_fdprintf(fd, "Atoms: %d\n", atom_table_size()); + erts_cbprintf(to, to_arg, "Taints: "); + erts_print_nif_taints(to, to_arg); + erts_cbprintf(to, to_arg, "Atoms: %d\n", atom_table_size()); #ifdef USE_THREADS /* We want to note which thread it was that called erts_exit */ if (erts_get_scheduler_data()) { - erts_fdprintf(fd, "Calling Thread: scheduler:%d\n", + erts_cbprintf(to, to_arg, "Calling Thread: scheduler:%d\n", erts_get_scheduler_data()->no); } else { if (!erts_thr_getname(erts_thr_self(), dumpnamebuf, MAXPATHLEN)) - erts_fdprintf(fd, "Calling Thread: %s\n", dumpnamebuf); + erts_cbprintf(to, to_arg, "Calling Thread: %s\n", dumpnamebuf); else - erts_fdprintf(fd, "Calling Thread: %p\n", erts_thr_self()); + erts_cbprintf(to, to_arg, "Calling Thread: %p\n", erts_thr_self()); } #else - erts_fdprintf(fd, "Calling Thread: scheduler:1\n"); + erts_cbprintf(to, to_arg, "Calling Thread: scheduler:1\n"); #endif #if defined(ERTS_HAVE_TRY_CATCH) @@ -816,8 +853,8 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) */ for (i = 0; i < erts_no_schedulers; i++) { ERTS_SYS_TRY_CATCH( - erts_print_scheduler_info(fd, NULL, ERTS_SCHEDULER_IX(i)), - erts_fdprintf(fd, "** crashed **\n")); + erts_print_scheduler_info(to, to_arg, ERTS_SCHEDULER_IX(i)), + erts_cbprintf(to, to_arg, "** crashed **\n")); } #endif @@ -848,48 +885,39 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) #ifndef ERTS_HAVE_TRY_CATCH /* This is safe to call here, as all schedulers are blocked */ for (i = 0; i < erts_no_schedulers; i++) { - erts_print_scheduler_info(fd, NULL, ERTS_SCHEDULER_IX(i)); + erts_print_scheduler_info(to, to_arg, ERTS_SCHEDULER_IX(i)); } #endif - info(fd, NULL); /* General system info */ + info(to, to_arg); /* General system info */ if (erts_ptab_initialized(&erts_proc)) - process_info(fd, NULL); /* Info about each process and port */ - db_info(fd, NULL, 0); - erts_print_bif_timer_info(fd, NULL); - distribution_info(fd, NULL); - erts_fdprintf(fd, "=loaded_modules\n"); - loaded(fd, NULL); - erts_dump_fun_entries(fd, NULL); - erts_deep_process_dump(fd, NULL); - erts_fdprintf(fd, "=atoms\n"); - dump_atoms(fd, NULL); + process_info(to, to_arg); /* Info about each process and port */ + db_info(to, to_arg, 0); + erts_print_bif_timer_info(to, to_arg); + distribution_info(to, to_arg); + erts_cbprintf(to, to_arg, "=loaded_modules\n"); + loaded(to, to_arg); + erts_dump_fun_entries(to, to_arg); + erts_deep_process_dump(to, to_arg); + erts_cbprintf(to, to_arg, "=atoms\n"); + dump_atoms(to, to_arg); /* Keep the instrumentation data at the end of the dump */ if (erts_instr_memory_map || erts_instr_stat) { - erts_fdprintf(fd, "=instr_data\n"); + erts_cbprintf(to, to_arg, "=instr_data\n"); if (erts_instr_stat) { - erts_fdprintf(fd, "=memory_status\n"); - erts_instr_dump_stat_to_fd(fd, 0); + erts_cbprintf(to, to_arg, "=memory_status\n"); + erts_instr_dump_stat_to(to, to_arg, 0); } if (erts_instr_memory_map) { - erts_fdprintf(fd, "=memory_map\n"); - erts_instr_dump_memory_map_to_fd(fd); + erts_cbprintf(to, to_arg, "=memory_map\n"); + erts_instr_dump_memory_map_to(to, to_arg); } } - erts_fdprintf(fd, "=end\n"); + erts_cbprintf(to, to_arg, "=end\n"); close(fd); erts_fprintf(stderr,"done\n"); } -void -erl_crash_dump(char* file, int line, char* fmt, ...) -{ - va_list args; - - va_start(args, fmt); - erl_crash_dump_v(file, line, fmt, args); - va_end(args); -} diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index 584a605771..1b451bf921 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -56,12 +56,49 @@ # endif # include "sys.h" #endif + +#include "beam_opcodes.h" + struct process; #define ERTS_NUM_CODE_IX 3 typedef unsigned ErtsCodeIndex; +typedef struct ErtsCodeMFA_ { + Eterm module; + Eterm function; + Uint arity; +} ErtsCodeMFA; + +/* + * The ErtsCodeInfo structure is used both in the Export entry + * and in the code as the function header. + */ + +/* If you change the size of this, you also have to update the code + in ops.tab to reflect the new func_info size */ +typedef struct ErtsCodeInfo_ { + BeamInstr op; /* OpCode(i_func_info) */ + BeamInstr native; /* Used by hipe and trace to store extra data */ + ErtsCodeMFA mfa; +} ErtsCodeInfo; + +/* Get the code associated with a ErtsCodeInfo ptr. */ +ERTS_GLB_INLINE +BeamInstr *erts_codeinfo_to_code(ErtsCodeInfo *ci); + +/* Get the ErtsCodeInfo for from a code ptr. */ +ERTS_GLB_INLINE +ErtsCodeInfo *erts_code_to_codeinfo(BeamInstr *I); + +/* Get the code associated with a ErtsCodeMFA ptr. */ +ERTS_GLB_INLINE +BeamInstr *erts_codemfa_to_code(ErtsCodeMFA *mfa); + +/* Get the ErtsCodeMFA from a code ptr. */ +ERTS_GLB_INLINE +ErtsCodeMFA *erts_code_to_codemfa(BeamInstr *I); /* Called once at emulator initialization. */ @@ -121,10 +158,47 @@ void erts_abort_staging_code_ix(void); int erts_has_code_write_permission(void); #endif - +/* module/function/arity can be NIL/NIL/-1 when the MFA is pointing to some + invalid code, for instance unloaded_fun. */ +#define ASSERT_MFA(MFA) \ + ASSERT((is_atom((MFA)->module) || is_nil((MFA)->module)) && \ + (is_atom((MFA)->function) || is_nil((MFA)->function)) && \ + (((MFA)->arity >= 0 && (MFA)->arity < 1024) || (MFA)->arity == -1)) #if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE +BeamInstr *erts_codeinfo_to_code(ErtsCodeInfo *ci) +{ + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op); + ASSERT_MFA(&ci->mfa); + return (BeamInstr*)(ci + 1); +} + +ERTS_GLB_INLINE +ErtsCodeInfo *erts_code_to_codeinfo(BeamInstr *I) +{ + ErtsCodeInfo *ci = ((ErtsCodeInfo *)(((char *)(I)) - sizeof(ErtsCodeInfo))); + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op); + ASSERT_MFA(&ci->mfa); + return ci; +} + +ERTS_GLB_INLINE +BeamInstr *erts_codemfa_to_code(ErtsCodeMFA *mfa) +{ + ASSERT_MFA(mfa); + return (BeamInstr*)(mfa + 1); +} + +ERTS_GLB_INLINE +ErtsCodeMFA *erts_code_to_codemfa(BeamInstr *I) +{ + ErtsCodeMFA *mfa = ((ErtsCodeMFA *)(((char *)(I)) - sizeof(ErtsCodeMFA))); + ASSERT_MFA(mfa); + return mfa; +} + extern erts_smp_atomic32_t the_active_code_index; extern erts_smp_atomic32_t the_staging_code_index; diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 09c83f1117..990edb274e 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -25,6 +25,7 @@ /* define this to get a lot of debug output */ /* #define ERTS_DIST_MSG_DBG */ +/* #define ERTS_RAW_DIST_MSG_DBG */ #ifdef HAVE_CONFIG_H # include "config.h" @@ -2402,13 +2403,13 @@ erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id) } struct print_to_data { - int to; + fmtfn_t to; void *arg; }; static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp) { - int to = ((struct print_to_data *) vptdp)->to; + fmtfn_t to = ((struct print_to_data *) vptdp)->to; void *arg = ((struct print_to_data *) vptdp)->arg; Process *rp; ErtsMonitor *rmon; @@ -2431,7 +2432,7 @@ static void doit_print_monitor_info(ErtsMonitor *mon, void *vptdp) } } -static void print_monitor_info(int to, void *arg, ErtsMonitor *mon) +static void print_monitor_info(fmtfn_t to, void *arg, ErtsMonitor *mon) { struct print_to_data ptd = {to, arg}; erts_doforall_monitors(mon,&doit_print_monitor_info,&ptd); @@ -2457,7 +2458,7 @@ static void doit_print_link_info(ErtsLink *lnk, void *vptdp) } } -static void print_link_info(int to, void *arg, ErtsLink *lnk) +static void print_link_info(fmtfn_t to, void *arg, ErtsLink *lnk) { struct print_to_data ptd = {to, arg}; erts_doforall_links(lnk, &doit_print_link_info, (void *) &ptd); @@ -2478,7 +2479,7 @@ static void doit_print_nodelink_info(ErtsLink *lnk, void *vpcontext) "Remote monitoring: %T %T\n", lnk->pid, pcontext->sysname); } -static void print_nodelink_info(int to, void *arg, ErtsLink *lnk, Eterm sysname) +static void print_nodelink_info(fmtfn_t to, void *arg, ErtsLink *lnk, Eterm sysname) { PrintNodeLinkContext context = {{to, arg}, sysname}; erts_doforall_links(lnk, &doit_print_nodelink_info, &context); @@ -2486,7 +2487,7 @@ static void print_nodelink_info(int to, void *arg, ErtsLink *lnk, Eterm sysname) static int -info_dist_entry(int to, void *arg, DistEntry *dep, int visible, int connected) +info_dist_entry(fmtfn_t to, void *arg, DistEntry *dep, int visible, int connected) { if (visible && connected) { @@ -2535,7 +2536,7 @@ info_dist_entry(int to, void *arg, DistEntry *dep, int visible, int connected) return 0; } -int distribution_info(int to, void *arg) /* Called by break handler */ +int distribution_info(fmtfn_t to, void *arg) /* Called by break handler */ { DistEntry *dep; diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index eda3ad870a..4ebe37ee1d 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -54,7 +54,7 @@ static void link_free_block (Allctr_t *, Block_t *); static void unlink_free_block (Allctr_t *, Block_t *); -static Eterm info_options (Allctr_t *, char *, int *, +static Eterm info_options (Allctr_t *, char *, fmtfn_t *, void *arg, Uint **, Uint *); static void init_atoms (void); @@ -227,7 +227,7 @@ add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2) static Eterm info_options(Allctr_t *allctr, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 3c2c9def3b..40a45c961f 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -373,10 +373,16 @@ set_default_exec_alloc_opts(struct au_init *ip) ip->init.util.rmbcmt = 0; ip->init.util.acul = 0; +# ifdef ERTS_HAVE_EXEC_MMAPPER ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc; ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc; ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc; ip->init.util.mseg_mmapper = &erts_exec_mmapper; +# else + ip->init.util.mseg_alloc = &erts_alcu_exec_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_exec_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_exec_mseg_dealloc; +# endif } #endif /* ERTS_ALC_A_EXEC */ @@ -1571,7 +1577,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) break; case 'X': if (has_prefix("scs", argv[i]+3)) { -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER init->mseg.exec_mmap.scs = #endif get_mb_value(argv[i]+6, argv, &i); @@ -2113,7 +2119,7 @@ add_fix_values(UWord *ap, UWord *up, ErtsAlcUFixInfo_t *fi, ErtsAlcType_t type) } Eterm -erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) +erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) { /* * NOTE! When updating this function, make sure to also update @@ -2476,7 +2482,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (print_to_p) { int i; - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; /* Print result... */ @@ -2530,7 +2536,7 @@ struct aa_values { }; Eterm -erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) +erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc) { #define MAX_AA_VALUES (24) struct aa_values values[MAX_AA_VALUES]; @@ -2665,7 +2671,7 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) if (print_to_p) { /* Print result... */ - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, "=allocated_areas\n"); @@ -2779,7 +2785,7 @@ erts_alloc_util_allocators(void *proc) } void -erts_allocator_info(int to, void *arg) +erts_allocator_info(fmtfn_t to, void *arg) { ErtsAlcType_t a; @@ -2852,7 +2858,7 @@ erts_allocator_info(int to, void *arg) erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n"); erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis); #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER erts_print(to, arg, "=allocator:erts_mmap.exec_mmap\n"); erts_mmap_info(&erts_exec_mmapper, &to, arg, NULL, NULL, &emis); #endif @@ -3010,7 +3016,7 @@ erts_allocator_options(void *proc) #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) terms[length++] = ERTS_MAKE_AM("literal_mmap"); #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER terms[length++] = ERTS_MAKE_AM("exec_mmap"); #endif features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; @@ -3102,7 +3108,7 @@ reply_alloc_info(void *vair) # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) struct erts_mmap_info_struct mmap_info_literal; # endif -# ifdef ERTS_ALC_A_EXEC +# ifdef ERTS_HAVE_EXEC_MMAPPER struct erts_mmap_info_struct mmap_info_exec; # endif #endif @@ -3110,7 +3116,7 @@ reply_alloc_info(void *vair) Eterm (*info_func)(Allctr_t *, int, int, - int *, + fmtfn_t *, void *, Uint **, Uint *) = (air->only_sz @@ -3232,7 +3238,7 @@ reply_alloc_info(void *vair) erts_bld_atom(hpp,szp,"literal_mmap"), ainfo); # endif -# ifdef ERTS_ALC_A_EXEC +# ifdef ERTS_HAVE_EXEC_MMAPPER ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list); ainfo = (air->only_sz ? NIL : @@ -4129,12 +4135,20 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *dptr; Uint size; + int free_pattern = n; ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE); - sys_memset((void *) dptr, n, size + FENCE_SZ); +#ifdef ERTS_ALC_A_EXEC +# if defined(__i386__) || defined(__x86_64__) + if (ERTS_ALC_T2A(ERTS_ALC_N2T(n)) == ERTS_ALC_A_EXEC) { + free_pattern = 0x0f; /* Illegal instruction */ + } +# endif +#endif + sys_memset((void *) dptr, free_pattern, size + FENCE_SZ); (*real_af->free)(n, real_af->extra, dptr); diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 925a081a02..56a3b73bf9 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -69,11 +69,11 @@ void *erts_sys_aligned_realloc(UWord alignment, void *ptr, UWord size, UWord old void erts_sys_aligned_free(UWord alignment, void *ptr); #endif -Eterm erts_memory(int *, void *, void *, Eterm); -Eterm erts_allocated_areas(int *, void *, void *); +Eterm erts_memory(fmtfn_t *, void *, void *, Eterm); +Eterm erts_allocated_areas(fmtfn_t *, void *, void *); Eterm erts_alloc_util_allocators(void *proc); -void erts_allocator_info(int, void *); +void erts_allocator_info(fmtfn_t, void *); Eterm erts_allocator_options(void *proc); struct process; diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 2995f2f822..61cce224dd 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -907,7 +907,9 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, #elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) -/* Used by literal allocator that has its own mmapper (super carrier) */ +/* For allocators that have their own mmapper (super carrier), + * like literal_alloc and exec_alloc on amd64 + */ void* erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { @@ -948,6 +950,50 @@ erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, } #endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ +#if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) + +/* + * For exec_alloc on non-amd64 that just need memory with PROT_EXEC + */ +void* +erts_alcu_exec_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +{ + void* res = erts_alcu_mseg_alloc(allctr, size_p, flags); + + if (res) { + int r = mprotect(res, *size_p, PROT_EXEC | PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + return res; +} + +void* +erts_alcu_exec_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) +{ + void *res; + + if (seg && old_size) { + int r = mprotect(seg, old_size, PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p); + if (res) { + int r = mprotect(res, *new_size_p, PROT_EXEC | PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + return res; +} + +void +erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) +{ + int r = mprotect(seg, size, PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + erts_alcu_mseg_dealloc(allctr, seg, size, flags); +} +#endif /* ERTS_ALC_A_EXEC && !ERTS_HAVE_EXEC_MMAPPER */ + #endif /* HAVE_ERTS_MSEG */ void* @@ -1361,6 +1407,7 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + ASSERT(size == fix->type_size); res = fix->list; if (res) { @@ -1372,8 +1419,6 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) fix_cpool_check_shrink(allctr, type, fix, NULL); return res; } - if (size < 2*sizeof(UWord)) - size += sizeof(UWord); if (size >= allctr->sbc_threshold) { Block_t *blk; blk = create_carrier(allctr, size, CFLG_SBC); @@ -1493,6 +1538,7 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + ASSERT(size == fix->type_size); ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); fix->u.nocpool.used++; @@ -1515,8 +1561,6 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); return res; } - if (size < 2*sizeof(UWord)) - size += sizeof(UWord); if (fix->u.nocpool.limit < fix->u.nocpool.used) fix->u.nocpool.limit = fix->u.nocpool.used; if (fix->u.nocpool.max_used < fix->u.nocpool.used) @@ -4153,9 +4197,9 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) ASSERT(IS_LAST_BLK(blk)); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - (*allctr->link_free_block)(allctr, blk, 0); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk, 0); + (*allctr->unlink_free_block)(allctr, blk); #endif } #endif @@ -4473,7 +4517,7 @@ add_fix_types(Allctr_t *allctr, int internal, Uint **hpp, Uint *szp, static Eterm sz_info_fix(Allctr_t *allctr, int internal, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4494,7 +4538,7 @@ sz_info_fix(Allctr_t *allctr, UWord used = fix->type_size * fix->u.cpool.used; if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, @@ -4522,7 +4566,7 @@ sz_info_fix(Allctr_t *allctr, UWord used = fix->type_size*fix->u.nocpool.used; if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, @@ -4548,7 +4592,7 @@ static Eterm sz_info_carriers(Allctr_t *allctr, CarriersStats_t *cs, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4557,7 +4601,7 @@ sz_info_carriers(Allctr_t *allctr, UWord curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, @@ -4598,7 +4642,7 @@ static Eterm info_cpool(Allctr_t *allctr, int sz_only, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4615,7 +4659,7 @@ info_cpool(Allctr_t *allctr, } if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; if (!sz_only) erts_print(to, arg, "%sblocks: %bpu\n", prefix, nob); @@ -4652,7 +4696,7 @@ static Eterm info_carriers(Allctr_t *allctr, CarriersStats_t *cs, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4664,7 +4708,7 @@ info_carriers(Allctr_t *allctr, curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; if (print_to_p) { - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; erts_print(to, arg, @@ -4790,7 +4834,7 @@ make_name_atoms(Allctr_t *allctr) static Eterm info_calls(Allctr_t *allctr, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -4807,7 +4851,7 @@ info_calls(Allctr_t *allctr, erts_print(TO, TOA, "%s%s calls: %b64u\n",PRFX,NAME,CC) char *prefix = allctr->name_prefix; - int to = *print_to_p; + fmtfn_t to = *print_to_p; void *arg = print_to_arg; PRINT_CC_5(to, arg, prefix, "alloc", allctr->calls.this_alloc); @@ -4883,7 +4927,7 @@ info_calls(Allctr_t *allctr, static Eterm info_options(Allctr_t *allctr, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -5035,7 +5079,7 @@ reset_max_values(CarriersStats_t *cs) \* */ Eterm -erts_alcu_au_info_options(int *print_to_p, void *print_to_arg, +erts_alcu_au_info_options(fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { Eterm res = THE_NON_VALUE; @@ -5078,7 +5122,7 @@ erts_alcu_au_info_options(int *print_to_p, void *print_to_arg, Eterm erts_alcu_info_options(Allctr_t *allctr, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -5110,7 +5154,7 @@ Eterm erts_alcu_sz_info(Allctr_t *allctr, int internal, int begin_max_period, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -5196,7 +5240,7 @@ Eterm erts_alcu_info(Allctr_t *allctr, int internal, int begin_max_period, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) @@ -6039,7 +6083,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) goto error; allctr->min_block_size = UNIT_CEILING(allctr->min_block_size + sizeof(FreeBlkFtr_t)); -#if ERTS_SMP +#ifdef ERTS_SMP if (init->tpref) { Uint sz = ABLK_HDR_SZ; sz += (init->fix ? @@ -6440,11 +6484,6 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) ASSERT(SBC2BLK(allctr, sbc) == iblk); ASSERT(CARRIER_SZ(sbc) - SBC_HEADER_SIZE >= SBC_BLK_SZ(iblk)); -#if HAVE_ERTS_MSEG - if (IS_MSEG_CARRIER(sbc)) { - ASSERT(CARRIER_SZ(sbc) % ERTS_SACRR_UNIT_SZ == 0); - } -#endif crr = sbc; cl = &allctr->sbc_list; } diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index f50f09907a..f570703f25 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -178,10 +178,10 @@ void * erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t, void *, void *, Uint); void erts_alcu_free_thr_pref(ErtsAlcType_t, void *, void *); #endif #endif -Eterm erts_alcu_au_info_options(int *, void *, Uint **, Uint *); -Eterm erts_alcu_info_options(Allctr_t *, int *, void *, Uint **, Uint *); -Eterm erts_alcu_sz_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *); -Eterm erts_alcu_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *); +Eterm erts_alcu_au_info_options(fmtfn_t *, void *, Uint **, Uint *); +Eterm erts_alcu_info_options(Allctr_t *, fmtfn_t *, void *, Uint **, Uint *); +Eterm erts_alcu_sz_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *); +Eterm erts_alcu_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *); void erts_alcu_init(AlcUInit_t *); void erts_alcu_current_size(Allctr_t *, AllctrSize_t *, ErtsAlcUFixInfo_t *, int); @@ -210,6 +210,12 @@ void* erts_alcu_mmapper_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); void erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); # endif + +# if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) +void* erts_alcu_exec_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); +void* erts_alcu_exec_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); +void erts_alcu_exec_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); +# endif #endif /* HAVE_ERTS_MSEG */ void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign); @@ -586,7 +592,7 @@ struct Allctr_t_ { Block_t *, Uint); void (*link_free_block) (Allctr_t *, Block_t *); void (*unlink_free_block) (Allctr_t *, Block_t *); - Eterm (*info_options) (Allctr_t *, char *, int *, + Eterm (*info_options) (Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); Uint (*get_next_mbc_size) (Allctr_t *); diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 7e239d1f5d..05ba1f9891 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -224,7 +224,7 @@ static AOFF_RBTree_t* rbt_search(AOFF_RBTree_t* root, Uint size); static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node); #endif -static Eterm info_options(Allctr_t *, char *, int *, void *, Uint **, Uint *); +static Eterm info_options(Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); static void init_atoms(void); @@ -1014,7 +1014,7 @@ add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2) static Eterm info_options(Allctr_t *allctr, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index 379cee39a1..6173c408e1 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -104,7 +104,7 @@ static void bf_link_free_block (Allctr_t *, Block_t *); static ERTS_INLINE void bf_unlink_free_block (Allctr_t *, Block_t *); -static Eterm info_options (Allctr_t *, char *, int *, +static Eterm info_options (Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); static void init_atoms (void); @@ -921,7 +921,7 @@ add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2) static Eterm info_options(Allctr_t *allctr, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index ef77201544..587859e413 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1099,7 +1099,7 @@ void erts_ddll_increment_port_count(DE_Handle *dh) void erts_ddll_decrement_port_count(DE_Handle *dh) { assert_drv_list_locked(); -#if DEBUG +#ifdef DEBUG ASSERT(erts_smp_atomic32_dec_read_nob(&dh->port_count) >= 0); #else erts_smp_atomic32_dec_nob(&dh->port_count); diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 75389107bb..0e0842e139 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -317,7 +317,7 @@ make_link_list(Process *p, ErtsLink *root, Eterm tail) } int -erts_print_system_version(int to, void *arg, Process *c_p) +erts_print_system_version(fmtfn_t to, void *arg, Process *c_p) { int i, rc = -1; char *rc_str = ""; @@ -1114,9 +1114,9 @@ process_info_aux(Process *BIF_P, case am_initial_call: hp = HAlloc(BIF_P, 3+4); res = TUPLE3(hp, - rp->u.initial[INITIAL_MOD], - rp->u.initial[INITIAL_FUN], - make_small(rp->u.initial[INITIAL_ARI])); + rp->u.initial.module, + rp->u.initial.function, + make_small(rp->u.initial.arity)); hp += 4; break; @@ -1563,9 +1563,9 @@ process_info_aux(Process *BIF_P, term = am_timeout; else { term = TUPLE3(hp, - scb->ct[j]->code[0], - scb->ct[j]->code[1], - make_small(scb->ct[j]->code[2])); + scb->ct[j]->info.mfa.module, + scb->ct[j]->info.mfa.function, + make_small(scb->ct[j]->info.mfa.arity)); hp += 4; } list = CONS(hp, term, list); @@ -1614,10 +1614,10 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) if (rp->current == NULL) { erts_lookup_function_info(&fi, rp->i, full_info); - rp->current = fi.current; + rp->current = fi.mfa; } else if (full_info) { erts_lookup_function_info(&fi, rp->i, full_info); - if (fi.current == NULL) { + if (fi.mfa == NULL) { /* Use the current function without location info */ erts_set_current_function(&fi, rp->current); } @@ -1633,9 +1633,9 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) * instead if it can be looked up. */ erts_lookup_function_info(&fi2, rp->cp, full_info); - if (fi2.current) { + if (fi2.mfa) { fi = fi2; - rp->current = fi2.current; + rp->current = fi2.mfa; } } @@ -1650,8 +1650,9 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) hp = erts_build_mfa_item(&fi, hp, am_true, &res); } else { hp = HAlloc(BIF_P, 3+4); - res = TUPLE3(hp, rp->current[0], - rp->current[1], make_small(rp->current[2])); + res = TUPLE3(hp, rp->current->module, + rp->current->function, + make_small(rp->current->arity)); hp += 4; } *hpp = hp; @@ -1692,7 +1693,7 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp) heap_size = 3; for (i = 0; i < depth; i++) { erts_lookup_function_info(stkp, s->trace[i], 1); - if (stkp->current) { + if (stkp->mfa) { heap_size += stkp->needed + 2; stkp++; } @@ -2859,6 +2860,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("ets_limit",BIF_ARG_1)) { BIF_RET(make_small(erts_db_get_max_tabs())); } + else if (ERTS_IS_ATOM_STR("atom_limit",BIF_ARG_1)) { + BIF_RET(make_small(erts_get_atom_limit())); + } else if (ERTS_IS_ATOM_STR("tolerant_timeofday",BIF_ARG_1)) { if (erts_has_time_correction() && erts_time_offset_state() == ERTS_TIME_OFFSET_FINAL) { @@ -3227,7 +3231,7 @@ fun_info_2(BIF_ALIST_2) break; case am_module: hp = HAlloc(p, 3); - val = exp->code[0]; + val = exp->info.mfa.module; break; case am_new_index: hp = HAlloc(p, 3); @@ -3255,11 +3259,11 @@ fun_info_2(BIF_ALIST_2) break; case am_arity: hp = HAlloc(p, 3); - val = make_small(exp->code[2]); + val = make_small(exp->info.mfa.arity); break; case am_name: hp = HAlloc(p, 3); - val = exp->code[1]; + val = exp->info.mfa.function; break; default: goto error; @@ -3285,7 +3289,9 @@ fun_info_mfa_1(BIF_ALIST_1) } else if (is_export(fun)) { Export* exp = (Export *) ((UWord) (export_val(fun))[1]); hp = HAlloc(p, 4); - BIF_RET(TUPLE3(hp,exp->code[0],exp->code[1],make_small(exp->code[2]))); + BIF_RET(TUPLE3(hp,exp->info.mfa.module, + exp->info.mfa.function, + make_small(exp->info.mfa.arity))); } BIF_ERROR(p, BADARG); } diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c index aecb8bf0c1..a594ec1493 100644 --- a/erts/emulator/beam/erl_bif_op.c +++ b/erts/emulator/beam/erl_bif_op.c @@ -260,7 +260,7 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2) } else if (is_export(arg1)) { Export* exp = (Export *) (export_val(arg1)[1]); - if (exp->code[2] == (Uint) arity) { + if (exp->info.mfa.arity == (Uint) arity) { BIF_RET(am_true); } } diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 66e5146da0..b1c2c427b0 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -125,7 +125,6 @@ erts_internal_trace_pattern_3(BIF_ALIST_3) static Eterm trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) { - DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */ int i; int matches = -1; int specified = 0; @@ -308,30 +307,30 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) } matches = 0; } else if (is_tuple(MFA)) { + ErtsCodeMFA mfa; Eterm *tp = tuple_val(MFA); if (tp[0] != make_arityval(3)) { goto error; } - mfa[0] = tp[1]; - mfa[1] = tp[2]; - mfa[2] = tp[3]; - if (!is_atom(mfa[0]) || !is_atom(mfa[1]) || - (!is_small(mfa[2]) && mfa[2] != am_Underscore)) { + if (!is_atom(tp[1]) || !is_atom(tp[2]) || + (!is_small(tp[3]) && tp[3] != am_Underscore)) { goto error; } - for (i = 0; i < 3 && mfa[i] != am_Underscore; i++, specified++) { + for (i = 0; i < 3 && tp[i+1] != am_Underscore; i++, specified++) { /* Empty loop body */ } for (i = specified; i < 3; i++) { - if (mfa[i] != am_Underscore) { + if (tp[i+1] != am_Underscore) { goto error; } } - if (is_small(mfa[2])) { - mfa[2] = signed_val(mfa[2]); + mfa.module = tp[1]; + mfa.function = tp[2]; + if (specified == 3) { + mfa.arity = signed_val(tp[3]); } - matches = erts_set_trace_pattern(p, mfa, specified, + matches = erts_set_trace_pattern(p, &mfa, specified, match_prog_set, match_prog_set, on, flags, meta_tracer, 0); } else if (is_atom(MFA)) { @@ -343,7 +342,6 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) error: MatchSetUnref(match_prog_set); - UnUseTmpHeap(3,p); ERTS_TRACER_CLEAR(&meta_tracer); @@ -512,7 +510,7 @@ start_trace(Process *c_p, ErtsTracer tracer, && !ERTS_TRACER_COMPARE(ERTS_TRACER(port), tracer)) { /* This tracee is already being traced, and not by the * tracer to be */ - if (erts_is_tracer_enabled(tracer, common)) { + if (erts_is_tracer_enabled(ERTS_TRACER(port), common)) { /* The tracer is still in use */ return 1; } @@ -715,8 +713,8 @@ Eterm erts_internal_trace_3(BIF_ALIST_3) Process* tracee_p = erts_pix2proc(i); if (! tracee_p) continue; - start_trace(p, tracer, &tracee_p->common, on, mask); - matches++; + if (!start_trace(p, tracer, &tracee_p->common, on, mask)) + matches++; } } if (ports || mods) { @@ -730,8 +728,8 @@ Eterm erts_internal_trace_3(BIF_ALIST_3) state = erts_atomic32_read_nob(&tracee_port->state); if (state & ERTS_PORT_SFLGS_DEAD) continue; - start_trace(p, tracer, &tracee_port->common, on, mask); - matches++; + if (!start_trace(p, tracer, &tracee_port->common, on, mask)) + matches++; } } } @@ -975,13 +973,14 @@ static int function_is_traced(Process *p, Export e; Export* ep; BeamInstr* pc; + ErtsCodeInfo *ci; /* First look for an export entry */ - e.code[0] = mfa[0]; - e.code[1] = mfa[1]; - e.code[2] = mfa[2]; + e.info.mfa.module = mfa[0]; + e.info.mfa.function = mfa[1]; + e.info.mfa.arity = mfa[2]; if ((ep = export_get(&e)) != NULL) { - pc = ep->code+3; + pc = ep->beam; if (ep->addressv[erts_active_code_ix()] == pc && *pc != (BeamInstr) em_call_error_handler) { @@ -990,17 +989,17 @@ static int function_is_traced(Process *p, ASSERT(*pc == (BeamInstr) em_apply_bif || *pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); - if (erts_is_trace_break(pc, ms, 0)) { + if (erts_is_trace_break(&ep->info, ms, 0)) { return FUNC_TRACE_GLOBAL_TRACE; } - if (erts_is_trace_break(pc, ms, 1)) { + if (erts_is_trace_break(&ep->info, ms, 1)) { r |= FUNC_TRACE_LOCAL_TRACE; } - if (erts_is_mtrace_break(pc, ms_meta, tracer_pid_meta)) { + if (erts_is_mtrace_break(&ep->info, ms_meta, tracer_pid_meta)) { r |= FUNC_TRACE_META_TRACE; } - if (erts_is_time_break(p, pc, call_time)) { + if (erts_is_time_break(p, &ep->info, call_time)) { r |= FUNC_TRACE_TIME_TRACE; } return r ? r : FUNC_TRACE_UNTRACED; @@ -1008,15 +1007,15 @@ static int function_is_traced(Process *p, } /* OK, now look for breakpoint tracing */ - if ((pc = erts_find_local_func(mfa)) != NULL) { + if ((ci = erts_find_local_func(&e.info.mfa)) != NULL) { int r = - (erts_is_trace_break(pc, ms, 1) + (erts_is_trace_break(ci, ms, 1) ? FUNC_TRACE_LOCAL_TRACE : 0) - | (erts_is_mtrace_break(pc, ms_meta, tracer_pid_meta) + | (erts_is_mtrace_break(ci, ms_meta, tracer_pid_meta) ? FUNC_TRACE_META_TRACE : 0) - | (erts_is_count_break(pc, count) + | (erts_is_count_break(ci, count) ? FUNC_TRACE_COUNT_TRACE : 0) - | (erts_is_time_break(p, pc, call_time) + | (erts_is_time_break(p, ci, call_time) ? FUNC_TRACE_TIME_TRACE : 0); return r ? r : FUNC_TRACE_UNTRACED; @@ -1350,7 +1349,7 @@ trace_info_event(Process* p, Eterm event, Eterm key) #undef FUNC_TRACE_LOCAL_TRACE int -erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, +erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, Binary* match_prog_set, Binary *meta_match_prog_set, int on, struct trace_pattern_flags flags, ErtsTracer meta_tracer, int is_blocking) @@ -1370,22 +1369,23 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, n = finish_bp.e.matched; for (i = 0; i < n; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + ErtsCodeInfo *ci = fp[i].ci; + BeamInstr* pc = erts_codeinfo_to_code(ci); + Export* ep = ErtsContainerStruct(ci, Export, info); if (on && !flags.breakpoint) { /* Turn on global call tracing */ if (ep->addressv[code_ix] != pc) { fp[i].mod->curr.num_traced_exports++; #ifdef DEBUG - pc[-5] = (BeamInstr) BeamOp(op_i_func_info_IaaI); + ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); #endif - pc[0] = (BeamInstr) BeamOp(op_jump_f); - pc[1] = (BeamInstr) ep->addressv[code_ix]; + ep->beam[0] = (BeamInstr) BeamOp(op_jump_f); + ep->beam[1] = (BeamInstr) ep->addressv[code_ix]; } - erts_set_call_trace_bif(pc, match_prog_set, 0); + erts_set_call_trace_bif(ci, match_prog_set, 0); if (ep->addressv[code_ix] != pc) { - pc[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint); + ep->beam[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint); } } else if (!on && flags.breakpoint) { /* Turn off breakpoint tracing -- nothing to do here. */ @@ -1394,9 +1394,9 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, * Turn off global tracing, either explicitly or implicitly * before turning on breakpoint tracing. */ - erts_clear_call_trace_bif(pc, 0); - if (pc[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - pc[0] = (BeamInstr) BeamOp(op_jump_f); + erts_clear_call_trace_bif(ci, 0); + if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { + ep->beam[0] = (BeamInstr) BeamOp(op_jump_f); } } } @@ -1406,68 +1406,76 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, */ for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - int j; - + if (!ExportIsBuiltIn(ep)) { continue; } - + if (bif_table[i].f == bif_table[i].traced) { /* Trace wrapper same as regular function - untraceable */ continue; } - - for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) { - /* Empty loop body */ - } - if (j == specified) { - BeamInstr* pc = (BeamInstr *)bif_export[i]->code + 3; - if (! flags.breakpoint) { /* Export entry call trace */ - if (on) { - erts_clear_call_trace_bif(pc, 1); - erts_clear_mtrace_bif(pc); - erts_set_call_trace_bif(pc, match_prog_set, 0); - } else { /* off */ - erts_clear_call_trace_bif(pc, 0); - } - matches++; - } else { /* Breakpoint call trace */ - int m = 0; - - if (on) { - if (flags.local) { - erts_clear_call_trace_bif(pc, 0); - erts_set_call_trace_bif(pc, match_prog_set, 1); - m = 1; - } - if (flags.meta) { - erts_set_mtrace_bif(pc, meta_match_prog_set, - meta_tracer); - m = 1; - } - if (flags.call_time) { - erts_set_time_trace_bif(pc, on); - /* I don't want to remove any other tracers */ - m = 1; - } - } else { /* off */ - if (flags.local) { - erts_clear_call_trace_bif(pc, 1); - m = 1; - } - if (flags.meta) { - erts_clear_mtrace_bif(pc); - m = 1; - } - if (flags.call_time) { - erts_clear_time_trace_bif(pc); - m = 1; - } - } - matches += m; - } - } + switch (specified) { + case 3: + if (mfa->arity != ep->info.mfa.arity) + continue; + case 2: + if (mfa->function != ep->info.mfa.function) + continue; + case 1: + if (mfa->module != ep->info.mfa.module) + continue; + case 0: + break; + default: + ASSERT(0); + } + + if (! flags.breakpoint) { /* Export entry call trace */ + if (on) { + erts_clear_call_trace_bif(&ep->info, 1); + erts_clear_mtrace_bif(&ep->info); + erts_set_call_trace_bif(&ep->info, match_prog_set, 0); + } else { /* off */ + erts_clear_call_trace_bif(&ep->info, 0); + } + matches++; + } else { /* Breakpoint call trace */ + int m = 0; + + if (on) { + if (flags.local) { + erts_clear_call_trace_bif(&ep->info, 0); + erts_set_call_trace_bif(&ep->info, match_prog_set, 1); + m = 1; + } + if (flags.meta) { + erts_set_mtrace_bif(&ep->info, meta_match_prog_set, + meta_tracer); + m = 1; + } + if (flags.call_time) { + erts_set_time_trace_bif(&ep->info, on); + /* I don't want to remove any other tracers */ + m = 1; + } + } else { /* off */ + if (flags.local) { + erts_clear_call_trace_bif(&ep->info, 1); + m = 1; + } + if (flags.meta) { + erts_clear_mtrace_bif(&ep->info); + m = 1; + } + if (flags.call_time) { + erts_clear_time_trace_bif(&ep->info); + m = 1; + } + } + matches += m; + } } /* @@ -1669,10 +1677,9 @@ install_exp_breakpoints(BpFunctions* f) Uint i; for (i = 0; i < ne; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - ep->addressv[code_ix] = pc; + ep->addressv[code_ix] = ep->beam; } } @@ -1685,14 +1692,13 @@ uninstall_exp_breakpoints(BpFunctions* f) Uint i; for (i = 0; i < ne; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - if (ep->addressv[code_ix] != pc) { + if (ep->addressv[code_ix] != ep->beam) { continue; } - ASSERT(*pc == (BeamInstr) BeamOp(op_jump_f)); - ep->addressv[code_ix] = (BeamInstr *) ep->code[4]; + ASSERT(ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)); + ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; } } @@ -1705,15 +1711,14 @@ clean_export_entries(BpFunctions* f) Uint i; for (i = 0; i < ne; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - if (ep->addressv[code_ix] == pc) { + if (ep->addressv[code_ix] == ep->beam) { continue; } - if (*pc == (BeamInstr) BeamOp(op_jump_f)) { - ep->code[3] = (BeamInstr) 0; - ep->code[4] = (BeamInstr) 0; + if (ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)) { + ep->beam[0] = (BeamInstr) 0; + ep->beam[1] = (BeamInstr) 0; } } } @@ -1725,11 +1730,11 @@ setup_bif_trace(void) for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - GenericBp* g = (GenericBp *) ep->fake_op_func_info_for_hipe[1]; + GenericBp* g = (GenericBp *) ep->info.native; if (g) { if (ExportIsBuiltIn(ep)) { - ASSERT(ep->code[4]); - ep->code[4] = (BeamInstr) bif_table[i].traced; + ASSERT(ep->beam[1]); + ep->beam[1] = (BeamInstr) bif_table[i].traced; } } } @@ -1743,12 +1748,11 @@ reset_bif_trace(void) for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - BeamInstr* pc = ep->code+3; - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ep->info.native; if (g && g->data[active].flags == 0) { if (ExportIsBuiltIn(ep)) { - ASSERT(ep->code[4]); - ep->code[4] = (BeamInstr) bif_table[i].f; + ASSERT(ep->beam[1]); + ep->beam[1] = (BeamInstr) bif_table[i].f; } } } diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 485f68ca98..fd092015bd 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -201,7 +201,7 @@ static int free_table_cont(Process *p, DbTable *tb, int first, int clean_meta_tab); -static void print_table(int to, void *to_arg, int show, DbTable* tb); +static void print_table(fmtfn_t to, void *to_arg, int show, DbTable* tb); static BIF_RETTYPE ets_select_delete_1(BIF_ALIST_1); static BIF_RETTYPE ets_select_count_1(BIF_ALIST_1); static BIF_RETTYPE ets_select_trap_1(BIF_ALIST_1); @@ -3871,7 +3871,7 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) return ret; } -static void print_table(int to, void *to_arg, int show, DbTable* tb) +static void print_table(fmtfn_t to, void *to_arg, int show, DbTable* tb) { erts_print(to, to_arg, "Table: %T\n", tb->common.id); erts_print(to, to_arg, "Name: %T\n", tb->common.the_name); @@ -3891,7 +3891,7 @@ static void print_table(int to, void *to_arg, int show, DbTable* tb) erts_print(to, to_arg, "Read Concurrency: %T\n", table_info(NULL, tb, am_read_concurrency)); } -void db_info(int to, void *to_arg, int show) /* Called by break handler */ +void db_info(fmtfn_t to, void *to_arg, int show) /* Called by break handler */ { int i; for (i=0; i < db_max_tabs; i++) diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index f7eb3dc45c..852440ff31 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -74,7 +74,7 @@ typedef enum { void init_db(ErtsDbSpinCount); int erts_db_process_exiting(Process *, ErtsProcLocks); -void db_info(int, void *, int); +void db_info(fmtfn_t, void *, int); void erts_db_foreach_table(void (*)(DbTable *, void *), void *); void erts_db_foreach_offheap(DbTable *, void (*func)(ErlOffHeap *, void *), diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 939ae57192..b2a231467b 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -412,7 +412,7 @@ static int db_select_count_continue_hash(Process *p, DbTable *tbl, static int db_select_delete_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); static int db_take_hash(Process *, DbTable *, Eterm, Eterm *); -static void db_print_hash(int to, +static void db_print_hash(fmtfn_t to, void *to_arg, int show, DbTable *tbl); @@ -2135,7 +2135,7 @@ int db_mark_all_deleted_hash(DbTable *tbl) /* Display hash table contents (for dump) */ -static void db_print_hash(int to, void *to_arg, int show, DbTable *tbl) +static void db_print_hash(fmtfn_t to, void *to_arg, int show, DbTable *tbl) { DbTableHash *tb = &tbl->hash; DbHashStats stats; diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 02d211a4bb..dd9403e132 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -385,7 +385,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, static int db_select_delete_continue_tree(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); static int db_take_tree(Process *, DbTable *, Eterm, Eterm *); -static void db_print_tree(int to, void *to_arg, +static void db_print_tree(fmtfn_t to, void *to_arg, int show, DbTable *tbl); static int db_free_table_tree(DbTable *tbl); @@ -1740,7 +1740,7 @@ static int db_take_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) /* Display tree contents (for dump) */ -static void db_print_tree(int to, void *to_arg, +static void db_print_tree(fmtfn_t to, void *to_arg, int show, DbTable *tbl) { diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 6732b708a8..4e987d5bee 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1769,7 +1769,7 @@ Eterm db_prog_match(Process *c_p, Eterm t; Eterm *esp; MatchVariable* variables; - BeamInstr *cp; + ErtsCodeMFA *cp; const UWord *pc = prog->text; Eterm *ehp; Eterm ret; @@ -2408,9 +2408,9 @@ restart: ehp = HAllocX(build_proc, 4, HEAP_XTRA); *esp++ = make_tuple(ehp); ehp[0] = make_arityval(3); - ehp[1] = cp[0]; - ehp[2] = cp[1]; - ehp[3] = make_small((Uint) cp[2]); + ehp[1] = cp->module; + ehp[2] = cp->function; + ehp[3] = make_small((Uint) cp->arity); } break; case matchSilent: diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 4acedbfed0..49e5f6b4cf 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -175,7 +175,7 @@ typedef struct db_table_method int (*db_free_table)(DbTable* db /* [in out] */ ); int (*db_free_table_continue)(DbTable* db); /* [in out] */ - void (*db_print)(int to, + void (*db_print)(fmtfn_t to, void* to_arg, int show, DbTable* tb /* [in out] */ ); diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index 3e3bfa03a2..3526bb684d 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -60,10 +60,10 @@ static const char dashes[PTR_SIZE+3] = { void pps(Process*, Eterm*); void ptd(Process*, Eterm); -void paranoid_display(int, void*, Process*, Eterm); +void paranoid_display(fmtfn_t, void*, Process*, Eterm); static int dcount; -static int pdisplay1(int to, void *to_arg, Process* p, Eterm obj); +static int pdisplay1(fmtfn_t to, void *to_arg, Process* p, Eterm obj); void ptd(Process* p, Eterm x) { @@ -77,14 +77,14 @@ void ptd(Process* p, Eterm x) */ void -paranoid_display(int to, void *to_arg, Process* p, Eterm obj) +paranoid_display(fmtfn_t to, void *to_arg, Process* p, Eterm obj) { dcount = 100000; pdisplay1(to, to_arg, p, obj); } static int -pdisplay1(int to, void *to_arg, Process* p, Eterm obj) +pdisplay1(fmtfn_t to, void *to_arg, Process* p, Eterm obj) { int i, k; Eterm* nobj; @@ -201,7 +201,7 @@ pdisplay1(int to, void *to_arg, Process* p, Eterm obj) void pps(Process* p, Eterm* stop) { - int to = ERTS_PRINT_STDOUT; + fmtfn_t to = ERTS_PRINT_STDOUT; void *to_arg = NULL; Eterm* sp = STACK_START(p) - 1; diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index 6ec5fbb895..2489099b5c 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -69,10 +69,6 @@ typedef enum { # define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG # undef SIZEOF_LONG_LONG #endif -#ifdef HALFWORD_HEAP_EMULATOR -# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR -# undef HALFWORD_HEAP_EMULATOR -#endif #include "erl_int_sizes_config.h" #if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR # error SIZEOF_CHAR mismatch diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index c639ba623f..41097203d4 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -31,6 +31,9 @@ static Hash erts_fun_table; #include "erl_smp.h" +#ifdef HIPE +# include "hipe_mode_switch.h" +#endif static erts_smp_rwmtx_t erts_fun_table_lock; @@ -49,8 +52,8 @@ static void fun_free(ErlFunEntry* obj); * to unloaded_fun[]. The -1 in unloaded_fun[0] will be interpreted * as an illegal arity when attempting to call a fun. */ -static BeamInstr unloaded_fun_code[3] = {NIL, -1, 0}; -static BeamInstr* unloaded_fun = unloaded_fun_code + 2; +static BeamInstr unloaded_fun_code[4] = {NIL, NIL, -1, 0}; +static BeamInstr* unloaded_fun = unloaded_fun_code + 3; void erts_init_fun_table(void) @@ -74,7 +77,7 @@ erts_init_fun_table(void) } void -erts_fun_info(int to, void *to_arg) +erts_fun_info(fmtfn_t to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) @@ -219,6 +222,10 @@ erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end) fe->pend_purge_address = addr; ERTS_SMP_WRITE_MEMORY_BARRIER; fe->address = unloaded_fun; +#ifdef HIPE + fe->pend_purge_native_address = fe->native_address; + hipe_set_closure_stub(fe); +#endif erts_purge_state_add_fun(fe); } b = b->next; @@ -234,9 +241,12 @@ erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no) for (ix = 0; ix < no; ix++) { ErlFunEntry *fe = funs[ix]; - if (fe->address == unloaded_fun) + if (fe->address == unloaded_fun) { fe->address = fe->pend_purge_address; - fe->pend_purge_address = NULL; +#ifdef HIPE + fe->native_address = fe->pend_purge_native_address; +#endif + } } } @@ -245,8 +255,12 @@ erts_fun_purge_abort_finalize(ErlFunEntry **funs, Uint no) { Uint ix; - for (ix = 0; ix < no; ix++) + for (ix = 0; ix < no; ix++) { funs[ix]->pend_purge_address = NULL; +#ifdef HIPE + funs[ix]->pend_purge_native_address = NULL; +#endif + } } void @@ -257,6 +271,9 @@ erts_fun_purge_complete(ErlFunEntry **funs, Uint no) for (ix = 0; ix < no; ix++) { ErlFunEntry *fe = funs[ix]; fe->pend_purge_address = NULL; +#ifdef HIPE + fe->pend_purge_native_address = NULL; +#endif if (erts_refc_dectest(&fe->refc, 0) == 0) erts_erase_fun_entry(fe); } @@ -264,7 +281,7 @@ erts_fun_purge_complete(ErlFunEntry **funs, Uint no) } void -erts_dump_fun_entries(int to, void *to_arg) +erts_dump_fun_entries(fmtfn_t to, void *to_arg) { int limit; HashBucket** bucket; @@ -325,6 +342,7 @@ fun_alloc(ErlFunEntry* template) obj->pend_purge_address = NULL; #ifdef HIPE obj->native_address = NULL; + obj->pend_purge_native_address = NULL; #endif return obj; } diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 73c3e19c1c..83899f24a6 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -45,6 +45,9 @@ typedef struct erl_fun_entry { erts_refc_t refc; /* Reference count: One for code + one for each fun object in each process. */ BeamInstr *pend_purge_address; /* address stored during a pending purge */ +#ifdef HIPE + UWord* pend_purge_native_address; +#endif } ErlFunEntry; /* @@ -57,9 +60,6 @@ typedef struct erl_fun_thing { Eterm thing_word; /* Subtag FUN_SUBTAG. */ ErlFunEntry* fe; /* Pointer to fun entry. */ struct erl_off_heap_header* next; -#ifdef HIPE - UWord* native_address; /* Native code for the fun. */ -#endif Uint arity; /* The arity of the fun. */ Uint num_free; /* Number of free variables (in env). */ /* -- The following may be compound Erlang terms ---------------------- */ @@ -71,7 +71,7 @@ typedef struct erl_fun_thing { #define ERL_FUN_SIZE ((sizeof(ErlFunThing)/sizeof(Eterm))-1) void erts_init_fun_table(void); -void erts_fun_info(int, void *); +void erts_fun_info(fmtfn_t, void *); int erts_fun_table_sz(void); ErlFunEntry* erts_put_fun_entry(Eterm mod, int uniq, int index); @@ -86,6 +86,6 @@ void erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end); void erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no); void erts_fun_purge_abort_finalize(ErlFunEntry **funs, Uint no); void erts_fun_purge_complete(ErlFunEntry **funs, Uint no); -void erts_dump_fun_entries(int, void *); +void erts_dump_fun_entries(fmtfn_t, void *); #endif diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index db262d35c5..28b2b02914 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -35,7 +35,7 @@ #include "error.h" #include "big.h" #include "erl_gc.h" -#if HIPE +#ifdef HIPE #include "hipe_stack.h" #include "hipe_mode_switch.h" #endif @@ -395,11 +395,11 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end, if (is_non_value(result)) { if (p->freason == TRAP) { - #if HIPE +#ifdef HIPE if (regs == NULL) { regs = erts_proc_sched_data(p)->x_reg_array; } - #endif +#endif cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls); } else { cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls); @@ -2317,6 +2317,11 @@ move_msgq_to_heap(Process *p) static Uint setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) { + /* + * NOTE! + * Remember to update offset_rootset() when changing + * this function. + */ Roots* roots; Uint n; @@ -2961,6 +2966,12 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, offset_heap_ptr(objv, nobj, offs, area, area_size); } offset_off_heap(p, offs, area, area_size); + if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) { + Eterm* argv; + int argc; + if (erts_setup_nif_gc(p, &argv, &argc)) + offset_heap_ptr(argv, argc, offs, area, area_size); + } } static void diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index 223ba193da..50aa41b4d2 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -168,7 +168,7 @@ static Block_t * get_free_block (Allctr_t *, Uint, static void link_free_block (Allctr_t *, Block_t *); static void unlink_free_block (Allctr_t *, Block_t *); static void update_last_aux_mbc (Allctr_t *, Carrier_t *); -static Eterm info_options (Allctr_t *, char *, int *, +static Eterm info_options (Allctr_t *, char *, fmtfn_t *, void *, Uint **, Uint *); static void init_atoms (void); @@ -551,7 +551,7 @@ add_2tup(Uint **hpp, Uint *szp, Eterm *lp, Eterm el1, Eterm el2) static Eterm info_options(Allctr_t *allctr, char *prefix, - int *print_to_p, + fmtfn_t *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index f1bef28186..d29d079fc5 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -2851,7 +2851,7 @@ erts_read_port_timer(Port *c_prt) */ typedef struct { - int to; + fmtfn_t to; void *to_arg; ErtsMonotonicTime now; } ErtsBTMPrint; @@ -2881,7 +2881,7 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp) } void -erts_print_bif_timer_info(int to, void *to_arg) +erts_print_bif_timer_info(fmtfn_t to, void *to_arg) { ErtsBTMPrint btmp; int six; diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h index 0931bb8965..705be94532 100644 --- a/erts/emulator/beam/erl_hl_timer.h +++ b/erts/emulator/beam/erl_hl_timer.h @@ -72,7 +72,7 @@ erts_handle_canceled_timers(void *vesdp, #endif Uint erts_bif_timer_memory_size(void); -void erts_print_bif_timer_info(int to, void *to_arg); +void erts_print_bif_timer_info(fmtfn_t to, void *to_arg); void erts_debug_bif_timer_foreach(void (*func)(Eterm, Eterm, diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index b299ac8d64..c5904b375e 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -795,9 +795,6 @@ early_init(int *argc, char **argv) /* erts_thr_progress_pre_init(); #endif -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init(); -#endif #ifdef ERTS_SMP erts_smp_atomic32_init_nob(&erts_writing_erl_crash_dump, 0L); erts_tsd_key_create(&erts_is_crash_dumping_key,"erts_is_crash_dumping_key"); @@ -2372,6 +2369,8 @@ system_cleanup(int flush_async) erts_exit_flush_async(); } +static int erts_exit_code; + static __decl_noreturn void __noreturn erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) { @@ -2383,12 +2382,21 @@ erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) if (fmt != NULL && *fmt != '\0') erl_error(fmt, args2); /* Print error message. */ - /* Produce an Erlang core dump if error */ + erts_exit_code = n; + + /* Produce an Erlang crash dump if error */ if (((n == ERTS_ERROR_EXIT && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) && erts_initialized) { erl_crash_dump_v((char*) NULL, 0, fmt, args1); } + erts_exit_epilogue(); +} + +__decl_noreturn void __noreturn erts_exit_epilogue(void) +{ + int n = erts_exit_code; + sys_tty_reset(n); if (n == ERTS_INTR_EXIT) diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index f84c63e7a4..4d4defd8b5 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -539,7 +539,7 @@ map_stat_free(ErtsAlcType_t n, void *extra, void *ptr) } -static void dump_memory_map_to_stream(FILE *fp) +static void dump_memory_map_to_stream(fmtfn_t to, void* to_arg) { ErtsAlcType_t n; MapStatBlock_t *bp; @@ -551,7 +551,7 @@ static void dump_memory_map_to_stream(FILE *fp) /* Write header */ - fprintf(fp, + erts_cbprintf(to, to_arg, "{instr_hdr,\n" " %lu,\n" " %lu,\n" @@ -574,7 +574,7 @@ static void dump_memory_map_to_stream(FILE *fp) else astr = ERTS_ALC_A2AD(ERTS_ALC_A_SYSTEM); - fprintf(fp, + erts_cbprintf(to, to_arg, "%s{%s,%s,%s}%s", (n == ERTS_ALC_N_MIN) ? "" : " ", ERTS_ALC_N2TD(n), @@ -583,12 +583,12 @@ static void dump_memory_map_to_stream(FILE *fp) (n == ERTS_ALC_N_MAX) ? "" : ",\n"); } - fprintf(fp, "}}.\n"); + erts_cbprintf(to, to_arg, "}}.\n"); /* Write memory data */ for (bp = mem_anchor; bp; bp = bp->next) { if (is_internal_pid(bp->pid)) - fprintf(fp, + erts_cbprintf(to, to_arg, "{%lu, %lu, %lu, {%lu,%lu,%lu}}.\n", (UWord) bp->type_no, (UWord) bp->mem, @@ -597,7 +597,7 @@ static void dump_memory_map_to_stream(FILE *fp) (UWord) pid_number(bp->pid), (UWord) pid_serial(bp->pid)); else - fprintf(fp, + erts_cbprintf(to, to_arg, "{%lu, %lu, %lu, undefined}.\n", (UWord) bp->type_no, (UWord) bp->mem, @@ -608,40 +608,29 @@ static void dump_memory_map_to_stream(FILE *fp) erts_mtx_unlock(&instr_mutex); } -int erts_instr_dump_memory_map_to_fd(int fd) +int erts_instr_dump_memory_map_to(fmtfn_t to, void* to_arg) { - char buf[BUFSIZ]; - FILE *f; - if (!erts_instr_memory_map) return 0; - f = fdopen(fd, "w"); - if (f == NULL) - return 0; - - /* Avoid allocating memory; we may have run out of it at this point. */ - setbuf(f, buf); - - dump_memory_map_to_stream(f); - fflush(f); + dump_memory_map_to_stream(to, to_arg); return 1; } int erts_instr_dump_memory_map(const char *name) { - FILE *f; + int fd; if (!erts_instr_memory_map) return 0; - f = fopen(name, "w"); - if (f == NULL) + fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, 0640); + if (fd < 0) return 0; - dump_memory_map_to_stream(f); + dump_memory_map_to_stream(erts_write_fd, (void*)&fd); - fclose(f); + close(fd); return 1; } @@ -998,19 +987,19 @@ erts_instr_get_stat(Process *proc, Eterm what, int begin_max_period) } static void -dump_stat_to_stream(FILE *fp, int begin_max_period) +dump_stat_to_stream(fmtfn_t to, void* to_arg, int begin_max_period) { ErtsAlcType_t i, a_max, a_min; erts_mtx_lock(&instr_mutex); - fprintf(fp, + erts_cbprintf(to, to_arg, "{instr_vsn,%lu}.\n", (unsigned long) ERTS_INSTR_VSN); update_max_ever_values(&stats->tot, 0, 0); - fprintf(fp, + erts_cbprintf(to, to_arg, "{total,[{total,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}]}.\n", (UWord) stats->tot.size, (UWord) stats->tot.max_size, @@ -1038,7 +1027,7 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { if (erts_allctrs_info[i].enabled) { - fprintf(fp, + erts_cbprintf(to, to_arg, "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", i == a_min ? "{allocators,\n [" : " ", ERTS_ALC_A2AD(i), @@ -1055,7 +1044,7 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) update_max_ever_values(stats->c, ERTS_ALC_C_MIN, ERTS_ALC_C_MAX); for (i = ERTS_ALC_C_MIN; i <= ERTS_ALC_C_MAX; i++) { - fprintf(fp, + erts_cbprintf(to, to_arg, "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", i == ERTS_ALC_C_MIN ? "{classes,\n [" : " ", ERTS_ALC_C2CD(i), @@ -1071,7 +1060,7 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) update_max_ever_values(stats->n, ERTS_ALC_N_MIN, ERTS_ALC_N_MAX); for (i = ERTS_ALC_N_MIN; i <= ERTS_ALC_N_MAX; i++) { - fprintf(fp, + erts_cbprintf(to, to_arg, "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s", i == ERTS_ALC_N_MIN ? "{types,\n [" : " ", ERTS_ALC_N2TD(i), @@ -1095,40 +1084,29 @@ dump_stat_to_stream(FILE *fp, int begin_max_period) } -int erts_instr_dump_stat_to_fd(int fd, int begin_max_period) +int erts_instr_dump_stat_to(fmtfn_t to, void* to_arg, int begin_max_period) { - char buf[BUFSIZ]; - FILE *fp; - if (!erts_instr_stat) return 0; - fp = fdopen(fd, "w"); - if (fp == NULL) - return 0; - - /* Avoid allocating memory; we may have run out of it at this point. */ - setbuf(fp, buf); - - dump_stat_to_stream(fp, begin_max_period); - fflush(fp); + dump_stat_to_stream(to, to_arg, begin_max_period); return 1; } int erts_instr_dump_stat(const char *name, int begin_max_period) { - FILE *file; + int fd; if (!erts_instr_stat) return 0; - file = fopen(name, "w"); - if (file == NULL) + fd = open(name, O_WRONLY | O_CREAT | O_TRUNC,0640); + if (fd < 0) return 0; - dump_stat_to_stream(file, begin_max_period); + dump_stat_to_stream(erts_write_fd, (void*)&fd, begin_max_period); - fclose(file); + close(fd); return 1; } diff --git a/erts/emulator/beam/erl_instrument.h b/erts/emulator/beam/erl_instrument.h index 1f04c91d5e..351172b2fa 100644 --- a/erts/emulator/beam/erl_instrument.h +++ b/erts/emulator/beam/erl_instrument.h @@ -29,10 +29,10 @@ extern int erts_instr_memory_map; extern int erts_instr_stat; Uint erts_instr_init(int stat, int map_stat); -int erts_instr_dump_memory_map_to_fd(int fd); +int erts_instr_dump_memory_map_to(fmtfn_t to, void* to_arg); int erts_instr_dump_memory_map(const char *name); Eterm erts_instr_get_memory_map(Process *process); -int erts_instr_dump_stat_to_fd(int fd, int begin_max_period); +int erts_instr_dump_stat_to(fmtfn_t to, void* to_arg, int begin_max_period); int erts_instr_dump_stat(const char *name, int begin_max_period); Eterm erts_instr_get_stat(Process *proc, Eterm what, int begin_max_period); Eterm erts_instr_get_type_info(Process *proc); diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index a00e0f0fff..481e92b2cd 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -274,11 +274,11 @@ void erts_lcnt_init() { lcnt_unlock(); - /* set start timer and zero statistics */ - erts_lcnt_clear_counters(); } void erts_lcnt_late_init() { + /* set start timer and zero statistics */ + erts_lcnt_clear_counters(); erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler); } diff --git a/erts/emulator/beam/erl_math.c b/erts/emulator/beam/erl_math.c index 990fa63bd4..1f270eb55f 100644 --- a/erts/emulator/beam/erl_math.c +++ b/erts/emulator/beam/erl_math.c @@ -256,3 +256,8 @@ BIF_RETTYPE math_floor_1(BIF_ALIST_1) { return math_call_1(BIF_P, floor, BIF_ARG_1); } + +BIF_RETTYPE math_fmod_2(BIF_ALIST_2) +{ + return math_call_2(BIF_P, fmod, BIF_ARG_1, BIF_ARG_2); +} diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index ad7c8c5eee..7e02d8e101 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -22,7 +22,7 @@ #define ERL_MSACC_H__ /* Can be enabled/disabled via configure */ -#if ERTS_ENABLE_MSACC == 2 +#if defined(ERTS_ENABLE_MSACC) && ERTS_ENABLE_MSACC == 2 #define ERTS_MSACC_EXTENDED_STATES 1 #endif @@ -66,7 +66,7 @@ #define ERTS_MSACC_STATE_COUNT 7 -#if ERTS_MSACC_STATE_STRINGS && ERTS_ENABLE_MSACC +#if defined(ERTS_MSACC_STATE_STRINGS) && defined(ERTS_ENABLE_MSACC) static char *erts_msacc_states[] = { "aux", "check_io", @@ -104,7 +104,7 @@ static char *erts_msacc_states[] = { #define ERTS_MSACC_STATE_COUNT ERTS_MSACC_STATIC_STATE_COUNT #endif -#if ERTS_MSACC_STATE_STRINGS +#ifdef ERTS_MSACC_STATE_STRINGS static char *erts_msacc_states[] = { "alloc", "aux", @@ -157,7 +157,7 @@ struct erl_msacc_t_ { }; -#if ERTS_ENABLE_MSACC +#ifdef ERTS_ENABLE_MSACC #ifdef USE_THREADS extern erts_tsd_key_t erts_msacc_key; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 7442e99cb3..202f713f67 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -71,10 +71,12 @@ struct erl_module_nif { void* priv_data; void* handle; /* "dlopen" */ - struct enif_entry_t* entry; + struct enif_entry_t entry; erts_refc_t rt_cnt; /* number of resource types */ erts_refc_t rt_dtor_cnt; /* number of resource types with destructors */ Module* mod; /* Can be NULL if orphan with dtor-resources left */ + + ErlNifFunc _funcs_copy_[1]; /* only used for old libs */ }; #ifdef DEBUG @@ -1958,10 +1960,10 @@ static void close_lib(struct erl_module_nif* lib) ASSERT(lib->handle != NULL); ASSERT(erts_refc_read(&lib->rt_dtor_cnt,0) == 0); - if (lib->entry != NULL && lib->entry->unload != NULL) { + if (lib->entry.unload != NULL) { struct enif_msg_environment_t msg_env; pre_nif_noproc(&msg_env, lib, NULL); - lib->entry->unload(&msg_env.env, lib->priv_data); + lib->entry.unload(&msg_env.env, lib->priv_data); post_nif_noproc(&msg_env); } if (!erts_is_static_nif(lib->handle)) @@ -2285,7 +2287,7 @@ typedef struct { Export exp; struct erl_module_nif* m; NativeFunPtr fp; - BeamInstr *saved_current; + ErtsCodeMFA *saved_current; int exception_thrown; int saved_argc; int rootset_extra; @@ -2312,6 +2314,17 @@ erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj) return gc; } +int +erts_check_nif_export_in_area(Process *p, char *start, Uint size) +{ + NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p); + if (!nep || !nep->saved_current) + return 0; + if (ErtsInArea(nep->saved_current, start, size)) + return 1; + return 0; +} + /* * Allocate a NifExport and set it in proc specific data */ @@ -2328,9 +2341,9 @@ allocate_nif_sched_data(Process* proc, int argc) ep->rootset_extra = argc; ep->rootset[0] = NIL; for (i=0; i<ERTS_NUM_CODE_IX; i++) { - ep->exp.addressv[i] = &ep->exp.code[3]; + ep->exp.addressv[i] = &ep->exp.beam[0]; } - ep->exp.code[3] = (BeamInstr) em_call_nif; + ep->exp.beam[0] = (BeamInstr) em_call_nif; (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep); return ep; } @@ -2363,6 +2376,7 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec Eterm* reg; NifExport* ep; int i, scheduler; + int orig_argc; execution_state(env, &proc, &scheduler); @@ -2373,11 +2387,14 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec reg = erts_proc_sched_data(proc)->x_reg_array; + ASSERT(!need_save || proc->current); + orig_argc = need_save ? (int) proc->current->arity : 0; + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); if (!ep) - ep = allocate_nif_sched_data(proc, argc); - else if (need_save && ep->rootset_extra < argc) { - NifExport* new_ep = allocate_nif_sched_data(proc, argc); + ep = allocate_nif_sched_data(proc, orig_argc); + else if (need_save && ep->rootset_extra < orig_argc) { + NifExport* new_ep = allocate_nif_sched_data(proc, orig_argc); destroy_nif_export(ep); ep = new_ep; } @@ -2390,21 +2407,19 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec } if (scheduler > 0) ERTS_VBUMP_ALL_REDS(proc); - for (i = 0; i < argc; i++) { - if (need_save) - ep->rootset[i+1] = reg[i]; - reg[i] = (Eterm) argv[i]; - } if (need_save) { - ASSERT(proc->current); ep->saved_current = proc->current; - ep->saved_argc = argc; + ep->saved_argc = orig_argc; + for (i = 0; i < orig_argc; i++) + ep->rootset[i+1] = reg[i]; } + for (i = 0; i < argc; i++) + reg[i] = (Eterm) argv[i]; proc->i = (BeamInstr*) ep->exp.addressv[0]; - ep->exp.code[0] = (BeamInstr) proc->current[0]; - ep->exp.code[1] = (BeamInstr) proc->current[1]; - ep->exp.code[2] = argc; - ep->exp.code[4] = (BeamInstr) direct_fp; + ep->exp.info.mfa.module = proc->current->module; + ep->exp.info.mfa.function = proc->current->function; + ep->exp.info.mfa.arity = argc; + ep->exp.beam[1] = (BeamInstr) direct_fp; ep->m = env->mod_nif; ep->fp = indirect_fp; proc->freason = TRAP; @@ -2426,7 +2441,7 @@ restore_nif_mfa(Process* proc, NifExport* ep, int exception) ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc) & ERTS_PROC_LOCK_MAIN); - ASSERT(ep->saved_current != &ep->exp.code[0]); + ASSERT(ep->saved_current != &ep->exp.info.mfa); proc->current = ep->saved_current; ep->saved_current = NULL; if (exception) { @@ -2500,7 +2515,8 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) execution_state(env, &proc, NULL); - fp = (NativeFunPtr) proc->current[6]; + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->fp; ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); @@ -2569,7 +2585,8 @@ schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[ erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); } - fp = (NativeFunPtr) proc->current[6]; + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->fp; ASSERT(fp); @@ -2629,7 +2646,8 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ERL_NIF_TERM result; execution_state(env, &proc, NULL); - fp = (NativeFunPtr) proc->current[6]; + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->fp; ASSERT(!env->exception_thrown); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); @@ -2697,7 +2715,7 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); - ep->exp.code[1] = (BeamInstr) fun_name_atom; + ep->exp.info.mfa.function = (BeamInstr) fun_name_atom; done: if (scheduler < 0) @@ -3011,17 +3029,16 @@ int enif_map_iterator_get_pair(ErlNifEnv *env, ***************************************************************************/ -static BeamInstr** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsigned arity) +static ErtsCodeInfo** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsigned arity) { int n = (int) mod_code->num_functions; int j; for (j = 0; j < n; ++j) { - BeamInstr* code_ptr = (BeamInstr*) mod_code->functions[j]; - ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - if (f_atom == ((Eterm) code_ptr[3]) - && arity == ((unsigned) code_ptr[4])) { - - return (BeamInstr**) &mod_code->functions[j]; + ErtsCodeInfo* ci = mod_code->functions[j]; + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + if (f_atom == ci->mfa.function + && arity == ci->mfa.arity) { + return mod_code->functions+j; } } return NULL; @@ -3071,16 +3088,16 @@ Eterm erts_nif_taints(Process* p) return list; } -void erts_print_nif_taints(int to, void* to_arg) +void erts_print_nif_taints(fmtfn_t to, void* to_arg) { struct tainted_module_t* t; const char* delim = ""; for (t=first_tainted_module ; t!=NULL; t=t->next) { const Atom* atom = atom_tab(atom_val(t->module_atom)); - erts_print(to,to_arg,"%s%.*s", delim, atom->len, atom->name); + erts_cbprintf(to,to_arg,"%s%.*s", delim, atom->len, atom->name); delim = ","; } - erts_print(to,to_arg,"\n"); + erts_cbprintf(to,to_arg,"\n"); } @@ -3113,33 +3130,52 @@ static Eterm load_nif_error(Process* p, const char* atom, const char* format, .. return ret; } +#define AT_LEAST_VERSION(E,MAJ,MIN) \ + (((E)->major * 0x100 + (E)->minor) >= ((MAJ) * 0x100 + (MIN))) + /* - * The function below is for looping through ErlNifFunc arrays, helping - * provide backwards compatibility across the version 2.7 change that added - * the "flags" field to ErlNifFunc. + * Allocate erl_module_nif and make a _modern_ copy of the lib entry. */ -static ErlNifFunc* next_func(ErlNifEntry* entry, int* incrp, ErlNifFunc* func) +static struct erl_module_nif* create_lib(const ErlNifEntry* src) { - ASSERT(incrp); - if (!*incrp) { - if (entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) - *incrp = sizeof(ErlNifFunc); - else { - /* - * ErlNifFuncV1 below is what ErlNifFunc was before the - * addition of the flags field for 2.7, and is needed to handle - * backward compatibility. - */ - typedef struct { - const char* name; - unsigned arity; - ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - }ErlNifFuncV1; - *incrp = sizeof(ErlNifFuncV1); - } + struct erl_module_nif* lib; + ErlNifEntry* dst; + Uint bytes = offsetof(struct erl_module_nif, _funcs_copy_); + + if (!AT_LEAST_VERSION(src, 2, 7)) + bytes += src->num_of_funcs * sizeof(ErlNifFunc); + + lib = erts_alloc(ERTS_ALC_T_NIF, bytes); + dst = &lib->entry; + + sys_memcpy(dst, src, offsetof(ErlNifEntry, vm_variant)); + + if (AT_LEAST_VERSION(src, 2, 1)) { + dst->vm_variant = src->vm_variant; + } else { + dst->vm_variant = "beam.vanilla"; } - return (ErlNifFunc*) ((char*)func + *incrp); -} + if (AT_LEAST_VERSION(src, 2, 7)) { + dst->options = src->options; + } else { + /* + * Make a modern copy of the ErlNifFunc array + */ + struct ErlNifFunc_V1 { + const char* name; + unsigned arity; + ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + }*src_funcs = (struct ErlNifFunc_V1*) src->funcs; + int i; + for (i = 0; i < src->num_of_funcs; ++i) { + sys_memcpy(&lib->_funcs_copy_[i], &src_funcs[i], sizeof(*src_funcs)); + lib->_funcs_copy_[i].flags = 0; + } + dst->funcs = lib->_funcs_copy_; + dst->options = 0; + } + return lib; +}; BIF_RETTYPE load_nif_2(BIF_ALIST_2) @@ -3156,7 +3192,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) Eterm mod_atom; const Atom* mod_atomp; Eterm f_atom; - BeamInstr* caller; + ErtsCodeMFA* caller; ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT; Eterm ret = am_ok; int veto; @@ -3164,6 +3200,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) struct erl_module_instance* this_mi; struct erl_module_instance* prev_mi; + if (BIF_P->flags & F_HIPE_MODE) { + ret = load_nif_error(BIF_P, "notsup", "Calling load_nif from HiPE compiled " + "modules not supported"); + BIF_RET(ret); + } + encoding = erts_get_native_filename_encoding(); if (encoding == ERL_FILENAME_WIN_WCHAR) { /* Do not convert the lib name to utf-16le yet, do that in win32 specific code */ @@ -3189,12 +3231,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) /* Find calling module */ ASSERT(BIF_P->current != NULL); - ASSERT(BIF_P->current[0] == am_erlang - && BIF_P->current[1] == am_load_nif - && BIF_P->current[2] == 2); + ASSERT(BIF_P->current->module == am_erlang + && BIF_P->current->function == am_load_nif + && BIF_P->current->arity == 2); caller = find_function_from_pc(BIF_P->cp); ASSERT(caller != NULL); - mod_atom = caller[0]; + mod_atom = caller->module; ASSERT(is_atom(mod_atom)); module_p = erts_get_module(mod_atom, erts_active_code_ix()); ASSERT(module_p != NULL); @@ -3253,7 +3295,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).", entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION); } - else if (entry->minor >= 1 + else if (AT_LEAST_VERSION(entry, 2, 1) && sys_strcmp(entry->vm_variant, ERL_NIF_VM_VARIANT) != 0) { ret = load_nif_error(BIF_P, bad_lib, "Library (%s) not compiled for " "this vm variant (%s).", @@ -3264,20 +3306,25 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) " match calling module '%T'", entry->name, mod_atom); } else { - /*erts_fprintf(stderr, "Found module %T\r\n", mod_atom);*/ - - int maybe_dirty_nifs = ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) - && (entry->options & ERL_NIF_DIRTY_NIF_OPTION)); - int incr = 0; - ErlNifFunc* f = entry->funcs; - for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { - BeamInstr** code_pp; + lib = create_lib(entry); + entry = &lib->entry; /* Use a guaranteed modern lib entry from now on */ + + lib->handle = handle; + erts_refc_init(&lib->rt_cnt, 0); + erts_refc_init(&lib->rt_dtor_cnt, 0); + ASSERT(opened_rt_list == NULL); + lib->mod = module_p; + + for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { + ErtsCodeInfo** ci_pp; + ErlNifFunc* f = &entry->funcs[i]; + if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1) - || (code_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) { + || (ci_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) { ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); } - else if (maybe_dirty_nifs && f->flags) { + else if (f->flags) { /* * If the flags field is non-zero and this emulator was * built with dirty scheduler support, check that the flags @@ -3295,9 +3342,9 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) #endif } #ifdef ERTS_DIRTY_SCHEDULERS - else if (code_pp[1] - code_pp[0] < (5+4)) + else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (4)) #else - else if (code_pp[1] - code_pp[0] < (5+3)) + else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (3)) #endif { ret = load_nif_error(BIF_P,bad_lib,"No explicit call to load_nif" @@ -3306,7 +3353,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } /*erts_fprintf(stderr, "Found NIF %T:%s/%u\r\n", mod_atom, f->name, f->arity);*/ - f = next_func(entry, &incr, f); } } @@ -3317,14 +3363,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) /* Call load or upgrade: */ - - lib = erts_alloc(ERTS_ALC_T_NIF, sizeof(struct erl_module_nif)); - lib->handle = handle; - lib->entry = entry; - erts_refc_init(&lib->rt_cnt, 0); - erts_refc_init(&lib->rt_dtor_cnt, 0); - ASSERT(opened_rt_list == NULL); - lib->mod = module_p; env.mod_nif = lib; lib->priv_data = NULL; @@ -3339,58 +3377,55 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_post_nif(&env); if (veto) { prev_mi->nif->priv_data = prev_old_data; - ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful."); + ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful (%d).", veto); } - else - commit_opened_resource_types(lib); } else if (entry->load != NULL) { /********* Initial load ***********/ erts_pre_nif(&env, BIF_P, lib, NULL); veto = entry->load(&env, &lib->priv_data, BIF_ARG_2); erts_post_nif(&env); if (veto) { - ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful."); + ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful (%d).", veto); } - else - commit_opened_resource_types(lib); } if (ret == am_ok) { + commit_opened_resource_types(lib); + /* ** Everything ok, patch the beam code with op_call_nif */ - int incr = 0; - ErlNifFunc* f = entry->funcs; - this_mi->nif = lib; for (i=0; i < entry->num_of_funcs; i++) { - BeamInstr* code_ptr; + ErlNifFunc* f = &entry->funcs[i]; + ErtsCodeInfo* ci; + BeamInstr *code_ptr; + erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1); - code_ptr = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); + ci = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); + code_ptr = erts_codeinfo_to_code(ci); - if (code_ptr[1] == 0) { - code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif); + if (ci->native == 0) { + code_ptr[0] = (BeamInstr) BeamOp(op_call_nif); } else { /* Function traced, patch the original instruction word */ - GenericBp* g = (GenericBp *) code_ptr[1]; - ASSERT(code_ptr[5+0] == + GenericBp* g = (GenericBp *) ci->native; + ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)); g->orig_instr = (BeamInstr) BeamOp(op_call_nif); } #ifdef ERTS_DIRTY_SCHEDULERS - if ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) - && (entry->options & ERL_NIF_DIRTY_NIF_OPTION) && f->flags) { - code_ptr[5+3] = (BeamInstr) f->fptr; - code_ptr[5+1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? + if (f->flags) { + code_ptr[3] = (BeamInstr) f->fptr; + code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? (BeamInstr) schedule_dirty_io_nif : (BeamInstr) schedule_dirty_cpu_nif; } else #endif - code_ptr[5+1] = (BeamInstr) f->fptr; - code_ptr[5+2] = (BeamInstr) lib; - f = next_func(entry, &incr, f); + code_ptr[1] = (BeamInstr) f->fptr; + code_ptr[2] = (BeamInstr) lib; } } else { @@ -3474,8 +3509,8 @@ void erl_nif_init() int erts_nif_get_funcs(struct erl_module_nif* mod, ErlNifFunc **funcs) { - *funcs = mod->entry->funcs; - return mod->entry->num_of_funcs; + *funcs = mod->entry.funcs; + return mod->entry.num_of_funcs; } Eterm erts_nif_call_function(Process *p, Process *tracee, @@ -3486,10 +3521,10 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, #ifdef DEBUG /* Verify that function is part of this module */ int i; - for (i = 0; i < mod->entry->num_of_funcs; i++) - if (fun == &(mod->entry->funcs[i])) + for (i = 0; i < mod->entry.num_of_funcs; i++) + if (fun == &(mod->entry.funcs[i])) break; - ASSERT(i < mod->entry->num_of_funcs); + ASSERT(i < mod->entry.num_of_funcs); if (p) ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN || erts_smp_thr_progress_is_blocking()); diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 494971e118..413b4f7343 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -49,7 +49,7 @@ ** 2.8: 18.0 add enif_has_pending_exception ** 2.9: 18.2 enif_getenv ** 2.10: Time API -** 2.11: 19.0 enif_snprintf +** 2.11: 19.0 enif_snprintf */ #define ERL_NIF_MAJOR_VERSION 2 #define ERL_NIF_MINOR_VERSION 11 @@ -116,12 +116,13 @@ typedef struct enif_entry_t int (*reload) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info); int (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); void (*unload) (ErlNifEnv*, void* priv_data); + + /* Added in 2.1 */ const char* vm_variant; - unsigned options; -}ErlNifEntry; -/* Field bits for ErlNifEntry options */ -#define ERL_NIF_DIRTY_NIF_OPTION 1 + /* Added in 2.7 */ + unsigned options; /* Unused. Can be set to 0 or 1 (dirty sched config) */ +}ErlNifEntry; typedef struct @@ -266,8 +267,6 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; # define ERL_NIF_INIT_DECL(MODNAME) ERL_NIF_INIT_EXPORT ErlNifEntry* nif_init(ERL_NIF_INIT_ARGS) #endif -#define ERL_NIF_ENTRY_OPTIONS ERL_NIF_DIRTY_NIF_OPTION - #ifdef __cplusplus } # define ERL_NIF_INIT_PROLOGUE extern "C" { @@ -293,7 +292,7 @@ ERL_NIF_INIT_DECL(NAME) \ FUNCS, \ LOAD, RELOAD, UPGRADE, UNLOAD, \ ERL_NIF_VM_VARIANT, \ - ERL_NIF_ENTRY_OPTIONS \ + 1 \ }; \ ERL_NIF_INIT_BODY; \ return &entry; \ diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 646f786651..70500ed6e1 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -188,7 +188,7 @@ dist_table_free(void *vdep) void -erts_dist_table_info(int to, void *to_arg) +erts_dist_table_info(fmtfn_t to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) @@ -564,7 +564,7 @@ erts_node_table_size(void) } void -erts_node_table_info(int to, void *to_arg) +erts_node_table_info(fmtfn_t to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) @@ -649,7 +649,7 @@ void erts_schedule_delete_node(ErlNode *enp) } struct pn_data { - int to; + fmtfn_t to; void *to_arg; Eterm sysname; int no_sysname; @@ -679,7 +679,7 @@ static void print_node(void *venp, void *vpndp) pndp->no_total++; } -void erts_print_node_info(int to, +void erts_print_node_info(fmtfn_t to, void *to_arg, Eterm sysname, int *no_sysname, diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index 7a4434acbf..47a6724c21 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -179,7 +179,7 @@ DistEntry *erts_find_or_insert_dist_entry(Eterm); DistEntry *erts_find_dist_entry(Eterm); void erts_schedule_delete_dist_entry(DistEntry *); Uint erts_dist_table_size(void); -void erts_dist_table_info(int, void *); +void erts_dist_table_info(fmtfn_t, void *); void erts_set_dist_entry_not_connected(DistEntry *); void erts_set_dist_entry_connected(DistEntry *, Eterm, Uint); ErlNode *erts_find_or_insert_node(Eterm, Uint32); @@ -187,8 +187,8 @@ void erts_schedule_delete_node(ErlNode *); void erts_set_this_node(Eterm, Uint); Uint erts_node_table_size(void); void erts_init_node_tables(int); -void erts_node_table_info(int, void *); -void erts_print_node_info(int, void *, Eterm, int*, int*); +void erts_node_table_info(fmtfn_t, void *); +void erts_print_node_info(fmtfn_t, void *, Eterm, int*, int*); Eterm erts_get_node_and_dist_references(struct process *); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_lc_is_de_rwlocked(DistEntry *); diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index f90844ccc8..206078903d 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -369,7 +369,7 @@ Eterm erts_request_io_bytes(Process *c_p); #define ERTS_PORT_REDS_INFO (CONTEXT_REDS/100) #define ERTS_PORT_REDS_TERMINATE (CONTEXT_REDS/50) -void print_port_info(Port *, int, void *); +void print_port_info(Port *, fmtfn_t, void *); void erts_port_free(Port *); #ifndef ERTS_SMP void erts_port_cleanup(Port *); @@ -733,7 +733,7 @@ erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl) if (lock_pdl && prt->port_data_lock) driver_pdl_lock(prt->port_data_lock); -#if ERTS_ENABLE_LOCK_CHECK +#ifdef ERTS_ENABLE_LOCK_CHECK if (!ERTS_IS_CRASH_DUMPING) { if (erts_lc_is_emu_thr()) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 1a579704a8..b43a1b0190 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -526,8 +526,8 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { case EXPORT_DEF: { Export* ep = *((Export **) (export_val(wobj) + 1)); - Atom* module = atom_tab(atom_val(ep->code[0])); - Atom* name = atom_tab(atom_val(ep->code[1])); + Atom* module = atom_tab(atom_val(ep->info.mfa.module)); + Atom* name = atom_tab(atom_val(ep->info.mfa.function)); PRINT_STRING(res, fn, arg, "#Fun<"); PRINT_BUF(res, fn, arg, module->name, module->len); @@ -535,7 +535,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { PRINT_BUF(res, fn, arg, name->name, name->len); PRINT_CHAR(res, fn, arg, '.'); PRINT_SWORD(res, fn, arg, 'd', 0, 1, - (ErlPfSWord) ep->code[2]); + (ErlPfSWord) ep->info.mfa.arity); PRINT_CHAR(res, fn, arg, '>'); } break; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 269518d8d6..4fb0f9e975 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -154,6 +154,7 @@ int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1; int ERTS_WRITE_UNLIKELY(erts_sched_compact_load); int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0; Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers); +Uint ERTS_WRITE_UNLIKELY(erts_no_total_schedulers); Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers) = 0; Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers) = 0; @@ -556,8 +557,8 @@ do { \ */ static void exec_misc_ops(ErtsRunQueue *); -static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); -static int stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg); +static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x); +static int stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg); static void aux_work_timeout(void *unused); static void aux_work_timeout_early_init(int no_schedulers); @@ -5907,9 +5908,12 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online n = (int) no_schedulers; erts_no_schedulers = n; + erts_no_total_schedulers = n; #ifdef ERTS_DIRTY_SCHEDULERS erts_no_dirty_cpu_schedulers = no_dirty_cpu_schedulers; + erts_no_total_schedulers += no_dirty_cpu_schedulers; erts_no_dirty_io_schedulers = no_dirty_io_schedulers; + erts_no_total_schedulers += no_dirty_io_schedulers; #endif /* Create and initialize scheduler sleep info */ @@ -6270,7 +6274,11 @@ check_dirty_enqueue_in_prio_queue(Process *c_p, return -1*queue; } - *newp |= ERTS_PSFLG_IN_RUNQ; + /* + * Enqueue using process struct. + */ + *newp &= ~ERTS_PSFLGS_PRQ_PRIO_MASK; + *newp |= ERTS_PSFLG_IN_RUNQ | (aprio << ERTS_PSFLGS_PRQ_PRIO_OFFSET); return queue; } @@ -6682,15 +6690,8 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, erts_aint32_t fail_state, state, a, n, enq_prio; int enqueue; /* < 0 -> use proxy */ unsigned int prof_runnable_procs; - int strict_fail_state; fail_state = *fail_state_p; - /* - * If fail state something other than just exiting process, - * ensure that the task wont be scheduled when the - * receiver is in the failure state. - */ - strict_fail_state = fail_state != ERTS_PSFLG_EXITING; res = 1; /* prepare for success */ st->next = st->prev = st; /* Prep for empty prio queue */ @@ -6772,7 +6773,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, /* Status lock prevents out of order "runnable proc" trace msgs */ ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - if (!prof_runnable_procs && !strict_fail_state) { + if (!prof_runnable_procs) { erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); locked = 0; } @@ -6783,11 +6784,6 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, erts_aint32_t e; n = e = a; - if (strict_fail_state && (a & fail_state)) { - *fail_state_p = (a & fail_state); - goto cleanup; - } - if (a & ERTS_PSFLG_FREE) goto cleanup; /* We don't want to schedule free processes... */ @@ -11464,9 +11460,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->schedule_count = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); - p->u.initial[INITIAL_MOD] = mod; - p->u.initial[INITIAL_FUN] = func; - p->u.initial[INITIAL_ARI] = (Uint) arity; + p->u.initial.module = mod; + p->u.initial.function = func; + p->u.initial.arity = (Uint) arity; /* * Must initialize binary lists here before copying binaries to process. @@ -11508,7 +11504,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). /* No need to initialize p->fcalls. */ - p->current = p->u.initial+INITIAL_MOD; + p->current = &p->u.initial; p->i = (BeamInstr *) beam_apply; p->cp = (BeamInstr *) beam_apply+1; @@ -11690,11 +11686,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_spawn)) { + ErtsCodeMFA cmfa = {mod, func, arity}; DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); - dtrace_fun_decode(p, mod, func, arity, process_name, mfa); - DTRACE2(process_spawn, process_name, mfa); + dtrace_fun_decode(p, &cmfa, process_name, mfa_buf); + DTRACE2(process_spawn, process_name, mfa_buf); } #endif return res; @@ -11769,9 +11766,9 @@ void erts_init_empty_process(Process *p) p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; p->seq_trace_token = NIL; - p->u.initial[0] = 0; - p->u.initial[1] = 0; - p->u.initial[2] = 0; + p->u.initial.module = 0; + p->u.initial.function = 0; + p->u.initial.arity = 0; p->catches = 0; p->cp = NULL; p->i = NULL; @@ -12403,6 +12400,8 @@ send_exit_signal(Process *c_p, /* current process if and only else if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) { /* Process not running ... */ ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL; + ErlHeapFragment *bp = NULL; + Eterm rsn_cpy; if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { /* ... but we havn't got all locks on it ... */ @@ -12415,12 +12414,32 @@ send_exit_signal(Process *c_p, /* current process if and only } /* ...and we have all locks on it... */ *rp_locks = ERTS_PROC_LOCKS_ALL; - set_proc_exiting(rp, - state, - (is_immed(rsn) - ? rsn - : copy_object(rsn, rp)), - NULL); + + state = erts_smp_atomic32_read_nob(&rp->state); + + if (is_immed(rsn)) + rsn_cpy = rsn; + else { + Eterm *hp; + ErlOffHeap *ohp; + Uint rsn_sz = size_object(rsn); +#ifdef ERTS_DIRTY_SCHEDULERS + if (state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + bp = new_message_buffer(rsn_sz); + ohp = &bp->off_heap; + hp = &bp->mem[0]; + } + else +#endif + { + hp = HAlloc(rp, rsn_sz); + ohp = &rp->off_heap; + } + rsn_cpy = copy_struct(rsn, rsn_sz, &hp, ohp); + } + + set_proc_exiting(rp, state, rsn_cpy, bp); } else { /* Process running... */ @@ -13173,7 +13192,7 @@ erts_continue_exit_process(Process *p) */ void -erts_stack_dump(int to, void *to_arg, Process *p) +erts_stack_dump(fmtfn_t to, void *to_arg, Process *p) { Eterm* sp; int yreg = -1; @@ -13188,7 +13207,7 @@ erts_stack_dump(int to, void *to_arg, Process *p) } void -erts_program_counter_info(int to, void *to_arg, Process *p) +erts_program_counter_info(fmtfn_t to, void *to_arg, Process *p) { erts_aint32_t state; int i; @@ -13218,10 +13237,10 @@ erts_program_counter_info(int to, void *to_arg, Process *p) } static void -print_function_from_pc(int to, void *to_arg, BeamInstr* x) +print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x) { - BeamInstr* addr = find_function_from_pc(x); - if (addr == NULL) { + ErtsCodeMFA *cmfa = find_function_from_pc(x); + if (cmfa == NULL) { if (x == beam_exit) { erts_print(to, to_arg, "<terminate process>"); } else if (x == beam_continue_exit) { @@ -13235,12 +13254,13 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x) } } else { erts_print(to, to_arg, "%T:%T/%d + %d", - addr[0], addr[1], addr[2], ((x-addr)-2) * sizeof(Eterm)); + cmfa->module, cmfa->function, cmfa->arity, + (x-(BeamInstr*)cmfa) * sizeof(Eterm)); } } static int -stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) +stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg) { Eterm x = *sp; @@ -13272,7 +13292,7 @@ stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) * Print scheduler information */ void -erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) { +erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { int i; erts_aint32_t flg; Process *p; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 3347a7a60e..1c4b9f149d 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -21,6 +21,8 @@ #ifndef __PROCESS_H__ #define __PROCESS_H__ +#include "sys.h" + #undef ERTS_INCLUDE_SCHEDULER_INTERNALS #if (defined(ERL_PROCESS_C__) \ || defined(ERL_PORT_TASK_C__) \ @@ -37,8 +39,6 @@ typedef struct process Process; -#include "sys.h" - #define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__ #include "erl_process_lock.h" /* Only pull out important types... */ #undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__ @@ -93,10 +93,6 @@ struct ErtsNodesMonitor_; #define ERTS_HEAP_FREE(Type, Ptr, Size) \ erts_free((Type), (Ptr)) -#define INITIAL_MOD 0 -#define INITIAL_FUN 1 -#define INITIAL_ARI 2 - #include "export.h" struct saved_calls { @@ -111,6 +107,7 @@ extern int erts_eager_check_io; extern int erts_sched_compact_load; extern int erts_sched_balance_util; extern Uint erts_no_schedulers; +extern Uint erts_no_total_schedulers; #ifdef ERTS_DIRTY_SCHEDULERS extern Uint erts_no_dirty_cpu_schedulers; extern Uint erts_no_dirty_io_schedulers; @@ -1022,15 +1019,16 @@ struct process { #endif union { void *terminate; - BeamInstr initial[3]; /* Initial module(0), function(1), arity(2), often used instead - of pointer to funcinfo instruction, hence the BeamInstr datatype */ + ErtsCodeMFA initial; /* Initial module(0), function(1), arity(2), + often used instead of pointer to funcinfo + instruction. */ } u; - BeamInstr* current; /* Current Erlang function, part of the funcinfo: + ErtsCodeMFA* current; /* Current Erlang function, part of the funcinfo: * module(0), function(1), arity(2) * (module and functions are tagged atoms; - * arity an untagged integer). BeamInstr * because it references code + * arity an untagged integer). */ - + /* * Information mainly for post-mortem use (erl crash dump). */ @@ -1586,6 +1584,7 @@ Uint64 erts_step_proc_interval(void); int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */ void erts_destroy_nif_export(void *); /* see erl_nif.c */ +int erts_check_nif_export_in_area(Process *p, char *start, Uint size); ErtsProcList *erts_proclist_create(Process *); ErtsProcList *erts_proclist_copy(ErtsProcList *); @@ -1837,12 +1836,12 @@ void erts_cleanup_empty_process(Process* p); #ifdef DEBUG void erts_debug_verify_clean_empty_process(Process* p); #endif -void erts_stack_dump(int to, void *to_arg, Process *); -void erts_limited_stack_trace(int to, void *to_arg, Process *); -void erts_program_counter_info(int to, void *to_arg, Process *); -void erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp); -void erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg); -void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg); +void erts_stack_dump(fmtfn_t to, void *to_arg, Process *); +void erts_limited_stack_trace(fmtfn_t to, void *to_arg, Process *); +void erts_program_counter_info(fmtfn_t to, void *to_arg, Process *); +void erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp); +void erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); +void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg); Eterm erts_get_process_priority(Process *p); Eterm erts_set_process_priority(Process *p, Eterm prio); @@ -1878,7 +1877,7 @@ void erts_handle_pending_exit(Process *, ErtsProcLocks); #define ERTS_PROC_PENDING_EXIT(P) 0 #endif -void erts_deep_process_dump(int, void *); +void erts_deep_process_dump(fmtfn_t, void *); Eterm erts_get_reader_groups_map(Process *c_p); Eterm erts_debug_reader_groups_map(Process *c_p, int groups); diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index d8c2eaba94..8311fde025 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -56,7 +56,7 @@ #define MAKE_HASH(Term) \ ((is_small(Term)) ? unsigned_val(Term) : \ ((is_atom(Term)) ? \ - (atom_tab(atom_val(Term))->slot.bucket.hvalue) : \ + atom_val(Term) : \ make_internal_hash(Term))) #define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm)) @@ -156,7 +156,7 @@ erts_pd_set_initial_size(int size) * Called from break handler */ void -erts_dictionary_dump(int to, void *to_arg, ProcDict *pd) +erts_dictionary_dump(fmtfn_t to, void *to_arg, ProcDict *pd) { unsigned int i; #ifdef DEBUG @@ -196,8 +196,8 @@ erts_dictionary_dump(int to, void *to_arg, ProcDict *pd) } void -erts_deep_dictionary_dump(int to, void *to_arg, - ProcDict* pd, void (*cb)(int, void *, Eterm)) +erts_deep_dictionary_dump(fmtfn_t to, void *to_arg, + ProcDict* pd, void (*cb)(fmtfn_t, void *, Eterm)) { unsigned int i; Eterm t; @@ -408,6 +408,11 @@ static void pd_hash_erase_all(Process *p) } } +Uint32 erts_pd_make_hx(Eterm key) +{ + return MAKE_HASH(key); +} + Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id) { unsigned int hval; diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h index 387562058c..ab58f3c239 100644 --- a/erts/emulator/beam/erl_process_dict.h +++ b/erts/emulator/beam/erl_process_dict.h @@ -37,12 +37,13 @@ typedef struct proc_dict { int erts_pd_set_initial_size(int size); Uint erts_dicts_mem_size(struct process *p); void erts_erase_dicts(struct process *p); -void erts_dictionary_dump(int to, void *to_arg, ProcDict *pd); -void erts_deep_dictionary_dump(int to, void *to_arg, - ProcDict* pd, void (*cb)(int, void *, Eterm obj)); +void erts_dictionary_dump(fmtfn_t to, void *to_arg, ProcDict *pd); +void erts_deep_dictionary_dump(fmtfn_t to, void *to_arg, + ProcDict* pd, void (*cb)(fmtfn_t, void *, Eterm obj)); Eterm erts_dictionary_copy(struct process *p, ProcDict *pd); Eterm erts_pd_hash_get(struct process *p, Eterm id); +Uint32 erts_pd_make_hx(Eterm key); Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id); #endif diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index a70dfb8e73..e534c33637 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -40,17 +40,17 @@ #define OUR_NIL _make_header(0,_TAG_HEADER_FLOAT) -static void dump_process_info(int to, void *to_arg, Process *p); -static void dump_element(int to, void *to_arg, Eterm x); -static void dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep); -static void dump_element_nl(int to, void *to_arg, Eterm x); -static int stack_element_dump(int to, void *to_arg, Eterm* sp, +static void dump_process_info(fmtfn_t to, void *to_arg, Process *p); +static void dump_element(fmtfn_t to, void *to_arg, Eterm x); +static void dump_dist_ext(fmtfn_t to, void *to_arg, ErtsDistExternal *edep); +static void dump_element_nl(fmtfn_t to, void *to_arg, Eterm x); +static int stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg); -static void stack_trace_dump(int to, void *to_arg, Eterm* sp); -static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); -static void heap_dump(int to, void *to_arg, Eterm x); -static void dump_binaries(int to, void *to_arg, Binary* root); -static void dump_externally(int to, void *to_arg, Eterm term); +static void stack_trace_dump(fmtfn_t to, void *to_arg, Eterm* sp); +static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x); +static void heap_dump(fmtfn_t to, void *to_arg, Eterm x); +static void dump_binaries(fmtfn_t to, void *to_arg, Binary* root); +static void dump_externally(fmtfn_t to, void *to_arg, Eterm term); static Binary* all_binaries; @@ -60,7 +60,7 @@ extern BeamInstr beam_continue_exit[]; void -erts_deep_process_dump(int to, void *to_arg) +erts_deep_process_dump(fmtfn_t to, void *to_arg) { int i, max = erts_ptab_max(&erts_proc); @@ -117,7 +117,7 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) { } static void -dump_process_info(int to, void *to_arg, Process *p) +dump_process_info(fmtfn_t to, void *to_arg, Process *p) { Eterm* sp; ErtsMessage* mp; @@ -176,7 +176,7 @@ dump_process_info(int to, void *to_arg, Process *p) } static void -dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep) +dump_dist_ext(fmtfn_t to, void *to_arg, ErtsDistExternal *edep) { if (!edep) erts_print(to, to_arg, "D0:E0:"); @@ -210,7 +210,7 @@ dump_dist_ext(int to, void *to_arg, ErtsDistExternal *edep) } static void -dump_element(int to, void *to_arg, Eterm x) +dump_element(fmtfn_t to, void *to_arg, Eterm x) { if (is_list(x)) { erts_print(to, to_arg, "H" PTR_FMT, list_val(x)); @@ -240,14 +240,14 @@ dump_element(int to, void *to_arg, Eterm x) } static void -dump_element_nl(int to, void *to_arg, Eterm x) +dump_element_nl(fmtfn_t to, void *to_arg, Eterm x) { dump_element(to, to_arg, x); erts_putc(to, to_arg, '\n'); } static void -stack_trace_dump(int to, void *to_arg, Eterm *sp) { +stack_trace_dump(fmtfn_t to, void *to_arg, Eterm *sp) { Eterm x = *sp; if (is_CP(x)) { erts_print(to, to_arg, "%p:", sp); @@ -258,7 +258,7 @@ stack_trace_dump(int to, void *to_arg, Eterm *sp) { } void -erts_limited_stack_trace(int to, void *to_arg, Process *p) +erts_limited_stack_trace(fmtfn_t to, void *to_arg, Process *p) { Eterm* sp; @@ -304,7 +304,7 @@ erts_limited_stack_trace(int to, void *to_arg, Process *p) } static int -stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) +stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg) { Eterm x = *sp; @@ -332,10 +332,10 @@ stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) } static void -print_function_from_pc(int to, void *to_arg, BeamInstr* x) +print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x) { - BeamInstr* addr = find_function_from_pc(x); - if (addr == NULL) { + ErtsCodeMFA* cmfa = find_function_from_pc(x); + if (cmfa == NULL) { if (x == beam_exit) { erts_print(to, to_arg, "<terminate process>"); } else if (x == beam_continue_exit) { @@ -347,12 +347,13 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x) } } else { erts_print(to, to_arg, "%T:%T/%bpu + %bpu", - addr[0], addr[1], addr[2], ((x-addr)-2) * sizeof(Eterm)); + cmfa->module, cmfa->function, cmfa->arity, + (x-(BeamInstr*)cmfa) * sizeof(Eterm)); } } static void -heap_dump(int to, void *to_arg, Eterm x) +heap_dump(fmtfn_t to, void *to_arg, Eterm x) { DeclareTmpHeapNoproc(last,1); Eterm* next = last; @@ -512,7 +513,7 @@ heap_dump(int to, void *to_arg, Eterm x) } static void -dump_binaries(int to, void *to_arg, Binary* current) +dump_binaries(fmtfn_t to, void *to_arg, Binary* current) { while (current) { long i; @@ -530,7 +531,7 @@ dump_binaries(int to, void *to_arg, Binary* current) } static void -dump_externally(int to, void *to_arg, Eterm term) +dump_externally(fmtfn_t to, void *to_arg, Eterm term) { byte sbuf[1024]; /* encode and hope for the best ... */ byte* s; @@ -573,7 +574,7 @@ dump_externally(int to, void *to_arg, Eterm term) } } -void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg) +void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) { char *s; switch (erts_process_state2status(psflg)) { @@ -591,7 +592,7 @@ void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg) } void -erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg) { +erts_dump_extended_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) { int i; diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 8c84303997..9be4741ec8 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -781,7 +781,8 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what) tmp = make_small(0); } else { hp = HAlloc(p, 4); - tmp = TUPLE3(hp,p->current[0],p->current[1],make_small(p->current[2])); + tmp = TUPLE3(hp,p->current->module,p->current->function, + make_small(p->current->arity)); hp += 4; } @@ -1027,14 +1028,14 @@ erts_trace_return_to(Process *p, BeamInstr *pc) { Eterm mfa; - BeamInstr *code_ptr = find_function_from_pc(pc); + ErtsCodeMFA *cmfa = find_function_from_pc(pc); - - if (!code_ptr) { + if (!cmfa) { mfa = am_undefined; } else { Eterm *hp = HAlloc(p, 4); - mfa = TUPLE3(hp, code_ptr[0], code_ptr[1], make_small(code_ptr[2])); + mfa = TUPLE3(hp, cmfa->module, cmfa->function, + make_small(cmfa->arity)); } send_to_tracer_nif(p, &p->common, p->common.id, NULL, TRACE_FUN_T_CALL, @@ -1046,11 +1047,11 @@ erts_trace_return_to(Process *p, BeamInstr *pc) * or {trace, Pid, return_from, {Mod, Name, Arity}, Retval} */ void -erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer) +erts_trace_return(Process* p, ErtsCodeMFA *mfa, + Eterm retval, ErtsTracer *tracer) { Eterm* hp; - Eterm mfa, mod, name; - int arity; + Eterm mfa_tuple; Uint meta_flags, *tracee_flags; ASSERT(tracer); @@ -1084,15 +1085,13 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer) tracee_flags = &meta_flags; } - mod = fi[0]; - name = fi[1]; - arity = fi[2]; - hp = HAlloc(p, 4); - mfa = TUPLE3(hp, mod, name, make_small(arity)); + mfa_tuple = TUPLE3(hp, mfa->module, mfa->function, + make_small(mfa->arity)); hp += 4; send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id, - NULL, TRACE_FUN_T_CALL, am_return_from, mfa, retval, am_true); + NULL, TRACE_FUN_T_CALL, am_return_from, mfa_tuple, + retval, am_true); } /* Send {trace_ts, Pid, exception_from, {Mod, Name, Arity}, {Class,Value}, @@ -1103,7 +1102,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer) * Where Class is atomic but Value is any term. */ void -erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, +erts_trace_exception(Process* p, ErtsCodeMFA *mfa, Eterm class, Eterm value, ErtsTracer *tracer) { Eterm* hp; @@ -1142,7 +1141,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, } hp = HAlloc(p, 7);; - mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm)mfa[2])); + mfa_tuple = TUPLE3(hp, mfa->module, mfa->function, make_small(mfa->arity)); hp += 4; cv = TUPLE2(hp, class, value); hp += 3; @@ -1165,7 +1164,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, * if it is a pid or port we do a meta trace. */ Uint32 -erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, +erts_call_trace(Process* p, ErtsCodeInfo *info, Binary *match_spec, Eterm* args, int local, ErtsTracer *tracer) { Eterm* hp; @@ -1244,7 +1243,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, * such as size_object() and copy_struct(), we must make sure that we * temporarily convert any match contexts to sub binaries. */ - arity = (Eterm) mfa[2]; + arity = info->mfa.arity; for (i = 0; i < arity; i++) { Eterm arg = args[i]; if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) { @@ -1339,7 +1338,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, hp += 2; } } - mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple); + mfa_tuple = TUPLE3(hp, info->mfa.module, info->mfa.function, mfa_tuple); hp += 4; /* @@ -1459,7 +1458,8 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg) } void -monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint time) +monitor_long_schedule_proc(Process *p, ErtsCodeMFA *in_fp, + ErtsCodeMFA *out_fp, Uint time) { ErlHeapFragment *bp; ErlOffHeap *off_heap; @@ -1490,11 +1490,13 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, monitor_p); tmo = erts_bld_uint(&hp, NULL, time); if (in_fp != NULL) { - in_mfa = TUPLE3(hp,(Eterm) in_fp[0], (Eterm) in_fp[1], make_small(in_fp[2])); + in_mfa = TUPLE3(hp, in_fp->module, in_fp->function, + make_small(in_fp->arity)); hp +=4; } if (out_fp != NULL) { - out_mfa = TUPLE3(hp,(Eterm) out_fp[0], (Eterm) out_fp[1], make_small(out_fp[2])); + out_mfa = TUPLE3(hp, out_fp->module, out_fp->function, + make_small(out_fp->arity)); hp +=4; } tmo_tpl = TUPLE2(hp,am_timeout, tmo); @@ -2129,7 +2131,7 @@ profile_runnable_proc(Process *p, Eterm status){ Eterm *hp, msg; Eterm where = am_undefined; ErlHeapFragment *bp = NULL; - BeamInstr *current = NULL; + ErtsCodeMFA *cmfa = NULL; #ifndef ERTS_SMP #define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) @@ -2151,14 +2153,14 @@ profile_runnable_proc(Process *p, Eterm status){ if (!ERTS_PROC_IS_EXITING(p)) { if (p->current) { - current = p->current; + cmfa = p->current; } else { - current = find_function_from_pc(p->i); + cmfa = find_function_from_pc(p->i); } } #ifdef ERTS_SMP - if (!current) { + if (!cmfa) { hsz -= 4; } @@ -2166,8 +2168,10 @@ profile_runnable_proc(Process *p, Eterm status){ hp = bp->mem; #endif - if (current) { - where = TUPLE3(hp, current[0], current[1], make_small(current[2])); hp += 4; + if (cmfa) { + where = TUPLE3(hp, cmfa->module, cmfa->function, + make_small(cmfa->arity)); + hp += 4; } else { where = make_small(0); } diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index 0095d4386b..378b6de49c 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -101,11 +101,11 @@ void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *); void trace_send(Process*, Eterm, Eterm); void trace_receive(Process*, Eterm, Eterm, ErtsTracingEvent*); -Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec, +Uint32 erts_call_trace(Process *p, ErtsCodeInfo *info, struct binary *match_spec, Eterm* args, int local, ErtsTracer *tracer); -void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, +void erts_trace_return(Process* p, ErtsCodeMFA *mfa, Eterm retval, ErtsTracer *tracer); -void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value, +void erts_trace_exception(Process* p, ErtsCodeMFA *mfa, Eterm class, Eterm value, ErtsTracer *tracer); void erts_trace_return_to(Process *p, BeamInstr *pc); void trace_sched(Process*, ErtsProcLocks, Eterm); @@ -134,7 +134,8 @@ void erts_system_profile_setup_active_schedulers(void); /* system_monitor */ void monitor_long_gc(Process *p, Uint time); -void monitor_long_schedule_proc(Process *p, BeamInstr *in_i, BeamInstr *out_i, Uint time); +void monitor_long_schedule_proc(Process *p, ErtsCodeMFA *in_i, + ErtsCodeMFA *out_i, Uint time); void monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time); void monitor_large_heap(Process *p); void monitor_generic(Process *p, Eterm type, Eterm spec); @@ -176,7 +177,7 @@ struct trace_pattern_flags { }; extern const struct trace_pattern_flags erts_trace_pattern_flags_off; extern int erts_call_time_breakpoint_tracing; -int erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, +int erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, struct binary* match_prog_set, struct binary *meta_match_prog_set, int on, struct trace_pattern_flags, diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 60c2349f36..93cfe08105 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -186,4 +186,11 @@ extern int erts_pd_initial_size;/* Initial Process dictionary table size */ #include "erl_term.h" +#ifdef NO_JUMP_TABLE +#define BeamOp(Op) (Op) +#else +extern void** beam_ops; +#define BeamOp(Op) beam_ops[(Op)] +#endif + #endif /* __ERL_VM_H__ */ diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index 6c33b12dd0..e431c3051b 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -21,6 +21,8 @@ #ifndef __ERROR_H__ #define __ERROR_H__ +#include "code_ix.h" + /* * There are three primary exception classes: * @@ -197,7 +199,7 @@ struct StackTrace { Eterm header; /* bignum header - must be first in struct */ Eterm freason; /* original exception reason is saved in the struct */ BeamInstr* pc; - BeamInstr* current; + ErtsCodeMFA* current; int depth; /* number of saved pointers in trace[] */ BeamInstr *trace[1]; /* varying size - must be last in struct */ }; diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 2a19211987..f397ab6b00 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -83,7 +83,7 @@ static struct export_blob* entry_to_blob(struct export_entry* ee) } void -export_info(int to, void *to_arg) +export_info(fmtfn_t to, void *to_arg) { #ifdef ERTS_SMP int lock = !ERTS_IS_CRASH_DUMPING; @@ -103,7 +103,8 @@ static HashValue export_hash(struct export_entry* ee) { Export* x = ee->ep; - return EXPORT_HASH(x->code[0], x->code[1], x->code[2]); + return EXPORT_HASH(x->info.mfa.module, x->info.mfa.function, + x->info.mfa.arity); } static int @@ -111,9 +112,9 @@ export_cmp(struct export_entry* tmpl_e, struct export_entry* obj_e) { Export* tmpl = tmpl_e->ep; Export* obj = obj_e->ep; - return !(tmpl->code[0] == obj->code[0] && - tmpl->code[1] == obj->code[1] && - tmpl->code[2] == obj->code[2]); + return !(tmpl->info.mfa.module == obj->info.mfa.module && + tmpl->info.mfa.function == obj->info.mfa.function && + tmpl->info.mfa.arity == obj->info.mfa.arity); } @@ -130,21 +131,23 @@ export_alloc(struct export_entry* tmpl_e) blob = (struct export_blob*) erts_alloc(ERTS_ALC_T_EXPORT, sizeof(*blob)); erts_smp_atomic_add_nob(&total_entries_bytes, sizeof(*blob)); obj = &blob->exp; - obj->fake_op_func_info_for_hipe[0] = 0; - obj->fake_op_func_info_for_hipe[1] = 0; - obj->code[0] = tmpl->code[0]; - obj->code[1] = tmpl->code[1]; - obj->code[2] = tmpl->code[2]; - obj->code[3] = (BeamInstr) em_call_error_handler; - obj->code[4] = 0; + obj->info.op = 0; + obj->info.native = 0; + obj->info.mfa.module = tmpl->info.mfa.module; + obj->info.mfa.function = tmpl->info.mfa.function; + obj->info.mfa.arity = tmpl->info.mfa.arity; + obj->beam[0] = (BeamInstr) em_call_error_handler; + obj->beam[1] = 0; for (ix=0; ix<ERTS_NUM_CODE_IX; ix++) { - obj->addressv[ix] = obj->code+3; + obj->addressv[ix] = obj->beam; blob->entryv[ix].slot.index = -1; blob->entryv[ix].ep = &blob->exp; } ix = 0; + + DBG_TRACE_MFA_P(&obj->info.mfa, "export allocation at %p", obj); } else { /* Existing entry in another table, use free entry in blob */ blob = entry_to_blob(tmpl_e); @@ -163,9 +166,12 @@ export_free(struct export_entry* obj) obj->slot.index = -1; for (i=0; i < ERTS_NUM_CODE_IX; i++) { if (blob->entryv[i].slot.index >= 0) { + DBG_TRACE_MFA_P(&blob->exp.info.mfa, "export entry slot %u freed for %p", + (obj - blob->entryv), &blob->exp); return; } } + DBG_TRACE_MFA_P(&blob->exp.info.mfa, "export blob deallocation at %p", &blob->exp); erts_free(ERTS_ALC_T_EXPORT, blob); erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob)); } @@ -224,7 +230,9 @@ erts_find_export_entry(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) while (b != (HashBucket*) 0) { Export* ep = ((struct export_entry*) b)->ep; - if (ep->code[0] == m && ep->code[1] == f && ep->code[2] == a) { + if (ep->info.mfa.module == m && + ep->info.mfa.function == f && + ep->info.mfa.arity == a) { return ep; } b = b->next; @@ -237,9 +245,9 @@ static struct export_entry* init_template(struct export_templ* templ, { templ->entry.ep = &templ->exp; templ->entry.slot.index = -1; - templ->exp.code[0] = m; - templ->exp.code[1] = f; - templ->exp.code[2] = a; + templ->exp.info.mfa.module = m; + templ->exp.info.mfa.function = f; + templ->exp.info.mfa.arity = a; return &templ->entry; } @@ -263,8 +271,8 @@ erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a)); if (ee == NULL || - (ee->ep->addressv[code_ix] == ee->ep->code+3 && - ee->ep->code[3] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) { + (ee->ep->addressv[code_ix] == ee->ep->beam && + ee->ep->beam[0] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) { return NULL; } return ee->ep; diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index 1e7bb8514b..7c812b306c 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -33,27 +33,25 @@ typedef struct export { void* addressv[ERTS_NUM_CODE_IX]; /* Pointer to code for function. */ - BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */ + ErtsCodeInfo info; /* MUST be just before beam[] */ + /* - * code[0]: Tagged atom for module. - * code[1]: Tagged atom for function. - * code[2]: Arity (untagged integer). - * code[3]: This entry is 0 unless the 'address' field points to it. + * beam[0]: This entry is 0 unless the 'addressv' field points to it. * Threaded code instruction to load function * (em_call_error_handler), execute BIF (em_apply_bif), * or a breakpoint instruction (op_i_generic_breakpoint). - * code[4]: Function pointer to BIF function (for BIFs only), + * beam[1]: Function pointer to BIF function (for BIFs only), * or pointer to threaded code if the module has an * on_load function that has not been run yet, or pointer - * to code for function code[3] is a breakpont instruction. + * to code if function beam[0] is a breakpoint instruction. * Otherwise: 0. */ - BeamInstr code[5]; + BeamInstr beam[2]; } Export; void init_export_table(void); -void export_info(int, void *); +void export_info(fmtfn_t, void *); ERTS_GLB_INLINE Export* erts_active_export_entry(Eterm m, Eterm f, unsigned a); Export* erts_export_put(Eterm mod, Eterm func, unsigned int arity); @@ -74,8 +72,8 @@ extern erts_smp_mtx_t export_staging_lock; #include "beam_load.h" /* For em_* extern declarations */ #define ExportIsBuiltIn(EntryPtr) \ -(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code + 3) && \ - ((EntryPtr)->code[3] == (BeamInstr) em_apply_bif)) +(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->beam) && \ + ((EntryPtr)->beam[0] == (BeamInstr) em_apply_bif)) #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index beed847578..a49b242d7c 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2826,9 +2826,10 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Export* exp = *((Export **) (export_val(obj) + 1)); if ((dflags & DFLAG_EXPORT_PTR_TAG) != 0) { *ep++ = EXPORT_EXT; - ep = enc_atom(acmp, exp->code[0], ep, dflags); - ep = enc_atom(acmp, exp->code[1], ep, dflags); - ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags, off_heap); + ep = enc_atom(acmp, exp->info.mfa.module, ep, dflags); + ep = enc_atom(acmp, exp->info.mfa.function, ep, dflags); + ep = enc_term(acmp, make_small(exp->info.mfa.arity), + ep, dflags, off_heap); } else { /* Tag, arity */ *ep++ = SMALL_TUPLE_EXT; @@ -2836,10 +2837,10 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep += 1; /* Module name */ - ep = enc_atom(acmp, exp->code[0], ep, dflags); + ep = enc_atom(acmp, exp->info.mfa.module, ep, dflags); /* Function name */ - ep = enc_atom(acmp, exp->code[1], ep, dflags); + ep = enc_atom(acmp, exp->info.mfa.function, ep, dflags); } break; } @@ -3747,9 +3748,8 @@ dec_term_atom_common: funp->arity = arity; #ifdef HIPE if (funp->fe->native_address == NULL) { - hipe_set_closure_stub(funp->fe, num_free); + hipe_set_closure_stub(funp->fe); } - funp->native_address = funp->fe->native_address; #endif hp = factory->hp; @@ -3821,9 +3821,6 @@ dec_term_atom_common: funp->fe = erts_put_fun_entry(module, old_uniq, old_index); funp->arity = funp->fe->address[-1] - num_free; -#ifdef HIPE - funp->native_address = funp->fe->native_address; -#endif hp = factory->hp; /* Environment */ @@ -4262,9 +4259,9 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, { Export* ep = *((Export **) (export_val(obj) + 1)); result += 1; - result += encode_size_struct2(acmp, ep->code[0], dflags); - result += encode_size_struct2(acmp, ep->code[1], dflags); - result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags); + result += encode_size_struct2(acmp, ep->info.mfa.module, dflags); + result += encode_size_struct2(acmp, ep->info.mfa.function, dflags); + result += encode_size_struct2(acmp, make_small(ep->info.mfa.arity), dflags); } break; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 93996e8b41..2b2f3c5cdc 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -68,7 +68,7 @@ extern void erts_pre_dirty_nif(ErtsSchedulerData *, extern void erts_post_dirty_nif(struct enif_environment_t* env); #endif extern Eterm erts_nif_taints(Process* p); -extern void erts_print_nif_taints(int to, void* to_arg); +extern void erts_print_nif_taints(fmtfn_t to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); extern void erl_nif_init(void); extern int erts_nif_get_funcs(struct erl_module_nif*, @@ -773,8 +773,8 @@ do { \ typedef struct ErtsPStack_ { byte* pstart; - byte* psp; - byte* pend; + int offs; /* "stack pointer" as byte offset from pstart */ + int size; /* allocated size in bytes */ ErtsAlcType_t alloc_type; }ErtsPStack; @@ -785,8 +785,8 @@ void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes); #define PSTACK_DECLARE(s, DEF_PSTACK_SIZE) \ PSTACK_TYPE PSTK_DEF_STACK(s)[DEF_PSTACK_SIZE]; \ ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */ \ - (byte*)(PSTK_DEF_STACK(s) - 1), /* psp */ \ - (byte*)(PSTK_DEF_STACK(s) + (DEF_PSTACK_SIZE)), /* pend */\ + -(int)sizeof(PSTACK_TYPE), /* offs */ \ + DEF_PSTACK_SIZE*sizeof(PSTACK_TYPE), /* size */ \ ERTS_ALC_T_ESTACK /* alloc_type */ \ } @@ -806,19 +806,21 @@ do { \ } \ } while(0) -#define PSTACK_IS_EMPTY(s) (s.psp < s.pstart) +#define PSTACK_IS_EMPTY(s) (s.offs < 0) -#define PSTACK_COUNT(s) (((PSTACK_TYPE*)s.psp + 1) - (PSTACK_TYPE*)s.pstart) +#define PSTACK_COUNT(s) ((s.offs + sizeof(PSTACK_TYPE)) / sizeof(PSTACK_TYPE)) -#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), (PSTACK_TYPE*)(s.psp)) +#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), \ + (PSTACK_TYPE*)(s.pstart + s.offs)) -#define PSTACK_PUSH(s) \ - (s.psp += sizeof(PSTACK_TYPE), \ - ((s.psp == s.pend) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s), \ - sizeof(PSTACK_TYPE)) : (void)0), \ - ((PSTACK_TYPE*) s.psp)) +#define PSTACK_PUSH(s) \ + (s.offs += sizeof(PSTACK_TYPE), \ + ((s.offs == s.size) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s), \ + sizeof(PSTACK_TYPE)) : (void)0), \ + ((PSTACK_TYPE*) (s.pstart + s.offs))) -#define PSTACK_POP(s) ((PSTACK_TYPE*) (s.psp -= sizeof(PSTACK_TYPE))) +#define PSTACK_POP(s) ((s.offs -= sizeof(PSTACK_TYPE)), \ + (PSTACK_TYPE*)(s.pstart + s.offs)) /* * Do not free the stack after this, it may have pointers into what @@ -831,8 +833,8 @@ do {\ (dst)->pstart = erts_alloc(s.alloc_type,\ sizeof(PSTK_DEF_STACK(s)));\ sys_memcpy((dst)->pstart, s.pstart, _pbytes);\ - (dst)->psp = (dst)->pstart + _pbytes - sizeof(PSTACK_TYPE);\ - (dst)->pend = (dst)->pstart + sizeof(PSTK_DEF_STACK(s));\ + (dst)->offs = s.offs;\ + (dst)->size = s.size;\ (dst)->alloc_type = s.alloc_type;\ } else\ *(dst) = s;\ @@ -847,8 +849,8 @@ do { \ ASSERT(s.pstart == (byte*)PSTK_DEF_STACK(s)); \ s = *(src); /* struct copy */ \ (src)->pstart = NULL; \ - ASSERT(s.psp >= (s.pstart - sizeof(PSTACK_TYPE))); \ - ASSERT(s.psp < s.pend); \ + ASSERT(s.offs >= -(int)sizeof(PSTACK_TYPE)); \ + ASSERT(s.offs < s.size); \ } while (0) #define PSTACK_DESTROY_SAVED(pstack)\ @@ -1024,7 +1026,7 @@ extern Process *erts_code_purger; /* beam_load.c */ typedef struct { - BeamInstr* current; /* Pointer to: Mod, Name, Arity */ + ErtsCodeMFA* mfa; /* Pointer to: Mod, Name, Arity */ Uint needed; /* Heap space needed for entire tuple */ Uint32 loc; /* Location in source code */ Eterm* fname_ptr; /* Pointer to fname table */ @@ -1041,13 +1043,14 @@ Eterm erts_finish_loading(Binary* loader_state, Process* c_p, Eterm erts_preload_module(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm* mod, byte* code, Uint size); void init_load(void); -BeamInstr* find_function_from_pc(BeamInstr* pc); +ErtsCodeMFA* find_function_from_pc(BeamInstr* pc); Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p); -void erts_set_current_function(FunctionInfo* fi, BeamInstr* current); +void erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa); Eterm erts_module_info_0(Process* p, Eterm module); Eterm erts_module_info_1(Process* p, Eterm module, Eterm what); Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); +int erts_commit_hipe_patch_load(Eterm hipe_magic_bin); /* beam_ranges.c */ void erts_init_ranges(void); @@ -1062,16 +1065,17 @@ void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info); void init_break_handler(void); void erts_set_ignore_break(void); void erts_replace_intr(void); -void process_info(int, void *); -void print_process_info(int, void *, Process*); -void info(int, void *); -void loaded(int, void *); +void process_info(fmtfn_t, void *); +void print_process_info(fmtfn_t, void *, Process*); +void info(fmtfn_t, void *); +void loaded(fmtfn_t, void *); /* erl_arith.c */ double erts_get_positive_zero_float(void); /* config.c */ +__decl_noreturn void __noreturn erts_exit_epilogue(void); __decl_noreturn void __noreturn erts_exit(int n, char*, ...); __decl_noreturn void __noreturn erts_flush_async_exit(int n, char*, ...); void erl_error(char*, va_list); @@ -1178,7 +1182,7 @@ extern void erts_delete_nodes_monitors(Process *, ErtsProcLocks); extern Eterm erts_monitor_nodes(Process *, Eterm, Eterm); extern Eterm erts_processes_monitoring_nodes(Process *); extern int erts_do_net_exits(DistEntry*, Eterm); -extern int distribution_info(int, void *); +extern int distribution_info(fmtfn_t, void *); extern int is_node_name_atom(Eterm a); extern int erts_net_message(Port *, DistEntry *, @@ -1367,7 +1371,7 @@ int erts_utf8_to_latin1(byte* dest, const byte* source, int slen); #define ERTS_UTF8_ANALYZE_MORE 3 #define ERTS_UTF8_OK_MAX_CHARS 4 -void bin_write(int, void*, byte*, size_t); +void bin_write(fmtfn_t, void*, byte*, size_t); Sint intlist_to_buf(Eterm, char*, Sint); /* most callers pass plain char*'s */ struct Sint_buf { @@ -1471,7 +1475,7 @@ Eterm erts_gc_bnot(Process* p, Eterm* reg, Uint live); Uint erts_current_reductions(Process* current, Process *p); -int erts_print_system_version(int to, void *arg, Process *c_p); +int erts_print_system_version(fmtfn_t to, void *arg, Process *c_p); int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg); @@ -1577,8 +1581,7 @@ int erts_beam_jump_table(void); ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf); ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf); ERTS_GLB_INLINE void dtrace_port_str(Port *port, char *port_buf); -ERTS_GLB_INLINE void dtrace_fun_decode(Process *process, - Eterm module, Eterm function, int arity, +ERTS_GLB_INLINE void dtrace_fun_decode(Process *process, ErtsCodeMFA *mfa, char *process_buf, char *mfa_buf); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -1612,8 +1615,7 @@ dtrace_port_str(Port *port, char *port_buf) } ERTS_GLB_INLINE void -dtrace_fun_decode(Process *process, - Eterm module, Eterm function, int arity, +dtrace_fun_decode(Process *process, ErtsCodeMFA *mfa, char *process_buf, char *mfa_buf) { if (process_buf) { @@ -1621,7 +1623,7 @@ dtrace_fun_decode(Process *process, } erts_snprintf(mfa_buf, DTRACE_TERM_BUF_SIZE, "%T:%T/%d", - module, function, arity); + mfa->module, mfa->function, mfa->arity); } #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c index cd038d100b..8548e30e8b 100644 --- a/erts/emulator/beam/hash.c +++ b/erts/emulator/beam/hash.c @@ -95,7 +95,7 @@ void hash_get_info(HashInfo *hi, Hash *h) ** */ -void hash_info(int to, void *arg, Hash* h) +void hash_info(fmtfn_t to, void *arg, Hash* h) { HashInfo hi; diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h index 4e769c0119..d319aaca83 100644 --- a/erts/emulator/beam/hash.h +++ b/erts/emulator/beam/hash.h @@ -37,7 +37,7 @@ typedef void (*HFREE_FUN)(void*); /* Meta functions */ typedef void* (*HMALLOC_FUN)(int,size_t); typedef void (*HMFREE_FUN)(int,void*); -typedef int (*HMPRINT_FUN)(int,void*,char*, ...); +typedef int (*HMPRINT_FUN)(fmtfn_t,void*,char*, ...); /* ** This bucket must be placed in top of @@ -89,7 +89,7 @@ Hash* hash_init(int, Hash*, char*, int, HashFunctions); void hash_delete(Hash*); void hash_get_info(HashInfo*, Hash*); -void hash_info(int, void *, Hash*); +void hash_info(fmtfn_t, void *, Hash*); int hash_table_sz(Hash *); void* hash_get(Hash*, void*); diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index 26d6c04ea0..c86a2122f6 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -27,7 +27,7 @@ #include "global.h" #include "index.h" -void index_info(int to, void *arg, IndexTable *t) +void index_info(fmtfn_t to, void *arg, IndexTable *t) { hash_info(to, arg, &t->htable); erts_print(to, arg, "=index_table:%s\n", t->htable.name); diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index 0a109d8699..b2e3c0eab5 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -51,7 +51,7 @@ typedef struct index_table #define INDEX_PAGE_MASK ((1 << INDEX_PAGE_SHIFT)-1) IndexTable *erts_index_init(ErtsAlcType_t,IndexTable*,char*,int,int,HashFunctions); -void index_info(int, void *, IndexTable*); +void index_info(fmtfn_t, void *, IndexTable*); int index_table_sz(IndexTable *); int index_get(IndexTable*, void*); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 77dbe92241..4f131c74de 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -5480,7 +5480,7 @@ erts_request_io_bytes(Process *c_p) typedef struct { - int to; + fmtfn_t to; void *arg; } prt_one_lnk_data; @@ -5497,7 +5497,7 @@ static void prt_one_lnk(ErtsLink *lnk, void *vprtd) } void -print_port_info(Port *p, int to, void *arg) +print_port_info(Port *p, fmtfn_t to, void *arg) { erts_aint32_t state = erts_atomic32_read_nob(&p->state); diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 4f36377450..8ab6c713d6 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -26,6 +26,7 @@ #include "erl_vm.h" #include "global.h" #include "module.h" +#include "beam_catches.h" #ifdef DEBUG # define IF_DEBUG(x) x @@ -50,7 +51,7 @@ static erts_smp_atomic_t tot_module_bytes; #include "erl_smp.h" -void module_info(int to, void *to_arg) +void module_info(fmtfn_t to, void *to_arg) { index_info(to, to_arg, &module_tables[erts_active_code_ix()]); } @@ -67,6 +68,18 @@ static int module_cmp(Module* tmpl, Module* obj) return tmpl->module != obj->module; } +void erts_module_instance_init(struct erl_module_instance* modi) +{ + modi->code_hdr = 0; + modi->code_length = 0; + modi->catches = BEAM_CATCHES_NIL; + modi->nif = NULL; + modi->num_breakpoints = 0; + modi->num_traced_exports = 0; +#ifdef HIPE + modi->hipe_code = NULL; +#endif +} static Module* module_alloc(Module* tmpl) { @@ -74,18 +87,11 @@ static Module* module_alloc(Module* tmpl) erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module)); obj->module = tmpl->module; - obj->curr.code_hdr = 0; - obj->old.code_hdr = 0; - obj->curr.code_length = 0; - obj->old.code_length = 0; obj->slot.index = -1; - obj->curr.nif = NULL; - obj->old.nif = NULL; - obj->curr.num_breakpoints = 0; - obj->old.num_breakpoints = 0; - obj->curr.num_traced_exports = 0; - obj->old.num_traced_exports = 0; + erts_module_instance_init(&obj->curr); + erts_module_instance_init(&obj->old); obj->on_load = 0; + DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc"); return obj; } @@ -119,6 +125,7 @@ void init_module_table(void) erts_smp_atomic_init_nob(&tot_module_bytes, 0); } + Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix) { @@ -139,19 +146,14 @@ erts_get_module(Eterm mod, ErtsCodeIndex code_ix) } } -Module* -erts_put_module(Eterm mod) + +static Module* put_module(Eterm mod, IndexTable* mod_tab) { Module e; - IndexTable* mod_tab; int oldsz, newsz; Module* res; ASSERT(is_atom(mod)); - ERTS_SMP_LC_ASSERT(erts_initialized == 0 - || erts_has_code_write_permission()); - - mod_tab = &module_tables[erts_staging_code_ix()]; e.module = atom_val(mod); oldsz = index_table_sz(mod_tab); res = (Module*) index_put_entry(mod_tab, (void*) &e); @@ -160,6 +162,15 @@ erts_put_module(Eterm mod) return res; } +Module* +erts_put_module(Eterm mod) +{ + ERTS_SMP_LC_ASSERT(erts_initialized == 0 + || erts_has_code_write_permission()); + + return put_module(mod, &module_tables[erts_staging_code_ix()]); +} + Module *module_code(int i, ErtsCodeIndex code_ix) { return (Module*) erts_index_lookup(&module_tables[code_ix], i); @@ -181,6 +192,13 @@ static ErtsCodeIndex dbg_load_code_ix = 0; static int entries_at_start_staging = 0; +static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod) +{ + dst_mod->curr = src_mod->curr; + dst_mod->old = src_mod->old; + dst_mod->on_load = src_mod->on_load; +} + void module_start_staging(void) { IndexTable* src = &module_tables[erts_active_code_ix()]; @@ -199,10 +217,7 @@ void module_start_staging(void) src_mod = (Module*) erts_index_lookup(src, i); dst_mod = (Module*) erts_index_lookup(dst, i); ASSERT(src_mod->module == dst_mod->module); - - dst_mod->curr = src_mod->curr; - dst_mod->old = src_mod->old; - dst_mod->on_load = src_mod->on_load; + copy_module(dst_mod, src_mod); } /* @@ -214,9 +229,7 @@ void module_start_staging(void) dst_mod = (Module*) index_put_entry(dst, src_mod); ASSERT(dst_mod != src_mod); - dst_mod->curr = src_mod->curr; - dst_mod->old = src_mod->old; - dst_mod->on_load = src_mod->on_load; + copy_module(dst_mod, src_mod); } newsz = index_table_sz(dst); erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 1c1afc8461..9d258d5dbf 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -23,6 +23,10 @@ #include "index.h" +#ifdef HIPE +#include "hipe_module.h" +#endif + struct erl_module_instance { BeamCodeHeader* code_hdr; int code_length; /* Length of loaded code in bytes. */ @@ -30,6 +34,9 @@ struct erl_module_instance { struct erl_module_nif* nif; int num_breakpoints; int num_traced_exports; +#ifdef HIPE + HipeModule *hipe_code; +#endif }; typedef struct erl_module { @@ -42,13 +49,14 @@ typedef struct erl_module { struct erl_module_instance* on_load; } Module; +void erts_module_instance_init(struct erl_module_instance* modi); Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix); Module* erts_put_module(Eterm mod); void init_module_table(void); void module_start_staging(void); void module_end_staging(int commit); -void module_info(int, void *); +void module_info(fmtfn_t, void *); Module *module_code(int, ErtsCodeIndex); int module_code_size(ErtsCodeIndex); diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index ac7096745e..acda51c9fc 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -102,7 +102,7 @@ is_proc_alive(Process *p) return !ERTS_PROC_IS_EXITING(p); } -void register_info(int to, void *to_arg) +void register_info(fmtfn_t to, void *to_arg) { int lock = !ERTS_IS_CRASH_DUMPING; if (lock) diff --git a/erts/emulator/beam/register.h b/erts/emulator/beam/register.h index d839f55d6b..27a314ca78 100644 --- a/erts/emulator/beam/register.h +++ b/erts/emulator/beam/register.h @@ -44,7 +44,7 @@ typedef struct reg_proc int process_reg_size(void); int process_reg_sz(void); void init_register_table(void); -void register_info(int, void *); +void register_info(fmtfn_t, void *); int erts_register_name(Process *, Eterm, Eterm); Eterm erts_whereis_name_to_id(Process *, Eterm); void erts_whereis_name(Process *, ErtsProcLocks, diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 16e6c33367..4b3ac594a0 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -384,6 +384,7 @@ typedef long Sint64; # ifdef LONG_MIN # define ERTS_SINT64_MIN LONG_MIN # endif +# define ErtsStrToSint64 strtol # elif SIZEOF_LONG_LONG == 8 # define HAVE_INT64 1 typedef unsigned long long Uint64; @@ -397,6 +398,7 @@ typedef long long Sint64; # ifdef LLONG_MIN # define ERTS_SINT64_MIN LLONG_MIN # endif +# define ErtsStrToSint64 strtoll # else # error "No 64-bit integer type found" # endif @@ -616,22 +618,21 @@ Uint erts_sys_misc_mem_sz(void); #include "erl_printf.h" /* Io constants to erts_print and erts_putc */ -#define ERTS_PRINT_STDERR (2) -#define ERTS_PRINT_STDOUT (1) -#define ERTS_PRINT_FILE (-1) -#define ERTS_PRINT_SBUF (-2) -#define ERTS_PRINT_SNBUF (-3) -#define ERTS_PRINT_DSBUF (-4) - -#define ERTS_PRINT_MIN ERTS_PRINT_DSBUF +#define ERTS_PRINT_STDERR ((fmtfn_t)0) +#define ERTS_PRINT_STDOUT ((fmtfn_t)1) +#define ERTS_PRINT_FILE ((fmtfn_t)2) +#define ERTS_PRINT_SBUF ((fmtfn_t)3) +#define ERTS_PRINT_SNBUF ((fmtfn_t)4) +#define ERTS_PRINT_DSBUF ((fmtfn_t)5) +#define ERTS_PRINT_FD ((fmtfn_t)6) typedef struct { char *buf; size_t size; } erts_print_sn_buf; -int erts_print(int to, void *arg, char *format, ...); /* in utils.c */ -int erts_putc(int to, void *arg, char); /* in utils.c */ +int erts_print(fmtfn_t to, void *arg, char *format, ...); /* in utils.c */ +int erts_putc(fmtfn_t to, void *arg, char); /* in utils.c */ /* logger stuff is declared here instead of in global.h, so sys files won't have to include global.h */ diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 6786657faf..ec502d5a78 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -204,9 +204,8 @@ erl_grow_wstack(ErtsWStack* s, Uint need) void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes) { - Uint old_size = s->pend - s->pstart; + Uint old_size = s->size; Uint new_size; - Uint sp_offs = s->psp - s->pstart; if (need_bytes < old_size) new_size = 2 * old_size; @@ -220,8 +219,7 @@ erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes) sys_memcpy(new_ptr, s->pstart, old_size); s->pstart = new_ptr; } - s->pend = s->pstart + new_size; - s->psp = s->pstart + sp_offs; + s->size = new_size; } /* @@ -348,40 +346,41 @@ int erts_fit_in_bits_uint(Uint value) } int -erts_print(int to, void *arg, char *format, ...) +erts_print(fmtfn_t to, void *arg, char *format, ...) { int res; va_list arg_list; va_start(arg_list, format); - if (to < ERTS_PRINT_MIN) - res = -EINVAL; - else { - switch (to) { - case ERTS_PRINT_STDOUT: + { + switch ((UWord)to) { + case (UWord)ERTS_PRINT_STDOUT: res = erts_vprintf(format, arg_list); break; - case ERTS_PRINT_STDERR: + case (UWord)ERTS_PRINT_STDERR: res = erts_vfprintf(stderr, format, arg_list); break; - case ERTS_PRINT_FILE: + case (UWord)ERTS_PRINT_FILE: res = erts_vfprintf((FILE *) arg, format, arg_list); break; - case ERTS_PRINT_SBUF: + case (UWord)ERTS_PRINT_SBUF: res = erts_vsprintf((char *) arg, format, arg_list); break; - case ERTS_PRINT_SNBUF: + case (UWord)ERTS_PRINT_SNBUF: res = erts_vsnprintf(((erts_print_sn_buf *) arg)->buf, ((erts_print_sn_buf *) arg)->size, format, arg_list); break; - case ERTS_PRINT_DSBUF: + case (UWord)ERTS_PRINT_DSBUF: res = erts_vdsprintf((erts_dsprintf_buf_t *) arg, format, arg_list); break; - default: - res = erts_vfdprintf((int) to, format, arg_list); + case (UWord)ERTS_PRINT_FD: + res = erts_vfdprintf((int)(SWord) arg, format, arg_list); break; + default: + res = erts_vcbprintf(to, arg, format, arg_list); + break; } } @@ -390,7 +389,7 @@ erts_print(int to, void *arg, char *format, ...) } int -erts_putc(int to, void *arg, char c) +erts_putc(fmtfn_t to, void *arg, char c) { return erts_print(to, arg, "%c", c); } @@ -893,11 +892,11 @@ tail_recur: { Export* ep = *((Export **) (export_val(term) + 1)); - hash = hash * FUNNY_NUMBER11 + ep->code[2]; + hash = hash * FUNNY_NUMBER11 + ep->info.mfa.arity; hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue); hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue); break; } @@ -1330,11 +1329,11 @@ make_hash2(Eterm term) { Export* ep = *((Export **) (export_val(term) + 1)); UINT32_HASH_2 - (ep->code[2], - atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue, + (ep->info.mfa.arity, + atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue, HCONST); UINT32_HASH - (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue, + (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue, HCONST_14); goto hash2_common; } @@ -2025,11 +2024,11 @@ tail_recur: { Export* ep = *((Export **) (export_val(term) + 1)); - hash = hash * FUNNY_NUMBER11 + ep->code[2]; + hash = hash * FUNNY_NUMBER11 + ep->info.mfa.arity; hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue); hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue); break; } @@ -3306,13 +3305,15 @@ tailrecur_ne: Export* a_exp = *((Export **) (export_val(a) + 1)); Export* b_exp = *((Export **) (export_val(b) + 1)); - if ((j = erts_cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) { + if ((j = erts_cmp_atoms(a_exp->info.mfa.module, + b_exp->info.mfa.module)) != 0) { RETURN_NEQ(j); } - if ((j = erts_cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) { + if ((j = erts_cmp_atoms(a_exp->info.mfa.function, + b_exp->info.mfa.function)) != 0) { RETURN_NEQ(j); } - ON_CMP_GOTO((Sint) a_exp->code[2] - (Sint) b_exp->code[2]); + ON_CMP_GOTO((Sint) a_exp->info.mfa.arity - (Sint) b_exp->info.mfa.arity); } break; case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE): @@ -3876,7 +3877,7 @@ store_external_or_ref_in_proc_(Process *proc, Eterm ns) return store_external_or_ref_(&hp, &MSO(proc), ns); } -void bin_write(int to, void *to_arg, byte* buf, size_t sz) +void bin_write(fmtfn_t to, void *to_arg, byte* buf, size_t sz) { size_t i; |