diff options
Diffstat (limited to 'erts/emulator')
148 files changed, 5885 insertions, 3947 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 8772befe27..ce50022683 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -52,7 +52,6 @@ OMIT_OMIT_FP=no TYPE_LIBS= DIRTY_SCHEDULER_SUPPORT=@DIRTY_SCHEDULER_SUPPORT@ -NEW_PURGE_STRATEGY=@NEW_PURGE_STRATEGY@ ifeq ($(TYPE),debug) PURIFY = @@ -566,9 +565,9 @@ $(TARGET)/erl_bif_table.c \ $(TARGET)/erl_bif_table.h \ $(TARGET)/erl_bif_wrap.c \ $(TARGET)/erl_bif_list.h \ +$(TARGET)/erl_gc_bifs.c \ $(TARGET)/erl_atom_table.c \ $(TARGET)/erl_atom_table.h \ -$(TARGET)/erl_pbifs.c \ : $(TARGET)/TABLES-GENERATED $(TARGET)/TABLES-GENERATED: $(ATOMS) $(BIFS) utils/make_tables $(gen_verbose)LANG=C $(PERL) utils/make_tables -src $(TARGET) -include $(TARGET)\ @@ -742,7 +741,6 @@ EMU_OBJS = \ $(OBJDIR)/beam_ranges.o RUN_OBJS = \ - $(OBJDIR)/erl_pbifs.o \ $(OBJDIR)/erl_alloc.o $(OBJDIR)/erl_mtrace.o \ $(OBJDIR)/erl_alloc_util.o $(OBJDIR)/erl_goodfit_alloc.o \ $(OBJDIR)/erl_bestfit_alloc.o $(OBJDIR)/erl_afit_alloc.o \ @@ -753,6 +751,7 @@ RUN_OBJS = \ $(OBJDIR)/erl_bif_os.o $(OBJDIR)/erl_bif_lists.o \ $(OBJDIR)/erl_bif_trace.o $(OBJDIR)/erl_bif_unique.o \ $(OBJDIR)/erl_bif_wrap.o \ + $(OBJDIR)/erl_gc_bifs.o \ $(OBJDIR)/erl_trace.o $(OBJDIR)/copy.o \ $(OBJDIR)/utils.o $(OBJDIR)/bif.o \ $(OBJDIR)/io.o $(OBJDIR)/erl_printf_term.o\ @@ -888,7 +887,9 @@ HIPE_OBJS= \ $(OBJDIR)/hipe_bif2.o \ $(OBJDIR)/hipe_debug.o \ $(OBJDIR)/hipe_gc.o \ + $(OBJDIR)/hipe_load.o \ $(OBJDIR)/hipe_mode_switch.o \ + $(OBJDIR)/hipe_module.o \ $(OBJDIR)/hipe_native_bif.o \ $(OBJDIR)/hipe_stack.o $(HIPE_ARCH_OBJS) ifdef HIPE_ENABLED diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index 2052afe52b..2b5ad097a0 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -199,7 +199,7 @@ atom_alloc(Atom* tmpl) static void atom_free(Atom* obj) { - erts_free(ERTS_ALC_T_ATOM, (void*) obj); + ASSERT(obj->slot.index == atom_val(am_ErtsSecretAtom)); } static void latin1_to_utf8(byte* conv_buf, const byte** srcp, int* lenp) @@ -467,6 +467,9 @@ init_atom_table(void) atom_space -= a.len; atom_tab(ix)->name = (byte*)erl_atom_names[i]; } + + /* Hide am_ErtsSecretAtom */ + hash_erase(&erts_atom_table.htable, atom_tab(atom_val(am_ErtsSecretAtom))); } void @@ -483,3 +486,9 @@ dump_atoms(fmtfn_t to, void *to_arg) } } } + +Uint +erts_get_atom_limit(void) +{ + return erts_atom_table.limit; +}
\ No newline at end of file diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index a82efabb9f..abd3b44993 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -133,11 +133,10 @@ int atom_table_sz(void); /* table size in bytes, excluding stored objects */ Eterm am_atom_put(const char*, int); /* ONLY 7-bit ascii! */ Eterm erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc); -int atom_erase(byte*, int); -int atom_static_put(byte*, int); void init_atom_table(void); void atom_info(fmtfn_t, void *); void dump_atoms(fmtfn_t, void *); +Uint erts_get_atom_limit(void); int erts_atom_get(const char* name, int len, Eterm* ap, ErtsAtomEncoding enc); void erts_atom_get_text_space_sizes(Uint *reserved, Uint *used); #endif diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 2b66bf6f4e..b1aeed7889 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -59,6 +59,9 @@ atom nocatch atom undefined_function atom undefined_lambda +# Secret internal atom that can never be found by string lookup +# and should never leak out to be seen by the user. +atom ErtsSecretAtom='3RT$' # All other atoms. Try to keep the order alphabetic. # @@ -366,6 +369,7 @@ atom long_schedule atom low atom Lt='<' atom machine +atom major atom match atom match_limit atom match_limit_recursion @@ -395,6 +399,7 @@ atom milli_seconds atom millisecond atom min_heap_size atom min_bin_vheap_size +atom minor atom minor_version atom Minus='-' atom module diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 7f5e9ff691..2d1aa13004 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -36,6 +36,12 @@ #include "erl_nif.h" #include "erl_bits.h" #include "erl_thr_progress.h" +#ifdef HIPE +# include "hipe_bif0.h" +# define IF_HIPE(X) (X) +#else +# define IF_HIPE(X) (0) +#endif #ifdef HIPE # include "hipe_stack.h" @@ -65,7 +71,6 @@ erts_smp_atomic_t erts_copy_literal_area__; #define ERTS_SET_COPY_LITERAL_AREA(LA) \ erts_smp_atomic_set_nob(&erts_copy_literal_area__, \ (erts_aint_t) (LA)) -#ifdef ERTS_NEW_PURGE_STRATEGY Process *erts_literal_area_collector = NULL; typedef struct ErtsLiteralAreaRef_ ErtsLiteralAreaRef; @@ -79,10 +84,9 @@ struct { ErtsLiteralAreaRef *first; ErtsLiteralAreaRef *last; } release_literal_areas; -#endif static void set_default_trace_pattern(Eterm module); -static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls); +static Eterm check_process_code(Process* rp, Module* modp, int *redsp, int fcalls); static void delete_code(Module* modp); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); @@ -113,11 +117,9 @@ init_purge_state(void) void erts_beam_bif_load_init(void) { -#ifdef ERTS_NEW_PURGE_STRATEGY erts_smp_mtx_init(&release_literal_areas.mtx, "release_literal_areas"); release_literal_areas.first = NULL; release_literal_areas.last = NULL; -#endif erts_smp_atomic_init_nob(&erts_copy_literal_area__, (erts_aint_t) NULL); @@ -147,8 +149,17 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) { +#if !defined(HIPE) + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else Module* modp; - Eterm res; + Eterm res, mod; + + if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1) || + is_not_atom(mod = erts_module_for_prepared_code + (((ProcBin*)binary_val(BIF_ARG_1))->val))) { + BIF_ERROR(BIF_P, BADARG); + } if (!erts_try_seize_code_write_permission(BIF_P)) { ERTS_BIF_YIELD3(bif_export[BIF_code_make_stub_module_3], @@ -158,7 +169,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); - modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); + modp = erts_get_module(mod, erts_active_code_ix()); if (modp && modp->curr.num_breakpoints > 0) { ASSERT(modp->curr.code_hdr != NULL); @@ -170,9 +181,12 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); - if (res == BIF_ARG_1) { + if (res == mod) { erts_end_staging_code_ix(); erts_commit_staging_code_ix(); + if (!modp) + modp = erts_get_module(mod, erts_active_code_ix()); + hipe_redirect_to_module(modp); } else { erts_abort_staging_code_ix(); @@ -181,6 +195,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); return res; +#endif } BIF_RETTYPE @@ -245,7 +260,7 @@ struct m { Uint exception; }; -static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int); +static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int, int); #ifdef ERTS_SMP static void smp_code_ix_commiter(void*); @@ -381,8 +396,9 @@ finish_loading_1(BIF_ALIST_1) for (i = 0; i < n; i++) { if (p[i].modp->curr.num_breakpoints > 0 || p[i].modp->curr.num_traced_exports > 0 || - erts_is_default_trace_enabled()) { - /* tracing involved, fallback with thread blocking */ + erts_is_default_trace_enabled() || + IF_HIPE(hipe_need_blocking(p[i].modp))) { + /* tracing or hipe need thread blocking */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; @@ -440,32 +456,36 @@ finish_loading_1(BIF_ALIST_1) } done: - return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n); + return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n, 1); } static Eterm staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, - struct m* loaded, int nloaded) + struct m* mods, int nmods, int free_mods) { #ifdef ERTS_SMP if (is_blocking || !commit) #endif { if (commit) { + int i; erts_end_staging_code_ix(); erts_commit_staging_code_ix(); - if (loaded) { - int i; - for (i=0; i < nloaded; i++) { - set_default_trace_pattern(loaded[i].module); + + for (i=0; i < nmods; i++) { + if (mods[i].modp->curr.code_hdr) { + set_default_trace_pattern(mods[i].module); } + #ifdef HIPE + hipe_redirect_to_module(mods[i].modp); + #endif } } else { erts_abort_staging_code_ix(); } - if (loaded) { - erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + if (free_mods) { + erts_free(ERTS_ALC_T_LOADER_TMP, mods); } if (is_blocking) { erts_smp_thr_progress_unblock(); @@ -478,8 +498,8 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, else { ASSERT(is_value(res)); - if (loaded) { - erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + if (free_mods) { + erts_free(ERTS_ALC_T_LOADER_TMP, mods); } erts_end_staging_code_ix(); /* @@ -547,7 +567,7 @@ check_old_code_1(BIF_ALIST_1) } Eterm -erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls) +erts_check_process_code(Process *c_p, Eterm module, int *redsp, int fcalls) { Module* modp; Eterm res; @@ -562,31 +582,23 @@ erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int if (!modp) return am_false; erts_rlock_old_code(code_ix); - res = (!modp->old.code_hdr ? am_false : - check_process_code(c_p, modp, flags, redsp, fcalls)); + res = (!modp->old.code_hdr + ? am_false + : check_process_code(c_p, modp, redsp, fcalls)); erts_runlock_old_code(code_ix); return res; } -BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2) +BIF_RETTYPE erts_internal_check_process_code_1(BIF_ALIST_1) { int reds = 0; - Uint flags; Eterm res; if (is_not_atom(BIF_ARG_1)) goto badarg; - if (is_not_small(BIF_ARG_2)) - goto badarg; - - flags = unsigned_val(BIF_ARG_2); - if (flags & ~ERTS_CPC_ALL) { - goto badarg; - } - - res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds, BIF_P->fcalls); + res = erts_check_process_code(BIF_P, BIF_ARG_1, &reds, BIF_P->fcalls); ASSERT(is_value(res)); @@ -622,7 +634,7 @@ BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2) if (!rp) BIF_RET(am_false); - res = erts_check_process_code(rp, BIF_ARG_2, 0, &reds, BIF_P->fcalls); + res = erts_check_process_code(rp, BIF_ARG_2, &reds, BIF_P->fcalls); if (BIF_P != rp) erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); @@ -665,8 +677,9 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } else { if (modp->curr.num_breakpoints > 0 || - modp->curr.num_traced_exports > 0) { - /* we have tracing, retry single threaded */ + modp->curr.num_traced_exports > 0 || + IF_HIPE(hipe_need_blocking(modp))) { + /* tracing or hipe need to go single threaded */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; @@ -680,7 +693,14 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) success = 1; } } - return staging_epilogue(BIF_P, success, res, is_blocking, NULL, 0); + { + struct m mod; + Eterm retval; + mod.module = BIF_ARG_1; + mod.modp = modp; + retval = staging_epilogue(BIF_P, success, res, is_blocking, &mod, 1, 0); + return retval; + } } BIF_RETTYPE module_loaded_1(BIF_ALIST_1) @@ -805,23 +825,26 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) num_exps = export_list_size(code_ix); for (i = 0; i < num_exps; i++) { Export *ep = export_list(i,code_ix); - if (ep == NULL || ep->code[0] != BIF_ARG_1) { + if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; } - if (ep->code[4] != 0) { - ep->addressv[code_ix] = (void *) ep->code[4]; - ep->code[4] = 0; + if (ep->beam[1] != 0) { + ep->addressv[code_ix] = (void *) ep->beam[1]; + ep->beam[1] = 0; } else { - if (ep->addressv[code_ix] == ep->code+3 && - ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep->addressv[code_ix] == ep->beam && + ep->beam[0] == (BeamInstr) em_apply_bif) { continue; } - ep->addressv[code_ix] = ep->code+3; - ep->code[3] = (BeamInstr) em_call_error_handler; + ep->addressv[code_ix] = ep->beam; + ep->beam[0] = (BeamInstr) em_call_error_handler; } } modp->curr.code_hdr->on_load_function_ptr = NULL; set_default_trace_pattern(BIF_ARG_1); + #ifdef HIPE + hipe_redirect_to_module(modp); + #endif } else if (BIF_ARG_2 == am_false) { int i, num_exps; @@ -833,13 +856,13 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) num_exps = export_list_size(code_ix); for (i = 0; i < num_exps; i++) { Export *ep = export_list(i,code_ix); - if (ep == NULL || ep->code[0] != BIF_ARG_1) { + if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; } - if (ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep->beam[0] == (BeamInstr) em_apply_bif) { continue; } - ep->code[4] = 0; + ep->beam[1] = 0; } } erts_smp_thr_progress_unblock(); @@ -863,9 +886,9 @@ set_default_trace_pattern(Eterm module) &trace_pattern_flags, &meta_tracer); if (trace_pattern_is_on) { - Eterm mfa[1]; - mfa[0] = module; - (void) erts_set_trace_pattern(0, mfa, 1, + ErtsCodeMFA mfa; + mfa.module = module; + (void) erts_set_trace_pattern(0, &mfa, 1, match_spec, meta_match_spec, 1, trace_pattern_flags, @@ -873,32 +896,12 @@ set_default_trace_pattern(Eterm module) } } -#ifndef ERTS_NEW_PURGE_STRATEGY - -static ERTS_INLINE int -check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size) -{ - struct erl_off_heap_header* oh; - for (oh = off_heap->first; oh; oh = oh->next) { - if (thing_subtag(oh->thing_word) == FUN_SUBTAG) { - ErlFunThing* funp = (ErlFunThing*) oh; - if (ErtsInArea(funp->fe->address, area, area_size)) - return !0; - } - } - return 0; -} - -#endif - static Uint hfrag_literal_size(Eterm* start, Eterm* end, char* lit_start, Uint lit_size); static void hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, Eterm *start, Eterm *end, char *lit_start, Uint lit_size); -#ifdef ERTS_NEW_PURGE_STRATEGY - Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed) { @@ -1071,12 +1074,16 @@ literal_gc: } static Eterm -check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) +check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) { BeamInstr* start; char* mod_start; Uint mod_size; Eterm* sp; +#ifdef HIPE + void *nat_start = NULL; + Uint nat_size = 0; +#endif *redsp += 1; @@ -1111,84 +1118,19 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } } - /* - * Check all continuation pointers stored in stackdump - * and clear exception stackdump if there is a pointer - * to the module. - */ - if (rp->ftrace != NIL) { - struct StackTrace *s; - ASSERT(is_list(rp->ftrace)); - s = (struct StackTrace *) big_val(CDR(list_val(rp->ftrace))); - if ((s->pc && ErtsInArea(s->pc, mod_start, mod_size)) || - (s->current && ErtsInArea(s->current, mod_start, mod_size))) { - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - } else { - int i; - for (i = 0; i < s->depth; i++) { - if (ErtsInArea(s->trace[i], mod_start, mod_size)) { - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - break; - } - } - } - } - - return am_false; -} - -#else /* !ERTS_NEW_PURGE_STRATEGY, i.e, old style purge... */ - -static Eterm -check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) -{ - BeamInstr* start; - char* literals; - Uint lit_bsize; - char* mod_start; - Uint mod_size; - Eterm* sp; - int done_gc = 0; - int need_gc = 0; - ErtsMessage *msgp; - ErlHeapFragment *hfrag; - -#define ERTS_ORDINARY_GC__ (1 << 0) -#define ERTS_LITERAL_GC__ (1 << 1) - - /* - * Pick up limits for the module. - */ - start = (BeamInstr*) modp->old.code_hdr; - mod_start = (char *) start; - mod_size = modp->old.code_length; - - /* - * Check if current instruction or continuation pointer points into module. - */ - if (ErtsInArea(rp->i, mod_start, mod_size) - || ErtsInArea(rp->cp, mod_start, mod_size)) { - return am_true; - } - - *redsp += 1; - - if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) - return am_true; - - +#ifdef HIPE /* - * Check all continuation pointers stored on the stack. + * Check all continuation pointers stored on the native stack if the module + * has native code. */ - for (sp = rp->stop; sp < STACK_START(rp); sp++) { - if (is_CP(*sp) && ErtsInArea(cp_val(*sp), mod_start, mod_size)) { + if (modp->old.hipe_code) { + nat_start = modp->old.hipe_code->text_segment; + nat_size = modp->old.hipe_code->text_segment_size; + if (nat_size && nstack_any_cps_in_segment(rp, nat_start, nat_size)) { return am_true; } } +#endif /* * Check all continuation pointers stored in stackdump @@ -1206,8 +1148,16 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls rp->ftrace = NIL; } else { int i; + char *area_start = mod_start; + Uint area_size = mod_size; +#ifdef HIPE + if (rp->freason & EXF_NATIVE) { + area_start = nat_start; + area_size = nat_size; + } +#endif for (i = 0; i < s->depth; i++) { - if (ErtsInArea(s->trace[i], mod_start, mod_size)) { + if (ErtsInArea(s->trace[i], area_start, area_size)) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; @@ -1217,188 +1167,9 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } } - if (rp->flags & F_DISABLE_GC) { - /* - * Cannot proceed. Process has disabled gc in order to - * safely leave inconsistent data on the heap and/or - * off heap lists. Need to wait for gc to be enabled - * again. - */ - return THE_NON_VALUE; - } - - /* - * Message queue can contains funs, and may contain - * literals. If we got references to this module from the message - * queue. - * - * If a literal is in the message queue we maka an explicit copy of - * and attach it to the heap fragment. Each message needs to be - * self contained, we cannot save the literal in the old_heap or - * any other heap than the message it self. - */ - - erts_smp_proc_lock(rp, ERTS_PROC_LOCK_MSGQ); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); - - if (modp->old.code_hdr->literal_area) { - literals = (char*) modp->old.code_hdr->literal_area->start; - lit_bsize = (char*) modp->old.code_hdr->literal_area->end - literals; - } - else { - literals = NULL; - lit_bsize = 0; - } - - for (msgp = rp->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) - hfrag = &msgp->hfrag; - else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag) - hfrag = msgp->data.heap_frag; - else - continue; - { - ErlHeapFragment *hf; - Uint lit_sz; - for (hf=hfrag; hf; hf = hf->next) { - if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)) - return am_true; - lit_sz = hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size], - literals, lit_bsize); - } - if (lit_sz > 0) { - ErlHeapFragment *bp = new_message_buffer(lit_sz); - Eterm *hp = bp->mem; - - for (hf=hfrag; hf; hf = hf->next) { - hfrag_literal_copy(&hp, &bp->off_heap, - &hf->mem[0], &hf->mem[hf->used_size], - literals, lit_bsize); - hfrag=hf; - } - /* link new hfrag last */ - ASSERT(hfrag->next == NULL); - hfrag->next = bp; - bp->next = NULL; - } - } - } - - while (1) { - - /* Check heap, stack etc... */ - if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size)) - goto try_gc; - if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, literals, lit_bsize)) { - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - } - if (any_heap_ref_ptrs(rp->stop, rp->hend, literals, lit_bsize)) - goto try_literal_gc; -#ifdef HIPE - if (nstack_any_heap_ref_ptrs(rp, literals, lit_bsize)) - goto try_literal_gc; -#endif - if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize)) - goto try_literal_gc; - if (rp->abandoned_heap) { - if (any_heap_refs(rp->abandoned_heap, rp->abandoned_heap + rp->heap_sz, - literals, lit_bsize)) - goto try_literal_gc; - } - if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize)) - goto try_literal_gc; - - /* Check dictionary */ - if (rp->dictionary) { - Eterm* start = ERTS_PD_START(rp->dictionary); - Eterm* end = start + ERTS_PD_SIZE(rp->dictionary); - - if (any_heap_ref_ptrs(start, end, literals, lit_bsize)) - goto try_literal_gc; - } - - /* Check heap fragments */ - for (hfrag = rp->mbuf; hfrag; hfrag = hfrag->next) { - Eterm *hp, *hp_end; - /* Off heap lists should already have been moved into process */ - ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); - - hp = &hfrag->mem[0]; - hp_end = &hfrag->mem[hfrag->used_size]; - if (any_heap_refs(hp, hp_end, literals, lit_bsize)) - goto try_literal_gc; - } - - /* - * Message buffer fragments (matched messages) - * - off heap lists should already have been moved into - * process off heap structure. - * - Check for literals - */ - for (msgp = rp->msg_frag; msgp; msgp = msgp->next) { - hfrag = erts_message_to_heap_frag(msgp); - for (; hfrag; hfrag = hfrag->next) { - Eterm *hp, *hp_end; - ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); - - hp = &hfrag->mem[0]; - hp_end = &hfrag->mem[hfrag->used_size]; - - if (any_heap_refs(hp, hp_end, literals, lit_bsize)) - goto try_literal_gc; - } - } - - return am_false; - - try_literal_gc: - need_gc |= ERTS_LITERAL_GC__; - - try_gc: - need_gc |= ERTS_ORDINARY_GC__; - - if ((done_gc & need_gc) == need_gc) - return am_true; - - if (!(flags & ERTS_CPC_ALLOW_GC)) - return am_aborted; - - need_gc &= ~done_gc; - - /* - * Try to get rid of literals by by garbage collecting. - * Clear both fvalue and ftrace. - */ - - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - - if (need_gc & ERTS_ORDINARY_GC__) { - FLAGS(rp) |= F_NEED_FULLSWEEP; - *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity, fcalls); - done_gc |= ERTS_ORDINARY_GC__; - } - if (need_gc & ERTS_LITERAL_GC__) { - struct erl_off_heap_header* oh; - oh = modp->old.code_hdr->literal_area->off_heap; - *redsp += lit_bsize / 64; /* Need, better value... */ - erts_garbage_collect_literals(rp, (Eterm*)literals, lit_bsize, oh); - done_gc |= ERTS_LITERAL_GC__; - } - need_gc = 0; - } - -#undef ERTS_ORDINARY_GC__ -#undef ERTS_LITERAL_GC__ - + return am_false; } -#endif /* !ERTS_NEW_PURGE_STRATEGY */ - static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) { @@ -1527,8 +1298,6 @@ hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, } } -#ifdef ERTS_NEW_PURGE_STRATEGY - #ifdef ERTS_SMP ErtsThrPrgrLaterOp later_literal_area_switch; @@ -1559,13 +1328,8 @@ complete_literal_area_switch(void *literal_area) } #endif -#endif /* ERTS_NEW_PURGE_STRATEGY */ - BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) { -#ifndef ERTS_NEW_PURGE_STRATEGY - BIF_ERROR(BIF_P, EXC_NOTSUP); -#else ErtsLiteralArea *unused_la; ErtsLiteralAreaRef *la_ref; @@ -1624,7 +1388,6 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) BIF_RET(am_true); #endif -#endif /* ERTS_NEW_PURGE_STRATEGY */ } void @@ -1818,10 +1581,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) code = (BeamInstr*) modp->old.code_hdr; end = (BeamInstr *)((char *)code + modp->old.code_length); erts_fun_purge_prepare(code, end); -#if !defined(ERTS_NEW_PURGE_STRATEGY) - ASSERT(!ERTS_COPY_LITERAL_AREA()); - ERTS_SET_COPY_LITERAL_AREA(modp->old.code_hdr->literal_area); -#endif } if (BIF_ARG_2 == am_prepare_on_load) { @@ -1866,10 +1625,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) erts_fun_purge_abort_prepare(purge_state.funs, purge_state.fe_ix); -#if !defined(ERTS_NEW_PURGE_STRATEGY) - ASSERT(ERTS_COPY_LITERAL_AREA()); - ERTS_SET_COPY_LITERAL_AREA(NULL); -#endif #ifndef ERTS_SMP erts_fun_purge_abort_finalize(purge_state.funs, purge_state.fe_ix); finalize_purge_operation(BIF_P, 0); @@ -1938,15 +1693,18 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) /* * Unload any NIF library */ - if (modp->old.nif != NULL) { + if (modp->old.nif != NULL + || IF_HIPE(hipe_purge_need_blocking(modp))) { /* ToDo: Do unload nif without blocking */ erts_rwunlock_old_code(code_ix); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; erts_rwlock_old_code(code_ix); - erts_unload_nif(modp->old.nif); - modp->old.nif = NULL; + if (modp->old.nif) { + erts_unload_nif(modp->old.nif); + modp->old.nif = NULL; + } } /* @@ -1965,7 +1723,9 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) modp->old.code_length = 0; modp->old.catches = BEAM_CATCHES_NIL; erts_remove_from_ranges(code); - +#ifdef HIPE + hipe_purge_module(modp, is_blocking); +#endif ERTS_BIF_PREP_RET(ret, am_true); } @@ -1984,14 +1744,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) finalize_purge_operation(BIF_P, ret == am_true); -#if !defined(ERTS_NEW_PURGE_STRATEGY) - - ASSERT(ERTS_COPY_LITERAL_AREA() == literals); - ERTS_SET_COPY_LITERAL_AREA(NULL); - erts_release_literal_area(literals); - -#else /* ERTS_NEW_PURGE_STRATEGY */ - if (literals) { ErtsLiteralAreaRef *ref; ref = erts_alloc(ERTS_ALC_T_LITERAL_REF, @@ -2015,8 +1767,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) BIF_P->common.id); } -#endif /* ERTS_NEW_PURGE_STRATEGY */ - return ret; } @@ -2039,34 +1789,34 @@ delete_code(Module* modp) for (i = 0; i < num_exps; i++) { Export *ep = export_list(i, code_ix); - if (ep != NULL && (ep->code[0] == module)) { - if (ep->addressv[code_ix] == ep->code+3) { - if (ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep != NULL && (ep->info.mfa.module == module)) { + if (ep->addressv[code_ix] == ep->beam) { + if (ep->beam[0] == (BeamInstr) em_apply_bif) { continue; } - else if (ep->code[3] == + else if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); - erts_clear_export_break(modp, ep->code+3); + DBG_TRACE_MFA_P(&ep->info.mfa, + "export trace cleared, code_ix=%d", code_ix); + erts_clear_export_break(modp, &ep->info); } - else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler + else ASSERT(ep->beam[0] == (BeamInstr) em_call_error_handler || !erts_initialized); } - ep->addressv[code_ix] = ep->code+3; - ep->code[3] = (BeamInstr) em_call_error_handler; - ep->code[4] = 0; + ep->addressv[code_ix] = ep->beam; + ep->beam[0] = (BeamInstr) em_call_error_handler; + ep->beam[1] = 0; + DBG_TRACE_MFA_P(&ep->info.mfa, + "export invalidation, code_ix=%d", code_ix); } } ASSERT(modp->curr.num_breakpoints == 0); ASSERT(modp->curr.num_traced_exports == 0); modp->old = modp->curr; - modp->curr.code_hdr = NULL; - modp->curr.code_length = 0; - modp->curr.catches = BEAM_CATCHES_NIL; - modp->curr.nif = NULL; - + erts_module_instance_init(&modp->curr); } @@ -2080,9 +1830,11 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) * if not, delete old code; error if old code already exists. */ - if (modp->curr.code_hdr && modp->old.code_hdr) { - return am_not_purged; - } else if (!modp->old.code_hdr) { /* Make the current version old. */ + if (modp->curr.code_hdr) { + if (modp->old.code_hdr) { + return am_not_purged; + } + /* Make the current version old. */ delete_code(modp); } return NIL; diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 0df2df0eaa..455362f5ae 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -120,27 +120,27 @@ release_bp_sched_ix(Uint32 ix) /* ** Helpers */ -static ErtsTracer do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, +static ErtsTracer do_call_trace(Process* c_p, ErtsCodeInfo *info, Eterm* reg, int local, Binary* ms, ErtsTracer tracer); static void set_break(BpFunctions* f, Binary *match_spec, Uint break_flags, enum erts_break_op count_op, ErtsTracer tracer); -static void set_function_break(BeamInstr *pc, +static void set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, enum erts_break_op count_op, ErtsTracer tracer); static void clear_break(BpFunctions* f, Uint break_flags); -static int clear_function_break(BeamInstr *pc, Uint break_flags); +static int clear_function_break(ErtsCodeInfo *ci, Uint break_flags); -static BpDataTime* get_time_break(BeamInstr *pc); -static GenericBpData* check_break(BeamInstr *pc, Uint break_flags); +static BpDataTime* get_time_break(ErtsCodeInfo *ci); +static GenericBpData* check_break(ErtsCodeInfo *ci, Uint break_flags); -static void bp_meta_unref(BpMetaTracer* bmt); -static void bp_count_unref(BpCount* bcp); -static void bp_time_unref(BpDataTime* bdt); -static void consolidate_bp_data(Module* modp, BeamInstr* pc, int local); -static void uninstall_breakpoint(BeamInstr* pc); +static void bp_meta_unref(BpMetaTracer *bmt); +static void bp_count_unref(BpCount *bcp); +static void bp_time_unref(BpDataTime *bdt); +static void consolidate_bp_data(Module *modp, ErtsCodeInfo *ci, int local); +static void uninstall_breakpoint(ErtsCodeInfo *ci); /* bp_hash */ #define BP_TIME_ADD(pi0, pi1) \ @@ -170,7 +170,7 @@ erts_bp_init(void) { void -erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) +erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified) { ErtsCodeIndex code_ix = erts_active_code_ix(); Uint max_funcs = 0; @@ -195,41 +195,44 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) i = 0; for (current = 0; current < num_modules; current++) { BeamCodeHeader* code_hdr = module[current]->curr.code_hdr; - BeamInstr* code; + ErtsCodeInfo* ci; Uint num_functions = (Uint)(UWord) code_hdr->num_functions; Uint fi; if (specified > 0) { - if (mfa[0] != make_atom(module[current]->module)) { + if (mfa->module != make_atom(module[current]->module)) { /* Wrong module name */ continue; } } for (fi = 0; fi < num_functions; fi++) { - BeamInstr* pc; - int wi; - code = code_hdr->functions[fi]; - ASSERT(code[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - pc = code+5; - if (erts_is_native_break(pc)) { + ci = code_hdr->functions[fi]; + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + if (erts_is_native_break(ci)) { continue; } - if (is_nil(code[3])) { /* Ignore BIF stub */ + if (is_nil(ci->mfa.module)) { /* Ignore BIF stub */ continue; } - for (wi = 0; - wi < specified && (Eterm) code[2+wi] == mfa[wi]; - wi++) { - /* Empty loop body */ - } - if (wi == specified) { - /* Store match */ - f->matching[i].pc = pc; - f->matching[i].mod = module[current]; - i++; - } + switch (specified) { + case 3: + if (ci->mfa.arity != mfa->arity) + continue; + case 2: + if (ci->mfa.function != mfa->function) + continue; + case 1: + if (ci->mfa.module != mfa->module) + continue; + case 0: + break; + } + /* Store match */ + f->matching[i].ci = ci; + f->matching[i].mod = module[current]; + i++; } } f->matched = i; @@ -237,7 +240,7 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) } void -erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified) +erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified) { ErtsCodeIndex code_ix = erts_active_code_ix(); int i; @@ -249,27 +252,36 @@ erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified) for (i = 0; i < num_exps; i++) { Export* ep = export_list(i, code_ix); BeamInstr* pc; - int j; - for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) { - /* Empty loop body */ - } - if (j < specified) { - continue; - } - pc = ep->code+3; + switch (specified) { + case 3: + if (mfa->arity != ep->info.mfa.arity) + continue; + case 2: + if (mfa->function != ep->info.mfa.function) + continue; + case 1: + if (mfa->module != ep->info.mfa.module) + continue; + case 0: + break; + default: + ASSERT(0); + } + + pc = ep->beam; if (ep->addressv[code_ix] == pc) { if ((*pc == (BeamInstr) em_apply_bif || *pc == (BeamInstr) em_call_error_handler)) { continue; } ASSERT(*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); - } else if (erts_is_native_break(ep->addressv[code_ix])) { + } else if (erts_is_native_break(erts_code_to_codeinfo(ep->addressv[code_ix]))) { continue; } - f->matching[ne].pc = pc; - f->matching[ne].mod = erts_get_module(ep->code[0], code_ix); + f->matching[ne].ci = &ep->info; + f->matching[ne].mod = erts_get_module(ep->info.mfa.module, code_ix); ne++; } @@ -295,7 +307,7 @@ erts_consolidate_bp_data(BpFunctions* f, int local) ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); for (i = 0; i < n; i++) { - consolidate_bp_data(fs[i].mod, fs[i].pc, local); + consolidate_bp_data(fs[i].mod, fs[i].ci, local); } } @@ -307,14 +319,14 @@ erts_consolidate_bif_bp_data(void) ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); for (i = 0; i < BIF_SIZE; i++) { Export *ep = bif_export[i]; - consolidate_bp_data(0, ep->code+3, 0); + consolidate_bp_data(0, &ep->info, 0); } } static void -consolidate_bp_data(Module* modp, BeamInstr* pc, int local) +consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; GenericBpData* src; GenericBpData* dst; Uint flags; @@ -360,9 +372,10 @@ consolidate_bp_data(Module* modp, BeamInstr* pc, int local) } ASSERT(modp->curr.num_breakpoints >= 0); ASSERT(modp->curr.num_traced_exports >= 0); - ASSERT(*pc != (BeamInstr) BeamOp(op_i_generic_breakpoint)); + ASSERT(*erts_codeinfo_to_code(ci) != + (BeamInstr) BeamOp(op_i_generic_breakpoint)); } - pc[-4] = 0; + ci->native = 0; Free(g); return; } @@ -411,8 +424,9 @@ erts_install_breakpoints(BpFunctions* f) BeamInstr br = (BeamInstr) BeamOp(op_i_generic_breakpoint); for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - GenericBp* g = (GenericBp *) pc[-4]; + ErtsCodeInfo* ci = f->matching[i].ci; + BeamInstr *pc = erts_codeinfo_to_code(ci); + GenericBp* g = (GenericBp *) ci->native; if (*pc != br && g) { Module* modp = f->matching[i].mod; @@ -444,16 +458,16 @@ erts_uninstall_breakpoints(BpFunctions* f) Uint n = f->matched; for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - uninstall_breakpoint(pc); + uninstall_breakpoint(f->matching[i].ci); } } static void -uninstall_breakpoint(BeamInstr* pc) +uninstall_breakpoint(ErtsCodeInfo *ci) { + BeamInstr *pc = erts_codeinfo_to_code(ci); if (*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; if (g->data[erts_active_bp_ix()].flags == 0) { /* * The following write is not protected by any lock. We @@ -480,30 +494,30 @@ erts_set_mtrace_break(BpFunctions* f, Binary *match_spec, ErtsTracer tracer) } void -erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local) +erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; - set_function_break(pc, match_spec, flags, 0, erts_tracer_nil); + set_function_break(ci, match_spec, flags, 0, erts_tracer_nil); } void -erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, ErtsTracer tracer) +erts_set_mtrace_bif(ErtsCodeInfo *ci, Binary *match_spec, ErtsTracer tracer) { - set_function_break(pc, match_spec, ERTS_BPF_META_TRACE, 0, tracer); + set_function_break(ci, match_spec, ERTS_BPF_META_TRACE, 0, tracer); } void -erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op count_op) +erts_set_time_trace_bif(ErtsCodeInfo *ci, enum erts_break_op count_op) { - set_function_break(pc, NULL, + set_function_break(ci, NULL, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE, count_op, erts_tracer_nil); } void -erts_clear_time_trace_bif(BeamInstr *pc) { - clear_function_break(pc, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE); +erts_clear_time_trace_bif(ErtsCodeInfo *ci) { + clear_function_break(ci, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE); } void @@ -532,14 +546,14 @@ erts_clear_trace_break(BpFunctions* f) } void -erts_clear_call_trace_bif(BeamInstr *pc, int local) +erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; if (g) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; if (g->data[erts_staging_bp_ix()].flags & flags) { - clear_function_break(pc, flags); + clear_function_break(ci, flags); } } } @@ -551,9 +565,9 @@ erts_clear_mtrace_break(BpFunctions* f) } void -erts_clear_mtrace_bif(BeamInstr *pc) +erts_clear_mtrace_bif(ErtsCodeInfo *ci) { - clear_function_break(pc, ERTS_BPF_META_TRACE); + clear_function_break(ci, ERTS_BPF_META_TRACE); } void @@ -595,52 +609,48 @@ erts_clear_module_break(Module *modp) { } n = (Uint)(UWord) code_hdr->num_functions; for (i = 0; i < n; ++i) { - BeamInstr* pc; - - pc = code_hdr->functions[i] + 5; - if (erts_is_native_break(pc)) { + ErtsCodeInfo *ci = code_hdr->functions[i]; + if (erts_is_native_break(ci)) continue; - } - clear_function_break(pc, ERTS_BPF_ALL); + clear_function_break(ci, ERTS_BPF_ALL); } erts_commit_staged_bp(); for (i = 0; i < n; ++i) { - BeamInstr* pc; - - pc = code_hdr->functions[i] + 5; - if (erts_is_native_break(pc)) { + ErtsCodeInfo *ci = code_hdr->functions[i]; + if (erts_is_native_break(ci)) continue; - } - uninstall_breakpoint(pc); - consolidate_bp_data(modp, pc, 1); - ASSERT(pc[-4] == 0); + uninstall_breakpoint(ci); + consolidate_bp_data(modp, ci, 1); + ASSERT(ci->native == 0); } return n; } void -erts_clear_export_break(Module* modp, BeamInstr* pc) +erts_clear_export_break(Module* modp, ErtsCodeInfo *ci) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - clear_function_break(pc, ERTS_BPF_ALL); + clear_function_break(ci, ERTS_BPF_ALL); erts_commit_staged_bp(); - *pc = (BeamInstr) 0; - consolidate_bp_data(modp, pc, 0); - ASSERT(pc[-4] == 0); + *erts_codeinfo_to_code(ci) = (BeamInstr) 0; + consolidate_bp_data(modp, ci, 0); + ASSERT(ci->native == 0); } BeamInstr -erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) +erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) { GenericBp* g; GenericBpData* bp; Uint bp_flags; ErtsBpIndex ix = erts_active_bp_ix(); - g = (GenericBp *) I[-4]; + ASSERT(info->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + + g = (GenericBp *) info->native; bp = &g->data[ix]; bp_flags = bp->flags; ASSERT((bp_flags & ~ERTS_BPF_ALL) == 0); @@ -659,9 +669,9 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) if (bp_flags & ERTS_BPF_LOCAL_TRACE) { ASSERT((bp_flags & ERTS_BPF_GLOBAL_TRACE) == 0); - (void) do_call_trace(c_p, I, reg, 1, bp->local_ms, erts_tracer_true); + (void) do_call_trace(c_p, info, reg, 1, bp->local_ms, erts_tracer_true); } else if (bp_flags & ERTS_BPF_GLOBAL_TRACE) { - (void) do_call_trace(c_p, I, reg, 0, bp->local_ms, erts_tracer_true); + (void) do_call_trace(c_p, info, reg, 0, bp->local_ms, erts_tracer_true); } if (bp_flags & ERTS_BPF_META_TRACE) { @@ -669,7 +679,8 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) old_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer); - new_tracer = do_call_trace(c_p, I, reg, 1, bp->meta_ms, old_tracer); + new_tracer = do_call_trace(c_p, info, reg, 1, bp->meta_ms, old_tracer); + if (!ERTS_TRACER_COMPARE(new_tracer, old_tracer)) { if (old_tracer == erts_smp_atomic_cmpxchg_acqb( &bp->meta_tracer->tracer, @@ -688,7 +699,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) { Eterm w; - erts_trace_time_call(c_p, I, bp->time); + erts_trace_time_call(c_p, info, bp->time); w = (BeamInstr) *c_p->cp; if (! (w == (BeamInstr) BeamOp(op_i_return_time_trace) || w == (BeamInstr) BeamOp(op_return_trace) || @@ -696,7 +707,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) Eterm* E = c_p->stop; ASSERT(c_p->htop <= E && E <= c_p->hend); if (E - 2 < c_p->htop) { - (void) erts_garbage_collect(c_p, 2, reg, I[-1]); + (void) erts_garbage_collect(c_p, 2, reg, info->mfa.arity); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); } E = c_p->stop; @@ -704,7 +715,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) ASSERT(c_p->htop <= E && E <= c_p->hend); E -= 2; - E[0] = make_cp(I); + E[0] = make_cp(erts_codeinfo_to_code(info)); E[1] = make_cp(c_p->cp); /* original return address */ c_p->cp = beam_return_time_trace; c_p->stop = E; @@ -732,9 +743,9 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) Export* ep = bif_export[bif_index]; Uint32 flags = 0, flags_meta = 0; ErtsTracer meta_tracer = erts_tracer_nil; - int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif - * is actually in the - * export entry */ + int applying = (I == ep->beam); /* Yup, the apply code for a bif + * is actually in the + * export entry */ BeamInstr *cp = p->cp; GenericBp* g; GenericBpData* bp = NULL; @@ -742,7 +753,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - g = (GenericBp *) ep->fake_op_func_info_for_hipe[1]; + g = (GenericBp *) ep->info.native; if (g) { bp = &g->data[erts_active_bp_ix()]; bp_flags = bp->flags; @@ -758,7 +769,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) if (bp_flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE) && IS_TRACED_FL(p, F_TRACE_CALLS)) { int local = !!(bp_flags & ERTS_BPF_LOCAL_TRACE); - flags = erts_call_trace(p, ep->code, bp->local_ms, args, + flags = erts_call_trace(p, &ep->info, bp->local_ms, args, local, &ERTS_TRACER(p)); } if (bp_flags & ERTS_BPF_META_TRACE) { @@ -766,7 +777,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) meta_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer); old_tracer = meta_tracer; - flags_meta = erts_call_trace(p, ep->code, bp->meta_ms, args, + flags_meta = erts_call_trace(p, &ep->info, bp->meta_ms, args, 0, &meta_tracer); if (!ERTS_TRACER_COMPARE(old_tracer, meta_tracer)) { @@ -784,8 +795,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && IS_TRACED_FL(p, F_TRACE_CALLS)) { - BeamInstr *pc = (BeamInstr *)ep->code+3; - erts_trace_time_call(p, pc, bp->time); + erts_trace_time_call(p, &ep->info, bp->time); } /* Restore original continuation pointer (if changed). */ @@ -848,11 +858,11 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) class = exception_tag[GET_EXC_CLASS(reason)]; if (flags_meta & MATCH_SET_EXCEPTION_TRACE) { - erts_trace_exception(p, ep->code, class, value, + erts_trace_exception(p, &ep->info.mfa, class, value, &meta_tracer); } if (flags & MATCH_SET_EXCEPTION_TRACE) { - erts_trace_exception(p, ep->code, class, value, + erts_trace_exception(p, &ep->info.mfa, class, value, &ERTS_TRACER(p)); } if ((flags & MATCH_SET_RETURN_TO_TRACE) && p->catches > 0) { @@ -883,11 +893,11 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } } else { if (flags_meta & MATCH_SET_RX_TRACE) { - erts_trace_return(p, ep->code, result, &meta_tracer); + erts_trace_return(p, &ep->info.mfa, result, &meta_tracer); } /* MATCH_SET_RETURN_TO_TRACE cannot occur if(meta) */ if (flags & MATCH_SET_RX_TRACE) { - erts_trace_return(p, ep->code, result, &ERTS_TRACER(p)); + erts_trace_return(p, &ep->info.mfa, result, &ERTS_TRACER(p)); } if (flags & MATCH_SET_RETURN_TO_TRACE && IS_TRACED_FL(p, F_TRACE_RETURN_TO)) { @@ -906,7 +916,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } static ErtsTracer -do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, +do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg, int local, Binary* ms, ErtsTracer tracer) { Eterm* cpp; @@ -947,7 +957,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, ASSERT(is_CP(*cpp)); } ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - flags = erts_call_trace(c_p, I-3, ms, reg, local, &tracer); + flags = erts_call_trace(c_p, info, ms, reg, local, &tracer); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); if (cpp) { c_p->cp = cp_save; @@ -963,7 +973,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, if (need) { ASSERT(c_p->htop <= E && E <= c_p->hend); if (E - need < c_p->htop) { - (void) erts_garbage_collect(c_p, need, reg, I[-1]); + (void) erts_garbage_collect(c_p, need, reg, info->mfa.arity); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); E = c_p->stop; } @@ -979,12 +989,12 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, E -= 3; c_p->stop = E; ASSERT(c_p->htop <= E && E <= c_p->hend); - ASSERT(is_CP((Eterm) (UWord) (I - 3))); + ASSERT(is_CP((Eterm) (UWord) (&info->mfa.module))); ASSERT(IS_TRACER_VALID(tracer)); E[2] = make_cp(c_p->cp); E[1] = copy_object(tracer, c_p); - E[0] = make_cp(I - 3); /* We ARE at the beginning of an - instruction, + E[0] = make_cp(&info->mfa.module); + /* We ARE at the beginning of an instruction, the funcinfo is above i. */ c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE) ? beam_exception_trace : beam_return_trace; @@ -997,7 +1007,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, } void -erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) +erts_trace_time_call(Process* c_p, ErtsCodeInfo *info, BpDataTime* bdt) { ErtsMonotonicTime time; process_breakpoint_time_t *pbt = NULL; @@ -1027,14 +1037,14 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) pbt = Alloc(sizeof(process_breakpoint_time_t)); (void) ERTS_PROC_SET_CALL_TIME(c_p, pbt); } else { - ASSERT(pbt->pc); + ASSERT(pbt->ci); /* add time to previous code */ sitem.time = time - pbt->time; sitem.pid = c_p->common.id; sitem.count = 0; /* previous breakpoint */ - pbdt = get_time_break(pbt->pc); + pbdt = get_time_break(pbt->ci); /* if null then the breakpoint was removed */ if (pbdt) { @@ -1071,14 +1081,14 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) BP_TIME_ADD(item, &sitem); } - pbt->pc = I; + pbt->ci = info; pbt->time = time; release_bp_sched_ix(six); } void -erts_trace_time_return(Process *p, BeamInstr *pc) +erts_trace_time_return(Process *p, ErtsCodeInfo *ci) { ErtsMonotonicTime time; process_breakpoint_time_t *pbt = NULL; @@ -1110,14 +1120,14 @@ erts_trace_time_return(Process *p, BeamInstr *pc) /* might have been removed due to * trace_pattern(false) */ - ASSERT(pbt->pc); + ASSERT(pbt->ci); sitem.time = time - pbt->time; sitem.pid = p->common.id; sitem.count = 0; /* previous breakpoint */ - pbdt = get_time_break(pbt->pc); + pbdt = get_time_break(pbt->ci); /* beware, the trace_pattern might have been removed */ if (pbdt) { @@ -1136,7 +1146,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc) } - pbt->pc = pc; + pbt->ci = ci; pbt->time = time; } @@ -1145,10 +1155,10 @@ erts_trace_time_return(Process *p, BeamInstr *pc) } int -erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local) +erts_is_trace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, int local) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; - GenericBpData* bp = check_break(pc, flags); + GenericBpData* bp = check_break(ci, flags); if (bp) { if (match_spec_ret) { @@ -1160,10 +1170,10 @@ erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local) } int -erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, +erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, ErtsTracer *tracer_ret) { - GenericBpData* bp = check_break(pc, ERTS_BPF_META_TRACE); + GenericBpData* bp = check_break(ci, ERTS_BPF_META_TRACE); if (bp) { if (match_spec_ret) { @@ -1178,20 +1188,20 @@ erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, } int -erts_is_native_break(BeamInstr *pc) { +erts_is_native_break(ErtsCodeInfo *ci) { #ifdef HIPE - ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - return pc[0] == (BeamInstr) BeamOp(op_hipe_trap_call) - || pc[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure); + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + return erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call) + || erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure); #else return 0; #endif } int -erts_is_count_break(BeamInstr *pc, Uint *count_ret) +erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret) { - GenericBpData* bp = check_break(pc, ERTS_BPF_COUNT); + GenericBpData* bp = check_break(ci, ERTS_BPF_COUNT); if (bp) { if (count_ret) { @@ -1202,13 +1212,13 @@ erts_is_count_break(BeamInstr *pc, Uint *count_ret) return 0; } -int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { +int erts_is_time_break(Process *p, ErtsCodeInfo *ci, Eterm *retval) { Uint i, ix; bp_time_hash_t hash; Uint size; Eterm *hp, t; bp_data_time_item_t *item = NULL; - BpDataTime *bdt = get_time_break(pc); + BpDataTime *bdt = get_time_break(ci); if (bdt) { if (retval) { @@ -1262,26 +1272,25 @@ int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { } -BeamInstr * -erts_find_local_func(Eterm mfa[3]) { +ErtsCodeInfo * +erts_find_local_func(ErtsCodeMFA *mfa) { Module *modp; BeamCodeHeader* code_hdr; - BeamInstr* code_ptr; + ErtsCodeInfo* ci; Uint i,n; - if ((modp = erts_get_module(mfa[0], erts_active_code_ix())) == NULL) + if ((modp = erts_get_module(mfa->module, erts_active_code_ix())) == NULL) return NULL; if ((code_hdr = modp->curr.code_hdr) == NULL) return NULL; n = (BeamInstr) code_hdr->num_functions; for (i = 0; i < n; ++i) { - code_ptr = code_hdr->functions[i]; - ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]); - ASSERT(mfa[0] == ((Eterm) code_ptr[2]) || - is_nil((Eterm) code_ptr[2])); - if (mfa[1] == ((Eterm) code_ptr[3]) && - ((BeamInstr) mfa[2]) == code_ptr[4]) { - return code_ptr + 5; + ci = code_hdr->functions[i]; + ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == ci->op); + ASSERT(mfa->module == ci->mfa.module || is_nil(ci->mfa.module)); + if (mfa->function == ci->mfa.function && + mfa->arity == ci->mfa.arity) { + return ci; } } return NULL; @@ -1411,7 +1420,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { * the previous breakpoint. */ - pbdt = get_time_break(pbt->pc); + pbdt = get_time_break(pbt->ci); if (pbdt) { sitem.time = get_mtime(p) - pbt->time; sitem.pid = p->common.id; @@ -1461,14 +1470,14 @@ set_break(BpFunctions* f, Binary *match_spec, Uint break_flags, n = f->matched; for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - set_function_break(pc, match_spec, break_flags, + set_function_break(f->matching[i].ci, + match_spec, break_flags, count_op, tracer); } } static void -set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, +set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, enum erts_break_op count_op, ErtsTracer tracer) { GenericBp* g; @@ -1477,7 +1486,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, ErtsBpIndex ix = erts_staging_bp_ix(); ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); - g = (GenericBp *) pc[-4]; + g = (GenericBp *) ci->native; if (g == 0) { int i; if (count_op == ERTS_BREAK_RESTART || count_op == ERTS_BREAK_PAUSE) { @@ -1485,11 +1494,11 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, return; } g = Alloc(sizeof(GenericBp)); - g->orig_instr = *pc; + g->orig_instr = *erts_codeinfo_to_code(ci); for (i = 0; i < ERTS_NUM_BP_IX; i++) { g->data[i].flags = 0; } - pc[-4] = (BeamInstr) g; + ci->native = (BeamInstr) g; } bp = &g->data[ix]; @@ -1585,13 +1594,12 @@ clear_break(BpFunctions* f, Uint break_flags) n = f->matched; for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - clear_function_break(pc, break_flags); + clear_function_break(f->matching[i].ci, break_flags); } } static int -clear_function_break(BeamInstr *pc, Uint break_flags) +clear_function_break(ErtsCodeInfo *ci, Uint break_flags) { GenericBp* g; GenericBpData* bp; @@ -1600,7 +1608,7 @@ clear_function_break(BeamInstr *pc, Uint break_flags) ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); - if ((g = (GenericBp *) pc[-4]) == 0) { + if ((g = (GenericBp *) ci->native) == 0) { return 1; } @@ -1686,19 +1694,19 @@ bp_time_unref(BpDataTime* bdt) } static BpDataTime* -get_time_break(BeamInstr *pc) +get_time_break(ErtsCodeInfo *ci) { - GenericBpData* bp = check_break(pc, ERTS_BPF_TIME_TRACE); + GenericBpData* bp = check_break(ci, ERTS_BPF_TIME_TRACE); return bp ? bp->time : 0; } static GenericBpData* -check_break(BeamInstr *pc, Uint break_flags) +check_break(ErtsCodeInfo *ci, Uint break_flags) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; - ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - if (erts_is_native_break(pc)) { + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + if (erts_is_native_break(ci)) { return 0; } if (g) { diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 4743e4fc2f..cccd395e0a 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -46,7 +46,7 @@ typedef struct bp_data_time { /* Call time */ typedef struct { ErtsMonotonicTime time; - BeamInstr *pc; + ErtsCodeInfo *ci; } process_breakpoint_time_t; /* used within psd */ typedef struct { @@ -93,7 +93,7 @@ enum erts_break_op{ typedef Uint32 ErtsBpIndex; typedef struct { - BeamInstr* pc; + ErtsCodeInfo *ci; Module* mod; } BpFunction; @@ -114,8 +114,8 @@ void erts_commit_staged_bp(void); ERTS_GLB_INLINE ErtsBpIndex erts_active_bp_ix(void); ERTS_GLB_INLINE ErtsBpIndex erts_staging_bp_ix(void); -void erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified); -void erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified); +void erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified); +void erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified); void erts_bp_free_matched_functions(BpFunctions* f); void erts_install_breakpoints(BpFunctions* f); @@ -126,15 +126,15 @@ void erts_consolidate_bif_bp_data(void); void erts_set_trace_break(BpFunctions *f, Binary *match_spec); void erts_clear_trace_break(BpFunctions *f); -void erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local); -void erts_clear_call_trace_bif(BeamInstr *pc, int local); +void erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local); +void erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local); void erts_set_mtrace_break(BpFunctions *f, Binary *match_spec, ErtsTracer tracer); void erts_clear_mtrace_break(BpFunctions *f); -void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, +void erts_set_mtrace_bif(ErtsCodeInfo *ci, Binary *match_spec, ErtsTracer tracer); -void erts_clear_mtrace_bif(BeamInstr *pc); +void erts_clear_mtrace_bif(ErtsCodeInfo *ci); void erts_set_debug_break(BpFunctions *f); void erts_clear_debug_break(BpFunctions *f); @@ -144,32 +144,32 @@ void erts_clear_count_break(BpFunctions *f); void erts_clear_all_breaks(BpFunctions* f); int erts_clear_module_break(Module *modp); -void erts_clear_export_break(Module *modp, BeamInstr* pc); +void erts_clear_export_break(Module *modp, ErtsCodeInfo* ci); -BeamInstr erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg); -BeamInstr erts_trace_break(Process *p, BeamInstr *pc, Eterm *args, +BeamInstr erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *ci, Eterm* reg); +BeamInstr erts_trace_break(Process *p, ErtsCodeInfo *ci, Eterm *args, Uint32 *ret_flags, ErtsTracer *tracer); -int erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local); -int erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, +int erts_is_trace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, int local); +int erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, ErtsTracer *tracer_ret); -int erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret, +int erts_is_mtrace_bif(ErtsCodeInfo *ci, Binary **match_spec_ret, ErtsTracer *tracer_ret); -int erts_is_native_break(BeamInstr *pc); -int erts_is_count_break(BeamInstr *pc, Uint *count_ret); -int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *call_time); +int erts_is_native_break(ErtsCodeInfo *ci); +int erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret); +int erts_is_time_break(Process *p, ErtsCodeInfo *ci, Eterm *call_time); -void erts_trace_time_call(Process* c_p, BeamInstr* pc, BpDataTime* bdt); -void erts_trace_time_return(Process* c_p, BeamInstr* pc); +void erts_trace_time_call(Process* c_p, ErtsCodeInfo *ci, BpDataTime* bdt); +void erts_trace_time_return(Process* c_p, ErtsCodeInfo *ci); void erts_schedule_time_break(Process *p, Uint out); void erts_set_time_break(BpFunctions *f, enum erts_break_op); void erts_clear_time_break(BpFunctions *f); -int erts_is_time_trace_bif(Process *p, BeamInstr *pc, Eterm *call_time); -void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op); -void erts_clear_time_trace_bif(BeamInstr *pc); +int erts_is_time_trace_bif(Process *p, ErtsCodeInfo *ci, Eterm *call_time); +void erts_set_time_trace_bif(ErtsCodeInfo *ci, enum erts_break_op); +void erts_clear_time_trace_bif(ErtsCodeInfo *ci); -BeamInstr *erts_find_local_func(Eterm mfa[3]); +ErtsCodeInfo *erts_find_local_func(ErtsCodeMFA *mfa); #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 21d336049f..e72d7f8de4 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -51,6 +51,7 @@ void dbg_bt(Process* p, Eterm* sp); void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg); static int print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr); +static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif); BIF_RETTYPE erts_debug_same_2(BIF_ALIST_2) @@ -115,7 +116,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) Eterm MFA = BIF_ARG_1; Eterm boolean = BIF_ARG_2; Eterm* tp; - Eterm mfa[3]; + ErtsCodeMFA mfa; int i; int specified = 0; Eterm res; @@ -131,23 +132,24 @@ erts_debug_breakpoint_2(BIF_ALIST_2) if (*tp != make_arityval(3)) { goto error; } - mfa[0] = tp[1]; - mfa[1] = tp[2]; - mfa[2] = tp[3]; - if (!is_atom(mfa[0]) || !is_atom(mfa[1]) || - (!is_small(mfa[2]) && mfa[2] != am_Underscore)) { + if (!is_atom(tp[1]) || !is_atom(tp[2]) || + (!is_small(tp[3]) && tp[3] != am_Underscore)) { goto error; } - for (i = 0; i < 3 && mfa[i] != am_Underscore; i++, specified++) { + for (i = 0; i < 3 && tp[i+1] != am_Underscore; i++, specified++) { /* Empty loop body */ } for (i = specified; i < 3; i++) { - if (mfa[i] != am_Underscore) { + if (tp[i+1] != am_Underscore) { goto error; } } - if (is_small(mfa[2])) { - mfa[2] = signed_val(mfa[2]); + + mfa.module = tp[1]; + mfa.function = tp[2]; + + if (is_small(tp[3])) { + mfa.arity = signed_val(tp[3]); } if (!erts_try_seize_code_write_permission(BIF_P)) { @@ -157,7 +159,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); - erts_bp_match_functions(&f, mfa, specified); + erts_bp_match_functions(&f, &mfa, specified); if (boolean == am_true) { erts_set_debug_break(&f); erts_install_breakpoints(&f); @@ -241,9 +243,9 @@ erts_debug_disassemble_1(BIF_ALIST_1) Eterm* tp; Eterm bin; Eterm mfa; - BeamInstr* funcinfo = NULL; /* Initialized to eliminate warning. */ + ErtsCodeMFA *cmfa = NULL; BeamCodeHeader* code_hdr; - BeamInstr* code_ptr = NULL; /* Initialized to eliminate warning. */ + BeamInstr *code_ptr; BeamInstr instr; BeamInstr uaddr; Uint hsz; @@ -251,7 +253,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) if (term_to_UWord(addr, &uaddr)) { code_ptr = (BeamInstr *) uaddr; - if ((funcinfo = find_function_from_pc(code_ptr)) == NULL) { + if ((cmfa = find_function_from_pc(code_ptr)) == NULL) { BIF_RET(am_false); } } else if (is_tuple(addr)) { @@ -282,24 +284,22 @@ erts_debug_disassemble_1(BIF_ALIST_1) * such as erts_debug:apply/4. Then search for it in the module. */ if ((ep = erts_find_function(mod, name, arity, code_ix)) != NULL) { - /* XXX: add "&& ep->address != ep->code+3" condition? + /* XXX: add "&& ep->address != ep->code" condition? * Consider a traced function. - * Its ep will have ep->address == ep->code+3. + * Its ep will have ep->address == ep->code. * erts_find_function() will return the non-NULL ep. * Below we'll try to derive a code_ptr from ep->address. * But this code_ptr will point to the start of the Export, * not the function's func_info instruction. BOOM !? */ - code_ptr = ((BeamInstr *) ep->addressv[code_ix]) - 5; - funcinfo = code_ptr+2; + cmfa = erts_code_to_codemfa(ep->addressv[code_ix]); } else if (modp == NULL || (code_hdr = modp->curr.code_hdr) == NULL) { BIF_RET(am_undef); } else { n = code_hdr->num_functions; for (i = 0; i < n; i++) { - code_ptr = code_hdr->functions[i]; - if (code_ptr[3] == name && code_ptr[4] == arity) { - funcinfo = code_ptr+2; + cmfa = &code_hdr->functions[i]->mfa; + if (cmfa->function == name && cmfa->arity == arity) { break; } } @@ -307,6 +307,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) BIF_RET(am_undef); } } + code_ptr = erts_codemfa_to_code(cmfa); } else { goto error; } @@ -332,9 +333,10 @@ erts_debug_disassemble_1(BIF_ALIST_1) (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr); hp = HAlloc(p, hsz); addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr); - ASSERT(is_atom(funcinfo[0]) || funcinfo[0] == NIL); - ASSERT(is_atom(funcinfo[1]) || funcinfo[1] == NIL); - mfa = TUPLE3(hp, (Eterm) funcinfo[0], (Eterm) funcinfo[1], make_small((Eterm) funcinfo[2])); + ASSERT(is_atom(cmfa->module) || is_nil(cmfa->module)); + ASSERT(is_atom(cmfa->function) || is_nil(cmfa->function)); + mfa = TUPLE3(hp, cmfa->module, cmfa->function, + make_small(cmfa->arity)); hp += 4; return TUPLE3(hp, addr, bin, mfa); } @@ -346,11 +348,12 @@ dbg_bt(Process* p, Eterm* sp) while (sp < stack) { if (is_CP(*sp)) { - BeamInstr* addr = find_function_from_pc(cp_val(*sp)); - if (addr) + ErtsCodeMFA* cmfa = find_function_from_pc(cp_val(*sp)); + if (cmfa) erts_fprintf(stderr, HEXF ": %T:%T/%bpu\n", - addr, (Eterm) addr[0], (Eterm) addr[1], addr[2]); + &cmfa->module, cmfa->module, + cmfa->function, cmfa->arity); } sp++; } @@ -359,17 +362,17 @@ dbg_bt(Process* p, Eterm* sp) void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg) { - BeamInstr* f = find_function_from_pc(addr); + ErtsCodeMFA* cmfa = find_function_from_pc(addr); - if (f == NULL) { + if (cmfa == NULL) { erts_fprintf(stderr, "???\n"); } else { int arity; int i; - addr = f; - arity = addr[2]; - erts_fprintf(stderr, HEXF ": %T:%T(", addr, (Eterm) addr[0], (Eterm) addr[1]); + arity = cmfa->arity; + erts_fprintf(stderr, HEXF ": %T:%T(", addr, + cmfa->module, cmfa->function); for (i = 0; i < arity; i++) erts_fprintf(stderr, i ? ", %T" : "%T", i ? reg[i] : x0); erts_fprintf(stderr, ")\n"); @@ -520,27 +523,49 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) break; case 'I': /* Untagged integer. */ case 't': - erts_print(to, to_arg, "%d", *ap); + switch (op) { + case op_i_gc_bif1_jIsId: + case op_i_gc_bif2_jIIssd: + case op_i_gc_bif3_jIIssd: + { + const ErtsGcBif* p; + BifFunction gcf = (BifFunction) *ap; + for (p = erts_gc_bifs; p->bif != 0; p++) { + if (p->gc_bif == gcf) { + print_bif_name(to, to_arg, p->bif); + break; + } + } + if (p->bif == 0) { + erts_print(to, to_arg, "%d", (Uint)gcf); + } + break; + } + default: + erts_print(to, to_arg, "%d", *ap); + } ap++; break; case 'f': /* Destination label */ { - BeamInstr* f = find_function_from_pc((BeamInstr *)*ap); - if (f+3 != (BeamInstr *) *ap) { + ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap); + if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { erts_print(to, to_arg, "f(" HEXF ")", *ap); } else { - erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]); + erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, + cmfa->function, cmfa->arity); } ap++; } break; case 'p': /* Pointer (to label) */ { - BeamInstr* f = find_function_from_pc((BeamInstr *)*ap); - if (f+3 != (BeamInstr *) *ap) { + ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap); + if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { erts_print(to, to_arg, "p(" HEXF ")", *ap); } else { - erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]); + erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, + cmfa->function, cmfa->arity); } ap++; } @@ -553,26 +578,16 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) { Export* ex = (Export *) *ap; erts_print(to, to_arg, - "%T:%T/%bpu", (Eterm) ex->code[0], (Eterm) ex->code[1], ex->code[2]); + "%T:%T/%bpu", (Eterm) ex->info.mfa.module, + (Eterm) ex->info.mfa.function, + ex->info.mfa.arity); ap++; } break; case 'F': /* Function definition */ break; case 'b': - for (i = 0; i < BIF_SIZE; i++) { - BifFunction bif = (BifFunction) *ap; - if (bif == bif_table[i].f) { - break; - } - } - if (i == BIF_SIZE) { - erts_print(to, to_arg, "b(%d)", (Uint) *ap); - } else { - Eterm name = bif_table[i].name; - unsigned arity = bif_table[i].arity; - erts_print(to, to_arg, "%T/%u", name, arity); - } + print_bif_name(to, to_arg, (BifFunction) *ap); ap++; break; case 'P': /* Byte offset into tuple (see beam_load.c) */ @@ -731,3 +746,21 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) return size; } + +static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif) +{ + int i; + + for (i = 0; i < BIF_SIZE; i++) { + if (bif == bif_table[i].f) { + break; + } + } + if (i == BIF_SIZE) { + erts_print(to, to_arg, "b(%d)", (Uint) bif); + } else { + Eterm name = bif_table[i].name; + unsigned arity = bif_table[i].arity; + erts_print(to, to_arg, "%T/%u", name, arity); + } +} diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index f392feb06b..c554bd73b6 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -116,10 +116,10 @@ do { \ #define MAX(x, y) (((x) > (y)) ? (x) : (y)) #endif -#define GET_BIF_MODULE(p) ((Eterm) (((Export *) p)->code[0])) -#define GET_BIF_FUNCTION(p) ((Eterm) (((Export *) p)->code[1])) -#define GET_BIF_ARITY(p) ((Eterm) (((Export *) p)->code[2])) -#define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4])) +#define GET_BIF_MODULE(p) (p->info.mfa.module) +#define GET_BIF_FUNCTION(p) (p->info.mfa.function) +#define GET_BIF_ARITY(p) (p->info.mfa.arity) +#define GET_BIF_ADDRESS(p) ((BifFunction) (p->beam[1])) #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) @@ -1045,7 +1045,7 @@ void** beam_ops; static BifFunction translate_gc_bif(void* gcf) NOINLINE; static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) NOINLINE; -static BeamInstr* call_error_handler(Process* p, BeamInstr* ip, +static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) NOINLINE; static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity, BeamInstr *I, Uint offs) NOINLINE; @@ -1080,7 +1080,7 @@ static Eterm make_arglist(Process* c_p, Eterm* reg, int a); void init_emulator(void) { - process_main(); + process_main(0, 0); } /* @@ -1108,98 +1108,91 @@ init_emulator(void) #ifdef USE_VM_CALL_PROBES -#define DTRACE_LOCAL_CALL(p, m, f, a) \ +#define DTRACE_LOCAL_CALL(p, mfa) \ if (DTRACE_ENABLED(local_function_entry)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(local_function_entry, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(local_function_entry, process_name, mfa_buf, depth); \ } -#define DTRACE_GLOBAL_CALL(p, m, f, a) \ +#define DTRACE_GLOBAL_CALL(p, mfa) \ if (DTRACE_ENABLED(global_function_entry)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(global_function_entry, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(global_function_entry, process_name, mfa_buf, depth); \ } -#define DTRACE_RETURN(p, m, f, a) \ +#define DTRACE_RETURN(p, mfa) \ if (DTRACE_ENABLED(function_return)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(function_return, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(function_return, process_name, mfa_buf, depth); \ } -#define DTRACE_BIF_ENTRY(p, m, f, a) \ - if (DTRACE_ENABLED(bif_entry)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(bif_entry, process_name, mfa); \ +#define DTRACE_BIF_ENTRY(p, mfa) \ + if (DTRACE_ENABLED(bif_entry)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(bif_entry, process_name, mfa_buf); \ } -#define DTRACE_BIF_RETURN(p, m, f, a) \ - if (DTRACE_ENABLED(bif_return)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(bif_return, process_name, mfa); \ +#define DTRACE_BIF_RETURN(p, mfa) \ + if (DTRACE_ENABLED(bif_return)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(bif_return, process_name, mfa_buf); \ } -#define DTRACE_NIF_ENTRY(p, m, f, a) \ - if (DTRACE_ENABLED(nif_entry)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(nif_entry, process_name, mfa); \ +#define DTRACE_NIF_ENTRY(p, mfa) \ + if (DTRACE_ENABLED(nif_entry)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(nif_entry, process_name, mfa_buf); \ } -#define DTRACE_NIF_RETURN(p, m, f, a) \ - if (DTRACE_ENABLED(nif_return)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(nif_return, process_name, mfa); \ +#define DTRACE_NIF_RETURN(p, mfa) \ + if (DTRACE_ENABLED(nif_return)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(nif_return, process_name, mfa_buf); \ } #define DTRACE_GLOBAL_CALL_FROM_EXPORT(p,e) \ do { \ if (DTRACE_ENABLED(global_function_entry)) { \ BeamInstr* fp = (BeamInstr *) (((Export *) (e))->addressv[erts_active_code_ix()]); \ - DTRACE_GLOBAL_CALL((p), (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); \ + DTRACE_GLOBAL_CALL((p), erts_code_to_codemfa(fp)); \ } \ } while(0) #define DTRACE_RETURN_FROM_PC(p) \ do { \ - BeamInstr* fp; \ - if (DTRACE_ENABLED(function_return) && (fp = find_function_from_pc((p)->cp))) { \ - DTRACE_RETURN((p), (Eterm)fp[0], (Eterm)fp[1], (Uint)fp[2]); \ + ErtsCodeMFA* cmfa; \ + if (DTRACE_ENABLED(function_return) && (cmfa = find_function_from_pc((p)->cp))) { \ + DTRACE_RETURN((p), cmfa); \ } \ } while(0) #else /* USE_VM_PROBES */ -#define DTRACE_LOCAL_CALL(p, m, f, a) do {} while (0) -#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0) +#define DTRACE_LOCAL_CALL(p, mfa) do {} while (0) +#define DTRACE_GLOBAL_CALL(p, mfa) do {} while (0) #define DTRACE_GLOBAL_CALL_FROM_EXPORT(p, e) do {} while (0) -#define DTRACE_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_RETURN(p, mfa) do {} while (0) #define DTRACE_RETURN_FROM_PC(p) do {} while (0) -#define DTRACE_BIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_BIF_RETURN(p, m, f, a) do {} while (0) -#define DTRACE_NIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_BIF_ENTRY(p, mfa) do {} while (0) +#define DTRACE_BIF_RETURN(p, mfa) do {} while (0) +#define DTRACE_NIF_ENTRY(p, mfa) do {} while (0) +#define DTRACE_NIF_RETURN(p, mfa) do {} while (0) #endif /* USE_VM_PROBES */ #ifdef DEBUG @@ -1227,7 +1220,7 @@ init_emulator(void) * the instructions' C labels to the loader. * The second call starts execution of BEAM code. This call never returns. */ -void process_main(void) +void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) { static int init_done = 0; Process* c_p = NULL; @@ -1239,7 +1232,7 @@ void process_main(void) /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC, * in all other cases x0 is used. */ - register Eterm* reg REG_xregs = NULL; + register Eterm* reg REG_xregs = x_reg_array; /* * Top of heap (next free location); grows upwards. @@ -1266,7 +1259,7 @@ void process_main(void) * X registers and floating point registers are located in * scheduler specific data. */ - register FloatDef *freg; + register FloatDef *freg = f_reg_array; /* * For keeping the negative old value of 'reds' when call saving is active. @@ -1326,8 +1319,8 @@ void process_main(void) if (start_time != 0) { Sint64 diff = erts_timestamp_millis() - start_time; if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule) { - BeamInstr *inptr = find_function_from_pc(start_time_i); - BeamInstr *outptr = find_function_from_pc(c_p->i); + ErtsCodeMFA *inptr = find_function_from_pc(start_time_i); + ErtsCodeMFA *outptr = find_function_from_pc(c_p->i); monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff); } } @@ -1352,8 +1345,6 @@ void process_main(void) start_time_i = c_p->i; } - reg = erts_proc_sched_data(c_p)->x_reg_array; - freg = erts_proc_sched_data(c_p)->f_reg_array; ERL_BITS_RELOAD_STATEP(c_p); { int reds; @@ -1403,10 +1394,9 @@ void process_main(void) if (ERTS_PROC_IS_EXITING(c_p)) { strcpy(fun_buf, "<exiting>"); } else { - BeamInstr *fptr = find_function_from_pc(c_p->i); - if (fptr) { - dtrace_fun_decode(c_p, (Eterm)fptr[0], - (Eterm)fptr[1], (Uint)fptr[2], + ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i); + if (cmfa) { + dtrace_fun_decode(c_p, cmfa, NULL, fun_buf); } else { erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), @@ -1587,7 +1577,7 @@ void process_main(void) /* FALL THROUGH */ OpCase(i_call_only_f): { SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } @@ -1599,7 +1589,7 @@ void process_main(void) RESTORE_CP(E); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } @@ -1611,7 +1601,7 @@ void process_main(void) OpCase(i_call_f): { SET_CP(c_p, I+2); SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } @@ -2838,25 +2828,27 @@ do { \ Eterm result; BeamInstr *next; ErlHeapFragment *live_hf_end; + Export *export = (Export*)Arg(0); if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { /* If we have run out of reductions, we do a context switch before calling the bif */ - c_p->arity = ((Export *)Arg(0))->code[2]; - c_p->current = ((Export *)Arg(0))->code; + c_p->arity = GET_BIF_ARITY(export); + c_p->current = &export->info.mfa; goto context_switch3; } - ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(Arg(0)), GET_BIF_ADDRESS(Arg(0))); + ERTS_MSACC_SET_BIF_STATE_CACHED_X( + GET_BIF_MODULE(export), GET_BIF_ADDRESS(export)); - bf = GET_BIF_ADDRESS(Arg(0)); + bf = GET_BIF_ADDRESS(export); PRE_BIF_SWAPOUT(c_p); ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS - 1; if (FCALLS <= 0) { - save_calls(c_p, (Export *) Arg(0)); + save_calls(c_p, export); } PreFetch(1, next); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -2870,7 +2862,7 @@ do { \ ERTS_HOLE_CHECK(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); if (ERTS_IS_GC_DESIRED(c_p)) { - Uint arity = ((Export *)Arg(0))->code[2]; + Uint arity = GET_BIF_ARITY(export); result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity); E = c_p->stop; } @@ -3363,14 +3355,15 @@ do { \ * called from I[-3], I[-2], and I[-1] respectively. */ context_switch_fun: - c_p->arity = I[-1] + 1; + /* Add one for the environment of the fun */ + c_p->arity = erts_code_to_codemfa(I)->arity + 1; goto context_switch2; context_switch: - c_p->arity = I[-1]; + c_p->arity = erts_code_to_codemfa(I)->arity; - context_switch2: /* Entry for fun calls. */ - c_p->current = I-3; /* Pointer to Mod, Func, Arity */ + context_switch2: /* Entry for fun calls. */ + c_p->current = erts_code_to_codemfa(I); context_switch3: @@ -3511,7 +3504,8 @@ do { \ * code[4]: Not used */ HEAVY_SWAPOUT; - I = call_error_handler(c_p, I-3, reg, am_undefined_function); + I = call_error_handler(c_p, erts_code_to_codemfa(I), + reg, am_undefined_function); HEAVY_SWAPIN; if (I) { Goto(*I); @@ -3548,17 +3542,25 @@ do { \ * I[1]: Function pointer to NIF function * I[2]: Pointer to erl_module_nif * I[3]: Function pointer to dirty NIF + * + * This layout is determined by the NifExport struct */ BifFunction vbf; ErlHeapFragment *live_hf_end; + ErtsCodeMFA *codemfa; ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); - DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); - c_p->current = I-3; /* current and vbf set to please handle_error */ + codemfa = erts_code_to_codemfa(I); + + c_p->current = codemfa; /* current and vbf set to please handle_error */ + + DTRACE_NIF_ENTRY(c_p, codemfa); + HEAVY_SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); - bif_nif_arity = I[-1]; + bif_nif_arity = codemfa->arity; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -3585,19 +3587,19 @@ do { \ ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } - DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_NIF_RETURN(c_p, codemfa); goto apply_bif_or_nif_epilogue; OpCase(apply_bif): /* - * At this point, I points to the code[3] in the export entry for + * At this point, I points to the code[0] in the export entry for * the BIF: * - * code[0]: Module - * code[1]: Function - * code[2]: Arity - * code[3]: &&apply_bif - * code[4]: Function pointer to BIF function + * code[-3]: Module + * code[-2]: Function + * code[-1]: Arity + * code[0]: &&apply_bif + * code[1]: Function pointer to BIF function */ if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { @@ -3606,21 +3608,25 @@ do { \ goto context_switch; } - ERTS_MSACC_SET_BIF_STATE_CACHED_X((Eterm)I[-3], (BifFunction)Arg(0)); + codemfa = erts_code_to_codemfa(I); + + ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)Arg(0)); - c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ + + /* In case we apply process_info/1,2 or load_nif/1 */ + c_p->current = codemfa; c_p->i = I; /* In case we apply check_process_code/2. */ c_p->arity = 0; /* To allow garbage collection on ourselves * (check_process_code/2). */ - DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_BIF_ENTRY(c_p, codemfa); SWAPOUT; ERTS_DBG_CHK_REDS(c_p, FCALLS - 1); c_p->fcalls = FCALLS - 1; vbf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); - bif_nif_arity = I[-1]; + bif_nif_arity = codemfa->arity; ASSERT(bif_nif_arity <= 4); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -3642,7 +3648,7 @@ do { \ if (ERTS_MSACC_IS_ENABLED_CACHED_X()) ERTS_MSACC_UPDATE_CACHE_X(); ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - DTRACE_BIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_BIF_RETURN(c_p, codemfa); apply_bif_or_nif_epilogue: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); @@ -3709,8 +3715,9 @@ do { \ goto find_func_info; OpCase(i_func_info_IaaI): { + ErtsCodeInfo *ci = (ErtsCodeInfo*)I; c_p->freason = EXC_FUNCTION_CLAUSE; - c_p->current = I + 2; + c_p->current = &ci->mfa; goto handle_error; } @@ -4701,11 +4708,11 @@ do { \ */ OpCase(return_trace): { - BeamInstr* code = (BeamInstr *) (UWord) E[0]; + ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[0]); SWAPOUT; /* Needed for shared heap */ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - erts_trace_return(c_p, code, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); + erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); SWAPIN; c_p->cp = NULL; @@ -4716,9 +4723,8 @@ do { \ OpCase(i_generic_breakpoint): { BeamInstr real_I; - ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); HEAVY_SWAPOUT; - real_I = erts_generic_breakpoint(c_p, I, reg); + real_I = erts_generic_breakpoint(c_p, erts_code_to_codeinfo(I), reg); HEAVY_SWAPIN; ASSERT(VALID_INSTR(real_I)); Goto(real_I); @@ -4727,7 +4733,7 @@ do { \ OpCase(i_return_time_trace): { BeamInstr *pc = (BeamInstr *) (UWord) E[0]; SWAPOUT; - erts_trace_time_return(c_p, pc); + erts_trace_time_return(c_p, erts_code_to_codeinfo(pc)); SWAPIN; c_p->cp = NULL; SET_I((BeamInstr *) cp_val(E[1])); @@ -4912,16 +4918,18 @@ do { \ * I[ 0]: &&lb_hipe_trap_call * ... remainder of original BEAM code */ - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); + c_p->hipe.u.ncallee = (void(*)(void)) ci->native; ++hipe_trap_count; - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8)); + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8)); } OpCase(hipe_trap_call_closure): { - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); + c_p->hipe.u.ncallee = (void(*)(void)) ci->native; ++hipe_trap_count; - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8)); + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8)); } OpCase(hipe_trap_return): { HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN); @@ -4990,8 +4998,9 @@ do { \ * I[ 0]: &&lb_hipe_call_count * ... remainder of original BEAM code */ - struct hipe_call_count *hcc = (struct hipe_call_count*)I[-4]; - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + struct hipe_call_count *hcc = (struct hipe_call_count*)ci->native; + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); ASSERT(hcc != NULL); ASSERT(VALID_INSTR(hcc->opcode)); ++(hcc->count); @@ -5040,7 +5049,7 @@ do { \ } else { TestHeap(ERTS_SINT64_HEAP_SIZE(ts),0); r(0) = make_big(HTOP); -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) if (ts >= (((Uint64) 1) << 32)) { *HTOP = make_pos_bignum_header(2); BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff)); @@ -5060,7 +5069,7 @@ do { \ OpCase(i_debug_breakpoint): { HEAVY_SWAPOUT; - I = call_error_handler(c_p, I-3, reg, am_breakpoint); + I = call_error_handler(c_p, erts_code_to_codemfa(I), reg, am_breakpoint); HEAVY_SWAPIN; if (I) { Goto(*I); @@ -5133,10 +5142,10 @@ do { \ bif_table[i].name, bif_table[i].arity); bif_export[i] = ep; - ep->code[3] = (BeamInstr) OpCode(apply_bif); - ep->code[4] = (BeamInstr) bif_table[i].f; + ep->beam[0] = (BeamInstr) OpCode(apply_bif); + ep->beam[1] = (BeamInstr) bif_table[i].f; /* XXX: set func info for bifs */ - ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); + ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); } return; @@ -5216,8 +5225,8 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) goto do_dirty_schedule; context_switch: - c_p->arity = I[-1]; - c_p->current = I-3; /* Pointer to Mod, Func, Arity */ + c_p->current = erts_code_to_codemfa(I); /* Pointer to Mod, Func, Arity */ + c_p->arity = c_p->current->arity; { int reds_used; @@ -5342,11 +5351,9 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) if (ERTS_PROC_IS_EXITING(c_p)) { strcpy(fun_buf, "<exiting>"); } else { - BeamInstr *fptr = find_function_from_pc(c_p->i); - if (fptr) { - dtrace_fun_decode(c_p, (Eterm)fptr[0], - (Eterm)fptr[1], (Uint)fptr[2], - NULL, fun_buf); + ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i); + if (cmfa) { + dtrace_fun_decode(c_p, cmfa, NULL, fun_buf); } else { erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), "<unknown/%p>", *I); @@ -5375,16 +5382,22 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) * I[1]: Function pointer to NIF function * I[2]: Pointer to erl_module_nif * I[3]: Function pointer to dirty NIF + * + * This layout is determined by the NifExport struct */ BifFunction vbf; + ErtsCodeMFA *codemfa; ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); - DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); - c_p->current = I-3; /* current and vbf set to please handle_error */ + codemfa = erts_code_to_codemfa(I); + + DTRACE_NIF_ENTRY(c_p, codemfa); + /* current and vbf set to please handle_error */ + c_p->current = codemfa; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); - arity = I[-1]; + arity = codemfa->arity; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -5419,7 +5432,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } - DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_NIF_RETURN(c_p, codemfa); ERTS_HOLE_CHECK(c_p); SWAPIN; I = c_p->i; @@ -5432,31 +5445,14 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) static BifFunction translate_gc_bif(void* gcf) { - if (gcf == erts_gc_length_1) { - return length_1; - } else if (gcf == erts_gc_size_1) { - return size_1; - } else if (gcf == erts_gc_bit_size_1) { - return bit_size_1; - } else if (gcf == erts_gc_byte_size_1) { - return byte_size_1; - } else if (gcf == erts_gc_map_size_1) { - return map_size_1; - } else if (gcf == erts_gc_abs_1) { - return abs_1; - } else if (gcf == erts_gc_float_1) { - return float_1; - } else if (gcf == erts_gc_round_1) { - return round_1; - } else if (gcf == erts_gc_trunc_1) { - return round_1; - } else if (gcf == erts_gc_binary_part_2) { - return binary_part_2; - } else if (gcf == erts_gc_binary_part_3) { - return binary_part_3; - } else { - erts_exit(ERTS_ERROR_EXIT, "bad gc bif"); + const ErtsGcBif* p; + + for (p = erts_gc_bifs; p->bif != 0; p++) { + if (p->gc_bif == gcf) { + return p->bif; + } } + erts_exit(ERTS_ERROR_EXIT, "bad gc bif"); } /* @@ -5616,7 +5612,8 @@ next_catch(Process* c_p, Eterm *reg) { /* Can not follow cp here - code may be unloaded */ BeamInstr *cpp = c_p->cp; if (cpp == beam_exception_trace) { - erts_trace_exception(c_p, cp_val(ptr[0]), + ErtsCodeMFA *mfa = (ErtsCodeMFA*)cp_val(ptr[0]); + erts_trace_exception(c_p, mfa, reg[1], reg[2], ERTS_TRACER_FROM_ETERM(ptr+1)); /* Skip return_trace parameters */ @@ -5644,7 +5641,8 @@ next_catch(Process* c_p, Eterm *reg) { if (is_catch(*ptr) && active_catches) goto found_catch; } if (cp_val(*prev) == beam_exception_trace) { - erts_trace_exception(c_p, cp_val(ptr[0]), + ErtsCodeMFA *mfa = (ErtsCodeMFA*)cp_val(ptr[0]); + erts_trace_exception(c_p, mfa, reg[1], reg[2], ERTS_TRACER_FROM_ETERM(ptr+1)); } @@ -5851,7 +5849,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, for (i = 0; i < BIF_SIZE; i++) { if (bf == bif_table[i].f || bf == bif_table[i].traced) { Export *ep = bif_export[i]; - s->current = ep->code; + s->current = &ep->info.mfa; a = bif_table[i].arity; break; } @@ -5865,7 +5863,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, */ ASSERT(c_p->current); s->current = c_p->current; - a = s->current[2]; + a = s->current->arity; } /* Save first stack entry */ ASSERT(pc); @@ -5890,7 +5888,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, (GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) ) { int a; ASSERT(s->current); - a = s->current[2]; + a = s->current->arity; args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */ /* Save first stack entry */ ASSERT(c_p->cp); @@ -6079,7 +6077,7 @@ build_stacktrace(Process* c_p, Eterm exc) { erts_lookup_function_info(&fi, s->pc, 1); } else if (GET_EXC_INDEX(s->freason) == GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) { - erts_lookup_function_info(&fi, s->current, 1); + erts_lookup_function_info(&fi, erts_codemfa_to_code(s->current), 1); } else { erts_set_current_function(&fi, s->current); } @@ -6090,9 +6088,9 @@ build_stacktrace(Process* c_p, Eterm exc) { * stack at all, default to the initial function * (e.g. spawn_link(erlang, abs, [1])). */ - if (fi.current == NULL) { + if (fi.mfa == NULL) { if (depth <= 0) - erts_set_current_function(&fi, c_p->u.initial); + erts_set_current_function(&fi, &c_p->u.initial); args = am_true; /* Just in case */ } else { args = get_args_from_exc(exc); @@ -6104,10 +6102,10 @@ build_stacktrace(Process* c_p, Eterm exc) { */ stk = stkp = (FunctionInfo *) erts_alloc(ERTS_ALC_T_TMP, depth*sizeof(FunctionInfo)); - heap_size = fi.current ? fi.needed + 2 : 0; + heap_size = fi.mfa ? fi.needed + 2 : 0; for (i = 0; i < depth; i++) { erts_lookup_function_info(stkp, s->trace[i], 1); - if (stkp->current) { + if (stkp->mfa) { heap_size += stkp->needed + 2; stkp++; } @@ -6123,7 +6121,7 @@ build_stacktrace(Process* c_p, Eterm exc) { res = CONS(hp, mfa, res); hp += 2; } - if (fi.current) { + if (fi.mfa) { hp = erts_build_mfa_item(&fi, hp, args, &mfa); res = CONS(hp, mfa, res); } @@ -6133,7 +6131,7 @@ build_stacktrace(Process* c_p, Eterm exc) { } static BeamInstr* -call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) +call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) { Eterm* hp; Export* ep; @@ -6142,13 +6140,14 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) Uint sz; int i; + DBG_TRACE_MFA_P(mfa, "call_error_handler"); /* * Search for the error_handler module. */ ep = erts_find_function(erts_proc_get_error_handler(p), func, 3, erts_active_code_ix()); if (ep == NULL) { /* No error handler */ - p->current = fi; + p->current = mfa; p->freason = EXC_UNDEF; return 0; } @@ -6157,7 +6156,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) * Create a list with all arguments in the x registers. */ - arity = fi[2]; + arity = mfa->arity; sz = 2 * arity; if (HeapWordsLeft(p) < sz) { erts_garbage_collect(p, sz, reg, arity); @@ -6173,8 +6172,8 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) /* * Set up registers for call to error_handler:<func>/3. */ - reg[0] = fi[0]; - reg[1] = fi[1]; + reg[0] = mfa->module; + reg[1] = mfa->function; reg[2] = args; return ep->addressv[erts_active_code_ix()]; } @@ -6234,7 +6233,7 @@ apply_bif_error_adjustment(Process *p, Export *ep, * and apply_last_IP. */ if (I - && ep->code[3] == (BeamInstr) em_apply_bif + && ep->beam[0] == (BeamInstr) em_apply_bif && (ep == bif_export[BIF_error_1] || ep == bif_export[BIF_error_2] || ep == bif_export[BIF_exit_1] @@ -6558,11 +6557,11 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_hibernate)) { + ErtsCodeMFA cmfa = { module, function, arity}; DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); - dtrace_fun_decode(c_p, module, function, arity, - process_name, mfa); - DTRACE2(process_hibernate, process_name, mfa); + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); + dtrace_fun_decode(c_p, &cmfa, process_name, mfa_buf); + DTRACE2(process_hibernate, process_name, mfa_buf); } #endif /* @@ -6609,7 +6608,7 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - c_p->current = bif_export[BIF_hibernate_3]->code; + c_p->current = &bif_export[BIF_hibernate_3]->info.mfa; c_p->flags |= F_HIBERNATE_SCHED; /* Needed also when woken! */ return 1; } @@ -6632,21 +6631,15 @@ call_fun(Process* p, /* Current process. */ if (is_fun_header(hdr)) { ErlFunThing* funp = (ErlFunThing *) fun_val(fun); - ErlFunEntry* fe; - BeamInstr* code_ptr; + ErlFunEntry* fe = funp->fe; + BeamInstr* code_ptr = fe->address; Eterm* var_ptr; - int actual_arity; - unsigned num_free; - - fe = funp->fe; - num_free = funp->num_free; - code_ptr = fe->address; - actual_arity = (int) code_ptr[-1]; + unsigned num_free = funp->num_free; + ErtsCodeMFA *mfa = erts_code_to_codemfa(code_ptr); + int actual_arity = mfa->arity; if (actual_arity == arity+num_free) { - DTRACE_LOCAL_CALL(p, (Eterm)code_ptr[-3], - (Eterm)code_ptr[-2], - code_ptr[-1]); + DTRACE_LOCAL_CALL(p, mfa); if (num_free == 0) { return code_ptr; } else { @@ -6751,10 +6744,10 @@ call_fun(Process* p, /* Current process. */ int actual_arity; ep = *((Export **) (export_val(fun) + 1)); - actual_arity = (int) ep->code[2]; + actual_arity = ep->info.mfa.arity; if (arity == actual_arity) { - DTRACE_GLOBAL_CALL(p, ep->code[0], ep->code[1], (Uint)ep->code[2]); + DTRACE_GLOBAL_CALL(p, &ep->info.mfa); return ep->addressv[erts_active_code_ix()]; } else { /* @@ -6842,9 +6835,6 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) funp->fe = fe; funp->num_free = num_free; funp->creator = p->common.id; -#ifdef HIPE - funp->native_address = fe->native_address; -#endif funp->arity = (int)fe->address[-1] - num_free; for (i = 0; i < num_free; i++) { *hp++ = reg[i]; @@ -7375,15 +7365,15 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) return 1; } - e.code[0] = Mod; - e.code[1] = Name; - e.code[2] = arity; + e.info.mfa.module = Mod; + e.info.mfa.function = Name; + e.info.mfa.arity = arity; if ((ep = export_get(&e)) == NULL) { return 0; } - return ep->addressv[erts_active_code_ix()] == ep->code+3 - && (ep->code[3] == (BeamInstr) em_apply_bif); + return ep->addressv[erts_active_code_ix()] == ep->beam + && (ep->beam[0] == (BeamInstr) em_apply_bif); } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 3f2bdf3f9d..309465bcd3 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -39,11 +39,13 @@ #include "erl_binary.h" #include "erl_zlib.h" #include "erl_map.h" +#include "erl_process_dict.h" #ifdef HIPE #include "hipe_bif0.h" #include "hipe_mode_switch.h" #include "hipe_arch.h" +#include "hipe_load.h" #endif ErlDrvBinary* erts_gzinflate_buffer(char*, int); @@ -480,9 +482,12 @@ static void free_loader_state(Binary* magic); static ErlHeapFragment* new_literal_fragment(Uint size); static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); +#ifdef HIPE static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code, Uint size); + BeamCodeHeader* code_hdr, Uint size, + HipeModule *hipe_code); +#endif static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -537,8 +542,6 @@ static Eterm compilation_info_for_module(Process* p, BeamCodeHeader*); static Eterm md5_of_module(Process* p, BeamCodeHeader*); static Eterm has_native(BeamCodeHeader*); static Eterm native_addresses(Process* p, BeamCodeHeader*); -int patch_funentries(Eterm Patchlist); -int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); static int must_swap_floats; @@ -806,21 +809,21 @@ erts_finish_loading(Binary* magic, Process* c_p, num_exps = export_list_size(code_ix); for (i = 0; i < num_exps; i++) { Export *ep = export_list(i, code_ix); - if (ep == NULL || ep->code[0] != module) { + if (ep == NULL || ep->info.mfa.module != module) { continue; } - if (ep->addressv[code_ix] == ep->code+3) { - if (ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep->addressv[code_ix] == ep->beam) { + if (ep->beam[0] == (BeamInstr) em_apply_bif) { continue; - } else if (ep->code[3] == + } else if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(mod_tab_p->curr.num_traced_exports > 0); - erts_clear_export_break(mod_tab_p, ep->code+3); - ep->addressv[code_ix] = (BeamInstr *) ep->code[4]; - ep->code[4] = 0; + erts_clear_export_break(mod_tab_p, &ep->info); + ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; + ep->beam[1] = 0; } - ASSERT(ep->code[4] == 0); + ASSERT(ep->beam[1] == 0); } } ASSERT(mod_tab_p->curr.num_breakpoints == 0); @@ -842,9 +845,7 @@ erts_finish_loading(Binary* magic, Process* c_p, erts_alloc(ERTS_ALC_T_PREPARED_CODE, sizeof(struct erl_module_instance)); inst_p = mod_tab_p->on_load; - inst_p->nif = 0; - inst_p->num_breakpoints = 0; - inst_p->num_traced_exports = 0; + erts_module_instance_init(inst_p); } inst_p->code_hdr = stp->hdr; @@ -943,6 +944,13 @@ erts_module_for_prepared_code(Binary* magic) LoaderState* stp; if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) { +#ifdef HIPE + HipeLoaderState *hipe_stp; + if ((hipe_stp = hipe_get_loader_state(magic)) + && hipe_stp->text_segment != 0) { + return hipe_stp->module; + } +#endif return NIL; } stp = ERTS_MAGIC_BIN_DATA(magic); @@ -1091,10 +1099,12 @@ loader_state_dtor(Binary* magic) ASSERT(stp->genop_blocks == 0); } +#ifdef HIPE static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code_hdr, Uint size) + BeamCodeHeader* code_hdr, Uint size, + HipeModule *hipe_code) { Module* modp; Eterm retval; @@ -1117,6 +1127,9 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modp->curr.code_hdr = code_hdr; modp->curr.code_length = size; modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code " + "first_hipe_ref = %p", hipe_code->first_hipe_ref); + modp->curr.hipe_code = hipe_code; /* * Update ranges (used for finding a function from a PC value). @@ -1125,6 +1138,7 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, erts_update_ranges((BeamInstr*)modp->curr.code_hdr, size); return NIL; } +#endif static int init_iff_file(LoaderState* stp, byte* code, Uint size) @@ -1420,8 +1434,8 @@ load_import_table(LoaderState* stp) * the BIF function. */ if ((e = erts_active_export_entry(mod, func, arity)) != NULL) { - if (e->code[3] == (BeamInstr) em_apply_bif) { - stp->import[i].bf = (BifFunction) e->code[4]; + if (e->beam[0] == (BeamInstr) em_apply_bif) { + stp->import[i].bf = (BifFunction) e->beam[1]; if (func == am_load_nif && mod == am_erlang && arity == 2) { stp->may_load_nif = 1; } @@ -1514,7 +1528,7 @@ is_bif(Eterm mod, Eterm func, unsigned arity) if (e == NULL) { return 0; } - if (e->code[3] != (BeamInstr) em_apply_bif) { + if (e->beam[0] != (BeamInstr) em_apply_bif) { return 0; } if (mod == am_erlang && func == am_apply && arity == 3) { @@ -1893,7 +1907,7 @@ load_code(LoaderState* stp) * by both the nif functionality and line instructions. */ enum { - FUNC_INFO_SZ = 5 + FUNC_INFO_SZ = sizeof(ErtsCodeInfo) / sizeof(Eterm) }; code = stp->codev; @@ -2532,15 +2546,10 @@ load_code(LoaderState* stp) if (stp->may_load_nif) { const int finfo_ix = ci - FUNC_INFO_SZ; -#ifdef ERTS_DIRTY_SCHEDULERS - enum { MIN_FUNC_SZ = 4 }; -#else - enum { MIN_FUNC_SZ = 3 }; -#endif - if (finfo_ix - last_func_start < MIN_FUNC_SZ && last_func_start) { + if (finfo_ix - last_func_start < BEAM_NIF_MIN_FUNC_SZ && last_func_start) { /* Must make room for call_nif op */ - int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start); - ASSERT(pad > 0 && pad < MIN_FUNC_SZ); + int pad = BEAM_NIF_MIN_FUNC_SZ - (finfo_ix - last_func_start); + ASSERT(pad > 0 && pad < BEAM_NIF_MIN_FUNC_SZ); CodeNeed(pad); sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FUNC_INFO_SZ*sizeof(BeamInstr)); @@ -2565,8 +2574,11 @@ load_code(LoaderState* stp) stp->function = code[ci-2]; stp->arity = code[ci-1]; + /* When this assert is triggered, it is normally a sign that + the size of the ops.tab i_func_info instruction is not + the same as FUNC_INFO_SZ */ ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ); - stp->hdr->functions[function_number] = (BeamInstr*) stp->labels[last_label].patches; + stp->hdr->functions[function_number] = (ErtsCodeInfo*) stp->labels[last_label].patches; offset = function_number; stp->labels[last_label].patches = offset; function_number++; @@ -4031,60 +4043,52 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx) op->next = NULL; return op; } + +static GenOp* +translate_gc_bif(LoaderState* stp, GenOp* op, GenOpArg Bif) +{ + const ErtsGcBif* p; + BifFunction bf; + + bf = stp->import[Bif.val].bf; + for (p = erts_gc_bifs; p->bif != 0; p++) { + if (p->bif == bf) { + op->a[1].type = TAG_u; + op->a[1].val = (BeamInstr) p->gc_bif; + return op; + } + } + + op->op = genop_unsupported_guard_bif_3; + op->arity = 3; + op->a[0].type = TAG_a; + op->a[0].val = stp->import[Bif.val].module; + op->a[1].type = TAG_a; + op->a[1].val = stp->import[Bif.val].function; + op->a[2].type = TAG_u; + op->a[2].val = stp->import[Bif.val].arity; + return op; +} + /* - * Rewrite gc_bifs with one parameter (the common case). Utilized - * in ops.tab to rewrite instructions calling bif's in guards - * to use a garbage collecting implementation. + * Rewrite gc_bifs with one parameter (the common case). */ static GenOp* gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, GenOpArg Src, GenOpArg Dst) { GenOp* op; - BifFunction bf; NEW_GENOP(stp, op); op->next = NULL; - bf = stp->import[Bif.val].bf; - /* The translations here need to have a reverse counterpart in - beam_emu.c:translate_gc_bif for error handling to work properly. */ - if (bf == length_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_length_1; - } else if (bf == size_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_size_1; - } else if (bf == bit_size_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_bit_size_1; - } else if (bf == byte_size_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_byte_size_1; - } else if (bf == map_size_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_map_size_1; - } else if (bf == abs_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_abs_1; - } else if (bf == float_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_float_1; - } else if (bf == round_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_round_1; - } else if (bf == trunc_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_trunc_1; - } else { - op->op = genop_unsupported_guard_bif_3; - op->arity = 3; - op->a[0].type = TAG_a; - op->a[0].val = stp->import[Bif.val].module; - op->a[1].type = TAG_a; - op->a[1].val = stp->import[Bif.val].function; - op->a[2].type = TAG_u; - op->a[2].val = stp->import[Bif.val].arity; - return op; - } op->op = genop_i_gc_bif1_5; op->arity = 5; op->a[0] = Fail; - op->a[1].type = TAG_u; + /* op->a[1] is set by translate_gc_bif() */ op->a[2] = Src; op->a[3] = Live; op->a[4] = Dst; - return op; + return translate_gc_bif(stp, op, Bif); } /* @@ -4095,35 +4099,18 @@ gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, GenOpArg S1, GenOpArg S2, GenOpArg Dst) { GenOp* op; - BifFunction bf; NEW_GENOP(stp, op); op->next = NULL; - bf = stp->import[Bif.val].bf; - /* The translations here need to have a reverse counterpart in - beam_emu.c:translate_gc_bif for error handling to work properly. */ - if (bf == binary_part_2) { - op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_2; - } else { - op->op = genop_unsupported_guard_bif_3; - op->arity = 3; - op->a[0].type = TAG_a; - op->a[0].val = stp->import[Bif.val].module; - op->a[1].type = TAG_a; - op->a[1].val = stp->import[Bif.val].function; - op->a[2].type = TAG_u; - op->a[2].val = stp->import[Bif.val].arity; - return op; - } op->op = genop_i_gc_bif2_6; op->arity = 6; op->a[0] = Fail; - op->a[1].type = TAG_u; + /* op->a[1] is set by translate_gc_bif() */ op->a[2] = Live; op->a[3] = S1; op->a[4] = S2; op->a[5] = Dst; - return op; + return translate_gc_bif(stp, op, Bif); } /* @@ -4134,37 +4121,19 @@ gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, GenOpArg S1, GenOpArg S2, GenOpArg S3, GenOpArg Dst) { GenOp* op; - BifFunction bf; NEW_GENOP(stp, op); op->next = NULL; - bf = stp->import[Bif.val].bf; - /* The translations here need to have a reverse counterpart in - beam_emu.c:translate_gc_bif for error handling to work properly. */ - if (bf == binary_part_3) { - op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_3; - } else { - op->op = genop_unsupported_guard_bif_3; - op->arity = 3; - op->a[0].type = TAG_a; - op->a[0].val = stp->import[Bif.val].module; - op->a[1].type = TAG_a; - op->a[1].val = stp->import[Bif.val].function; - op->a[2].type = TAG_u; - op->a[2].val = stp->import[Bif.val].arity; - return op; - } op->op = genop_ii_gc_bif3_7; op->arity = 7; op->a[0] = Fail; - op->a[1].type = TAG_u; + /* op->a[1] is set by translate_gc_bif() */ op->a[2] = Live; op->a[3] = S1; op->a[4] = S2; op->a[5] = S3; op->a[6] = Dst; - op->next = NULL; - return op; + return translate_gc_bif(stp, op, Bif); } static GenOp* @@ -4386,22 +4355,25 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src, static int hash_internal_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx) { + Eterm key_term; switch (Key.type) { case TAG_a: - *hx = atom_tab(atom_val(Key.val))->slot.bucket.hvalue; - return 1; + key_term = Key.val; + break; case TAG_i: - *hx = Key.val; - return 1; + key_term = make_small(Key.val); + break; case TAG_n: - *hx = make_internal_hash(NIL); - return 1; + key_term = NIL; + break; case TAG_q: - *hx = make_internal_hash(stp->literals[Key.val].term); - return 1; + key_term = stp->literals[Key.val].term; + break; default: return 0; } + *hx = erts_pd_make_hx(key_term); + return 1; } @@ -4556,7 +4528,7 @@ freeze_code(LoaderState* stp) * function table in the beginning of the file. */ - code_hdr->functions[stp->num_functions] = (codev + stp->ci - 1); + code_hdr->functions[stp->num_functions] = (ErtsCodeInfo*)(codev + stp->ci - 1); CHKBLK(ERTS_ALC_T_CODE,code_hdr); /* @@ -4816,16 +4788,16 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) } ep = erts_export_put(stp->module, stp->export[i].function, stp->export[i].arity); - if (!on_load) { - ep->addressv[erts_staging_code_ix()] = address; - } else { + if (on_load) { /* * on_load: Don't make any of the exported functions * callable yet. Keep any function in the current * code callable. */ - ep->code[4] = (BeamInstr) address; + ep->beam[1] = (BeamInstr) address; } + else + ep->addressv[erts_staging_code_ix()] = address; } /* @@ -4871,7 +4843,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) } fe->address = code_ptr; #ifdef HIPE - hipe_set_closure_stub(fe, stp->lambdas[i].num_free); + hipe_set_closure_stub(fe); #endif } } @@ -4882,7 +4854,7 @@ transform_engine(LoaderState* st) { Uint op; int ap; /* Current argument. */ - Uint* restart; /* Where to restart if current match fails. */ + const Uint* restart; /* Where to restart if current match fails. */ GenOpArg var[TE_MAX_VARS]; /* Buffer for variables. */ GenOpArg* rest_args = NULL; int num_rest_args = 0; @@ -4891,7 +4863,7 @@ transform_engine(LoaderState* st) GenOp* instr; GenOp* first = st->genop; GenOp* keep = NULL; - Uint* pc; + const Uint* pc; static Uint restart_fail[1] = {TOP_fail}; ASSERT(gen_opc[first->op].transform != -1); @@ -5002,7 +4974,7 @@ transform_engine(LoaderState* st) if (i >= st->num_imports || st->import[i].bf == NULL) goto restart; if (bif_number != -1 && - bif_export[bif_number]->code[4] != (BeamInstr) st->import[i].bf) { + bif_export[bif_number]->beam[1] != (BeamInstr) st->import[i].bf) { goto restart; } } @@ -5627,18 +5599,16 @@ functions_in_module(Process* p, /* Process whose heap to use. */ hp = HAlloc(p, need); hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { - BeamInstr* func_info = code_hdr->functions[i]; - Eterm name = (Eterm) func_info[3]; - int arity = (int) func_info[4]; + ErtsCodeInfo* ci = code_hdr->functions[i]; Eterm tuple; /* * If the function name is [], this entry is a stub for * a BIF that should be ignored. */ - ASSERT(is_atom(name) || is_nil(name)); - if (is_atom(name)) { - tuple = TUPLE2(hp, name, make_small(arity)); + ASSERT(is_atom(ci->mfa.function) || is_nil(ci->mfa.function)); + if (is_atom(ci->mfa.function)) { + tuple = TUPLE2(hp, ci->mfa.function, make_small(ci->mfa.arity)); hp += 3; result = CONS(hp, tuple, result); hp += 2; @@ -5726,17 +5696,17 @@ native_addresses(Process* p, BeamCodeHeader* code_hdr) hp = HAlloc(p, need); hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { - BeamInstr* func_info = code_hdr->functions[i]; - Eterm name = (Eterm) func_info[3]; - int arity = (int) func_info[4]; + ErtsCodeInfo *ci = code_hdr->functions[i]; Eterm tuple; - ASSERT(is_atom(name) || is_nil(name)); /* [] if BIF stub */ - if (func_info[1] != 0) { - Eterm addr; - ASSERT(is_atom(name)); - addr = erts_bld_uint(&hp, NULL, func_info[1]); - tuple = erts_bld_tuple(&hp, NULL, 3, name, make_small(arity), addr); + ASSERT(is_atom(ci->mfa.function) + || is_nil(ci->mfa.function)); /* [] if BIF stub */ + if (ci->native != 0) { + Eterm addr; + ASSERT(is_atom(ci->mfa.function)); + addr = erts_bld_uint(&hp, NULL, ci->native); + tuple = erts_bld_tuple(&hp, NULL, 3, ci->mfa.function, + make_small(ci->mfa.arity), addr); result = erts_bld_cons(&hp, NULL, tuple, result); } } @@ -5763,11 +5733,11 @@ exported_from_module(Process* p, /* Process whose heap to use. */ for (i = 0; i < num_exps; i++) { Export* ep = export_list(i,code_ix); - if (ep->code[0] == mod) { + if (ep->info.mfa.module == mod) { Eterm tuple; - if (ep->addressv[code_ix] == ep->code+3 && - ep->code[3] == (BeamInstr) em_call_error_handler) { + if (ep->addressv[code_ix] == ep->beam && + ep->beam[0] == (BeamInstr) em_call_error_handler) { /* There is a call to the function, but it does not exist. */ continue; } @@ -5777,7 +5747,8 @@ exported_from_module(Process* p, /* Process whose heap to use. */ hp = HAlloc(p, need); hend = hp + need; } - tuple = TUPLE2(hp, ep->code[1], make_small(ep->code[2])); + tuple = TUPLE2(hp, ep->info.mfa.function, + make_small(ep->info.mfa.arity)); hp += 3; result = CONS(hp, tuple, result); hp += 2; @@ -5851,7 +5822,6 @@ md5_of_module(Process* p, /* Process whose heap to use. */ Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) { - BeamInstr* current = fi->current; Eterm loc = NIL; if (fi->loc != LINE_INVALID_LOCATION) { @@ -5861,7 +5831,7 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) Eterm file_term = NIL; if (file == 0) { - Atom* ap = atom_tab(atom_val(fi->current[0])); + Atom* ap = atom_tab(atom_val(fi->mfa->module)); file_term = buf_to_intlist(&hp, ".erl", 4, NIL); file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, file_term); } else { @@ -5880,10 +5850,12 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) } if (is_list(args) || is_nil(args)) { - *mfa_p = TUPLE4(hp, current[0], current[1], args, loc); + *mfa_p = TUPLE4(hp, fi->mfa->module, fi->mfa->function, + args, loc); } else { - Eterm arity = make_small(current[2]); - *mfa_p = TUPLE4(hp, current[0], current[1], arity, loc); + Eterm arity = make_small(fi->mfa->arity); + *mfa_p = TUPLE4(hp, fi->mfa->module, fi->mfa->function, + arity, loc); } return hp + 5; } @@ -5894,9 +5866,9 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) * the function. */ void -erts_set_current_function(FunctionInfo* fi, BeamInstr* current) +erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa) { - fi->current = current; + fi->mfa = mfa; fi->needed = 5; fi->loc = LINE_INVALID_LOCATION; } @@ -5905,13 +5877,13 @@ erts_set_current_function(FunctionInfo* fi, BeamInstr* current) /* * Returns a pointer to {module, function, arity}, or NULL if not found. */ -BeamInstr* +ErtsCodeMFA* find_function_from_pc(BeamInstr* pc) { FunctionInfo fi; erts_lookup_function_info(&fi, pc, 0); - return fi.current; + return fi.mfa; } /* @@ -6028,24 +6000,21 @@ code_module_md5_1(BIF_ALIST_1) return res; } -#define WORDS_PER_FUNCTION 6 +#ifdef HIPE +#define WORDS_PER_FUNCTION (sizeof(ErtsCodeInfo) / sizeof(UWord) + 1) static BeamInstr* -make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode) +make_stub(ErtsCodeInfo* info, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode) { - fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); - fp[1] = native; - fp[2] = mod; - fp[3] = func; - fp[4] = arity; -#ifdef HIPE - if (native) { - fp[5] = BeamOpCode(op_move_return_n); - hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5); - } -#endif - fp[5] = OpCode; - return fp + WORDS_PER_FUNCTION; + DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", erts_codeinfo_to_code(info)); + ASSERT(WORDS_PER_FUNCTION == 6); + info->op = (BeamInstr) BeamOp(op_i_func_info_IaaI); + info->native = native; + info->mfa.module = mod; + info->mfa.function = func; + info->mfa.arity = arity; + erts_codeinfo_to_code(info)[0] = OpCode; + return erts_codeinfo_to_code(info)+1; } static byte* @@ -6104,22 +6073,17 @@ stub_read_export_table(LoaderState* stp) } static void -stub_final_touch(LoaderState* stp, BeamInstr* fp) +stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci) { unsigned int i; unsigned int n = stp->num_exps; - Eterm mod = fp[2]; - Eterm function = fp[3]; - int arity = fp[4]; -#ifdef HIPE Lambda* lp; -#endif - if (is_bif(mod, function, arity)) { - fp[1] = 0; - fp[2] = 0; - fp[3] = 0; - fp[4] = 0; + if (is_bif(ci->mfa.module, ci->mfa.function, ci->mfa.arity)) { + ci->native = 0; + ci->mfa.module = 0; + ci->mfa.function = 0; + ci->mfa.arity = 0; return; } @@ -6128,9 +6092,14 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) */ for (i = 0; i < n; i++) { - if (stp->export[i].function == function && stp->export[i].arity == arity) { - Export* ep = erts_export_put(mod, function, arity); - ep->addressv[erts_staging_code_ix()] = fp+5; + if (stp->export[i].function == ci->mfa.function && + stp->export[i].arity == ci->mfa.arity) { + Export* ep = erts_export_put(ci->mfa.module, + ci->mfa.function, + ci->mfa.arity); + ep->addressv[erts_staging_code_ix()] = erts_codeinfo_to_code(ci); + DBG_TRACE_MFA_P(&ci->mfa,"set beam stub at %p in export at %p (code_ix=%d)", + erts_codeinfo_to_code(ci), ep, erts_staging_code_ix()); return; } } @@ -6140,16 +6109,14 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) * Search the lambda table to find out which. */ -#ifdef HIPE n = stp->num_lambdas; for (i = 0, lp = stp->lambdas; i < n; i++, lp++) { ErlFunEntry* fe = stp->lambdas[i].fe; - if (lp->function == function && lp->arity == arity) { - fp[5] = (Eterm) BeamOpCode(op_hipe_trap_call_closure); - fe->address = &(fp[5]); + if (lp->function == ci->mfa.function && lp->arity == ci->mfa.arity) { + *erts_codeinfo_to_code(ci) = (Eterm) BeamOpCode(op_hipe_trap_call_closure); + fe->address = erts_codeinfo_to_code(ci); } } -#endif return; } @@ -6158,10 +6125,9 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) [{Adr, Patchtyppe} | Addresses] and the address of a fun_entry. */ -int +static int patch(Eterm Addresses, Uint fe) { -#ifdef HIPE Eterm* listp; Eterm tuple; Eterm* tp; @@ -6197,15 +6163,13 @@ patch(Eterm Addresses, Uint fe) } -#endif return 1; } -int +static int patch_funentries(Eterm Patchlist) { -#ifdef HIPE while (!is_nil(Patchlist)) { Eterm Info; Eterm MFA; @@ -6294,23 +6258,23 @@ patch_funentries(Eterm Patchlist) return 0; } -#endif return 1; /* Signal that all went well */ } - /* * Do a dummy load of a module. No threaded code will be loaded. * Used for loading native code. * Will also patch all references to fun_entries to point to * the new fun_entries created. */ - Eterm -erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) +erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) { Binary* magic; + Binary* hipe_magic; LoaderState* stp; + HipeLoaderState* hipe_stp; + HipeModule *hipe_code; BeamInstr Funcs; BeamInstr Patchlist; Eterm MD5Bin; @@ -6333,8 +6297,12 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) */ magic = erts_alloc_loader_state(); stp = ERTS_MAGIC_BIN_DATA(magic); + hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); - if (is_not_atom(Mod)) { + if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) || + !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val, + hipe_stp = hipe_get_loader_state(hipe_magic)) || + hipe_stp->module == NIL || hipe_stp->text_segment == 0) { goto error; } if (is_not_tuple(Info)) { @@ -6362,7 +6330,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Scan the Beam binary and read the interesting sections. */ - stp->module = Mod; + stp->module = hipe_stp->module; stp->group_leader = p->group_leader; stp->num_functions = n; if (!init_iff_file(stp, bytes, size)) { @@ -6470,20 +6438,17 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Set the pointer and make the stub. Put a return instruction * as the body until we know what kind of trap we should put there. */ - code_hdr->functions[i] = fp; -#ifdef HIPE + code_hdr->functions[i] = (ErtsCodeInfo*)fp; op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */ -#else - op = (Eterm) BeamOpCode(op_move_return_n); -#endif - fp = make_stub(fp, Mod, func, arity, (Uint)native_address, op); + fp = make_stub((ErtsCodeInfo*)fp, hipe_stp->module, func, arity, + (Uint)native_address, op); } /* * Insert the last pointer and the int_code_end instruction. */ - code_hdr->functions[i] = fp; + code_hdr->functions[i] = (ErtsCodeInfo*)fp; *fp++ = (BeamInstr) BeamOp(op_int_code_end); /* @@ -6516,11 +6481,20 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) } /* + * Initialise HiPE module + */ + hipe_code->text_segment = hipe_stp->text_segment; + hipe_code->text_segment_size = hipe_stp->text_segment_size; + hipe_code->data_segment = hipe_stp->data_segment; + hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs; + hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc; + + /* * Insert the module in the module table. */ - rval = stub_insert_new_code(p, 0, p->group_leader, Mod, - code_hdr, code_size); + rval = stub_insert_new_code(p, 0, p->group_leader, hipe_stp->module, + code_hdr, code_size, hipe_code); if (rval != NIL) { goto error; } @@ -6531,23 +6505,74 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) fp = code_base; for (i = 0; i < n; i++) { - stub_final_touch(stp, fp); + stub_final_touch(stp, (ErtsCodeInfo*)fp); fp += WORDS_PER_FUNCTION; } if (patch_funentries(Patchlist)) { + Eterm mod = hipe_stp->module; + /* Prevent code from being freed */ + hipe_stp->text_segment = 0; + hipe_stp->data_segment = 0; + hipe_stp->new_hipe_refs = NULL; + hipe_stp->new_hipe_sdesc = NULL; + erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); - return Mod; + hipe_free_loader_state(hipe_stp); + + return mod; } error: + erts_free(ERTS_ALC_T_HIPE, hipe_code); erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); BIF_ERROR(p, BADARG); } +int erts_commit_hipe_patch_load(Eterm hipe_magic_bin) +{ + Binary* hipe_magic; + HipeLoaderState* hipe_stp; + HipeModule *hipe_code; + Module* modp; + + if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) || + !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val, + hipe_stp = hipe_get_loader_state(hipe_magic)) || + hipe_stp->module == NIL || hipe_stp->text_segment == 0) { + return 0; + } + + modp = erts_get_module(hipe_stp->module, erts_active_code_ix()); + if (!modp) + return 0; + + /* + * Initialise HiPE module + */ + hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); + hipe_code->text_segment = hipe_stp->text_segment; + hipe_code->text_segment_size = hipe_stp->text_segment_size; + hipe_code->data_segment = hipe_stp->data_segment; + hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs; + hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc; + + modp->curr.hipe_code = hipe_code; + + /* Prevent code from being freed */ + hipe_stp->text_segment = 0; + hipe_stp->data_segment = 0; + hipe_stp->new_hipe_refs = NULL; + hipe_stp->new_hipe_sdesc = NULL; + + return 1; +} + #undef WORDS_PER_FUNCTION +#endif /* HIPE */ + static int safe_mul(UWord a, UWord b, UWord* resp) { @@ -6561,3 +6586,46 @@ static int safe_mul(UWord a, UWord b, UWord* resp) } } +#ifdef ENABLE_DBG_TRACE_MFA + +#define MFA_MAX 10 +Eterm dbg_trace_m[MFA_MAX]; +Eterm dbg_trace_f[MFA_MAX]; +Uint dbg_trace_a[MFA_MAX]; +unsigned int dbg_trace_ix = 0; + +void dbg_set_traced_mfa(const char* m, const char* f, Uint a) +{ + unsigned i = dbg_trace_ix++; + ASSERT(i < MFA_MAX); + dbg_trace_m[i] = am_atom_put(m, strlen(m)); + dbg_trace_f[i] = am_atom_put(f, strlen(f)); + dbg_trace_a[i] = a; +} + +int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a) +{ + unsigned int i; + for (i = 0; i < dbg_trace_ix; ++i) { + if (m == dbg_trace_m[i] && + (!f || (f == dbg_trace_f[i] && a == dbg_trace_a[i]))) { + + return i+1; + } + } + return 0; +} + +void dbg_vtrace_mfa(unsigned ix, const char* format, ...) +{ + va_list arglist; + va_start(arglist, format); + ASSERT(--ix < MFA_MAX); + erts_fprintf(stderr, "MFA TRACE %T:%T/%u: ", + dbg_trace_m[ix], dbg_trace_f[ix], (int)dbg_trace_a[ix]); + + erts_vfprintf(stderr, format, arglist); + va_end(arglist); +} + +#endif /* ENABLE_DBG_TRACE_MFA */ diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 9be5e14e40..659b9c303f 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -35,15 +35,7 @@ typedef struct gen_op_entry { int transform; } GenOpEntry; -extern GenOpEntry gen_opc[]; - -#ifdef NO_JUMP_TABLE -#define BeamOp(Op) (Op) -#else -extern void** beam_ops; -#define BeamOp(Op) beam_ops[(Op)] -#endif - +extern const GenOpEntry gen_opc[]; extern BeamInstr beam_debug_apply[]; extern BeamInstr* em_call_error_handler; @@ -115,10 +107,16 @@ typedef struct beam_code_header { * The actual loaded code (for the first function) start just beyond * this table. */ - BeamInstr* functions[1]; + ErtsCodeInfo* functions[1]; }BeamCodeHeader; +#ifdef ERTS_DIRTY_SCHEDULERS +# define BEAM_NIF_MIN_FUNC_SZ 4 +#else +# define BEAM_NIF_MIN_FUNC_SZ 3 +#endif + void erts_release_literal_area(struct ErtsLiteralArea_* literal_area); int erts_is_module_native(BeamCodeHeader* code); void erts_beam_bif_load_init(void); @@ -152,4 +150,34 @@ struct BeamCodeLineTab_ { #define LOC_FILE(Loc) ((Loc) >> 24) #define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) + +/* + * MFA event debug "tracing" usage: + * + * #define ENABLE_DBG_TRACE_MFA + * call dbg_set_traced_mfa("mymod","myfunc",arity) + * for the function(s) to trace, in some init function. + * + * Run and get stderr printouts when interesting things happen to your MFA. + */ +#ifdef ENABLE_DBG_TRACE_MFA + +void dbg_set_traced_mfa(const char* m, const char* f, Uint a); +int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a); +void dbg_vtrace_mfa(unsigned ix, const char* format, ...); +#define DBG_TRACE_MFA(M,F,A,FMT, ...) do {\ + unsigned ix;\ + if ((ix=dbg_is_traced_mfa(M,F,A))) \ + dbg_vtrace_mfa(ix, FMT"\n", ##__VA_ARGS__);\ + }while(0) + +#define DBG_TRACE_MFA_P(MFA, FMT, ...) \ + DBG_TRACE_MFA((MFA)->module, (MFA)->function, (MFA)->arity, FMT, ##__VA_ARGS__) + +#else +# define dbg_set_traced_mfa(M,F,A) +# define DBG_TRACE_MFA(M,F,A,FMT, ...) +# define DBG_TRACE_MFA_P(MFA,FMT, ...) +#endif /* ENABLE_DBG_TRACE_MFA */ + #endif /* _BEAM_LOAD_H */ diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c index 55342a38c6..9b0335e83d 100644 --- a/erts/emulator/beam/beam_ranges.c +++ b/erts/emulator/beam/beam_ranges.c @@ -221,13 +221,13 @@ erts_ranges_sz(void) void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) { - BeamInstr** low; - BeamInstr** high; - BeamInstr** mid; + ErtsCodeInfo** low; + ErtsCodeInfo** high; + ErtsCodeInfo** mid; Range* rp; BeamCodeHeader* hdr; - fi->current = NULL; + fi->mfa = NULL; fi->needed = 5; fi->loc = LINE_INVALID_LOCATION; rp = find_range(pc); @@ -240,12 +240,12 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) high = low + hdr->num_functions; while (low < high) { mid = low + (high-low) / 2; - if (pc < mid[0]) { + if (pc < (BeamInstr*)(mid[0])) { high = mid; - } else if (pc < mid[1]) { - fi->current = mid[0]+2; + } else if (pc < (BeamInstr*)(mid[1])) { + fi->mfa = &mid[0]->mfa; if (full_info) { - BeamInstr** fp = hdr->functions; + ErtsCodeInfo** fp = hdr->functions; int idx = mid - fp; lookup_loc(fi, pc, hdr, idx); } @@ -316,7 +316,7 @@ lookup_loc(FunctionInfo* fi, const BeamInstr* pc, file = LOC_FILE(fi->loc); if (file == 0) { /* Special case: Module name with ".erl" appended */ - Atom* mod_atom = atom_tab(atom_val(fi->current[0])); + Atom* mod_atom = atom_tab(atom_val(fi->mfa->module)); fi->needed += 2*(mod_atom->len+4); } else { Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index d9048065c8..d886c2985e 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -3869,7 +3869,24 @@ BIF_RETTYPE garbage_collect_0(BIF_ALIST_0) { FLAGS(BIF_P) |= F_NEED_FULLSWEEP; erts_garbage_collect(BIF_P, 0, NULL, 0); - BIF_RET(am_true); + return am_true; +} + +/* + * Pass atom 'minor' for relaxed generational GC run. This is only + * recommendation, major run may still be chosen by VM. + * Pass atom 'major' for default behaviour - major GC run (fullsweep) + */ +BIF_RETTYPE +erts_internal_garbage_collect_1(BIF_ALIST_1) +{ + switch (BIF_ARG_1) { + case am_minor: break; + case am_major: FLAGS(BIF_P) |= F_NEED_FULLSWEEP; break; + default: BIF_ERROR(BIF_P, BADARG); + } + erts_garbage_collect(BIF_P, 0, NULL, 0); + return am_true; } /**********************************************************************/ @@ -4946,13 +4963,13 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, int i; sys_memset((void *) ep, 0, sizeof(Export)); for (i=0; i<ERTS_NUM_CODE_IX; i++) { - ep->addressv[i] = &ep->code[3]; + ep->addressv[i] = ep->beam; } - ep->code[0] = m; - ep->code[1] = f; - ep->code[2] = a; - ep->code[3] = (BeamInstr) em_apply_bif; - ep->code[4] = (BeamInstr) bif; + ep->info.mfa.module = m; + ep->info.mfa.function = f; + ep->info.mfa.arity = a; + ep->beam[0] = (BeamInstr) em_apply_bif; + ep->beam[1] = (BeamInstr) bif; } void erts_init_bif(void) diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 2203182a0d..0c85e19ef0 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -159,7 +159,7 @@ do { \ #define ERTS_BIF_ERROR_TRAPPED0(Proc, Reason, Bif) \ do { \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ return THE_NON_VALUE; \ } while (0) @@ -167,7 +167,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ return THE_NON_VALUE; \ } while (0) @@ -176,7 +176,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ return THE_NON_VALUE; \ @@ -186,7 +186,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ reg[2] = (Eterm) (A2); \ @@ -202,7 +202,7 @@ do { \ #define ERTS_BIF_PREP_ERROR_TRAPPED0(Ret, Proc, Reason, Bif) \ do { \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -210,7 +210,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -219,7 +219,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ (Ret) = THE_NON_VALUE; \ @@ -229,7 +229,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ reg[2] = (Eterm) (A2); \ diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 80db4eb6ff..32600f4338 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -23,22 +23,24 @@ # # Lines starting with '#' are ignored. # -# <bif-decl> ::= "bif" <bif> <C-name>* | "ubif" <bif> <C-name>* +# <bif-decl> ::= "bif" <bif> <C-name>* | +# "ubif" <bif> <C-name>* | +# "gcbif" <bif> <C-name>* # <bif> ::= <module> ":" <name> "/" <arity> # -# "ubif" is an unwrapped bif, i.e. a bif without a trace wrapper, -# or rather; the trace entry point in the export entry is the same -# as the normal entry point, and no trace wrapper is generated. +# ubif: Use for operators and guard BIFs that never build anything +# on the heap (such as tuple_size/1) and operators. # -# Important: Use "ubif" for guard BIFs and operators; use "bif" for ordinary BIFs. +# gcbif: Use for guard BIFs that may build on the heap (such as abs/1). +# +# bif: Use for all other BIFs. # # Add new BIFs to the end of the file. # -# Note: Guards BIFs require special support in the compiler (to be able to actually -# call them from within a guard). +# Note: Guards BIFs usually require special support in the compiler. # -ubif erlang:abs/1 +gcbif erlang:abs/1 bif erlang:adler32/1 bif erlang:adler32/2 bif erlang:adler32_combine/3 @@ -62,11 +64,12 @@ bif erlang:exit/1 bif erlang:exit/2 bif erlang:external_size/1 bif erlang:external_size/2 -ubif erlang:float/1 +gcbif erlang:float/1 bif erlang:float_to_list/1 bif erlang:float_to_list/2 bif erlang:fun_info/2 bif erlang:garbage_collect/0 +bif erts_internal:garbage_collect/1 bif erlang:get/0 bif erlang:get/1 bif erlang:get_keys/1 @@ -79,7 +82,7 @@ bif erlang:phash2/2 ubif erlang:hd/1 bif erlang:integer_to_list/1 bif erlang:is_alive/0 -ubif erlang:length/1 +gcbif erlang:length/1 bif erlang:link/1 bif erlang:list_to_atom/1 bif erlang:list_to_binary/1 @@ -126,10 +129,10 @@ bif erlang:processes/0 bif erlang:put/2 bif erlang:register/2 bif erlang:registered/0 -ubif erlang:round/1 +gcbif erlang:round/1 ubif erlang:self/0 bif erlang:setelement/3 -ubif erlang:size/1 +gcbif erlang:size/1 bif erlang:spawn/3 bif erlang:spawn_link/3 bif erlang:split_binary/2 @@ -139,7 +142,7 @@ bif erlang:term_to_binary/2 bif erlang:throw/1 bif erlang:time/0 ubif erlang:tl/1 -ubif erlang:trunc/1 +gcbif erlang:trunc/1 bif erlang:tuple_to_list/1 bif erlang:universaltime/0 bif erlang:universaltime_to_localtime/1 @@ -162,7 +165,7 @@ bif erts_internal:port_connect/2 bif erts_internal:request_system_task/3 bif erts_internal:request_system_task/4 -bif erts_internal:check_process_code/2 +bif erts_internal:check_process_code/1 bif erts_internal:map_to_tuple_keys/1 bif erts_internal:term_type/1 @@ -464,8 +467,8 @@ bif erlang:list_to_existing_atom/1 # ubif erlang:is_bitstring/1 ubif erlang:tuple_size/1 -ubif erlang:byte_size/1 -ubif erlang:bit_size/1 +gcbif erlang:byte_size/1 +gcbif erlang:bit_size/1 bif erlang:list_to_bitstring/1 bif erlang:bitstring_to_list/1 @@ -517,8 +520,8 @@ bif erlang:binary_to_term/2 # # The searching/splitting/substituting thingies # -ubif erlang:binary_part/2 -ubif erlang:binary_part/3 +gcbif erlang:binary_part/2 +gcbif erlang:binary_part/3 bif binary:compile_pattern/1 bif binary:match/2 @@ -610,7 +613,7 @@ bif os:unsetenv/1 bif re:inspect/2 ubif erlang:is_map/1 -ubif erlang:map_size/1 +gcbif erlang:map_size/1 bif maps:to_list/1 bif maps:find/2 bif maps:get/2 @@ -657,6 +660,16 @@ bif erlang:has_prepared_code_on_load/1 bif maps:take/2 # +# New in 20.0 +# + +gcbif erlang:floor/1 +gcbif erlang:ceil/1 +bif math:floor/1 +bif math:ceil/1 +bif math:fmod/2 + +# # Obsolete # diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index dfbe1ced47..6e1e94b95b 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -230,9 +230,9 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) * Display the initial function name */ erts_print(to, to_arg, "Spawned as: %T:%T/%bpu\n", - p->u.initial[INITIAL_MOD], - p->u.initial[INITIAL_FUN], - p->u.initial[INITIAL_ARI]); + p->u.initial.module, + p->u.initial.function, + p->u.initial.arity); if (p->current != NULL) { if (running) { @@ -241,9 +241,9 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) erts_print(to, to_arg, "Current call: "); } erts_print(to, to_arg, "%T:%T/%bpu\n", - p->current[0], - p->current[1], - p->current[2]); + p->current->module, + p->current->function, + p->current->arity); } erts_print(to, to_arg, "Spawned by: %T\n", p->parent); @@ -290,9 +290,9 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) erts_print(to, to_arg, "timeout"); else erts_print(to, to_arg, "%T:%T/%bpu\n", - scb->ct[j]->code[0], - scb->ct[j]->code[1], - scb->ct[j]->code[2]); + scb->ct[j]->info.mfa.module, + scb->ct[j]->info.mfa.function, + scb->ct[j]->info.mfa.arity); } erts_print(to, to_arg, "\n"); } diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index 584a605771..1b451bf921 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -56,12 +56,49 @@ # endif # include "sys.h" #endif + +#include "beam_opcodes.h" + struct process; #define ERTS_NUM_CODE_IX 3 typedef unsigned ErtsCodeIndex; +typedef struct ErtsCodeMFA_ { + Eterm module; + Eterm function; + Uint arity; +} ErtsCodeMFA; + +/* + * The ErtsCodeInfo structure is used both in the Export entry + * and in the code as the function header. + */ + +/* If you change the size of this, you also have to update the code + in ops.tab to reflect the new func_info size */ +typedef struct ErtsCodeInfo_ { + BeamInstr op; /* OpCode(i_func_info) */ + BeamInstr native; /* Used by hipe and trace to store extra data */ + ErtsCodeMFA mfa; +} ErtsCodeInfo; + +/* Get the code associated with a ErtsCodeInfo ptr. */ +ERTS_GLB_INLINE +BeamInstr *erts_codeinfo_to_code(ErtsCodeInfo *ci); + +/* Get the ErtsCodeInfo for from a code ptr. */ +ERTS_GLB_INLINE +ErtsCodeInfo *erts_code_to_codeinfo(BeamInstr *I); + +/* Get the code associated with a ErtsCodeMFA ptr. */ +ERTS_GLB_INLINE +BeamInstr *erts_codemfa_to_code(ErtsCodeMFA *mfa); + +/* Get the ErtsCodeMFA from a code ptr. */ +ERTS_GLB_INLINE +ErtsCodeMFA *erts_code_to_codemfa(BeamInstr *I); /* Called once at emulator initialization. */ @@ -121,10 +158,47 @@ void erts_abort_staging_code_ix(void); int erts_has_code_write_permission(void); #endif - +/* module/function/arity can be NIL/NIL/-1 when the MFA is pointing to some + invalid code, for instance unloaded_fun. */ +#define ASSERT_MFA(MFA) \ + ASSERT((is_atom((MFA)->module) || is_nil((MFA)->module)) && \ + (is_atom((MFA)->function) || is_nil((MFA)->function)) && \ + (((MFA)->arity >= 0 && (MFA)->arity < 1024) || (MFA)->arity == -1)) #if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE +BeamInstr *erts_codeinfo_to_code(ErtsCodeInfo *ci) +{ + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op); + ASSERT_MFA(&ci->mfa); + return (BeamInstr*)(ci + 1); +} + +ERTS_GLB_INLINE +ErtsCodeInfo *erts_code_to_codeinfo(BeamInstr *I) +{ + ErtsCodeInfo *ci = ((ErtsCodeInfo *)(((char *)(I)) - sizeof(ErtsCodeInfo))); + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op); + ASSERT_MFA(&ci->mfa); + return ci; +} + +ERTS_GLB_INLINE +BeamInstr *erts_codemfa_to_code(ErtsCodeMFA *mfa) +{ + ASSERT_MFA(mfa); + return (BeamInstr*)(mfa + 1); +} + +ERTS_GLB_INLINE +ErtsCodeMFA *erts_code_to_codemfa(BeamInstr *I) +{ + ErtsCodeMFA *mfa = ((ErtsCodeMFA *)(((char *)(I)) - sizeof(ErtsCodeMFA))); + ASSERT_MFA(mfa); + return mfa; +} + extern erts_smp_atomic32_t the_active_code_index; extern erts_smp_atomic32_t the_staging_code_index; diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index ccc4cbad43..c4dcd6a3cc 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -40,7 +40,8 @@ static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*, int); /* * Copy object "obj" to process p. */ -Eterm copy_object_x(Eterm obj, Process* to, Uint extra) { +Eterm copy_object_x(Eterm obj, Process* to, Uint extra) +{ if (!is_immed(obj)) { Uint size = size_object(obj); Eterm* hp = HAllocX(to, size, extra); @@ -70,33 +71,46 @@ Eterm copy_object_x(Eterm obj, Process* to, Uint extra) { * Return the "flat" size of the object. */ -Uint size_object(Eterm obj) +#define in_literal_purge_area(PTR) \ + (lit_purge_ptr && ( \ + (lit_purge_ptr <= (PTR) && \ + (PTR) < (lit_purge_ptr + lit_purge_sz)))) + +Uint size_object_x(Eterm obj, erts_literal_area_t *litopt) { Uint sum = 0; Eterm* ptr; int arity; + Eterm *lit_purge_ptr = litopt ? litopt->lit_purge_ptr : NULL; + Uint lit_purge_sz = litopt ? litopt->lit_purge_sz : 0; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); #endif - DECLARE_ESTACK(s); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size_object %p\n", mypid, obj)); for (;;) { switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: - sum += 2; ptr = list_val(obj); + if (litopt && erts_is_literal(obj,ptr) && !in_literal_purge_area(ptr)) { + goto pop_next; + } + sum += 2; obj = *ptr++; if (!IS_CONST(obj)) { ESTACK_PUSH(s, obj); - } + } obj = *ptr; break; case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val(obj); + Eterm hdr; + ptr = boxed_val(obj); + if (litopt && erts_is_literal(obj,ptr) && !in_literal_purge_area(ptr)) { + goto pop_next; + } + hdr = *ptr; ASSERT(is_header(hdr)); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: @@ -279,10 +293,6 @@ do { \ #define COUNT_OFF_HEAP (0) -#define IN_LITERAL_PURGE_AREA(info, ptr) \ - ((info)->range_ptr && ( \ - (info)->range_ptr <= (ptr) && \ - (ptr) < ((info)->range_ptr + (info)->range_sz))) /* * Return the real size of an object and find sharing information * This currently returns the same as erts_debug:size/1. @@ -599,7 +609,7 @@ cleanup: /* * Copy a structure to a heap. */ -Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz) +Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz, erts_literal_area_t *litopt) { char* hstart; Uint hsize; @@ -616,6 +626,8 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint Eterm hdr; Eterm *hend; int i; + Eterm *lit_purge_ptr = litopt ? litopt->lit_purge_ptr : NULL; + Uint lit_purge_sz = litopt ? litopt->lit_purge_sz : 0; #ifdef DEBUG Eterm org_obj = obj; Uint org_sz = sz; @@ -651,7 +663,6 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint L_copy: while (hp != htop) { obj = *hp; - switch (primary_tag(obj)) { case TAG_PRIMARY_IMMED1: hp++; @@ -667,6 +678,10 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint L_copy_list: tailp = argp; + if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) { + *tailp = obj; + goto L_copy; + } for (;;) { tp = tailp; elem = CAR(objp); @@ -674,18 +689,23 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint hbot -= 2; CAR(hbot) = elem; tailp = &CDR(hbot); - } - else { + } else { CAR(htop) = elem; tailp = &CDR(htop); htop += 2; } *tp = make_list(tailp - 1); obj = CDR(objp); + if (!is_list(obj)) { break; } objp = list_val(obj); + + if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) { + *tailp = obj; + goto L_copy; + } } switch (primary_tag(obj)) { case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy; @@ -695,7 +715,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint "%s, line %d: Internal error in copy_struct: 0x%08x\n", __FILE__, __LINE__,obj); } - + case TAG_PRIMARY_BOXED: if (ErtsInArea(boxed_val(obj),hstart,hsize)) { hp++; @@ -705,6 +725,10 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint L_copy_boxed: objp = boxed_val(obj); + if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) { + *argp = obj; + break; + } hdr = *objp; switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: @@ -765,7 +789,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint extra_bytes = 1; } else { extra_bytes = 0; - } + } real_size = size+extra_bytes; objp = binary_val(real_bin); if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) { @@ -780,7 +804,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint } else { ProcBin* from = (ProcBin *) objp; ProcBin* to; - + ASSERT(thing_subtag(*objp) == REFC_BINARY_SUBTAG); if (from->flags) { erts_emasculate_writable_binary(from); @@ -900,6 +924,12 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz = hend - hbot; } else { #ifdef DEBUG + if (!eq(org_obj, res)) { + erts_exit(ERTS_ABORT_EXIT, + "Internal error in copy_struct() when copying %T:" + " not equal to copy %T\n", + org_obj, res); + } if (htop != hbot) erts_exit(ERTS_ABORT_EXIT, "Internal error in copy_struct() when copying %T:" @@ -1036,6 +1066,8 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) Uint e; unsigned sz; Eterm* ptr; + Eterm *lit_purge_ptr = info->lit_purge_ptr; + Uint lit_purge_sz = info->lit_purge_sz; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); #endif @@ -1081,7 +1113,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) /* off heap list pointers are copied verbatim */ if (erts_is_literal(obj,ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); - if (IN_LITERAL_PURGE_AREA(info,ptr)) + if (in_literal_purge_area(ptr)) info->literal_size += size_object(obj); goto pop_next; } @@ -1132,7 +1164,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) /* off heap pointers to boxes are copied verbatim */ if (erts_is_literal(obj,ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); - if (IN_LITERAL_PURGE_AREA(info,ptr)) + if (in_literal_purge_area(ptr)) info->literal_size += size_object(obj); goto pop_next; } @@ -1298,6 +1330,8 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, Eterm* resp; Eterm *hbot, *hend; unsigned remaining; + Eterm *lit_purge_ptr = info->lit_purge_ptr; + Uint lit_purge_sz = info->lit_purge_sz; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); Eterm saved_obj = obj; @@ -1347,11 +1381,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = list_val(obj); /* off heap list pointers are copied verbatim */ if (erts_is_literal(obj,ptr)) { - if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + if (!in_literal_purge_area(ptr)) { *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL); /* copy literal */ hbot -= bsz; } goto cleanup_next; @@ -1415,11 +1449,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ if (erts_is_literal(obj,ptr)) { - if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + if (!in_literal_purge_area(ptr)) { *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL); /* copy literal */ hbot -= bsz; } goto cleanup_next; @@ -1923,7 +1957,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite if (is_header(val)) { struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp; ASSERT(ptr + header_arity(val) < end); - MOVE_BOXED(ptr, val, hp, &dummy_ref); + MOVE_BOXED(ptr, val, hp, &dummy_ref); switch (val & _HEADER_SUBTAG_MASK) { case REFC_BINARY_SUBTAG: case FUN_SUBTAG: diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index d79245e0e6..990edb274e 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -25,6 +25,7 @@ /* define this to get a lot of debug output */ /* #define ERTS_DIST_MSG_DBG */ +/* #define ERTS_RAW_DIST_MSG_DBG */ #ifdef HAVE_CONFIG_H # include "config.h" diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 214fb1f2af..40a45c961f 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -373,10 +373,16 @@ set_default_exec_alloc_opts(struct au_init *ip) ip->init.util.rmbcmt = 0; ip->init.util.acul = 0; +# ifdef ERTS_HAVE_EXEC_MMAPPER ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc; ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc; ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc; ip->init.util.mseg_mmapper = &erts_exec_mmapper; +# else + ip->init.util.mseg_alloc = &erts_alcu_exec_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_exec_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_exec_mseg_dealloc; +# endif } #endif /* ERTS_ALC_A_EXEC */ @@ -1571,7 +1577,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) break; case 'X': if (has_prefix("scs", argv[i]+3)) { -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER init->mseg.exec_mmap.scs = #endif get_mb_value(argv[i]+6, argv, &i); @@ -2852,7 +2858,7 @@ erts_allocator_info(fmtfn_t to, void *arg) erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n"); erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis); #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER erts_print(to, arg, "=allocator:erts_mmap.exec_mmap\n"); erts_mmap_info(&erts_exec_mmapper, &to, arg, NULL, NULL, &emis); #endif @@ -3010,7 +3016,7 @@ erts_allocator_options(void *proc) #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) terms[length++] = ERTS_MAKE_AM("literal_mmap"); #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER terms[length++] = ERTS_MAKE_AM("exec_mmap"); #endif features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; @@ -3102,7 +3108,7 @@ reply_alloc_info(void *vair) # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) struct erts_mmap_info_struct mmap_info_literal; # endif -# ifdef ERTS_ALC_A_EXEC +# ifdef ERTS_HAVE_EXEC_MMAPPER struct erts_mmap_info_struct mmap_info_exec; # endif #endif @@ -3232,7 +3238,7 @@ reply_alloc_info(void *vair) erts_bld_atom(hpp,szp,"literal_mmap"), ainfo); # endif -# ifdef ERTS_ALC_A_EXEC +# ifdef ERTS_HAVE_EXEC_MMAPPER ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list); ainfo = (air->only_sz ? NIL : @@ -4129,12 +4135,20 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *dptr; Uint size; + int free_pattern = n; ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE); - sys_memset((void *) dptr, n, size + FENCE_SZ); +#ifdef ERTS_ALC_A_EXEC +# if defined(__i386__) || defined(__x86_64__) + if (ERTS_ALC_T2A(ERTS_ALC_N2T(n)) == ERTS_ALC_A_EXEC) { + free_pattern = 0x0f; /* Illegal instruction */ + } +# endif +#endif + sys_memset((void *) dptr, free_pattern, size + FENCE_SZ); (*real_af->free)(n, real_af->extra, dptr); diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 230ca6ccbb..d346f8c8b6 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -907,7 +907,9 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, #elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) -/* Used by literal allocator that has its own mmapper (super carrier) */ +/* For allocators that have their own mmapper (super carrier), + * like literal_alloc and exec_alloc on amd64 + */ void* erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { @@ -948,6 +950,50 @@ erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, } #endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ +#if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) + +/* + * For exec_alloc on non-amd64 that just need memory with PROT_EXEC + */ +void* +erts_alcu_exec_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +{ + void* res = erts_alcu_mseg_alloc(allctr, size_p, flags); + + if (res) { + int r = mprotect(res, *size_p, PROT_EXEC | PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + return res; +} + +void* +erts_alcu_exec_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) +{ + void *res; + + if (seg && old_size) { + int r = mprotect(seg, old_size, PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p); + if (res) { + int r = mprotect(res, *new_size_p, PROT_EXEC | PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + return res; +} + +void +erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) +{ + int r = mprotect(seg, size, PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + erts_alcu_mseg_dealloc(allctr, seg, size, flags); +} +#endif /* ERTS_ALC_A_EXEC && !ERTS_HAVE_EXEC_MMAPPER */ + #endif /* HAVE_ERTS_MSEG */ void* @@ -1361,6 +1407,7 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + ASSERT(size == fix->type_size); res = fix->list; if (res) { @@ -1372,8 +1419,6 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) fix_cpool_check_shrink(allctr, type, fix, NULL); return res; } - if (size < 2*sizeof(UWord)) - size += sizeof(UWord); if (size >= allctr->sbc_threshold) { Block_t *blk; blk = create_carrier(allctr, size, CFLG_SBC); @@ -1493,6 +1538,7 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + ASSERT(size == fix->type_size); ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); fix->u.nocpool.used++; @@ -1515,8 +1561,6 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); return res; } - if (size < 2*sizeof(UWord)) - size += sizeof(UWord); if (fix->u.nocpool.limit < fix->u.nocpool.used) fix->u.nocpool.limit = fix->u.nocpool.used; if (fix->u.nocpool.max_used < fix->u.nocpool.used) @@ -6039,7 +6083,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) goto error; allctr->min_block_size = UNIT_CEILING(allctr->min_block_size + sizeof(FreeBlkFtr_t)); -#if ERTS_SMP +#ifdef ERTS_SMP if (init->tpref) { Uint sz = ABLK_HDR_SZ; sz += (init->fix ? @@ -6090,18 +6134,18 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef USE_THREADS if (init->ts) { allctr->thread_safe = 1; - + #ifdef ERTS_ENABLE_LOCK_COUNT erts_mtx_init_x_opt(&allctr->mutex, "alcu_allocator", make_small(allctr->alloc_no), - ERTS_LCNT_LT_ALLOC,1); + ERTS_LCNT_LT_ALLOC); #else erts_mtx_init_x(&allctr->mutex, "alcu_allocator", - make_small(allctr->alloc_no),1); + make_small(allctr->alloc_no)); #endif /*ERTS_ENABLE_LOCK_COUNT*/ - + #ifdef DEBUG allctr->debug.saved_tid = 0; #endif diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 81180382af..f570703f25 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -210,6 +210,12 @@ void* erts_alcu_mmapper_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); void erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); # endif + +# if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) +void* erts_alcu_exec_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); +void* erts_alcu_exec_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); +void erts_alcu_exec_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); +# endif #endif /* HAVE_ERTS_MSEG */ void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign); diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index ef77201544..587859e413 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1099,7 +1099,7 @@ void erts_ddll_increment_port_count(DE_Handle *dh) void erts_ddll_decrement_port_count(DE_Handle *dh) { assert_drv_list_locked(); -#if DEBUG +#ifdef DEBUG ASSERT(erts_smp_atomic32_dec_read_nob(&dh->port_count) >= 0); #else erts_smp_atomic32_dec_nob(&dh->port_count); diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index b42d2dc28b..458315f293 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -141,6 +141,39 @@ BIF_RETTYPE trunc_1(BIF_ALIST_1) BIF_RET(res); } +BIF_RETTYPE floor_1(BIF_ALIST_1) +{ + Eterm res; + FloatDef f; + + if (is_not_float(BIF_ARG_1)) { + if (is_integer(BIF_ARG_1)) + BIF_RET(BIF_ARG_1); + BIF_ERROR(BIF_P, BADARG); + } + GET_DOUBLE(BIF_ARG_1, f); + res = double_to_integer(BIF_P, floor(f.fd)); + BIF_RET(res); +} + +BIF_RETTYPE ceil_1(BIF_ALIST_1) +{ + Eterm res; + FloatDef f; + + /* check arg */ + if (is_not_float(BIF_ARG_1)) { + if (is_integer(BIF_ARG_1)) + BIF_RET(BIF_ARG_1); + BIF_ERROR(BIF_P, BADARG); + } + /* get the float */ + GET_DOUBLE(BIF_ARG_1, f); + + res = double_to_integer(BIF_P, ceil(f.fd)); + BIF_RET(res); +} + BIF_RETTYPE round_1(BIF_ALIST_1) { Eterm res; @@ -621,6 +654,38 @@ Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live) reg, live); } +Eterm erts_gc_floor_1(Process* p, Eterm* reg, Uint live) +{ + Eterm arg; + FloatDef f; + + arg = reg[live]; + if (is_not_float(arg)) { + if (is_integer(arg)) { + return arg; + } + BIF_ERROR(p, BADARG); + } + GET_DOUBLE(arg, f); + return gc_double_to_integer(p, floor(f.fd), reg, live); +} + +Eterm erts_gc_ceil_1(Process* p, Eterm* reg, Uint live) +{ + Eterm arg; + FloatDef f; + + arg = reg[live]; + if (is_not_float(arg)) { + if (is_integer(arg)) { + return arg; + } + BIF_ERROR(p, BADARG); + } + GET_DOUBLE(arg, f); + return gc_double_to_integer(p, ceil(f.fd), reg, live); +} + static Eterm gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live) { diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 88a052cad7..009d2b72d3 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1114,9 +1114,9 @@ process_info_aux(Process *BIF_P, case am_initial_call: hp = HAlloc(BIF_P, 3+4); res = TUPLE3(hp, - rp->u.initial[INITIAL_MOD], - rp->u.initial[INITIAL_FUN], - make_small(rp->u.initial[INITIAL_ARI])); + rp->u.initial.module, + rp->u.initial.function, + make_small(rp->u.initial.arity)); hp += 4; break; @@ -1563,9 +1563,9 @@ process_info_aux(Process *BIF_P, term = am_timeout; else { term = TUPLE3(hp, - scb->ct[j]->code[0], - scb->ct[j]->code[1], - make_small(scb->ct[j]->code[2])); + scb->ct[j]->info.mfa.module, + scb->ct[j]->info.mfa.function, + make_small(scb->ct[j]->info.mfa.arity)); hp += 4; } list = CONS(hp, term, list); @@ -1614,10 +1614,10 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) if (rp->current == NULL) { erts_lookup_function_info(&fi, rp->i, full_info); - rp->current = fi.current; + rp->current = fi.mfa; } else if (full_info) { erts_lookup_function_info(&fi, rp->i, full_info); - if (fi.current == NULL) { + if (fi.mfa == NULL) { /* Use the current function without location info */ erts_set_current_function(&fi, rp->current); } @@ -1633,9 +1633,9 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) * instead if it can be looked up. */ erts_lookup_function_info(&fi2, rp->cp, full_info); - if (fi2.current) { + if (fi2.mfa) { fi = fi2; - rp->current = fi2.current; + rp->current = fi2.mfa; } } @@ -1650,8 +1650,9 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) hp = erts_build_mfa_item(&fi, hp, am_true, &res); } else { hp = HAlloc(BIF_P, 3+4); - res = TUPLE3(hp, rp->current[0], - rp->current[1], make_small(rp->current[2])); + res = TUPLE3(hp, rp->current->module, + rp->current->function, + make_small(rp->current->arity)); hp += 4; } *hpp = hp; @@ -1692,7 +1693,7 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp) heap_size = 3; for (i = 0; i < depth; i++) { erts_lookup_function_info(stkp, s->trace[i], 1); - if (stkp->current) { + if (stkp->mfa) { heap_size += stkp->needed + 2; stkp++; } @@ -2391,7 +2392,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) ERTS_ATOM_ENC_LATIN1, 1), erts_bld_uint(hpp, hszp, - opc[i].count)), + erts_instr_count[i])), res); } @@ -2859,6 +2860,12 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("ets_limit",BIF_ARG_1)) { BIF_RET(make_small(erts_db_get_max_tabs())); } + else if (ERTS_IS_ATOM_STR("atom_limit",BIF_ARG_1)) { + BIF_RET(make_small(erts_get_atom_limit())); + } + else if (ERTS_IS_ATOM_STR("atom_count",BIF_ARG_1)) { + BIF_RET(make_small(atom_table_size())); + } else if (ERTS_IS_ATOM_STR("tolerant_timeofday",BIF_ARG_1)) { if (erts_has_time_correction() && erts_time_offset_state() == ERTS_TIME_OFFSET_FINAL) { @@ -2883,27 +2890,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(AM_tag); #endif } - else if (ERTS_IS_ATOM_STR("check_process_code",BIF_ARG_1)) { - Eterm terms[3]; - Sint length = 1; - Uint sz = 0; - Eterm *hp, res; - DECL_AM(direct_references); - - terms[0] = AM_direct_references; -#if !defined(ERTS_NEW_PURGE_STRATEGY) - { - DECL_AM(indirect_references); - terms[1] = AM_indirect_references; - terms[2] = am_copy_literals; - length = 3; - } -#endif - erts_bld_list(NULL, &sz, length, terms); - hp = HAlloc(BIF_P, sz); - res = erts_bld_list(&hp, NULL, length, terms); - BIF_RET(res); - } BIF_ERROR(BIF_P, BADARG); } @@ -3248,7 +3234,7 @@ fun_info_2(BIF_ALIST_2) break; case am_module: hp = HAlloc(p, 3); - val = exp->code[0]; + val = exp->info.mfa.module; break; case am_new_index: hp = HAlloc(p, 3); @@ -3276,11 +3262,11 @@ fun_info_2(BIF_ALIST_2) break; case am_arity: hp = HAlloc(p, 3); - val = make_small(exp->code[2]); + val = make_small(exp->info.mfa.arity); break; case am_name: hp = HAlloc(p, 3); - val = exp->code[1]; + val = exp->info.mfa.function; break; default: goto error; @@ -3306,7 +3292,9 @@ fun_info_mfa_1(BIF_ALIST_1) } else if (is_export(fun)) { Export* exp = (Export *) ((UWord) (export_val(fun))[1]); hp = HAlloc(p, 4); - BIF_RET(TUPLE3(hp,exp->code[0],exp->code[1],make_small(exp->code[2]))); + BIF_RET(TUPLE3(hp,exp->info.mfa.module, + exp->info.mfa.function, + make_small(exp->info.mfa.arity))); } BIF_ERROR(p, BADARG); } @@ -3563,8 +3551,14 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("DbTable_words", BIF_ARG_1)) { /* Used by ets_SUITE (stdlib) */ size_t words = (sizeof(DbTable) + sizeof(Uint) - 1)/sizeof(Uint); - BIF_RET(make_small((Uint) words)); + Eterm* hp = HAlloc(BIF_P ,3); + BIF_RET(TUPLE2(hp, make_small((Uint) words), + erts_ets_hash_sizeof_ext_segtab())); } + else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) { + /* Used by ets_SUITE (stdlib) */ + BIF_RET(erts_ets_get_meta_state(BIF_P)); + } else if (ERTS_IS_ATOM_STR("check_io_debug", BIF_ARG_1)) { /* Used by driver_SUITE (emulator) */ Uint sz, *szp; @@ -4298,6 +4292,10 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } BIF_RET(am_ok); } + else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) { + /* Used by ets_SUITE (stdlib) */ + BIF_RET(erts_ets_restore_meta_state(BIF_P, BIF_ARG_2)); + } } BIF_ERROR(BIF_P, BADARG); diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c index aecb8bf0c1..a594ec1493 100644 --- a/erts/emulator/beam/erl_bif_op.c +++ b/erts/emulator/beam/erl_bif_op.c @@ -260,7 +260,7 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2) } else if (is_export(arg1)) { Export* exp = (Export *) (export_val(arg1)[1]); - if (exp->code[2] == (Uint) arity) { + if (exp->info.mfa.arity == (Uint) arity) { BIF_RET(am_true); } } diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 0627526d7e..a480754bdb 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -125,7 +125,6 @@ erts_internal_trace_pattern_3(BIF_ALIST_3) static Eterm trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) { - DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */ int i; int matches = -1; int specified = 0; @@ -308,30 +307,30 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) } matches = 0; } else if (is_tuple(MFA)) { + ErtsCodeMFA mfa; Eterm *tp = tuple_val(MFA); if (tp[0] != make_arityval(3)) { goto error; } - mfa[0] = tp[1]; - mfa[1] = tp[2]; - mfa[2] = tp[3]; - if (!is_atom(mfa[0]) || !is_atom(mfa[1]) || - (!is_small(mfa[2]) && mfa[2] != am_Underscore)) { + if (!is_atom(tp[1]) || !is_atom(tp[2]) || + (!is_small(tp[3]) && tp[3] != am_Underscore)) { goto error; } - for (i = 0; i < 3 && mfa[i] != am_Underscore; i++, specified++) { + for (i = 0; i < 3 && tp[i+1] != am_Underscore; i++, specified++) { /* Empty loop body */ } for (i = specified; i < 3; i++) { - if (mfa[i] != am_Underscore) { + if (tp[i+1] != am_Underscore) { goto error; } } - if (is_small(mfa[2])) { - mfa[2] = signed_val(mfa[2]); + mfa.module = tp[1]; + mfa.function = tp[2]; + if (specified == 3) { + mfa.arity = signed_val(tp[3]); } - matches = erts_set_trace_pattern(p, mfa, specified, + matches = erts_set_trace_pattern(p, &mfa, specified, match_prog_set, match_prog_set, on, flags, meta_tracer, 0); } else if (is_atom(MFA)) { @@ -343,7 +342,6 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) error: MatchSetUnref(match_prog_set); - UnUseTmpHeap(3,p); ERTS_TRACER_CLEAR(&meta_tracer); @@ -975,13 +973,14 @@ static int function_is_traced(Process *p, Export e; Export* ep; BeamInstr* pc; + ErtsCodeInfo *ci; /* First look for an export entry */ - e.code[0] = mfa[0]; - e.code[1] = mfa[1]; - e.code[2] = mfa[2]; + e.info.mfa.module = mfa[0]; + e.info.mfa.function = mfa[1]; + e.info.mfa.arity = mfa[2]; if ((ep = export_get(&e)) != NULL) { - pc = ep->code+3; + pc = ep->beam; if (ep->addressv[erts_active_code_ix()] == pc && *pc != (BeamInstr) em_call_error_handler) { @@ -990,17 +989,17 @@ static int function_is_traced(Process *p, ASSERT(*pc == (BeamInstr) em_apply_bif || *pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); - if (erts_is_trace_break(pc, ms, 0)) { + if (erts_is_trace_break(&ep->info, ms, 0)) { return FUNC_TRACE_GLOBAL_TRACE; } - if (erts_is_trace_break(pc, ms, 1)) { + if (erts_is_trace_break(&ep->info, ms, 1)) { r |= FUNC_TRACE_LOCAL_TRACE; } - if (erts_is_mtrace_break(pc, ms_meta, tracer_pid_meta)) { + if (erts_is_mtrace_break(&ep->info, ms_meta, tracer_pid_meta)) { r |= FUNC_TRACE_META_TRACE; } - if (erts_is_time_break(p, pc, call_time)) { + if (erts_is_time_break(p, &ep->info, call_time)) { r |= FUNC_TRACE_TIME_TRACE; } return r ? r : FUNC_TRACE_UNTRACED; @@ -1008,15 +1007,15 @@ static int function_is_traced(Process *p, } /* OK, now look for breakpoint tracing */ - if ((pc = erts_find_local_func(mfa)) != NULL) { + if ((ci = erts_find_local_func(&e.info.mfa)) != NULL) { int r = - (erts_is_trace_break(pc, ms, 1) + (erts_is_trace_break(ci, ms, 1) ? FUNC_TRACE_LOCAL_TRACE : 0) - | (erts_is_mtrace_break(pc, ms_meta, tracer_pid_meta) + | (erts_is_mtrace_break(ci, ms_meta, tracer_pid_meta) ? FUNC_TRACE_META_TRACE : 0) - | (erts_is_count_break(pc, count) + | (erts_is_count_break(ci, count) ? FUNC_TRACE_COUNT_TRACE : 0) - | (erts_is_time_break(p, pc, call_time) + | (erts_is_time_break(p, ci, call_time) ? FUNC_TRACE_TIME_TRACE : 0); return r ? r : FUNC_TRACE_UNTRACED; @@ -1357,7 +1356,7 @@ trace_info_event(Process* p, Eterm event, Eterm key) #undef FUNC_TRACE_LOCAL_TRACE int -erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, +erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, Binary* match_prog_set, Binary *meta_match_prog_set, int on, struct trace_pattern_flags flags, ErtsTracer meta_tracer, int is_blocking) @@ -1377,22 +1376,23 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, n = finish_bp.e.matched; for (i = 0; i < n; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + ErtsCodeInfo *ci = fp[i].ci; + BeamInstr* pc = erts_codeinfo_to_code(ci); + Export* ep = ErtsContainerStruct(ci, Export, info); if (on && !flags.breakpoint) { /* Turn on global call tracing */ if (ep->addressv[code_ix] != pc) { fp[i].mod->curr.num_traced_exports++; #ifdef DEBUG - pc[-5] = (BeamInstr) BeamOp(op_i_func_info_IaaI); + ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); #endif - pc[0] = (BeamInstr) BeamOp(op_jump_f); - pc[1] = (BeamInstr) ep->addressv[code_ix]; + ep->beam[0] = (BeamInstr) BeamOp(op_jump_f); + ep->beam[1] = (BeamInstr) ep->addressv[code_ix]; } - erts_set_call_trace_bif(pc, match_prog_set, 0); + erts_set_call_trace_bif(ci, match_prog_set, 0); if (ep->addressv[code_ix] != pc) { - pc[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint); + ep->beam[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint); } } else if (!on && flags.breakpoint) { /* Turn off breakpoint tracing -- nothing to do here. */ @@ -1401,9 +1401,9 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, * Turn off global tracing, either explicitly or implicitly * before turning on breakpoint tracing. */ - erts_clear_call_trace_bif(pc, 0); - if (pc[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - pc[0] = (BeamInstr) BeamOp(op_jump_f); + erts_clear_call_trace_bif(ci, 0); + if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { + ep->beam[0] = (BeamInstr) BeamOp(op_jump_f); } } } @@ -1413,68 +1413,76 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, */ for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - int j; - + if (!ExportIsBuiltIn(ep)) { continue; } - + if (bif_table[i].f == bif_table[i].traced) { /* Trace wrapper same as regular function - untraceable */ continue; } - - for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) { - /* Empty loop body */ - } - if (j == specified) { - BeamInstr* pc = (BeamInstr *)bif_export[i]->code + 3; - if (! flags.breakpoint) { /* Export entry call trace */ - if (on) { - erts_clear_call_trace_bif(pc, 1); - erts_clear_mtrace_bif(pc); - erts_set_call_trace_bif(pc, match_prog_set, 0); - } else { /* off */ - erts_clear_call_trace_bif(pc, 0); - } - matches++; - } else { /* Breakpoint call trace */ - int m = 0; - - if (on) { - if (flags.local) { - erts_clear_call_trace_bif(pc, 0); - erts_set_call_trace_bif(pc, match_prog_set, 1); - m = 1; - } - if (flags.meta) { - erts_set_mtrace_bif(pc, meta_match_prog_set, - meta_tracer); - m = 1; - } - if (flags.call_time) { - erts_set_time_trace_bif(pc, on); - /* I don't want to remove any other tracers */ - m = 1; - } - } else { /* off */ - if (flags.local) { - erts_clear_call_trace_bif(pc, 1); - m = 1; - } - if (flags.meta) { - erts_clear_mtrace_bif(pc); - m = 1; - } - if (flags.call_time) { - erts_clear_time_trace_bif(pc); - m = 1; - } - } - matches += m; - } - } + switch (specified) { + case 3: + if (mfa->arity != ep->info.mfa.arity) + continue; + case 2: + if (mfa->function != ep->info.mfa.function) + continue; + case 1: + if (mfa->module != ep->info.mfa.module) + continue; + case 0: + break; + default: + ASSERT(0); + } + + if (! flags.breakpoint) { /* Export entry call trace */ + if (on) { + erts_clear_call_trace_bif(&ep->info, 1); + erts_clear_mtrace_bif(&ep->info); + erts_set_call_trace_bif(&ep->info, match_prog_set, 0); + } else { /* off */ + erts_clear_call_trace_bif(&ep->info, 0); + } + matches++; + } else { /* Breakpoint call trace */ + int m = 0; + + if (on) { + if (flags.local) { + erts_clear_call_trace_bif(&ep->info, 0); + erts_set_call_trace_bif(&ep->info, match_prog_set, 1); + m = 1; + } + if (flags.meta) { + erts_set_mtrace_bif(&ep->info, meta_match_prog_set, + meta_tracer); + m = 1; + } + if (flags.call_time) { + erts_set_time_trace_bif(&ep->info, on); + /* I don't want to remove any other tracers */ + m = 1; + } + } else { /* off */ + if (flags.local) { + erts_clear_call_trace_bif(&ep->info, 1); + m = 1; + } + if (flags.meta) { + erts_clear_mtrace_bif(&ep->info); + m = 1; + } + if (flags.call_time) { + erts_clear_time_trace_bif(&ep->info); + m = 1; + } + } + matches += m; + } } /* @@ -1676,10 +1684,9 @@ install_exp_breakpoints(BpFunctions* f) Uint i; for (i = 0; i < ne; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - ep->addressv[code_ix] = pc; + ep->addressv[code_ix] = ep->beam; } } @@ -1692,14 +1699,13 @@ uninstall_exp_breakpoints(BpFunctions* f) Uint i; for (i = 0; i < ne; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - if (ep->addressv[code_ix] != pc) { + if (ep->addressv[code_ix] != ep->beam) { continue; } - ASSERT(*pc == (BeamInstr) BeamOp(op_jump_f)); - ep->addressv[code_ix] = (BeamInstr *) ep->code[4]; + ASSERT(ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)); + ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; } } @@ -1712,15 +1718,14 @@ clean_export_entries(BpFunctions* f) Uint i; for (i = 0; i < ne; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - if (ep->addressv[code_ix] == pc) { + if (ep->addressv[code_ix] == ep->beam) { continue; } - if (*pc == (BeamInstr) BeamOp(op_jump_f)) { - ep->code[3] = (BeamInstr) 0; - ep->code[4] = (BeamInstr) 0; + if (ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)) { + ep->beam[0] = (BeamInstr) 0; + ep->beam[1] = (BeamInstr) 0; } } } @@ -1732,11 +1737,11 @@ setup_bif_trace(void) for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - GenericBp* g = (GenericBp *) ep->fake_op_func_info_for_hipe[1]; + GenericBp* g = (GenericBp *) ep->info.native; if (g) { if (ExportIsBuiltIn(ep)) { - ASSERT(ep->code[4]); - ep->code[4] = (BeamInstr) bif_table[i].traced; + ASSERT(ep->beam[1]); + ep->beam[1] = (BeamInstr) bif_table[i].traced; } } } @@ -1750,12 +1755,11 @@ reset_bif_trace(void) for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - BeamInstr* pc = ep->code+3; - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ep->info.native; if (g && g->data[active].flags == 0) { if (ExportIsBuiltIn(ep)) { - ASSERT(ep->code[4]); - ep->code[4] = (BeamInstr) bif_table[i].f; + ASSERT(ep->beam[1]); + ep->beam[1] = (BeamInstr) bif_table[i].f; } } } diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index dceadc46f4..fd092015bd 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -3978,6 +3978,36 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt) return list; } +/* + * For testing only + * Retreive meta table size state + */ +Eterm erts_ets_get_meta_state(Process* p) +{ + Eterm* hp = HAlloc(p, 3); + return TUPLE2(hp, + erts_ets_hash_get_memstate(p, &meta_pid_to_tab->hash), + erts_ets_hash_get_memstate(p, &meta_pid_to_fixed_tab->hash)); +} +/* + * For testing only + * Restore a previously retrieved meta table size state. + * We do this to suppress failed memory checks + * caused by the hysteresis of meta tables grow/shrink limits. + */ +Eterm erts_ets_restore_meta_state(Process* p, Eterm meta_state) +{ + Eterm* tv; + Eterm* hp; + if (!is_tuple_arity(meta_state, 2)) + return am_badarg; + + tv = tuple_val(meta_state); + hp = HAlloc(p, 3); + return TUPLE2(hp, + erts_ets_hash_restore_memstate(&meta_pid_to_tab->hash, tv[1]), + erts_ets_hash_restore_memstate(&meta_pid_to_fixed_tab->hash, tv[2])); +} #ifdef HARDDEBUG /* Here comes some debug functions */ diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index b0508f2e74..852440ff31 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -90,6 +90,8 @@ extern Export ets_select_continue_exp; extern erts_smp_atomic_t erts_ets_misc_mem_size; Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt); +Eterm erts_ets_get_meta_state(Process* p); +Eterm erts_ets_restore_meta_state(Process* p, Eterm target_state); Uint erts_db_get_max_tabs(void); @@ -267,7 +269,5 @@ erts_db_free_nt(ErtsAlcType_t type, void *ptr, Uint size) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#undef ERTS_DB_ALC_MEM_UPDATE_ - #endif /* #if defined(ERTS_WANT_DB_INTERNAL__) && !defined(ERTS_HAVE_DB_INTERNAL__) */ diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 390369fdb9..b2a231467b 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -84,25 +84,28 @@ #include "erl_db_hash.h" -#ifdef MYDEBUG /* Will fail test case ets_SUITE:memory */ -# define IF_DEBUG(x) x -# define MY_ASSERT(x) ASSERT(x) -#else -# define IF_DEBUG(x) -# define MY_ASSERT(x) -#endif - /* * The following symbols can be manipulated to "tune" the linear hash array */ -#define CHAIN_LEN 6 /* Medium bucket chain len */ +#define GROW_LIMIT(NACTIVE) ((NACTIVE)*1) +#define SHRINK_LIMIT(NACTIVE) ((NACTIVE) / 2) -/* Number of slots per segment */ -#define SEGSZ_EXP 8 -#define SEGSZ (1 << SEGSZ_EXP) -#define SEGSZ_MASK (SEGSZ-1) +/* +** We want the first mandatory segment to be small (to reduce minimal footprint) +** and larger extra segments (to reduce number of alloc/free calls). +*/ -#define NSEG_1 2 /* Size of first segment table (must be at least 2) */ +/* Number of slots in first segment */ +#define FIRST_SEGSZ_EXP 8 +#define FIRST_SEGSZ (1 << FIRST_SEGSZ_EXP) +#define FIRST_SEGSZ_MASK (FIRST_SEGSZ - 1) + +/* Number of slots per extra segment */ +#define EXT_SEGSZ_EXP 11 +#define EXT_SEGSZ (1 << EXT_SEGSZ_EXP) +#define EXT_SEGSZ_MASK (EXT_SEGSZ-1) + +#define NSEG_1 (ErtsSizeofMember(DbTableHash,first_segtab) / sizeof(struct segment*)) #define NSEG_2 256 /* Size of second segment table */ #define NSEG_INC 128 /* Number of segments to grow after that */ @@ -123,7 +126,9 @@ #define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive)) #define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems)) -#define BUCKET(tb, i) SEGTAB(tb)[(i) >> SEGSZ_EXP]->buckets[(i) & SEGSZ_MASK] +#define SLOT_IX_TO_SEG_IX(i) (((i)+(EXT_SEGSZ-FIRST_SEGSZ)) >> EXT_SEGSZ_EXP) + +#define BUCKET(tb, i) SEGTAB(tb)[SLOT_IX_TO_SEG_IX(i)]->buckets[(i) & EXT_SEGSZ_MASK] /* * When deleting a table, the number of records to delete. @@ -189,6 +194,7 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix, #ifdef ERTS_SMP # define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1) # define GET_LOCK(tb,hval) (&(tb)->locks->lck_vec[(hval) & DB_HASH_LOCK_MASK].lck) +# define GET_LOCK_MAYBE(tb,hval) ((tb)->common.is_thread_safe ? NULL : GET_LOCK(tb,hval)) /* Fine grained read lock */ static ERTS_INLINE erts_smp_rwmtx_t* RLOCK_HASH(DbTableHash* tb, HashValue hval) @@ -318,27 +324,23 @@ struct mp_info { /* A table segment */ struct segment { - HashDbTerm* buckets[SEGSZ]; -#ifdef MYDEBUG - int is_ext_segment; -#endif + HashDbTerm* buckets[1]; }; +#define SIZEOF_SEGMENT(N) \ + (offsetof(struct segment,buckets) + sizeof(HashDbTerm*)*(N)) -/* A segment that also contains a segment table */ -struct ext_segment { - struct segment s; /* The segment itself. Must be first */ - +/* An extended segment table */ +struct ext_segtab { +#ifdef ERTS_SMP + ErtsThrPrgrLaterOp lop; +#endif struct segment** prev_segtab; /* Used when table is shrinking */ - int nsegs; /* Size of segtab */ + int prev_nsegs; /* Size of prev_segtab */ + int nsegs; /* Size of this segtab */ struct segment* segtab[1]; /* The segment table */ }; -#define SIZEOF_EXTSEG(NSEGS) \ - (offsetof(struct ext_segment,segtab) + sizeof(struct segment*)*(NSEGS)) - -#if defined(DEBUG) || defined(VALGRIND) -# define EXTSEG(SEGTAB_PTR) \ - ((struct ext_segment*) (((char*)(SEGTAB_PTR)) - offsetof(struct ext_segment,segtab))) -#endif +#define SIZEOF_EXT_SEGTAB(NSEGS) \ + (offsetof(struct ext_segtab,segtab) + sizeof(struct segment*)*(NSEGS)) static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb, @@ -348,49 +350,22 @@ static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb, erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) segtab); else erts_smp_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab); -#ifdef VALGRIND - tb->top_ptr_to_segment_with_active_segtab = EXTSEG(segtab); -#endif } -/* How the table segments relate to each other: - - ext_segment: ext_segment: "plain" segment - #=================# #================# #=============# - | bucket[0] |<--+ +------->| bucket[256] | +->| bucket[512] | - | bucket[1] | | | | [257] | | | [513] | - : : | | : : | : : - | bucket[255] | | | | [511] | | | [767] | - |-----------------| | | |----------------| | #=============# - | prev_segtab=NULL| | | +--<---prev_segtab | | - | nsegs = 2 | | | | | nsegs = 256 | | -+->| segtab[0] -->-------+---|---|--<---segtab[0] |<-+ | -| | segtab[1] -->-----------+---|--<---segtab[1] | | | -| #=================# | | segtab[2] -->-----|--+ ext_segment: -| | : : | #================# -+----------------<---------------+ | segtab[255] ->----|----->| bucket[255*256]| - #================# | | | - | : : - | |----------------| - +----<---prev_segtab | - : : -*/ - /* ** Forward decl's (static functions) */ -static struct ext_segment* alloc_ext_seg(DbTableHash* tb, unsigned seg_ix, - struct segment** old_segtab); -static int alloc_seg(DbTableHash *tb); +static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix); +static void alloc_seg(DbTableHash *tb); static int free_seg(DbTableHash *tb, int free_records); static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr, HashDbTerm *list); static HashDbTerm* search_list(DbTableHash* tb, Eterm key, HashValue hval, HashDbTerm *list); -static void shrink(DbTableHash* tb, int nactive); -static void grow(DbTableHash* tb, int nactive); +static void shrink(DbTableHash* tb, int nitems); +static void grow(DbTableHash* tb, int nitems); static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, Uint sz, DbTableHash*); static int analyze_pattern(DbTableHash *tb, Eterm pattern, @@ -463,9 +438,10 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle); static ERTS_INLINE void try_shrink(DbTableHash* tb) { int nactive = NACTIVE(tb); - if (nactive > SEGSZ && NITEMS(tb) < (nactive * CHAIN_LEN) + int nitems = NITEMS(tb); + if (nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive) && !IS_FIXED(tb)) { - shrink(tb, nactive); + shrink(tb, nitems); } } @@ -662,16 +638,20 @@ int db_create_hash(Process *p, DbTable *tbl) { DbTableHash *tb = &tbl->hash; - erts_smp_atomic_init_nob(&tb->szm, SEGSZ_MASK); - erts_smp_atomic_init_nob(&tb->nactive, SEGSZ); + erts_smp_atomic_init_nob(&tb->szm, FIRST_SEGSZ_MASK); + erts_smp_atomic_init_nob(&tb->nactive, FIRST_SEGSZ); erts_smp_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL); erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL); - SET_SEGTAB(tb, alloc_ext_seg(tb,0,NULL)->segtab); + SET_SEGTAB(tb, tb->first_segtab); tb->nsegs = NSEG_1; - tb->nslots = SEGSZ; + tb->nslots = FIRST_SEGSZ; + tb->first_segtab[0] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG, + (DbTable *) tb, + SIZEOF_SEGMENT(FIRST_SEGSZ)); + sys_memset(tb->first_segtab[0], 0, SIZEOF_SEGMENT(FIRST_SEGSZ)); - erts_smp_atomic_init_nob(&tb->is_resizing, 0); #ifdef ERTS_SMP + erts_smp_atomic_init_nob(&tb->is_resizing, 0); if (tb->common.type & DB_FINE_LOCKED) { erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; int i; @@ -686,7 +666,7 @@ int db_create_hash(Process *p, DbTable *tbl) erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt, "db_hash_slot", make_small(i)); } - /* This important property is needed to guarantee that the buckets + /* This important property is needed to guarantee the two buckets * involved in a grow/shrink operation it protected by the same lock: */ ASSERT(erts_smp_atomic_read_nob(&tb->nactive) % DB_HASH_LOCK_CNT == 0); @@ -862,8 +842,8 @@ Lnew: WUNLOCK_HASH(lck); { int nactive = NACTIVE(tb); - if (nitems > nactive * (CHAIN_LEN+1) && !IS_FIXED(tb)) { - grow(tb, nactive); + if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { + grow(tb, nitems); } } CHECK_TABLES(); @@ -1514,6 +1494,7 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, match_list = CONS(hp, match_res, match_list); ++got; } + --num_left; } current = current->next; } @@ -1531,7 +1512,6 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, } } else { /* Key is variable */ - --num_left; if ((slot_ix=next_slot(tb,slot_ix,&lck)) == 0) { slot_ix = -1; @@ -2250,12 +2230,12 @@ static int db_free_table_continue_hash(DbTable *tbl) done /= 2; while(tb->nslots != 0) { - free_seg(tb, 1); + done += 1 + EXT_SEGSZ/64 + free_seg(tb, 1); /* * If we have done enough work, get out here. */ - if (++done >= (DELETE_RECORD_LIMIT / CHAIN_LEN / SEGSZ)) { + if (done >= DELETE_RECORD_LIMIT) { return 0; /* Not done */ } } @@ -2414,90 +2394,81 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern, return DB_ERROR_NONE; } -static struct ext_segment* alloc_ext_seg(DbTableHash* tb, unsigned seg_ix, - struct segment** old_segtab) +static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix) { - int nsegs; - struct ext_segment* eseg; + struct segment** old_segtab = SEGTAB(tb); + int nsegs = 0; + struct ext_segtab* est; + ASSERT(seg_ix >= NSEG_1); switch (seg_ix) { - case 0: nsegs = NSEG_1; break; - case 1: nsegs = NSEG_2; break; - default: nsegs = seg_ix + NSEG_INC; break; - } - eseg = (struct ext_segment*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, - (DbTable *) tb, - SIZEOF_EXTSEG(nsegs)); - ASSERT(eseg != NULL); - sys_memset(&eseg->s, 0, sizeof(struct segment)); - IF_DEBUG(eseg->s.is_ext_segment = 1); - eseg->prev_segtab = old_segtab; - eseg->nsegs = nsegs; - if (old_segtab) { - ASSERT(nsegs > tb->nsegs); - sys_memcpy(eseg->segtab, old_segtab, tb->nsegs*sizeof(struct segment*)); - } + case NSEG_1: nsegs = NSEG_2; break; + default: nsegs = seg_ix + NSEG_INC; break; + } + ASSERT(nsegs > tb->nsegs); + est = (struct ext_segtab*) erts_db_alloc(ERTS_ALC_T_DB_SEG, + (DbTable *) tb, + SIZEOF_EXT_SEGTAB(nsegs)); + est->nsegs = nsegs; + est->prev_segtab = old_segtab; + est->prev_nsegs = tb->nsegs; + sys_memcpy(est->segtab, old_segtab, tb->nsegs*sizeof(struct segment*)); #ifdef DEBUG - sys_memset(&eseg->segtab[seg_ix], 0, (nsegs-seg_ix)*sizeof(struct segment*)); + sys_memset(&est->segtab[seg_ix], 0, (nsegs-seg_ix)*sizeof(struct segment*)); #endif - eseg->segtab[seg_ix] = &eseg->s; - return eseg; + return est; } /* Extend table with one new segment */ -static int alloc_seg(DbTableHash *tb) +static void alloc_seg(DbTableHash *tb) { - int seg_ix = tb->nslots >> SEGSZ_EXP; - - if (seg_ix+1 == tb->nsegs) { /* New segtab needed (extended segment) */ - struct segment** segtab = SEGTAB(tb); - struct ext_segment* seg = alloc_ext_seg(tb, seg_ix, segtab); - if (seg == NULL) return 0; - segtab[seg_ix] = &seg->s; - /* We don't use the new segtab until next call (see "shrink race") */ - } - else { /* Just a new plain segment */ - struct segment** segtab; - if (seg_ix == tb->nsegs) { /* Time to start use segtab from last call */ - struct ext_segment* eseg; - eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1]; - MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment); - SET_SEGTAB(tb, eseg->segtab); - tb->nsegs = eseg->nsegs; - } - ASSERT(seg_ix < tb->nsegs); - segtab = SEGTAB(tb); - ASSERT(segtab[seg_ix] == NULL); - segtab[seg_ix] = (struct segment*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, - (DbTable *) tb, - sizeof(struct segment)); - if (segtab[seg_ix] == NULL) return 0; - sys_memset(segtab[seg_ix], 0, sizeof(struct segment)); - } - tb->nslots += SEGSZ; - return 1; + int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots); + struct segment** segtab; + + ASSERT(seg_ix > 0); + if (seg_ix == tb->nsegs) { /* New segtab needed */ + struct ext_segtab* est = alloc_ext_segtab(tb, seg_ix); + SET_SEGTAB(tb, est->segtab); + tb->nsegs = est->nsegs; + } + ASSERT(seg_ix < tb->nsegs); + segtab = SEGTAB(tb); + segtab[seg_ix] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG, + (DbTable *) tb, + SIZEOF_SEGMENT(EXT_SEGSZ)); + sys_memset(segtab[seg_ix], 0, SIZEOF_SEGMENT(EXT_SEGSZ)); + tb->nslots += EXT_SEGSZ; +} + +#ifdef ERTS_SMP +static void dealloc_ext_segtab(void* lop_data) +{ + struct ext_segtab* est = (struct ext_segtab*) lop_data; + + erts_free(ERTS_ALC_T_DB_SEG, est); } +#endif /* Shrink table by freeing the top segment ** free_records: 1=free any records in segment, 0=assume segment is empty */ static int free_seg(DbTableHash *tb, int free_records) { - const int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1; + const int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots) - 1; struct segment** const segtab = SEGTAB(tb); - struct ext_segment* const top = (struct ext_segment*) segtab[seg_ix]; - int bytes; + struct segment* const segp = segtab[seg_ix]; + Uint seg_sz; int nrecords = 0; - ASSERT(top != NULL); + ASSERT(segp != NULL); #ifndef DEBUG if (free_records) #endif { - int i; - for (i=0; i<SEGSZ; ++i) { - HashDbTerm* p = top->s.buckets[i]; + int i = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ; + while (i--) { + HashDbTerm* p = segp->buckets[i]; while(p != 0) { HashDbTerm* nxt = p->next; ASSERT(free_records); /* segment not empty as assumed? */ @@ -2507,55 +2478,46 @@ static int free_seg(DbTableHash *tb, int free_records) } } } - - /* The "shrink race": - * We must avoid deallocating an extended segment while its segtab may - * still be used by other threads. - * The trick is to stop use a segtab one call earlier. That is, stop use - * a segtab when the segment above it is deallocated. When the segtab is - * later deallocated, it has not been used for a very long time. - * It is even theoretically safe as we have by then rehashed the entire - * segment, seizing *all* locks, so there cannot exist any retarded threads - * still hanging in BUCKET macro with an old segtab pointer. - * For this to work, we must of course allocate a new segtab one call - * earlier in alloc_seg() as well. And this is also the reason why - * the minimum size of the first segtab is 2 and not 1 (NSEG_1). - */ - if (seg_ix == tb->nsegs-1 || seg_ix==0) { /* Dealloc extended segment */ - MY_ASSERT(top->s.is_ext_segment); - ASSERT(segtab != top->segtab || seg_ix==0); - bytes = SIZEOF_EXTSEG(top->nsegs); - } - else { /* Dealloc plain segment */ - struct ext_segment* newtop = (struct ext_segment*) segtab[seg_ix-1]; - MY_ASSERT(!top->s.is_ext_segment); - - if (segtab == newtop->segtab) { /* New top segment is extended */ - MY_ASSERT(newtop->s.is_ext_segment); - if (newtop->prev_segtab != NULL) { - /* Time to use a smaller segtab */ - SET_SEGTAB(tb, newtop->prev_segtab); - tb->nsegs = seg_ix; - ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs); - } - else { - ASSERT(NSEG_1 > 2 && seg_ix==1); - } - } - bytes = sizeof(struct segment); + if (seg_ix >= NSEG_1) { + struct ext_segtab* est = ErtsContainerStruct_(segtab,struct ext_segtab,segtab); + + if (seg_ix == est->prev_nsegs) { /* Dealloc extended segtab */ + ASSERT(est->prev_segtab != NULL); + SET_SEGTAB(tb, est->prev_segtab); + tb->nsegs = est->prev_nsegs; + +#ifdef ERTS_SMP + if (!tb->common.is_thread_safe) { + /* + * Table is doing a graceful shrink operation and we must avoid + * deallocating this segtab while it may still be read by other + * threads. Schedule deallocation with thread progress to make + * sure no lingering threads are still hanging in BUCKET macro + * with an old segtab pointer. + */ + Uint sz = SIZEOF_EXT_SEGTAB(est->nsegs); + ASSERT(sz == ERTS_ALC_DBG_BLK_SZ(est)); + ERTS_DB_ALC_MEM_UPDATE_(tb, sz, 0); + erts_schedule_thr_prgr_later_cleanup_op(dealloc_ext_segtab, + est, + &est->lop, + sz); + } + else +#endif + erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est, + SIZEOF_EXT_SEGTAB(est->nsegs)); + } } + seg_sz = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ; + erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, segp, SIZEOF_SEGMENT(seg_sz)); - erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, - (void*)top, bytes); #ifdef DEBUG - if (seg_ix > 0) { - segtab[seg_ix] = NULL; - } else { - SET_SEGTAB(tb, NULL); - } + if (seg_ix < tb->nsegs) + SEGTAB(tb)[seg_ix] = NULL; #endif - tb->nslots -= SEGSZ; + tb->nslots -= seg_sz; ASSERT(tb->nslots >= 0); return nrecords; } @@ -2604,104 +2566,111 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, static ERTS_INLINE int begin_resizing(DbTableHash* tb) { +#ifdef ERTS_SMP if (DB_USING_FINE_LOCKING(tb)) - return !erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1); - else { - if (erts_smp_atomic_read_nob(&tb->is_resizing)) - return 0; - erts_smp_atomic_set_nob(&tb->is_resizing, 1); - return 1; - } + return !erts_atomic_xchg_acqb(&tb->is_resizing, 1); + else + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock)); +#endif + return 1; } static ERTS_INLINE void done_resizing(DbTableHash* tb) { +#ifdef ERTS_SMP if (DB_USING_FINE_LOCKING(tb)) - erts_smp_atomic_set_relb(&tb->is_resizing, 0); - else - erts_smp_atomic_set_nob(&tb->is_resizing, 0); + erts_atomic_set_relb(&tb->is_resizing, 0); +#endif } -/* Grow table with one new bucket. +/* Grow table with one or more new buckets. ** Allocate new segment if needed. */ -static void grow(DbTableHash* tb, int nactive) +static void grow(DbTableHash* tb, int nitems) { HashDbTerm** pnext; HashDbTerm** to_pnext; HashDbTerm* p; erts_smp_rwmtx_t* lck; - int from_ix; + int nactive; + int from_ix, to_ix; int szm; + int loop_limit = 5; - if (!begin_resizing(tb)) - return; /* already in progress */ - if (NACTIVE(tb) != nactive) { - goto abort; /* already done (race) */ - } - - /* Ensure that the slot nactive exists */ - if (nactive == tb->nslots) { - /* Time to get a new segment */ - ASSERT((nactive & SEGSZ_MASK) == 0); - if (!alloc_seg(tb)) goto abort; - } - ASSERT(nactive < tb->nslots); + do { + if (!begin_resizing(tb)) + return; /* already in progress */ + nactive = NACTIVE(tb); + if (nitems <= GROW_LIMIT(nactive)) { + goto abort; /* already done (race) */ + } - szm = erts_smp_atomic_read_nob(&tb->szm); - if (nactive <= szm) { - from_ix = nactive & (szm >> 1); - } else { - ASSERT(nactive == szm+1); - from_ix = 0; - szm = (szm<<1) | 1; - } + /* Ensure that the slot nactive exists */ + if (nactive == tb->nslots) { + /* Time to get a new segment */ + ASSERT(((nactive-FIRST_SEGSZ) & EXT_SEGSZ_MASK) == 0); + alloc_seg(tb); + } + ASSERT(nactive < tb->nslots); - lck = WLOCK_HASH(tb, from_ix); - /* Now a final double check (with the from_ix lock held) - * that we did not get raced by a table fixer. - */ - if (IS_FIXED(tb)) { - WUNLOCK_HASH(lck); - goto abort; - } - erts_smp_atomic_inc_nob(&tb->nactive); - if (from_ix == 0) { - if (DB_USING_FINE_LOCKING(tb)) - erts_smp_atomic_set_relb(&tb->szm, szm); - else - erts_smp_atomic_set_nob(&tb->szm, szm); - } - done_resizing(tb); + szm = erts_smp_atomic_read_nob(&tb->szm); + if (nactive <= szm) { + from_ix = nactive & (szm >> 1); + } else { + ASSERT(nactive == szm+1); + from_ix = 0; + szm = (szm<<1) | 1; + } + to_ix = nactive; + + lck = WLOCK_HASH(tb, from_ix); + ERTS_SMP_ASSERT(lck == GET_LOCK_MAYBE(tb,to_ix)); + /* Now a final double check (with the from_ix lock held) + * that we did not get raced by a table fixer. + */ + if (IS_FIXED(tb)) { + WUNLOCK_HASH(lck); + goto abort; + } + erts_smp_atomic_set_nob(&tb->nactive, ++nactive); + if (from_ix == 0) { + if (DB_USING_FINE_LOCKING(tb)) + erts_smp_atomic_set_relb(&tb->szm, szm); + else + erts_smp_atomic_set_nob(&tb->szm, szm); + } + done_resizing(tb); + + /* Finally, let's split the bucket. We try to do it in a smart way + to keep link order and avoid unnecessary updates of next-pointers */ + pnext = &BUCKET(tb, from_ix); + p = *pnext; + to_pnext = &BUCKET(tb, to_ix); + while (p != NULL) { + if (p->hvalue == INVALID_HASH) { /* rare but possible with fine locking */ + *pnext = p->next; + free_term(tb, p); + p = *pnext; + } + else { + int ix = p->hvalue & szm; + if (ix != from_ix) { + ASSERT(ix == (from_ix ^ ((szm+1)>>1))); + *to_pnext = p; + /* Swap "from" and "to": */ + from_ix = ix; + to_pnext = pnext; + } + pnext = &p->next; + p = *pnext; + } + } + *to_pnext = NULL; + WUNLOCK_HASH(lck); - /* Finally, let's split the bucket. We try to do it in a smart way - to keep link order and avoid unnecessary updates of next-pointers */ - pnext = &BUCKET(tb, from_ix); - p = *pnext; - to_pnext = &BUCKET(tb, nactive); - while (p != NULL) { - if (p->hvalue == INVALID_HASH) { /* rare but possible with fine locking */ - *pnext = p->next; - free_term(tb, p); - p = *pnext; - } - else { - int ix = p->hvalue & szm; - if (ix != from_ix) { - ASSERT(ix == (from_ix ^ ((szm+1)>>1))); - *to_pnext = p; - /* Swap "from" and "to": */ - from_ix = ix; - to_pnext = pnext; - } - pnext = &p->next; - p = *pnext; - } - } - *to_pnext = NULL; + }while (--loop_limit && nitems > GROW_LIMIT(nactive)); - WUNLOCK_HASH(lck); return; abort: @@ -2712,60 +2681,78 @@ abort: /* Shrink table by joining top bucket. ** Remove top segment if it gets empty. */ -static void shrink(DbTableHash* tb, int nactive) -{ - if (!begin_resizing(tb)) - return; /* already in progress */ - if (NACTIVE(tb) == nactive) { - erts_smp_rwmtx_t* lck; - int src_ix = nactive - 1; - int low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1; - int dst_ix = src_ix & low_szm; - - ASSERT(dst_ix < src_ix); - ASSERT(nactive > SEGSZ); - lck = WLOCK_HASH(tb, dst_ix); - /* Double check for racing table fixers */ - if (!IS_FIXED(tb)) { - HashDbTerm** src_bp = &BUCKET(tb, src_ix); - HashDbTerm** dst_bp = &BUCKET(tb, dst_ix); - HashDbTerm** bp = src_bp; - - /* Q: Why join lists by appending "dst" at the end of "src"? - A: Must step through "src" anyway to purge pseudo deleted. */ - while(*bp != NULL) { - if ((*bp)->hvalue == INVALID_HASH) { - HashDbTerm* deleted = *bp; - *bp = deleted->next; - free_term(tb, deleted); - } else { - bp = &(*bp)->next; - } - } - *bp = *dst_bp; - *dst_bp = *src_bp; - *src_bp = NULL; - - erts_smp_atomic_set_nob(&tb->nactive, src_ix); - if (dst_ix == 0) { - erts_smp_atomic_set_relb(&tb->szm, low_szm); - } - WUNLOCK_HASH(lck); - - if (tb->nslots - src_ix >= SEGSZ) { - free_seg(tb, 0); - } - } - else { - WUNLOCK_HASH(lck); - } +static void shrink(DbTableHash* tb, int nitems) +{ + HashDbTerm** src_bp; + HashDbTerm** dst_bp; + HashDbTerm** bp; + erts_smp_rwmtx_t* lck; + int src_ix, dst_ix, low_szm; + int nactive; + int loop_limit = 5; - } - /*else already done */ + do { + if (!begin_resizing(tb)) + return; /* already in progress */ + nactive = NACTIVE(tb); + if (!(nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive))) { + goto abort; /* already done (race) */ + } + src_ix = nactive - 1; + low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1; + dst_ix = src_ix & low_szm; + + ASSERT(dst_ix < src_ix); + ASSERT(nactive > FIRST_SEGSZ); + lck = WLOCK_HASH(tb, dst_ix); + ERTS_SMP_ASSERT(lck == GET_LOCK_MAYBE(tb,src_ix)); + /* Double check for racing table fixers */ + if (IS_FIXED(tb)) { + WUNLOCK_HASH(lck); + goto abort; + } + + src_bp = &BUCKET(tb, src_ix); + dst_bp = &BUCKET(tb, dst_ix); + bp = src_bp; + + /* + * We join lists by appending "dst" at the end of "src" + * as we must step through "src" anyway to purge pseudo deleted. + */ + while(*bp != NULL) { + if ((*bp)->hvalue == INVALID_HASH) { + HashDbTerm* deleted = *bp; + *bp = deleted->next; + free_term(tb, deleted); + } else { + bp = &(*bp)->next; + } + } + *bp = *dst_bp; + *dst_bp = *src_bp; + *src_bp = NULL; + + nactive = src_ix; + erts_smp_atomic_set_nob(&tb->nactive, nactive); + if (dst_ix == 0) { + erts_smp_atomic_set_relb(&tb->szm, low_szm); + } + WUNLOCK_HASH(lck); + + if (tb->nslots - src_ix >= EXT_SEGSZ) { + free_seg(tb, 0); + } + done_resizing(tb); + + } while (--loop_limit + && nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive)); + return; + +abort: done_resizing(tb); } - /* Search a list of tuples for a matching key */ static HashDbTerm* search_list(DbTableHash* tb, Eterm key, @@ -2931,8 +2918,8 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) WUNLOCK_HASH(lck); nactive = NACTIVE(tb); - if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) { - grow(tb, nactive); + if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { + grow(tb, nitems); } } else { WUNLOCK_HASH(lck); @@ -3016,6 +3003,51 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats) stats->std_dev_expected = sqrt(stats->avg_chain_len * (1 - 1.0/NACTIVE(tb))); stats->kept_items = kept_items; } + +/* For testing only */ +Eterm erts_ets_hash_sizeof_ext_segtab(void) +{ + return make_small(((SIZEOF_EXT_SEGTAB(0)-1) / sizeof(UWord)) + 1); +} +/* For testing only */ +Eterm erts_ets_hash_get_memstate(Process* p, DbTableHash* tb) +{ + Eterm seg_cnt; + while (!begin_resizing(tb)) + /*spinn*/; + + seg_cnt = make_small(SLOT_IX_TO_SEG_IX(tb->nslots)); + done_resizing(tb); + return seg_cnt; +} +/* For testing only */ +Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate) +{ + int seg_cnt, target; + + if (!is_small(memstate)) + return make_small(__LINE__); + + target = signed_val(memstate); + if (target < 1) + return make_small(__LINE__); + while (1) { + while (!begin_resizing(tb)) + /*spin*/; + seg_cnt = SLOT_IX_TO_SEG_IX(tb->nslots); + done_resizing(tb); + + if (target == seg_cnt) + return am_ok; + if (IS_FIXED(tb)) + return make_small(__LINE__); + if (target < seg_cnt) + shrink(tb, 0); + else + grow(tb, INT_MAX); + } +} + #ifdef HARDDEBUG void db_check_table_hash(DbTable *tbl) @@ -3024,7 +3056,7 @@ void db_check_table_hash(DbTable *tbl) HashDbTerm* list; int j; - for (j = 0; j < tb->nactive; j++) { + for (j = 0; j < NACTIVE(tb); j++) { if ((list = BUCKET(tb,j)) != 0) { while (list != 0) { if (!is_tuple(make_tuple(list->dbterm.tpl))) { diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index e654363cd5..6d25c73549 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -50,23 +50,23 @@ typedef struct db_table_hash_fine_locks { typedef struct db_table_hash { DbTableCommon common; - erts_smp_atomic_t segtab; /* The segment table (struct segment**) */ + /* SMP: szm and nactive are write-protected by is_resizing or table write lock */ erts_smp_atomic_t szm; /* current size mask. */ - + erts_smp_atomic_t nactive; /* Number of "active" slots */ + + erts_smp_atomic_t segtab; /* The segment table (struct segment**) */ + struct segment* first_segtab[1]; + /* SMP: nslots and nsegs are protected by is_resizing or table write lock */ int nslots; /* Total number of slots */ int nsegs; /* Size of segment table */ /* List of slots where elements have been deleted while table was fixed */ erts_smp_atomic_t fixdel; /* (FixedDeletion*) */ - erts_smp_atomic_t nactive; /* Number of "active" slots */ - erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ #ifdef ERTS_SMP + erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ DbTableHashFineLocks* locks; #endif -#ifdef VALGRIND - struct ext_segment* top_ptr_to_segment_with_active_segtab; -#endif } DbTableHash; @@ -109,5 +109,8 @@ typedef struct { }DbHashStats; void db_calc_stats_hash(DbTableHash* tb, DbHashStats*); +Eterm erts_ets_hash_sizeof_ext_segtab(void); +Eterm erts_ets_hash_get_memstate(Process*, DbTableHash* tb); +Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate); #endif /* _DB_HASH_H */ diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 6732b708a8..4e987d5bee 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1769,7 +1769,7 @@ Eterm db_prog_match(Process *c_p, Eterm t; Eterm *esp; MatchVariable* variables; - BeamInstr *cp; + ErtsCodeMFA *cp; const UWord *pc = prog->text; Eterm *ehp; Eterm ret; @@ -2408,9 +2408,9 @@ restart: ehp = HAllocX(build_proc, 4, HEAP_XTRA); *esp++ = make_tuple(ehp); ehp[0] = make_arityval(3); - ehp[1] = cp[0]; - ehp[2] = cp[1]; - ehp[3] = make_small((Uint) cp[2]); + ehp[1] = cp->module; + ehp[2] = cp->function; + ehp[3] = make_small((Uint) cp->arity); } break; case matchSilent: diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index 6ec5fbb895..2489099b5c 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -69,10 +69,6 @@ typedef enum { # define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG # undef SIZEOF_LONG_LONG #endif -#ifdef HALFWORD_HEAP_EMULATOR -# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR -# undef HALFWORD_HEAP_EMULATOR -#endif #include "erl_int_sizes_config.h" #if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR # error SIZEOF_CHAR mismatch diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 92edce5176..0e6aadf568 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -54,6 +54,9 @@ fatal_error(int err, char *func) struct ErlDrvMutex_ { ethr_mutex mtx; +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_t lcnt; +#endif char *name; }; @@ -64,6 +67,9 @@ struct ErlDrvCond_ { struct ErlDrvRWLock_ { ethr_rwmutex rwmtx; +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_t lcnt; +#endif char *name; }; @@ -161,14 +167,17 @@ erl_drv_mutex_create(char *name) opt.posix_compliant = 1; if (ethr_mutex_init_opt(&dmtx->mtx, &opt) != 0) { erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx); - dmtx = NULL; + return NULL; } - else if (!name) - dmtx->name = no_name; - else { + if (name) { dmtx->name = ((char *) dmtx) + sizeof(ErlDrvMutex); sys_strcpy(dmtx->name, name); + } else { + dmtx->name = no_name; } +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_init_lock(&dmtx->lcnt, dmtx->name, ERTS_LCNT_LT_MUTEX); +#endif } return dmtx; #else @@ -180,7 +189,11 @@ void erl_drv_mutex_destroy(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_destroy(&dmtx->mtx) : EINVAL; + int res; +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_destroy_lock(&dmtx->lcnt); +#endif + res = dmtx ? ethr_mutex_destroy(&dmtx->mtx) : EINVAL; if (res != 0) fatal_error(res, "erl_drv_mutex_destroy()"); erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx); @@ -202,9 +215,14 @@ int erl_drv_mutex_trylock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS + int res; if (!dmtx) fatal_error(EINVAL, "erl_drv_mutex_trylock()"); - return ethr_mutex_trylock(&dmtx->mtx); + res = ethr_mutex_trylock(&dmtx->mtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_trylock(&dmtx->lcnt, res); +#endif + return res; #else return 0; #endif @@ -216,8 +234,14 @@ erl_drv_mutex_lock(ErlDrvMutex *dmtx) #ifdef USE_THREADS if (!dmtx) fatal_error(EINVAL, "erl_drv_mutex_lock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock(&dmtx->lcnt); +#endif ethr_mutex_lock(&dmtx->mtx); #endif +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_post(&dmtx->lcnt); +#endif } void @@ -226,6 +250,9 @@ erl_drv_mutex_unlock(ErlDrvMutex *dmtx) #ifdef USE_THREADS if (!dmtx) fatal_error(EINVAL, "erl_drv_mutex_unlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_unlock(&dmtx->lcnt); +#endif ethr_mutex_unlock(&dmtx->mtx); #endif } @@ -306,10 +333,18 @@ erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx) if (!dcnd || !dmtx) { fatal_error(EINVAL, "erl_drv_cond_wait()"); } +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_unlock(&dmtx->lcnt); +#endif while (1) { int res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx); - if (res == 0) + if (res == 0) { +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock(&dmtx->lcnt); + erts_lcnt_lock_post(&dmtx->lcnt); +#endif break; + } } #endif } @@ -324,14 +359,17 @@ erl_drv_rwlock_create(char *name) if (drwlck) { if (ethr_rwmutex_init(&drwlck->rwmtx) != 0) { erts_free(ERTS_ALC_T_DRV_RWLCK, (void *) drwlck); - drwlck = NULL; + return NULL; } - else if (!name) - drwlck->name = no_name; - else { + if (name) { drwlck->name = ((char *) drwlck) + sizeof(ErlDrvRWLock); sys_strcpy(drwlck->name, name); + } else { + drwlck->name = no_name; } +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_init_lock(&drwlck->lcnt, drwlck->name, ERTS_LCNT_LT_RWMUTEX); +#endif } return drwlck; #else @@ -343,7 +381,11 @@ void erl_drv_rwlock_destroy(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_destroy(&drwlck->rwmtx) : EINVAL; + int res; +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_destroy_lock(&drwlck->lcnt); +#endif + res = drwlck ? ethr_rwmutex_destroy(&drwlck->rwmtx) : EINVAL; if (res != 0) fatal_error(res, "erl_drv_rwlock_destroy()"); erts_free(ERTS_ALC_T_DRV_RWLCK, (void *) drwlck); @@ -364,9 +406,14 @@ int erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS + int res; if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()"); - return ethr_rwmutex_tryrlock(&drwlck->rwmtx); + res = ethr_rwmutex_tryrlock(&drwlck->rwmtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LCNT_LO_READ); +#endif + return res; #else return 0; #endif @@ -378,7 +425,13 @@ erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck) #ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_rlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ); +#endif ethr_rwmutex_rlock(&drwlck->rwmtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_post(&drwlck->lcnt); +#endif #endif } @@ -388,6 +441,9 @@ erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck) #ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_runlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ); +#endif ethr_rwmutex_runlock(&drwlck->rwmtx); #endif } @@ -396,9 +452,14 @@ int erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS + int res; if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()"); - return ethr_rwmutex_tryrwlock(&drwlck->rwmtx); + res = ethr_rwmutex_tryrwlock(&drwlck->rwmtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LCNT_LO_READ_WRITE); +#endif + return res; #else return 0; #endif @@ -410,7 +471,13 @@ erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck) #ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_rwlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ_WRITE); +#endif ethr_rwmutex_rwlock(&drwlck->rwmtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_post(&drwlck->lcnt); +#endif #endif } @@ -420,6 +487,9 @@ erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck) #ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ_WRITE); +#endif ethr_rwmutex_rwunlock(&drwlck->rwmtx); #endif } diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index d0a57f0ad0..41097203d4 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -31,6 +31,9 @@ static Hash erts_fun_table; #include "erl_smp.h" +#ifdef HIPE +# include "hipe_mode_switch.h" +#endif static erts_smp_rwmtx_t erts_fun_table_lock; @@ -49,8 +52,8 @@ static void fun_free(ErlFunEntry* obj); * to unloaded_fun[]. The -1 in unloaded_fun[0] will be interpreted * as an illegal arity when attempting to call a fun. */ -static BeamInstr unloaded_fun_code[3] = {NIL, -1, 0}; -static BeamInstr* unloaded_fun = unloaded_fun_code + 2; +static BeamInstr unloaded_fun_code[4] = {NIL, NIL, -1, 0}; +static BeamInstr* unloaded_fun = unloaded_fun_code + 3; void erts_init_fun_table(void) @@ -219,6 +222,10 @@ erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end) fe->pend_purge_address = addr; ERTS_SMP_WRITE_MEMORY_BARRIER; fe->address = unloaded_fun; +#ifdef HIPE + fe->pend_purge_native_address = fe->native_address; + hipe_set_closure_stub(fe); +#endif erts_purge_state_add_fun(fe); } b = b->next; @@ -234,8 +241,12 @@ erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no) for (ix = 0; ix < no; ix++) { ErlFunEntry *fe = funs[ix]; - if (fe->address == unloaded_fun) + if (fe->address == unloaded_fun) { fe->address = fe->pend_purge_address; +#ifdef HIPE + fe->native_address = fe->pend_purge_native_address; +#endif + } } } @@ -244,8 +255,12 @@ erts_fun_purge_abort_finalize(ErlFunEntry **funs, Uint no) { Uint ix; - for (ix = 0; ix < no; ix++) + for (ix = 0; ix < no; ix++) { funs[ix]->pend_purge_address = NULL; +#ifdef HIPE + funs[ix]->pend_purge_native_address = NULL; +#endif + } } void @@ -256,6 +271,9 @@ erts_fun_purge_complete(ErlFunEntry **funs, Uint no) for (ix = 0; ix < no; ix++) { ErlFunEntry *fe = funs[ix]; fe->pend_purge_address = NULL; +#ifdef HIPE + fe->pend_purge_native_address = NULL; +#endif if (erts_refc_dectest(&fe->refc, 0) == 0) erts_erase_fun_entry(fe); } @@ -324,6 +342,7 @@ fun_alloc(ErlFunEntry* template) obj->pend_purge_address = NULL; #ifdef HIPE obj->native_address = NULL; + obj->pend_purge_native_address = NULL; #endif return obj; } diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index caa55c730c..83899f24a6 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -45,6 +45,9 @@ typedef struct erl_fun_entry { erts_refc_t refc; /* Reference count: One for code + one for each fun object in each process. */ BeamInstr *pend_purge_address; /* address stored during a pending purge */ +#ifdef HIPE + UWord* pend_purge_native_address; +#endif } ErlFunEntry; /* @@ -57,9 +60,6 @@ typedef struct erl_fun_thing { Eterm thing_word; /* Subtag FUN_SUBTAG. */ ErlFunEntry* fe; /* Pointer to fun entry. */ struct erl_off_heap_header* next; -#ifdef HIPE - UWord* native_address; /* Native code for the fun. */ -#endif Uint arity; /* The arity of the fun. */ Uint num_free; /* Number of free variables (in env). */ /* -- The following may be compound Erlang terms ---------------------- */ diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index a33a81babd..643b46c861 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -35,7 +35,7 @@ #include "error.h" #include "big.h" #include "erl_gc.h" -#if HIPE +#ifdef HIPE #include "hipe_stack.h" #include "hipe_mode_switch.h" #endif @@ -407,11 +407,11 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end, if (is_non_value(result)) { if (p->freason == TRAP) { - #if HIPE +#ifdef HIPE if (regs == NULL) { regs = erts_proc_sched_data(p)->x_reg_array; } - #endif +#endif cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls); } else { cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls); @@ -2221,26 +2221,18 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, *hp++ = val; break; case TAG_PRIMARY_LIST: -#ifdef SHCOPY_SEND if (erts_is_literal(val,list_val(val))) { *hp++ = val; } else { *hp++ = offset_ptr(val, offs); } -#else - *hp++ = offset_ptr(val, offs); -#endif break; case TAG_PRIMARY_BOXED: -#ifdef SHCOPY_SEND if (erts_is_literal(val,boxed_val(val))) { *hp++ = val; } else { *hp++ = offset_ptr(val, offs); } -#else - *hp++ = offset_ptr(val, offs); -#endif break; case TAG_PRIMARY_HEADER: *hp++ = val; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 2fd97208cc..c5904b375e 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2261,7 +2261,6 @@ erl_start(int argc, char **argv) ASSERT(erts_code_purger && erts_code_purger->common.id == pid); erts_proc_inc_refc(erts_code_purger); -#ifdef ERTS_NEW_PURGE_STRATEGY pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector"); erts_literal_area_collector = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, @@ -2269,7 +2268,6 @@ erl_start(int argc, char **argv) ASSERT(erts_literal_area_collector && erts_literal_area_collector->common.id == pid); erts_proc_inc_refc(erts_literal_area_collector); -#endif #ifdef ERTS_DIRTY_SCHEDULERS pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker"); @@ -2299,7 +2297,7 @@ erl_start(int argc, char **argv) #endif set_main_stack_size(); erts_sched_init_time_sup(esdp); - process_main(); + process_main(esdp->x_reg_array, esdp->f_reg_array); } #endif } diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 5d4823c077..08dcbed91c 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -113,10 +113,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "export_tab", NULL }, { "fun_tab", NULL }, { "environ", NULL }, -#ifdef ERTS_NEW_PURGE_STRATEGY { "release_literal_areas", NULL }, #endif -#endif { "efile_drv", "address" }, { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index 481e92b2cd..6354fc8663 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -55,11 +55,11 @@ static erts_lcnt_thread_data_t *lcnt_thread_data[2048]; /* local functions */ static ERTS_INLINE void lcnt_lock(void) { - ethr_mutex_lock(&lcnt_data_lock); + ethr_mutex_lock(&lcnt_data_lock); } static ERTS_INLINE void lcnt_unlock(void) { - ethr_mutex_unlock(&lcnt_data_lock); + ethr_mutex_unlock(&lcnt_data_lock); } const int log2_tab64[64] = { @@ -159,7 +159,7 @@ static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) { lcnt_thread_data[eltd->id] = eltd; return eltd; -} +} static erts_lcnt_thread_data_t *lcnt_get_thread_data(void) { return (erts_lcnt_thread_data_t *)ethr_tsd_get(lcnt_thr_data_key); @@ -254,9 +254,9 @@ void erts_lcnt_init() { /* init lock */ if (ethr_mutex_init(&lcnt_data_lock) != 0) abort(); - /* init tsd */ + /* init tsd */ lcnt_n_thr = 0; - ethr_tsd_key_create(&lcnt_thr_data_key,"lcnt_data"); + ethr_tsd_key_create(&lcnt_thr_data_key, "lcnt_data"); lcnt_lock(); @@ -352,11 +352,11 @@ void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) /* interface to erl_threads.h */ /* only lock on init and destroy, all others should use atomics */ -void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { +void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { erts_lcnt_init_lock_x(lock, name, flag, NIL); } -void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { +void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { int i; if (name == NULL) { ERTS_LCNT_CLEAR_FLAG(lock); return; } lcnt_lock(); @@ -382,7 +382,10 @@ void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eter erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock); lcnt_unlock(); } -/* init empty, instead of zero struct */ + +/* init empty, instead of zero struct + * used by process locks probes + */ void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock) { lock->next = NULL; lock->prev = NULL; @@ -444,7 +447,7 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { } /* we cannot acquire w_lock if either w or r are taken */ - /* we cannot acquire r_lock if w_lock is taken */ + /* we cannot acquire r_lock if w_lock is taken */ if ((w_state > 0) || (r_state > 0)) { eltd->lock_in_conflict = 1; @@ -561,7 +564,7 @@ void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; #ifdef DEBUG { - erts_aint_t w_state; + erts_aint_t w_state; erts_aint_t flowstate; /* flowstate */ @@ -647,7 +650,7 @@ Uint16 erts_lcnt_set_rt_opt(Uint16 opt) { return prev; } -Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) { +Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) { Uint16 prev; prev = (erts_lcnt_rt_options & opt); erts_lcnt_rt_options &= ~opt; diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index 3e8dcefe69..4f838f7faa 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -148,13 +148,13 @@ typedef struct erts_lcnt_lock_stats_s { typedef struct erts_lcnt_lock_s { char *name; /* lock name */ Uint16 flag; /* lock type */ - Eterm id; /* id if possible */ + Eterm id; /* id if possible */ #ifdef DEBUG ethr_atomic_t flowstate; #endif - /* lock states */ + /* lock states */ ethr_atomic_t w_state; /* 0 not taken, otherwise n threads waiting */ ethr_atomic_t r_state; /* 0 not taken, > 0 -> writes will wait */ diff --git a/erts/emulator/beam/erl_math.c b/erts/emulator/beam/erl_math.c index fc0aaed18a..1f270eb55f 100644 --- a/erts/emulator/beam/erl_math.c +++ b/erts/emulator/beam/erl_math.c @@ -247,6 +247,17 @@ BIF_RETTYPE math_pow_2(BIF_ALIST_2) return math_call_2(BIF_P, pow, BIF_ARG_1, BIF_ARG_2); } +BIF_RETTYPE math_ceil_1(BIF_ALIST_1) +{ + return math_call_1(BIF_P, ceil, BIF_ARG_1); +} +BIF_RETTYPE math_floor_1(BIF_ALIST_1) +{ + return math_call_1(BIF_P, floor, BIF_ARG_1); +} - +BIF_RETTYPE math_fmod_2(BIF_ALIST_2) +{ + return math_call_2(BIF_P, fmod, BIF_ARG_1, BIF_ARG_2); +} diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index f45e6974cd..792b69bb37 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -696,6 +696,9 @@ erts_send_message(Process* sender, erts_aint32_t receiver_state; #ifdef SHCOPY_SEND erts_shcopy_t info; +#else + erts_literal_area_t litarea; + INITIALIZE_LITERAL_PURGE_AREA(litarea); #endif #ifdef USE_VM_PROBES @@ -725,7 +728,7 @@ erts_send_message(Process* sender, */ if (have_seqtrace(stoken)) { seq_trace_update_send(sender); - seq_trace_output(stoken, message, SEQ_TRACE_SEND, + seq_trace_output(stoken, message, SEQ_TRACE_SEND, receiver->common.id, sender); seq_trace_size = 6; /* TUPLE5 */ } @@ -741,7 +744,7 @@ erts_send_message(Process* sender, INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else - msize = size_object(message); + msize = size_object_litopt(message, &litarea); #endif mp = erts_alloc_message_heap_state(receiver, &receiver_state, @@ -760,7 +763,7 @@ erts_send_message(Process* sender, DESTROY_SHCOPY(info); #else if (is_not_immed(message)) - message = copy_struct(message, msize, &hp, ohp); + message = copy_struct_litopt(message, msize, &hp, ohp, &litarea); #endif if (is_immed(stoken)) token = stoken; @@ -796,7 +799,7 @@ erts_send_message(Process* sender, INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else - msize = size_object(message); + msize = size_object_litopt(message, &litarea); #endif mp = erts_alloc_message_heap_state(receiver, &receiver_state, @@ -810,7 +813,7 @@ erts_send_message(Process* sender, DESTROY_SHCOPY(info); #else if (is_not_immed(message)) - message = copy_struct(message, msize, &hp, ohp); + message = copy_struct_litopt(message, msize, &hp, ohp, &litarea); #endif } #ifdef USE_VM_PROBES @@ -998,8 +1001,8 @@ erts_move_messages_off_heap(Process *c_p) hp = hfrag->mem; if (is_not_immed(ERL_MESSAGE_TERM(mp))) ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp), - msg_sz, &hp, - &hfrag->off_heap); + msg_sz, &hp, + &hfrag->off_heap); if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp), token_sz, &hp, diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index 4c8e1c8e22..9ef86a1293 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -22,7 +22,7 @@ #define ERL_MSACC_H__ /* Can be enabled/disabled via configure */ -#if ERTS_ENABLE_MSACC == 2 +#if defined(ERTS_ENABLE_MSACC) && ERTS_ENABLE_MSACC == 2 #define ERTS_MSACC_EXTENDED_STATES 1 #endif @@ -66,7 +66,7 @@ #define ERTS_MSACC_STATE_COUNT 7 -#if ERTS_MSACC_STATE_STRINGS && ERTS_ENABLE_MSACC +#if defined(ERTS_MSACC_STATE_STRINGS) && defined(ERTS_ENABLE_MSACC) static char *erts_msacc_states[] = { "aux", "check_io", @@ -104,7 +104,7 @@ static char *erts_msacc_states[] = { #define ERTS_MSACC_STATE_COUNT ERTS_MSACC_STATIC_STATE_COUNT #endif -#if ERTS_MSACC_STATE_STRINGS +#ifdef ERTS_MSACC_STATE_STRINGS static char *erts_msacc_states[] = { "alloc", "aux", @@ -157,7 +157,7 @@ struct erl_msacc_t_ { }; -#if ERTS_ENABLE_MSACC +#ifdef ERTS_ENABLE_MSACC #ifdef USE_THREADS extern erts_tsd_key_t erts_msacc_key; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 6b265a8b80..5499512dd1 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -71,10 +71,12 @@ struct erl_module_nif { void* priv_data; void* handle; /* "dlopen" */ - struct enif_entry_t* entry; + struct enif_entry_t entry; erts_refc_t rt_cnt; /* number of resource types */ erts_refc_t rt_dtor_cnt; /* number of resource types with destructors */ Module* mod; /* Can be NULL if orphan with dtor-resources left */ + + ErlNifFunc _funcs_copy_[1]; /* only used for old libs */ }; #ifdef DEBUG @@ -687,9 +689,12 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, MBUF(&menv->phony_proc) = NULL; } } else { - Uint sz = size_object(msg); + erts_literal_area_t litarea; ErlOffHeap *ohp; Eterm *hp; + Uint sz; + INITIALIZE_LITERAL_PURGE_AREA(litarea); + sz = size_object_litopt(msg, &litarea); if (env && !env->tracee) { full_flush_env(env); mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); @@ -709,7 +714,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ohp = &bp->off_heap; } } - msg = copy_struct(msg, sz, &hp, ohp); + msg = copy_struct_litopt(msg, sz, &hp, ohp, &litarea); } ERL_MESSAGE_TERM(mp) = msg; @@ -1955,10 +1960,10 @@ static void close_lib(struct erl_module_nif* lib) ASSERT(lib->handle != NULL); ASSERT(erts_refc_read(&lib->rt_dtor_cnt,0) == 0); - if (lib->entry != NULL && lib->entry->unload != NULL) { + if (lib->entry.unload != NULL) { struct enif_msg_environment_t msg_env; pre_nif_noproc(&msg_env, lib, NULL); - lib->entry->unload(&msg_env.env, lib->priv_data); + lib->entry.unload(&msg_env.env, lib->priv_data); post_nif_noproc(&msg_env); } if (!erts_is_static_nif(lib->handle)) @@ -2282,7 +2287,7 @@ typedef struct { Export exp; struct erl_module_nif* m; NativeFunPtr fp; - BeamInstr *saved_current; + ErtsCodeMFA *saved_current; int exception_thrown; int saved_argc; int rootset_extra; @@ -2336,9 +2341,9 @@ allocate_nif_sched_data(Process* proc, int argc) ep->rootset_extra = argc; ep->rootset[0] = NIL; for (i=0; i<ERTS_NUM_CODE_IX; i++) { - ep->exp.addressv[i] = &ep->exp.code[3]; + ep->exp.addressv[i] = &ep->exp.beam[0]; } - ep->exp.code[3] = (BeamInstr) em_call_nif; + ep->exp.beam[0] = (BeamInstr) em_call_nif; (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep); return ep; } @@ -2383,7 +2388,7 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec reg = erts_proc_sched_data(proc)->x_reg_array; ASSERT(!need_save || proc->current); - orig_argc = need_save ? (int) proc->current[2] : 0; + orig_argc = need_save ? (int) proc->current->arity : 0; ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); if (!ep) @@ -2411,10 +2416,10 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec for (i = 0; i < argc; i++) reg[i] = (Eterm) argv[i]; proc->i = (BeamInstr*) ep->exp.addressv[0]; - ep->exp.code[0] = (BeamInstr) proc->current[0]; - ep->exp.code[1] = (BeamInstr) proc->current[1]; - ep->exp.code[2] = argc; - ep->exp.code[4] = (BeamInstr) direct_fp; + ep->exp.info.mfa.module = proc->current->module; + ep->exp.info.mfa.function = proc->current->function; + ep->exp.info.mfa.arity = argc; + ep->exp.beam[1] = (BeamInstr) direct_fp; ep->m = env->mod_nif; ep->fp = indirect_fp; proc->freason = TRAP; @@ -2436,7 +2441,7 @@ restore_nif_mfa(Process* proc, NifExport* ep, int exception) ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc) & ERTS_PROC_LOCK_MAIN); - ASSERT(ep->saved_current != &ep->exp.code[0]); + ASSERT(ep->saved_current != &ep->exp.info.mfa); proc->current = ep->saved_current; ep->saved_current = NULL; if (exception) { @@ -2510,7 +2515,8 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) execution_state(env, &proc, NULL); - fp = (NativeFunPtr) proc->current[6]; + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->fp; ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); @@ -2579,7 +2585,8 @@ schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[ erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); } - fp = (NativeFunPtr) proc->current[6]; + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->fp; ASSERT(fp); @@ -2639,7 +2646,8 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ERL_NIF_TERM result; execution_state(env, &proc, NULL); - fp = (NativeFunPtr) proc->current[6]; + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->fp; ASSERT(!env->exception_thrown); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); @@ -2707,7 +2715,7 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); - ep->exp.code[1] = (BeamInstr) fun_name_atom; + ep->exp.info.mfa.function = (BeamInstr) fun_name_atom; done: if (scheduler < 0) @@ -3021,17 +3029,16 @@ int enif_map_iterator_get_pair(ErlNifEnv *env, ***************************************************************************/ -static BeamInstr** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsigned arity) +static ErtsCodeInfo** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsigned arity) { int n = (int) mod_code->num_functions; int j; for (j = 0; j < n; ++j) { - BeamInstr* code_ptr = (BeamInstr*) mod_code->functions[j]; - ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - if (f_atom == ((Eterm) code_ptr[3]) - && arity == ((unsigned) code_ptr[4])) { - - return (BeamInstr**) &mod_code->functions[j]; + ErtsCodeInfo* ci = mod_code->functions[j]; + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + if (f_atom == ci->mfa.function + && arity == ci->mfa.arity) { + return mod_code->functions+j; } } return NULL; @@ -3123,39 +3130,57 @@ static Eterm load_nif_error(Process* p, const char* atom, const char* format, .. return ret; } +#define AT_LEAST_VERSION(E,MAJ,MIN) \ + (((E)->major * 0x100 + (E)->minor) >= ((MAJ) * 0x100 + (MIN))) + /* - * The function below is for looping through ErlNifFunc arrays, helping - * provide backwards compatibility across the version 2.7 change that added - * the "flags" field to ErlNifFunc. + * Allocate erl_module_nif and make a _modern_ copy of the lib entry. */ -static ErlNifFunc* next_func(ErlNifEntry* entry, int* incrp, ErlNifFunc* func) +static struct erl_module_nif* create_lib(const ErlNifEntry* src) { - ASSERT(incrp); - if (!*incrp) { - if (entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) - *incrp = sizeof(ErlNifFunc); - else { - /* - * ErlNifFuncV1 below is what ErlNifFunc was before the - * addition of the flags field for 2.7, and is needed to handle - * backward compatibility. - */ - typedef struct { - const char* name; - unsigned arity; - ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); - }ErlNifFuncV1; - *incrp = sizeof(ErlNifFuncV1); - } + struct erl_module_nif* lib; + ErlNifEntry* dst; + Uint bytes = offsetof(struct erl_module_nif, _funcs_copy_); + + if (!AT_LEAST_VERSION(src, 2, 7)) + bytes += src->num_of_funcs * sizeof(ErlNifFunc); + + lib = erts_alloc(ERTS_ALC_T_NIF, bytes); + dst = &lib->entry; + + sys_memcpy(dst, src, offsetof(ErlNifEntry, vm_variant)); + + if (AT_LEAST_VERSION(src, 2, 1)) { + dst->vm_variant = src->vm_variant; + } else { + dst->vm_variant = "beam.vanilla"; } - return (ErlNifFunc*) ((char*)func + *incrp); -} + if (AT_LEAST_VERSION(src, 2, 7)) { + dst->options = src->options; + } else { + /* + * Make a modern copy of the ErlNifFunc array + */ + struct ErlNifFunc_V1 { + const char* name; + unsigned arity; + ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + }*src_funcs = (struct ErlNifFunc_V1*) src->funcs; + int i; + for (i = 0; i < src->num_of_funcs; ++i) { + sys_memcpy(&lib->_funcs_copy_[i], &src_funcs[i], sizeof(*src_funcs)); + lib->_funcs_copy_[i].flags = 0; + } + dst->funcs = lib->_funcs_copy_; + dst->options = 0; + } + return lib; +}; BIF_RETTYPE load_nif_2(BIF_ALIST_2) { static const char bad_lib[] = "bad_lib"; - static const char reload[] = "reload"; static const char upgrade[] = "upgrade"; char* lib_name = NULL; void* handle = NULL; @@ -3167,15 +3192,20 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) Eterm mod_atom; const Atom* mod_atomp; Eterm f_atom; - BeamInstr* caller; + ErtsCodeMFA* caller; ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT; Eterm ret = am_ok; int veto; struct erl_module_nif* lib = NULL; - int reload_warning = 0; struct erl_module_instance* this_mi; struct erl_module_instance* prev_mi; + if (BIF_P->flags & F_HIPE_MODE) { + ret = load_nif_error(BIF_P, "notsup", "Calling load_nif from HiPE compiled " + "modules not supported"); + BIF_RET(ret); + } + encoding = erts_get_native_filename_encoding(); if (encoding == ERL_FILENAME_WIN_WCHAR) { /* Do not convert the lib name to utf-16le yet, do that in win32 specific code */ @@ -3201,12 +3231,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) /* Find calling module */ ASSERT(BIF_P->current != NULL); - ASSERT(BIF_P->current[0] == am_erlang - && BIF_P->current[1] == am_load_nif - && BIF_P->current[2] == 2); + ASSERT(BIF_P->current->module == am_erlang + && BIF_P->current->function == am_load_nif + && BIF_P->current->arity == 2); caller = find_function_from_pc(BIF_P->cp); ASSERT(caller != NULL); - mod_atom = caller[0]; + mod_atom = caller->module; ASSERT(is_atom(mod_atom)); module_p = erts_get_module(mod_atom, erts_active_code_ix()); ASSERT(module_p != NULL); @@ -3232,8 +3262,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) this_mi = module_p->on_load; } - if (init_func == NULL && - (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { + if (this_mi->nif != NULL) { + ret = load_nif_error(BIF_P,"reload","NIF library already loaded" + " (reload disallowed since OTP 20)."); + } + else if (init_func == NULL && + (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { const char slogan[] = "Failed to load NIF library"; if (strstr(errdesc.str, lib_name) != NULL) { ret = load_nif_error(BIF_P, "load_failed", "%s: '%s'", slogan, errdesc.str); @@ -3261,7 +3295,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).", entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION); } - else if (entry->minor >= 1 + else if (AT_LEAST_VERSION(entry, 2, 1) && sys_strcmp(entry->vm_variant, ERL_NIF_VM_VARIANT) != 0) { ret = load_nif_error(BIF_P, bad_lib, "Library (%s) not compiled for " "this vm variant (%s).", @@ -3272,20 +3306,25 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) " match calling module '%T'", entry->name, mod_atom); } else { - /*erts_fprintf(stderr, "Found module %T\r\n", mod_atom);*/ - - int maybe_dirty_nifs = ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) - && (entry->options & ERL_NIF_DIRTY_NIF_OPTION)); - int incr = 0; - ErlNifFunc* f = entry->funcs; - for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { - BeamInstr** code_pp; + lib = create_lib(entry); + entry = &lib->entry; /* Use a guaranteed modern lib entry from now on */ + + lib->handle = handle; + erts_refc_init(&lib->rt_cnt, 0); + erts_refc_init(&lib->rt_dtor_cnt, 0); + ASSERT(opened_rt_list == NULL); + lib->mod = module_p; + + for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { + ErtsCodeInfo** ci_pp; + ErlNifFunc* f = &entry->funcs[i]; + if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1) - || (code_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) { + || (ci_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) { ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); } - else if (maybe_dirty_nifs && f->flags) { + else if (f->flags) { /* * If the flags field is non-zero and this emulator was * built with dirty scheduler support, check that the flags @@ -3302,11 +3341,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) mod_atom, f->name, f->arity); #endif } -#ifdef ERTS_DIRTY_SCHEDULERS - else if (code_pp[1] - code_pp[0] < (5+4)) -#else - else if (code_pp[1] - code_pp[0] < (5+3)) -#endif + else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) + < BEAM_NIF_MIN_FUNC_SZ) { ret = load_nif_error(BIF_P,bad_lib,"No explicit call to load_nif" " in module (%T:%s/%u too small)", @@ -3314,7 +3350,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) } /*erts_fprintf(stderr, "Found NIF %T:%s/%u\r\n", mod_atom, f->name, f->arity);*/ - f = next_func(entry, &incr, f); } } @@ -3322,132 +3357,72 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) goto error; } - /* Call load, reload or upgrade: + /* Call load or upgrade: */ - - lib = erts_alloc(ERTS_ALC_T_NIF, sizeof(struct erl_module_nif)); - lib->handle = handle; - lib->entry = entry; - erts_refc_init(&lib->rt_cnt, 0); - erts_refc_init(&lib->rt_dtor_cnt, 0); - ASSERT(opened_rt_list == NULL); - lib->mod = module_p; env.mod_nif = lib; - if (this_mi->nif != NULL) { /*************** Reload ******************/ - /* - * Repeated load_nif calls from same Erlang module instance ("reload") - * is deprecated and was only ment as a development feature not to - * be used in production systems. (See warning below) - */ - int k, old_incr = 0; - ErlNifFunc* old_func; - lib->priv_data = this_mi->nif->priv_data; - - ASSERT(this_mi->nif->entry != NULL); - if (entry->reload == NULL) { - ret = load_nif_error(BIF_P,reload,"Reload not supported by this NIF library."); - goto error; - } - /* Check that no NIF is removed */ - old_func = this_mi->nif->entry->funcs; - for (k=0; k < this_mi->nif->entry->num_of_funcs; k++) { - int incr = 0; - ErlNifFunc* f = entry->funcs; - for (i=0; i < entry->num_of_funcs; i++) { - if (old_func->arity == f->arity - && sys_strcmp(old_func->name, f->name) == 0) { - break; - } - f = next_func(entry, &incr, f); - } - if (i == entry->num_of_funcs) { - ret = load_nif_error(BIF_P,reload,"Reloaded library missing " - "function %T:%s/%u\r\n", mod_atom, - old_func->name, old_func->arity); - goto error; - } - old_func = next_func(this_mi->nif->entry, &old_incr, old_func); - } - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->reload(&env, &lib->priv_data, BIF_ARG_2); - erts_post_nif(&env); - if (veto) { - ret = load_nif_error(BIF_P, reload, "Library reload-call unsuccessful (%d).", veto); - } - else { - commit_opened_resource_types(lib); - this_mi->nif->entry = NULL; /* to prevent 'unload' callback */ - erts_unload_nif(this_mi->nif); - reload_warning = 1; - } + + lib->priv_data = NULL; + if (prev_mi->nif != NULL) { /**************** Upgrade ***************/ + void* prev_old_data = prev_mi->nif->priv_data; + if (entry->upgrade == NULL) { + ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); + goto error; + } + erts_pre_nif(&env, BIF_P, lib, NULL); + veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2); + erts_post_nif(&env); + if (veto) { + prev_mi->nif->priv_data = prev_old_data; + ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful (%d).", veto); + } } - else { - lib->priv_data = NULL; - if (prev_mi->nif != NULL) { /**************** Upgrade ***************/ - void* prev_old_data = prev_mi->nif->priv_data; - if (entry->upgrade == NULL) { - ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); - goto error; - } - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2); - erts_post_nif(&env); - if (veto) { - prev_mi->nif->priv_data = prev_old_data; - ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful (%d).", veto); - } - else - commit_opened_resource_types(lib); - } - else if (entry->load != NULL) { /********* Initial load ***********/ - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->load(&env, &lib->priv_data, BIF_ARG_2); - erts_post_nif(&env); - if (veto) { - ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful (%d).", veto); - } - else - commit_opened_resource_types(lib); - } + else if (entry->load != NULL) { /********* Initial load ***********/ + erts_pre_nif(&env, BIF_P, lib, NULL); + veto = entry->load(&env, &lib->priv_data, BIF_ARG_2); + erts_post_nif(&env); + if (veto) { + ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful (%d).", veto); + } } if (ret == am_ok) { + commit_opened_resource_types(lib); + /* ** Everything ok, patch the beam code with op_call_nif */ - int incr = 0; - ErlNifFunc* f = entry->funcs; - this_mi->nif = lib; for (i=0; i < entry->num_of_funcs; i++) { - BeamInstr* code_ptr; + ErlNifFunc* f = &entry->funcs[i]; + ErtsCodeInfo* ci; + BeamInstr *code_ptr; + erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1); - code_ptr = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); + ci = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); + code_ptr = erts_codeinfo_to_code(ci); - if (code_ptr[1] == 0) { - code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif); + if (ci->native == 0) { + code_ptr[0] = (BeamInstr) BeamOp(op_call_nif); } else { /* Function traced, patch the original instruction word */ - GenericBp* g = (GenericBp *) code_ptr[1]; - ASSERT(code_ptr[5+0] == + GenericBp* g = (GenericBp *) ci->native; + ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)); g->orig_instr = (BeamInstr) BeamOp(op_call_nif); } #ifdef ERTS_DIRTY_SCHEDULERS - if ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) - && (entry->options & ERL_NIF_DIRTY_NIF_OPTION) && f->flags) { - code_ptr[5+3] = (BeamInstr) f->fptr; - code_ptr[5+1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? + if (f->flags) { + code_ptr[3] = (BeamInstr) f->fptr; + code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? (BeamInstr) schedule_dirty_io_nif : (BeamInstr) schedule_dirty_cpu_nif; } else #endif - code_ptr[5+1] = (BeamInstr) f->fptr; - code_ptr[5+2] = (BeamInstr) lib; - f = next_func(entry, &incr, f); + code_ptr[1] = (BeamInstr) f->fptr; + code_ptr[2] = (BeamInstr) lib; } } else { @@ -3468,14 +3443,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_release_code_write_permission(); erts_free(ERTS_ALC_T_TMP, lib_name); - if (reload_warning) { - erts_dsprintf_buf_t* dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, - "Repeated calls to erlang:load_nif from module '%T'.\n\n" - "The NIF reload mechanism is deprecated and must not " - "be used in production systems.\n", mod_atom); - erts_send_warning_to_logger(BIF_P->group_leader, dsbufp); - } BIF_RET(ret); } @@ -3539,8 +3506,8 @@ void erl_nif_init() int erts_nif_get_funcs(struct erl_module_nif* mod, ErlNifFunc **funcs) { - *funcs = mod->entry->funcs; - return mod->entry->num_of_funcs; + *funcs = mod->entry.funcs; + return mod->entry.num_of_funcs; } Eterm erts_nif_call_function(Process *p, Process *tracee, @@ -3551,10 +3518,10 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, #ifdef DEBUG /* Verify that function is part of this module */ int i; - for (i = 0; i < mod->entry->num_of_funcs; i++) - if (fun == &(mod->entry->funcs[i])) + for (i = 0; i < mod->entry.num_of_funcs; i++) + if (fun == &(mod->entry.funcs[i])) break; - ASSERT(i < mod->entry->num_of_funcs); + ASSERT(i < mod->entry.num_of_funcs); if (p) ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN || erts_smp_thr_progress_is_blocking()); diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 494971e118..413b4f7343 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -49,7 +49,7 @@ ** 2.8: 18.0 add enif_has_pending_exception ** 2.9: 18.2 enif_getenv ** 2.10: Time API -** 2.11: 19.0 enif_snprintf +** 2.11: 19.0 enif_snprintf */ #define ERL_NIF_MAJOR_VERSION 2 #define ERL_NIF_MINOR_VERSION 11 @@ -116,12 +116,13 @@ typedef struct enif_entry_t int (*reload) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info); int (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); void (*unload) (ErlNifEnv*, void* priv_data); + + /* Added in 2.1 */ const char* vm_variant; - unsigned options; -}ErlNifEntry; -/* Field bits for ErlNifEntry options */ -#define ERL_NIF_DIRTY_NIF_OPTION 1 + /* Added in 2.7 */ + unsigned options; /* Unused. Can be set to 0 or 1 (dirty sched config) */ +}ErlNifEntry; typedef struct @@ -266,8 +267,6 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; # define ERL_NIF_INIT_DECL(MODNAME) ERL_NIF_INIT_EXPORT ErlNifEntry* nif_init(ERL_NIF_INIT_ARGS) #endif -#define ERL_NIF_ENTRY_OPTIONS ERL_NIF_DIRTY_NIF_OPTION - #ifdef __cplusplus } # define ERL_NIF_INIT_PROLOGUE extern "C" { @@ -293,7 +292,7 @@ ERL_NIF_INIT_DECL(NAME) \ FUNCS, \ LOAD, RELOAD, UPGRADE, UNLOAD, \ ERL_NIF_VM_VARIANT, \ - ERL_NIF_ENTRY_OPTIONS \ + 1 \ }; \ ERL_NIF_INIT_BODY; \ return &entry; \ diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index c59b42cdae..206078903d 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -733,7 +733,7 @@ erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl) if (lock_pdl && prt->port_data_lock) driver_pdl_lock(prt->port_data_lock); -#if ERTS_ENABLE_LOCK_CHECK +#ifdef ERTS_ENABLE_LOCK_CHECK if (!ERTS_IS_CRASH_DUMPING) { if (erts_lc_is_emu_thr()) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index 2a6bd165a3..e3550e878e 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -188,13 +188,11 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id) ptsp->taskq.in.last = NULL; erts_smp_atomic32_init_nob(&ptsp->flags, 0); #ifdef ERTS_SMP - erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id, #ifdef ERTS_ENABLE_LOCK_COUNT - (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) -#else - 1 + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)) + lock_str = NULL; #endif - ); + erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id); #endif } diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 1a579704a8..b43a1b0190 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -526,8 +526,8 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { case EXPORT_DEF: { Export* ep = *((Export **) (export_val(wobj) + 1)); - Atom* module = atom_tab(atom_val(ep->code[0])); - Atom* name = atom_tab(atom_val(ep->code[1])); + Atom* module = atom_tab(atom_val(ep->info.mfa.module)); + Atom* name = atom_tab(atom_val(ep->info.mfa.function)); PRINT_STRING(res, fn, arg, "#Fun<"); PRINT_BUF(res, fn, arg, module->name, module->len); @@ -535,7 +535,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { PRINT_BUF(res, fn, arg, name->name, name->len); PRINT_CHAR(res, fn, arg, '.'); PRINT_SWORD(res, fn, arg, 'd', 0, 1, - (ErlPfSWord) ep->code[2]); + (ErlPfSWord) ep->info.mfa.arity); PRINT_CHAR(res, fn, arg, '>'); } break; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 6affbd794c..6928b282ee 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -447,11 +447,10 @@ int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP; #endif typedef enum { - ERTS_PSTT_GC, /* Garbage Collect */ + ERTS_PSTT_GC_MAJOR, /* Garbage Collect: Fullsweep */ + ERTS_PSTT_GC_MINOR, /* Garbage Collect: Generational */ ERTS_PSTT_CPC, /* Check Process Code */ -#ifdef ERTS_NEW_PURGE_STRATEGY ERTS_PSTT_CLA, /* Copy Literal Area */ -#endif ERTS_PSTT_COHMQ, /* Change off heap message queue */ ERTS_PSTT_FTMQ /* Flush trace msg queue */ } ErtsProcSysTaskType; @@ -8203,7 +8202,8 @@ sched_thread_func(void *vesdp) ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); #endif - process_main(); + process_main(esdp->x_reg_array, esdp->f_reg_array); + /* No schedulers should *ever* terminate */ erts_exit(ERTS_ABORT_EXIT, "Scheduler thread number %beu terminated\n", @@ -10467,7 +10467,8 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) break; switch (st->type) { - case ERTS_PSTT_GC: + case ERTS_PSTT_GC_MAJOR: + case ERTS_PSTT_GC_MINOR: if (c_p->flags & F_DISABLE_GC) { save_gc_task(c_p, st, st_prio); st = NULL; @@ -10475,7 +10476,9 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } else { if (!garbage_collected) { - FLAGS(c_p) |= F_NEED_FULLSWEEP; + if (st->type == ERTS_PSTT_GC_MAJOR) { + FLAGS(c_p) |= F_NEED_FULLSWEEP; + } reds -= scheduler_gc_proc(c_p, reds); garbage_collected = 1; } @@ -10491,7 +10494,6 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) fcalls = reds - CONTEXT_REDS; st_res = erts_check_process_code(c_p, st->arg[0], - unsigned_val(st->arg[1]), &cpc_reds, fcalls); reds -= cpc_reds; @@ -10502,7 +10504,6 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } break; } -#ifdef ERTS_NEW_PURGE_STRATEGY case ERTS_PSTT_CLA: { int fcalls; int cla_reds = 0; @@ -10522,7 +10523,6 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } break; } -#endif case ERTS_PSTT_COHMQ: reds -= erts_complete_off_heap_message_queue_change(c_p); st_res = am_true; @@ -10572,16 +10572,15 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) break; switch (st->type) { - case ERTS_PSTT_GC: + case ERTS_PSTT_GC_MAJOR: + case ERTS_PSTT_GC_MINOR: case ERTS_PSTT_CPC: case ERTS_PSTT_COHMQ: st_res = am_false; break; -#ifdef ERTS_NEW_PURGE_STRATEGY case ERTS_PSTT_CLA: st_res = am_ok; break; -#endif #ifdef ERTS_SMP case ERTS_PSTT_FTMQ: reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN); @@ -10747,17 +10746,19 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, switch (req_type) { case am_garbage_collect: - st->type = ERTS_PSTT_GC; - noproc_res = am_false; - if (!rp) + switch (st->arg[0]) { + case am_minor: st->type = ERTS_PSTT_GC_MINOR; break; + case am_major: st->type = ERTS_PSTT_GC_MAJOR; break; + default: goto badarg; + } + noproc_res = am_false; + if (!rp) goto noproc; break; case am_check_process_code: if (is_not_atom(st->arg[0])) goto badarg; - if (is_not_small(st->arg[1]) || (unsigned_val(st->arg[1]) & ~ERTS_CPC_ALL)) - goto badarg; noproc_res = am_false; st->type = ERTS_PSTT_CPC; if (!rp) @@ -10773,7 +10774,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, #endif break; -#ifdef ERTS_NEW_PURGE_STRATEGY case am_copy_literals: if (st->arg[0] != am_true && st->arg[0] != am_false) goto badarg; @@ -10782,7 +10782,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, if (!rp) goto noproc; break; -#endif default: goto badarg; @@ -11432,6 +11431,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef SHCOPY_SPAWN erts_shcopy_t info; INITIALIZE_SHCOPY(info); +#else + erts_literal_area_t litarea; + INITIALIZE_LITERAL_PURGE_AREA(litarea); #endif erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); @@ -11490,7 +11492,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef SHCOPY_SPAWN arg_size = copy_shared_calculate(args, &info); #else - arg_size = size_object(args); + arg_size = size_object_litopt(args, &litarea); #endif heap_need = arg_size; @@ -11514,10 +11516,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). } p->schedule_count = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); - - p->u.initial[INITIAL_MOD] = mod; - p->u.initial[INITIAL_FUN] = func; - p->u.initial[INITIAL_ARI] = (Uint) arity; + + p->u.initial.module = mod; + p->u.initial.function = func; + p->u.initial.arity = (Uint) arity; /* * Must initialize binary lists here before copying binaries to process. @@ -11559,7 +11561,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). /* No need to initialize p->fcalls. */ - p->current = p->u.initial+INITIAL_MOD; + p->current = &p->u.initial; p->i = (BeamInstr *) beam_apply; p->cp = (BeamInstr *) beam_apply+1; @@ -11572,7 +11574,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap); DESTROY_SHCOPY(info); #else - p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap); + p->arg_reg[2] = copy_struct_litopt(args, arg_size, &p->htop, &p->off_heap, &litarea); #endif p->arity = 3; @@ -11741,11 +11743,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_spawn)) { + ErtsCodeMFA cmfa = {mod, func, arity}; DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); - dtrace_fun_decode(p, mod, func, arity, process_name, mfa); - DTRACE2(process_spawn, process_name, mfa); + dtrace_fun_decode(p, &cmfa, process_name, mfa_buf); + DTRACE2(process_spawn, process_name, mfa_buf); } #endif return res; @@ -11820,9 +11823,9 @@ void erts_init_empty_process(Process *p) p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; p->seq_trace_token = NIL; - p->u.initial[0] = 0; - p->u.initial[1] = 0; - p->u.initial[2] = 0; + p->u.initial.module = 0; + p->u.initial.function = 0; + p->u.initial.arity = 0; p->catches = 0; p->cp = NULL; p->i = NULL; @@ -13292,8 +13295,8 @@ erts_program_counter_info(fmtfn_t to, void *to_arg, Process *p) static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x) { - BeamInstr* addr = find_function_from_pc(x); - if (addr == NULL) { + ErtsCodeMFA *cmfa = find_function_from_pc(x); + if (cmfa == NULL) { if (x == beam_exit) { erts_print(to, to_arg, "<terminate process>"); } else if (x == beam_continue_exit) { @@ -13307,7 +13310,8 @@ print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x) } } else { erts_print(to, to_arg, "%T:%T/%d + %d", - addr[0], addr[1], addr[2], ((x-addr)-2) * sizeof(Eterm)); + cmfa->module, cmfa->function, cmfa->arity, + (x-(BeamInstr*)cmfa) * sizeof(Eterm)); } } diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 68fbb10602..7193b7d1db 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -21,6 +21,8 @@ #ifndef __PROCESS_H__ #define __PROCESS_H__ +#include "sys.h" + #undef ERTS_INCLUDE_SCHEDULER_INTERNALS #if (defined(ERL_PROCESS_C__) \ || defined(ERL_PORT_TASK_C__) \ @@ -37,8 +39,6 @@ typedef struct process Process; -#include "sys.h" - #define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__ #include "erl_process_lock.h" /* Only pull out important types... */ #undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__ @@ -93,10 +93,6 @@ struct ErtsNodesMonitor_; #define ERTS_HEAP_FREE(Type, Ptr, Size) \ erts_free((Type), (Ptr)) -#define INITIAL_MOD 0 -#define INITIAL_FUN 1 -#define INITIAL_ARI 2 - #include "export.h" struct saved_calls { @@ -1023,15 +1019,16 @@ struct process { #endif union { void *terminate; - BeamInstr initial[3]; /* Initial module(0), function(1), arity(2), often used instead - of pointer to funcinfo instruction, hence the BeamInstr datatype */ + ErtsCodeMFA initial; /* Initial module(0), function(1), arity(2), + often used instead of pointer to funcinfo + instruction. */ } u; - BeamInstr* current; /* Current Erlang function, part of the funcinfo: + ErtsCodeMFA* current; /* Current Erlang function, part of the funcinfo: * module(0), function(1), arity(2) * (module and functions are tagged atoms; - * arity an untagged integer). BeamInstr * because it references code + * arity an untagged integer). */ - + /* * Information mainly for post-mortem use (erl crash dump). */ diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 42654604cb..8311fde025 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -56,7 +56,7 @@ #define MAKE_HASH(Term) \ ((is_small(Term)) ? unsigned_val(Term) : \ ((is_atom(Term)) ? \ - (atom_tab(atom_val(Term))->slot.bucket.hvalue) : \ + atom_val(Term) : \ make_internal_hash(Term))) #define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm)) @@ -408,6 +408,11 @@ static void pd_hash_erase_all(Process *p) } } +Uint32 erts_pd_make_hx(Eterm key) +{ + return MAKE_HASH(key); +} + Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id) { unsigned int hval; diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h index b50a2af72c..ab58f3c239 100644 --- a/erts/emulator/beam/erl_process_dict.h +++ b/erts/emulator/beam/erl_process_dict.h @@ -43,6 +43,7 @@ void erts_deep_dictionary_dump(fmtfn_t to, void *to_arg, Eterm erts_dictionary_copy(struct process *p, ProcDict *pd); Eterm erts_pd_hash_get(struct process *p, Eterm id); +Uint32 erts_pd_make_hx(Eterm key); Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id); #endif diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index a19db74763..7c23b5c76a 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -337,8 +337,8 @@ stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg) static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x) { - BeamInstr* addr = find_function_from_pc(x); - if (addr == NULL) { + ErtsCodeMFA* cmfa = find_function_from_pc(x); + if (cmfa == NULL) { if (x == beam_exit) { erts_print(to, to_arg, "<terminate process>"); } else if (x == beam_continue_exit) { @@ -350,7 +350,8 @@ print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x) } } else { erts_print(to, to_arg, "%T:%T/%bpu + %bpu", - addr[0], addr[1], addr[2], ((x-addr)-2) * sizeof(Eterm)); + cmfa->module, cmfa->function, cmfa->arity, + (x-(BeamInstr*)cmfa) * sizeof(Eterm)); } } diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index a69185bc5c..180df229eb 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -123,7 +123,7 @@ erts_init_proc_lock(int cpus) for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) { #ifdef ERTS_ENABLE_LOCK_COUNT erts_mtx_init_x(&erts_pix_locks[i].u.mtx, - "pix_lock", make_small(i), 1); + "pix_lock", make_small(i)); #else erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock"); #endif @@ -1027,39 +1027,32 @@ erts_proc_lock_init(Process *p) #endif #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL -#ifdef ERTS_ENABLE_LOCK_COUNT - int do_lock_count = 1; -#else - int do_lock_count = 0; -#endif - - erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id); ethr_mutex_lock(&p->lock.main.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.main.lc); #endif - erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id); ethr_mutex_lock(&p->lock.link.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.link.lc); #endif - erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id); ethr_mutex_lock(&p->lock.msgq.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.msgq.lc); #endif - erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id); ethr_mutex_lock(&p->lock.btm.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.btm.lc); #endif - erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id, - do_lock_count); + erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id); ethr_mutex_lock(&p->lock.status.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.status.lc); #endif - erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id); ethr_mutex_lock(&p->lock.trace.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.trace.lc); diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 713ed50b86..14be511f86 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1065,7 +1065,7 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra) { #ifdef ERTS_SMP - erts_mtx_init_x(mtx, name, extra, 1); + erts_mtx_init_x(mtx, name, extra); #endif } @@ -1073,7 +1073,7 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra) { #ifdef ERTS_SMP - erts_mtx_init_locked_x(mtx, name, extra, 1); + erts_mtx_init_locked_x(mtx, name, extra); #endif } diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index eccd49f2a9..9e75f6fee5 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -479,14 +479,9 @@ ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void)); ERTS_GLB_INLINE erts_tid_t erts_thr_self(void); ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len); ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y); -ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, - int enable_lcnt); -ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, - Uint16 opt, int enable_lcnt); -ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, - char *name, - Eterm extra, - int enable_lcnt); +ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra); +ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt); +ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra); ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx); @@ -2164,7 +2159,7 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y) } ERTS_GLB_INLINE void -erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt) +erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra) { #ifdef USE_THREADS int res = ethr_mutex_init(&mtx->mtx); @@ -2174,17 +2169,13 @@ erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt) erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - if (enable_lcnt) - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); - else - erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra); + erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); #endif #endif } ERTS_GLB_INLINE void -erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt, - int enable_lcnt) +erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt) { #ifdef USE_THREADS int res = ethr_mutex_init(&mtx->mtx); @@ -2194,17 +2185,14 @@ erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt, erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - if (enable_lcnt) - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra); - else - erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX | opt, extra); + erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra); #endif #endif } ERTS_GLB_INLINE void -erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt) +erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra) { #ifdef USE_THREADS int res = ethr_mutex_init(&mtx->mtx); @@ -2214,10 +2202,7 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - if (enable_lcnt) - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); - else - erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra); + erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); #endif ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 8c84303997..9be4741ec8 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -781,7 +781,8 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what) tmp = make_small(0); } else { hp = HAlloc(p, 4); - tmp = TUPLE3(hp,p->current[0],p->current[1],make_small(p->current[2])); + tmp = TUPLE3(hp,p->current->module,p->current->function, + make_small(p->current->arity)); hp += 4; } @@ -1027,14 +1028,14 @@ erts_trace_return_to(Process *p, BeamInstr *pc) { Eterm mfa; - BeamInstr *code_ptr = find_function_from_pc(pc); + ErtsCodeMFA *cmfa = find_function_from_pc(pc); - - if (!code_ptr) { + if (!cmfa) { mfa = am_undefined; } else { Eterm *hp = HAlloc(p, 4); - mfa = TUPLE3(hp, code_ptr[0], code_ptr[1], make_small(code_ptr[2])); + mfa = TUPLE3(hp, cmfa->module, cmfa->function, + make_small(cmfa->arity)); } send_to_tracer_nif(p, &p->common, p->common.id, NULL, TRACE_FUN_T_CALL, @@ -1046,11 +1047,11 @@ erts_trace_return_to(Process *p, BeamInstr *pc) * or {trace, Pid, return_from, {Mod, Name, Arity}, Retval} */ void -erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer) +erts_trace_return(Process* p, ErtsCodeMFA *mfa, + Eterm retval, ErtsTracer *tracer) { Eterm* hp; - Eterm mfa, mod, name; - int arity; + Eterm mfa_tuple; Uint meta_flags, *tracee_flags; ASSERT(tracer); @@ -1084,15 +1085,13 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer) tracee_flags = &meta_flags; } - mod = fi[0]; - name = fi[1]; - arity = fi[2]; - hp = HAlloc(p, 4); - mfa = TUPLE3(hp, mod, name, make_small(arity)); + mfa_tuple = TUPLE3(hp, mfa->module, mfa->function, + make_small(mfa->arity)); hp += 4; send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id, - NULL, TRACE_FUN_T_CALL, am_return_from, mfa, retval, am_true); + NULL, TRACE_FUN_T_CALL, am_return_from, mfa_tuple, + retval, am_true); } /* Send {trace_ts, Pid, exception_from, {Mod, Name, Arity}, {Class,Value}, @@ -1103,7 +1102,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer) * Where Class is atomic but Value is any term. */ void -erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, +erts_trace_exception(Process* p, ErtsCodeMFA *mfa, Eterm class, Eterm value, ErtsTracer *tracer) { Eterm* hp; @@ -1142,7 +1141,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, } hp = HAlloc(p, 7);; - mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm)mfa[2])); + mfa_tuple = TUPLE3(hp, mfa->module, mfa->function, make_small(mfa->arity)); hp += 4; cv = TUPLE2(hp, class, value); hp += 3; @@ -1165,7 +1164,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, * if it is a pid or port we do a meta trace. */ Uint32 -erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, +erts_call_trace(Process* p, ErtsCodeInfo *info, Binary *match_spec, Eterm* args, int local, ErtsTracer *tracer) { Eterm* hp; @@ -1244,7 +1243,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, * such as size_object() and copy_struct(), we must make sure that we * temporarily convert any match contexts to sub binaries. */ - arity = (Eterm) mfa[2]; + arity = info->mfa.arity; for (i = 0; i < arity; i++) { Eterm arg = args[i]; if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) { @@ -1339,7 +1338,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, hp += 2; } } - mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple); + mfa_tuple = TUPLE3(hp, info->mfa.module, info->mfa.function, mfa_tuple); hp += 4; /* @@ -1459,7 +1458,8 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg) } void -monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint time) +monitor_long_schedule_proc(Process *p, ErtsCodeMFA *in_fp, + ErtsCodeMFA *out_fp, Uint time) { ErlHeapFragment *bp; ErlOffHeap *off_heap; @@ -1490,11 +1490,13 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, monitor_p); tmo = erts_bld_uint(&hp, NULL, time); if (in_fp != NULL) { - in_mfa = TUPLE3(hp,(Eterm) in_fp[0], (Eterm) in_fp[1], make_small(in_fp[2])); + in_mfa = TUPLE3(hp, in_fp->module, in_fp->function, + make_small(in_fp->arity)); hp +=4; } if (out_fp != NULL) { - out_mfa = TUPLE3(hp,(Eterm) out_fp[0], (Eterm) out_fp[1], make_small(out_fp[2])); + out_mfa = TUPLE3(hp, out_fp->module, out_fp->function, + make_small(out_fp->arity)); hp +=4; } tmo_tpl = TUPLE2(hp,am_timeout, tmo); @@ -2129,7 +2131,7 @@ profile_runnable_proc(Process *p, Eterm status){ Eterm *hp, msg; Eterm where = am_undefined; ErlHeapFragment *bp = NULL; - BeamInstr *current = NULL; + ErtsCodeMFA *cmfa = NULL; #ifndef ERTS_SMP #define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) @@ -2151,14 +2153,14 @@ profile_runnable_proc(Process *p, Eterm status){ if (!ERTS_PROC_IS_EXITING(p)) { if (p->current) { - current = p->current; + cmfa = p->current; } else { - current = find_function_from_pc(p->i); + cmfa = find_function_from_pc(p->i); } } #ifdef ERTS_SMP - if (!current) { + if (!cmfa) { hsz -= 4; } @@ -2166,8 +2168,10 @@ profile_runnable_proc(Process *p, Eterm status){ hp = bp->mem; #endif - if (current) { - where = TUPLE3(hp, current[0], current[1], make_small(current[2])); hp += 4; + if (cmfa) { + where = TUPLE3(hp, cmfa->module, cmfa->function, + make_small(cmfa->arity)); + hp += 4; } else { where = make_small(0); } diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index 0095d4386b..378b6de49c 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -101,11 +101,11 @@ void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *); void trace_send(Process*, Eterm, Eterm); void trace_receive(Process*, Eterm, Eterm, ErtsTracingEvent*); -Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec, +Uint32 erts_call_trace(Process *p, ErtsCodeInfo *info, struct binary *match_spec, Eterm* args, int local, ErtsTracer *tracer); -void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, +void erts_trace_return(Process* p, ErtsCodeMFA *mfa, Eterm retval, ErtsTracer *tracer); -void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value, +void erts_trace_exception(Process* p, ErtsCodeMFA *mfa, Eterm class, Eterm value, ErtsTracer *tracer); void erts_trace_return_to(Process *p, BeamInstr *pc); void trace_sched(Process*, ErtsProcLocks, Eterm); @@ -134,7 +134,8 @@ void erts_system_profile_setup_active_schedulers(void); /* system_monitor */ void monitor_long_gc(Process *p, Uint time); -void monitor_long_schedule_proc(Process *p, BeamInstr *in_i, BeamInstr *out_i, Uint time); +void monitor_long_schedule_proc(Process *p, ErtsCodeMFA *in_i, + ErtsCodeMFA *out_i, Uint time); void monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time); void monitor_large_heap(Process *p); void monitor_generic(Process *p, Eterm type, Eterm spec); @@ -176,7 +177,7 @@ struct trace_pattern_flags { }; extern const struct trace_pattern_flags erts_trace_pattern_flags_off; extern int erts_call_time_breakpoint_tracing; -int erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, +int erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, struct binary* match_prog_set, struct binary *meta_match_prog_set, int on, struct trace_pattern_flags, diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index f97716d030..93cfe08105 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -146,11 +146,12 @@ typedef struct op_entry { int sz; /* Number of loaded words. */ char* pack; /* Instructions for packing engine. */ char* sign; /* Signature string. */ - unsigned count; /* Number of times executed. */ } OpEntry; -extern OpEntry opc[]; /* Description of all instructions. */ -extern int num_instructions; /* Number of instruction in opc[]. */ +extern const OpEntry opc[]; /* Description of all instructions. */ +extern const int num_instructions; /* Number of instruction in opc[]. */ + +extern Uint erts_instr_count[]; /* some constants for various table sizes etc */ @@ -185,4 +186,11 @@ extern int erts_pd_initial_size;/* Initial Process dictionary table size */ #include "erl_term.h" +#ifdef NO_JUMP_TABLE +#define BeamOp(Op) (Op) +#else +extern void** beam_ops; +#define BeamOp(Op) beam_ops[(Op)] +#endif + #endif /* __ERL_VM_H__ */ diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index 6c33b12dd0..e431c3051b 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -21,6 +21,8 @@ #ifndef __ERROR_H__ #define __ERROR_H__ +#include "code_ix.h" + /* * There are three primary exception classes: * @@ -197,7 +199,7 @@ struct StackTrace { Eterm header; /* bignum header - must be first in struct */ Eterm freason; /* original exception reason is saved in the struct */ BeamInstr* pc; - BeamInstr* current; + ErtsCodeMFA* current; int depth; /* number of saved pointers in trace[] */ BeamInstr *trace[1]; /* varying size - must be last in struct */ }; diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 00fa452f79..33ed6d7ec1 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -103,7 +103,8 @@ static HashValue export_hash(struct export_entry* ee) { Export* x = ee->ep; - return EXPORT_HASH(x->code[0], x->code[1], x->code[2]); + return EXPORT_HASH(x->info.mfa.module, x->info.mfa.function, + x->info.mfa.arity); } static int @@ -111,9 +112,9 @@ export_cmp(struct export_entry* tmpl_e, struct export_entry* obj_e) { Export* tmpl = tmpl_e->ep; Export* obj = obj_e->ep; - return !(tmpl->code[0] == obj->code[0] && - tmpl->code[1] == obj->code[1] && - tmpl->code[2] == obj->code[2]); + return !(tmpl->info.mfa.module == obj->info.mfa.module && + tmpl->info.mfa.function == obj->info.mfa.function && + tmpl->info.mfa.arity == obj->info.mfa.arity); } @@ -130,21 +131,23 @@ export_alloc(struct export_entry* tmpl_e) blob = (struct export_blob*) erts_alloc(ERTS_ALC_T_EXPORT, sizeof(*blob)); erts_smp_atomic_add_nob(&total_entries_bytes, sizeof(*blob)); obj = &blob->exp; - obj->fake_op_func_info_for_hipe[0] = 0; - obj->fake_op_func_info_for_hipe[1] = 0; - obj->code[0] = tmpl->code[0]; - obj->code[1] = tmpl->code[1]; - obj->code[2] = tmpl->code[2]; - obj->code[3] = (BeamInstr) em_call_error_handler; - obj->code[4] = 0; + obj->info.op = 0; + obj->info.native = 0; + obj->info.mfa.module = tmpl->info.mfa.module; + obj->info.mfa.function = tmpl->info.mfa.function; + obj->info.mfa.arity = tmpl->info.mfa.arity; + obj->beam[0] = (BeamInstr) em_call_error_handler; + obj->beam[1] = 0; for (ix=0; ix<ERTS_NUM_CODE_IX; ix++) { - obj->addressv[ix] = obj->code+3; + obj->addressv[ix] = obj->beam; blob->entryv[ix].slot.index = -1; blob->entryv[ix].ep = &blob->exp; } ix = 0; + + DBG_TRACE_MFA_P(&obj->info.mfa, "export allocation at %p", obj); } else { /* Existing entry in another table, use free entry in blob */ blob = entry_to_blob(tmpl_e); @@ -163,9 +166,12 @@ export_free(struct export_entry* obj) obj->slot.index = -1; for (i=0; i < ERTS_NUM_CODE_IX; i++) { if (blob->entryv[i].slot.index >= 0) { + DBG_TRACE_MFA_P(&blob->exp.info.mfa, "export entry slot %u freed for %p", + (obj - blob->entryv), &blob->exp); return; } } + DBG_TRACE_MFA_P(&blob->exp.info.mfa, "export blob deallocation at %p", &blob->exp); erts_free(ERTS_ALC_T_EXPORT, blob); erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob)); } @@ -224,7 +230,9 @@ erts_find_export_entry(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) while (b != (HashBucket*) 0) { Export* ep = ((struct export_entry*) b)->ep; - if (ep->code[0] == m && ep->code[1] == f && ep->code[2] == a) { + if (ep->info.mfa.module == m && + ep->info.mfa.function == f && + ep->info.mfa.arity == a) { return ep; } b = b->next; @@ -237,9 +245,9 @@ static struct export_entry* init_template(struct export_templ* templ, { templ->entry.ep = &templ->exp; templ->entry.slot.index = -1; - templ->exp.code[0] = m; - templ->exp.code[1] = f; - templ->exp.code[2] = a; + templ->exp.info.mfa.module = m; + templ->exp.info.mfa.function = f; + templ->exp.info.mfa.arity = a; return &templ->entry; } @@ -263,8 +271,8 @@ erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a)); if (ee == NULL || - (ee->ep->addressv[code_ix] == ee->ep->code+3 && - ee->ep->code[3] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) { + (ee->ep->addressv[code_ix] == ee->ep->beam && + ee->ep->beam[0] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) { return NULL; } return ee->ep; diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index 17fc4828ca..7c812b306c 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -33,22 +33,20 @@ typedef struct export { void* addressv[ERTS_NUM_CODE_IX]; /* Pointer to code for function. */ - BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */ + ErtsCodeInfo info; /* MUST be just before beam[] */ + /* - * code[0]: Tagged atom for module. - * code[1]: Tagged atom for function. - * code[2]: Arity (untagged integer). - * code[3]: This entry is 0 unless the 'address' field points to it. + * beam[0]: This entry is 0 unless the 'addressv' field points to it. * Threaded code instruction to load function * (em_call_error_handler), execute BIF (em_apply_bif), * or a breakpoint instruction (op_i_generic_breakpoint). - * code[4]: Function pointer to BIF function (for BIFs only), + * beam[1]: Function pointer to BIF function (for BIFs only), * or pointer to threaded code if the module has an * on_load function that has not been run yet, or pointer - * to code for function code[3] is a breakpont instruction. + * to code if function beam[0] is a breakpoint instruction. * Otherwise: 0. */ - BeamInstr code[5]; + BeamInstr beam[2]; } Export; @@ -74,8 +72,8 @@ extern erts_smp_mtx_t export_staging_lock; #include "beam_load.h" /* For em_* extern declarations */ #define ExportIsBuiltIn(EntryPtr) \ -(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code + 3) && \ - ((EntryPtr)->code[3] == (BeamInstr) em_apply_bif)) +(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->beam) && \ + ((EntryPtr)->beam[0] == (BeamInstr) em_apply_bif)) #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index beed847578..10e452fa25 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2163,12 +2163,8 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) * We use this atom as sysname in local pid/port/refs * for the ETS compressed format (DFLAG_INTERNAL_TAGS). * - * We used atom '' earlier but that turned out to cause problems - * for buggy erl_interface/ic usage of c-nodes with empty node names. - * A long atom reduces risk of nodes actually called this and the length - * does not matter anyway as it's encoded with atom index (ATOM_INTERNAL_REF2). */ -#define INTERNAL_LOCAL_SYSNAME am_await_microstate_accounting_modifications +#define INTERNAL_LOCAL_SYSNAME am_ErtsSecretAtom static byte* enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) @@ -2826,9 +2822,10 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Export* exp = *((Export **) (export_val(obj) + 1)); if ((dflags & DFLAG_EXPORT_PTR_TAG) != 0) { *ep++ = EXPORT_EXT; - ep = enc_atom(acmp, exp->code[0], ep, dflags); - ep = enc_atom(acmp, exp->code[1], ep, dflags); - ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags, off_heap); + ep = enc_atom(acmp, exp->info.mfa.module, ep, dflags); + ep = enc_atom(acmp, exp->info.mfa.function, ep, dflags); + ep = enc_term(acmp, make_small(exp->info.mfa.arity), + ep, dflags, off_heap); } else { /* Tag, arity */ *ep++ = SMALL_TUPLE_EXT; @@ -2836,10 +2833,10 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep += 1; /* Module name */ - ep = enc_atom(acmp, exp->code[0], ep, dflags); + ep = enc_atom(acmp, exp->info.mfa.module, ep, dflags); /* Function name */ - ep = enc_atom(acmp, exp->code[1], ep, dflags); + ep = enc_atom(acmp, exp->info.mfa.function, ep, dflags); } break; } @@ -3747,9 +3744,8 @@ dec_term_atom_common: funp->arity = arity; #ifdef HIPE if (funp->fe->native_address == NULL) { - hipe_set_closure_stub(funp->fe, num_free); + hipe_set_closure_stub(funp->fe); } - funp->native_address = funp->fe->native_address; #endif hp = factory->hp; @@ -3821,9 +3817,6 @@ dec_term_atom_common: funp->fe = erts_put_fun_entry(module, old_uniq, old_index); funp->arity = funp->fe->address[-1] - num_free; -#ifdef HIPE - funp->native_address = funp->fe->native_address; -#endif hp = factory->hp; /* Environment */ @@ -4262,9 +4255,9 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, { Export* ep = *((Export **) (export_val(obj) + 1)); result += 1; - result += encode_size_struct2(acmp, ep->code[0], dflags); - result += encode_size_struct2(acmp, ep->code[1], dflags); - result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags); + result += encode_size_struct2(acmp, ep->info.mfa.module, dflags); + result += encode_size_struct2(acmp, ep->info.mfa.function, dflags); + result += encode_size_struct2(acmp, make_small(ep->info.mfa.arity), dflags); } break; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index d6df85034c..2b2f3c5cdc 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -773,8 +773,8 @@ do { \ typedef struct ErtsPStack_ { byte* pstart; - byte* psp; - byte* pend; + int offs; /* "stack pointer" as byte offset from pstart */ + int size; /* allocated size in bytes */ ErtsAlcType_t alloc_type; }ErtsPStack; @@ -785,8 +785,8 @@ void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes); #define PSTACK_DECLARE(s, DEF_PSTACK_SIZE) \ PSTACK_TYPE PSTK_DEF_STACK(s)[DEF_PSTACK_SIZE]; \ ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */ \ - (byte*)(PSTK_DEF_STACK(s) - 1), /* psp */ \ - (byte*)(PSTK_DEF_STACK(s) + (DEF_PSTACK_SIZE)), /* pend */\ + -(int)sizeof(PSTACK_TYPE), /* offs */ \ + DEF_PSTACK_SIZE*sizeof(PSTACK_TYPE), /* size */ \ ERTS_ALC_T_ESTACK /* alloc_type */ \ } @@ -806,19 +806,21 @@ do { \ } \ } while(0) -#define PSTACK_IS_EMPTY(s) (s.psp < s.pstart) +#define PSTACK_IS_EMPTY(s) (s.offs < 0) -#define PSTACK_COUNT(s) (((PSTACK_TYPE*)s.psp + 1) - (PSTACK_TYPE*)s.pstart) +#define PSTACK_COUNT(s) ((s.offs + sizeof(PSTACK_TYPE)) / sizeof(PSTACK_TYPE)) -#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), (PSTACK_TYPE*)(s.psp)) +#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), \ + (PSTACK_TYPE*)(s.pstart + s.offs)) -#define PSTACK_PUSH(s) \ - (s.psp += sizeof(PSTACK_TYPE), \ - ((s.psp == s.pend) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s), \ - sizeof(PSTACK_TYPE)) : (void)0), \ - ((PSTACK_TYPE*) s.psp)) +#define PSTACK_PUSH(s) \ + (s.offs += sizeof(PSTACK_TYPE), \ + ((s.offs == s.size) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s), \ + sizeof(PSTACK_TYPE)) : (void)0), \ + ((PSTACK_TYPE*) (s.pstart + s.offs))) -#define PSTACK_POP(s) ((PSTACK_TYPE*) (s.psp -= sizeof(PSTACK_TYPE))) +#define PSTACK_POP(s) ((s.offs -= sizeof(PSTACK_TYPE)), \ + (PSTACK_TYPE*)(s.pstart + s.offs)) /* * Do not free the stack after this, it may have pointers into what @@ -831,8 +833,8 @@ do {\ (dst)->pstart = erts_alloc(s.alloc_type,\ sizeof(PSTK_DEF_STACK(s)));\ sys_memcpy((dst)->pstart, s.pstart, _pbytes);\ - (dst)->psp = (dst)->pstart + _pbytes - sizeof(PSTACK_TYPE);\ - (dst)->pend = (dst)->pstart + sizeof(PSTK_DEF_STACK(s));\ + (dst)->offs = s.offs;\ + (dst)->size = s.size;\ (dst)->alloc_type = s.alloc_type;\ } else\ *(dst) = s;\ @@ -847,8 +849,8 @@ do { \ ASSERT(s.pstart == (byte*)PSTK_DEF_STACK(s)); \ s = *(src); /* struct copy */ \ (src)->pstart = NULL; \ - ASSERT(s.psp >= (s.pstart - sizeof(PSTACK_TYPE))); \ - ASSERT(s.psp < s.pend); \ + ASSERT(s.offs >= -(int)sizeof(PSTACK_TYPE)); \ + ASSERT(s.offs < s.size); \ } while (0) #define PSTACK_DESTROY_SAVED(pstack)\ @@ -1000,12 +1002,8 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg); Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); /* beam_bif_load.c */ -#define ERTS_CPC_ALLOW_GC (1 << 0) -#define ERTS_CPC_ALL ERTS_CPC_ALLOW_GC -Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls); -#ifdef ERTS_NEW_PURGE_STRATEGY +Eterm erts_check_process_code(Process *c_p, Eterm module, int *redsp, int fcalls); Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed); -#endif typedef struct ErtsLiteralArea_ { struct erl_off_heap_header *off_heap; @@ -1019,10 +1017,7 @@ typedef struct ErtsLiteralArea_ { extern erts_smp_atomic_t erts_copy_literal_area__; #define ERTS_COPY_LITERAL_AREA() \ ((ErtsLiteralArea *) erts_smp_atomic_read_nob(&erts_copy_literal_area__)) - -#ifdef ERTS_NEW_PURGE_STRATEGY extern Process *erts_literal_area_collector; -#endif #ifdef ERTS_DIRTY_SCHEDULERS extern Process *erts_dirty_process_code_checker; #endif @@ -1031,7 +1026,7 @@ extern Process *erts_code_purger; /* beam_load.c */ typedef struct { - BeamInstr* current; /* Pointer to: Mod, Name, Arity */ + ErtsCodeMFA* mfa; /* Pointer to: Mod, Name, Arity */ Uint needed; /* Heap space needed for entire tuple */ Uint32 loc; /* Location in source code */ Eterm* fname_ptr; /* Pointer to fname table */ @@ -1048,13 +1043,14 @@ Eterm erts_finish_loading(Binary* loader_state, Process* c_p, Eterm erts_preload_module(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm* mod, byte* code, Uint size); void init_load(void); -BeamInstr* find_function_from_pc(BeamInstr* pc); +ErtsCodeMFA* find_function_from_pc(BeamInstr* pc); Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p); -void erts_set_current_function(FunctionInfo* fi, BeamInstr* current); +void erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa); Eterm erts_module_info_0(Process* p, Eterm module); Eterm erts_module_info_1(Process* p, Eterm module, Eterm what); Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); +int erts_commit_hipe_patch_load(Eterm hipe_magic_bin); /* beam_ranges.c */ void erts_init_ranges(void); @@ -1105,26 +1101,26 @@ typedef struct { Eterm* shtable_start; ErtsAlcType_t shtable_alloc_type; Uint literal_size; - Eterm *range_ptr; - Uint range_sz; + Eterm *lit_purge_ptr; + Uint lit_purge_sz; } erts_shcopy_t; -#define INITIALIZE_SHCOPY(info) \ -do { \ - ErtsLiteralArea *larea__ = ERTS_COPY_LITERAL_AREA();\ - info.queue_start = info.queue_default; \ - info.bitstore_start = info.bitstore_default; \ - info.shtable_start = info.shtable_default; \ - info.literal_size = 0; \ - if (larea__) { \ - info.range_ptr = &larea__->start[0]; \ - info.range_sz = larea__->end - info.range_ptr; \ - } \ - else { \ - info.range_ptr = NULL; \ - info.range_sz = 0; \ - } \ -} while(0) +#define INITIALIZE_SHCOPY(info) \ + do { \ + ErtsLiteralArea *larea__ = ERTS_COPY_LITERAL_AREA(); \ + info.queue_start = info.queue_default; \ + info.bitstore_start = info.bitstore_default; \ + info.shtable_start = info.shtable_default; \ + info.literal_size = 0; \ + if (larea__) { \ + info.lit_purge_ptr = &larea__->start[0]; \ + info.lit_purge_sz = larea__->end - info.lit_purge_ptr; \ + } \ + else { \ + info.lit_purge_ptr = NULL; \ + info.lit_purge_sz = 0; \ + } \ + } while(0) #define DESTROY_SHCOPY(info) \ do { \ @@ -1140,18 +1136,42 @@ do { \ } while(0) /* copy.c */ +typedef struct { + Eterm *lit_purge_ptr; + Uint lit_purge_sz; +} erts_literal_area_t; + +#define INITIALIZE_LITERAL_PURGE_AREA(Area) \ + do { \ + ErtsLiteralArea *larea__ = ERTS_COPY_LITERAL_AREA(); \ + if (larea__) { \ + (Area).lit_purge_ptr = &larea__->start[0]; \ + (Area).lit_purge_sz = larea__->end - (Area).lit_purge_ptr; \ + } \ + else { \ + (Area).lit_purge_ptr = NULL; \ + (Area).lit_purge_sz = 0; \ + } \ + } while(0) + Eterm copy_object_x(Eterm, Process*, Uint); #define copy_object(Term, Proc) copy_object_x(Term,Proc,0) -Uint size_object(Eterm); +Uint size_object_x(Eterm, erts_literal_area_t*); +#define size_object(Term) size_object_x(Term,NULL) +#define size_object_litopt(Term,LitArea) size_object_x(Term,LitArea) + Uint copy_shared_calculate(Eterm, erts_shcopy_t*); Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*); Uint size_shared(Eterm); -Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz); +Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint*, erts_literal_area_t*); #define copy_struct(Obj,Sz,HPP,OH) \ - copy_struct_x(Obj,Sz,HPP,OH,NULL) + copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL) +#define copy_struct_litopt(Obj,Sz,HPP,OH,LitArea) \ + copy_struct_x(Obj,Sz,HPP,OH,NULL,LitArea) + Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, @@ -1180,7 +1200,7 @@ void print_pass_through(int, byte*, int); /* beam_emu.c */ int catchlevel(Process*); void init_emulator(void); -void process_main(void); +void process_main(Eterm* x_reg_array, FloatDef* f_reg_array); void erts_dirty_process_main(ErtsSchedulerData *); Eterm build_stacktrace(Process* c_p, Eterm exc); Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value); @@ -1453,18 +1473,6 @@ Eterm erts_gc_bor(Process* p, Eterm* reg, Uint live); Eterm erts_gc_bxor(Process* p, Eterm* reg, Uint live); Eterm erts_gc_bnot(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_length_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_size_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_bit_size_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_float_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live); - Uint erts_current_reductions(Process* current, Process *p); int erts_print_system_version(fmtfn_t to, void *arg, Process *c_p); @@ -1573,8 +1581,7 @@ int erts_beam_jump_table(void); ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf); ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf); ERTS_GLB_INLINE void dtrace_port_str(Port *port, char *port_buf); -ERTS_GLB_INLINE void dtrace_fun_decode(Process *process, - Eterm module, Eterm function, int arity, +ERTS_GLB_INLINE void dtrace_fun_decode(Process *process, ErtsCodeMFA *mfa, char *process_buf, char *mfa_buf); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -1608,8 +1615,7 @@ dtrace_port_str(Port *port, char *port_buf) } ERTS_GLB_INLINE void -dtrace_fun_decode(Process *process, - Eterm module, Eterm function, int arity, +dtrace_fun_decode(Process *process, ErtsCodeMFA *mfa, char *process_buf, char *mfa_buf) { if (process_buf) { @@ -1617,7 +1623,7 @@ dtrace_fun_decode(Process *process, } erts_snprintf(mfa_buf, DTRACE_TERM_BUF_SIZE, "%T:%T/%d", - module, function, arity); + mfa->module, mfa->function, mfa->arity); } #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 4f131c74de..33b74f30b7 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -259,13 +259,11 @@ static ERTS_INLINE void port_init_instr(Port *prt ASSERT(prt->drv_ptr && prt->lock); if (!prt->drv_ptr->lock) { char *lock_str = "port_lock"; - erts_mtx_init_locked_x(prt->lock, lock_str, id, #ifdef ERTS_ENABLE_LOCK_COUNT - (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) -#else - 0 + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)) + lock_str = NULL; #endif - ); + erts_mtx_init_locked_x(prt->lock, lock_str, id); } #endif erts_port_task_init_sched(&prt->sched, id); @@ -7107,7 +7105,7 @@ driver_pdl_create(ErlDrvPort dp) return NULL; pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK, sizeof(struct erl_drv_port_data_lock)); - erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id, 1); + erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id); pdl_init_refc(pdl); erts_port_inc_refc(pp); pdl->prt = pp; @@ -8290,14 +8288,13 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) erts_mtx_init_x(drv->lock, "driver_lock", #if defined(ERTS_ENABLE_LOCK_CHECK) || defined(ERTS_ENABLE_LOCK_COUNT) - erts_atom_put((byte *) drv->name, - sys_strlen(drv->name), - ERTS_ATOM_ENC_LATIN1, - 1), + erts_atom_put((byte *) drv->name, + sys_strlen(drv->name), + ERTS_ATOM_ENC_LATIN1, + 1) #else - NIL, + NIL #endif - 1 ); } #endif diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 5978478323..8ab6c713d6 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -26,6 +26,7 @@ #include "erl_vm.h" #include "global.h" #include "module.h" +#include "beam_catches.h" #ifdef DEBUG # define IF_DEBUG(x) x @@ -67,6 +68,18 @@ static int module_cmp(Module* tmpl, Module* obj) return tmpl->module != obj->module; } +void erts_module_instance_init(struct erl_module_instance* modi) +{ + modi->code_hdr = 0; + modi->code_length = 0; + modi->catches = BEAM_CATCHES_NIL; + modi->nif = NULL; + modi->num_breakpoints = 0; + modi->num_traced_exports = 0; +#ifdef HIPE + modi->hipe_code = NULL; +#endif +} static Module* module_alloc(Module* tmpl) { @@ -74,18 +87,11 @@ static Module* module_alloc(Module* tmpl) erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module)); obj->module = tmpl->module; - obj->curr.code_hdr = 0; - obj->old.code_hdr = 0; - obj->curr.code_length = 0; - obj->old.code_length = 0; obj->slot.index = -1; - obj->curr.nif = NULL; - obj->old.nif = NULL; - obj->curr.num_breakpoints = 0; - obj->old.num_breakpoints = 0; - obj->curr.num_traced_exports = 0; - obj->old.num_traced_exports = 0; + erts_module_instance_init(&obj->curr); + erts_module_instance_init(&obj->old); obj->on_load = 0; + DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc"); return obj; } @@ -119,6 +125,7 @@ void init_module_table(void) erts_smp_atomic_init_nob(&tot_module_bytes, 0); } + Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix) { @@ -139,19 +146,14 @@ erts_get_module(Eterm mod, ErtsCodeIndex code_ix) } } -Module* -erts_put_module(Eterm mod) + +static Module* put_module(Eterm mod, IndexTable* mod_tab) { Module e; - IndexTable* mod_tab; int oldsz, newsz; Module* res; ASSERT(is_atom(mod)); - ERTS_SMP_LC_ASSERT(erts_initialized == 0 - || erts_has_code_write_permission()); - - mod_tab = &module_tables[erts_staging_code_ix()]; e.module = atom_val(mod); oldsz = index_table_sz(mod_tab); res = (Module*) index_put_entry(mod_tab, (void*) &e); @@ -160,6 +162,15 @@ erts_put_module(Eterm mod) return res; } +Module* +erts_put_module(Eterm mod) +{ + ERTS_SMP_LC_ASSERT(erts_initialized == 0 + || erts_has_code_write_permission()); + + return put_module(mod, &module_tables[erts_staging_code_ix()]); +} + Module *module_code(int i, ErtsCodeIndex code_ix) { return (Module*) erts_index_lookup(&module_tables[code_ix], i); @@ -181,6 +192,13 @@ static ErtsCodeIndex dbg_load_code_ix = 0; static int entries_at_start_staging = 0; +static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod) +{ + dst_mod->curr = src_mod->curr; + dst_mod->old = src_mod->old; + dst_mod->on_load = src_mod->on_load; +} + void module_start_staging(void) { IndexTable* src = &module_tables[erts_active_code_ix()]; @@ -199,10 +217,7 @@ void module_start_staging(void) src_mod = (Module*) erts_index_lookup(src, i); dst_mod = (Module*) erts_index_lookup(dst, i); ASSERT(src_mod->module == dst_mod->module); - - dst_mod->curr = src_mod->curr; - dst_mod->old = src_mod->old; - dst_mod->on_load = src_mod->on_load; + copy_module(dst_mod, src_mod); } /* @@ -214,9 +229,7 @@ void module_start_staging(void) dst_mod = (Module*) index_put_entry(dst, src_mod); ASSERT(dst_mod != src_mod); - dst_mod->curr = src_mod->curr; - dst_mod->old = src_mod->old; - dst_mod->on_load = src_mod->on_load; + copy_module(dst_mod, src_mod); } newsz = index_table_sz(dst); erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index f105b3f401..9d258d5dbf 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -23,6 +23,10 @@ #include "index.h" +#ifdef HIPE +#include "hipe_module.h" +#endif + struct erl_module_instance { BeamCodeHeader* code_hdr; int code_length; /* Length of loaded code in bytes. */ @@ -30,6 +34,9 @@ struct erl_module_instance { struct erl_module_nif* nif; int num_breakpoints; int num_traced_exports; +#ifdef HIPE + HipeModule *hipe_code; +#endif }; typedef struct erl_module { @@ -42,6 +49,7 @@ typedef struct erl_module { struct erl_module_instance* on_load; } Module; +void erts_module_instance_init(struct erl_module_instance* modi); Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix); Module* erts_put_module(Eterm mod); diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 879daaca0a..4ef04d020a 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -792,14 +792,14 @@ allocate_init t I y ################################################################# # -# The BIFs erts_internal:check_process_code/2 must be called like a function, +# The BIFs erts_internal:check_process_code/1 must be called like a function, # to ensure that c_p->i (program counter) is set correctly (an ordinary # BIF call doesn't set it). # -call_ext u==2 Bif=u$bif:erts_internal:check_process_code/2 => i_call_ext Bif -call_ext_last u==2 Bif=u$bif:erts_internal:check_process_code/2 D => i_call_ext_last Bif D -call_ext_only u==2 Bif=u$bif:erts_internal:check_process_code/2 => i_call_ext_only Bif +call_ext u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext Bif +call_ext_last u==1 Bif=u$bif:erts_internal:check_process_code/1 D => i_call_ext_last Bif D +call_ext_only u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext_only Bif # # The BIFs erlang:garbage_collect/0 must be called like a function, @@ -811,6 +811,10 @@ call_ext u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext Bif call_ext_last u==0 Bif=u$bif:erlang:garbage_collect/0 D => i_call_ext_last Bif D call_ext_only u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext_only Bif +call_ext u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext Bif +call_ext_last u==1 Bif=u$bif:erts_internal:garbage_collect/1 D => i_call_ext_last Bif D +call_ext_only u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext_only Bif + # # put/2 and erase/1 must be able to do garbage collection, so we must call # them like functions. diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 7740dd4373..4b3ac594a0 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -99,6 +99,12 @@ #define ErtsContainerStruct(ptr, type, member) \ ((type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member))) +/* Use this variant when the member is an array */ +#define ErtsContainerStruct_(ptr, type, memberv) \ + ((type *)((char *)(1 ? (ptr) : ((type *)0)->memberv) - offsetof(type, memberv))) + +#define ErtsSizeofMember(type, member) sizeof(((type *)0)->member) + #if defined (__WIN32__) # include "erl_win_sys.h" #else diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 87ea4f05a1..ec502d5a78 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -204,9 +204,8 @@ erl_grow_wstack(ErtsWStack* s, Uint need) void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes) { - Uint old_size = s->pend - s->pstart; + Uint old_size = s->size; Uint new_size; - Uint sp_offs = s->psp - s->pstart; if (need_bytes < old_size) new_size = 2 * old_size; @@ -220,8 +219,7 @@ erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes) sys_memcpy(new_ptr, s->pstart, old_size); s->pstart = new_ptr; } - s->pend = s->pstart + new_size; - s->psp = s->pstart + sp_offs; + s->size = new_size; } /* @@ -894,11 +892,11 @@ tail_recur: { Export* ep = *((Export **) (export_val(term) + 1)); - hash = hash * FUNNY_NUMBER11 + ep->code[2]; + hash = hash * FUNNY_NUMBER11 + ep->info.mfa.arity; hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue); hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue); break; } @@ -1331,11 +1329,11 @@ make_hash2(Eterm term) { Export* ep = *((Export **) (export_val(term) + 1)); UINT32_HASH_2 - (ep->code[2], - atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue, + (ep->info.mfa.arity, + atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue, HCONST); UINT32_HASH - (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue, + (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue, HCONST_14); goto hash2_common; } @@ -2026,11 +2024,11 @@ tail_recur: { Export* ep = *((Export **) (export_val(term) + 1)); - hash = hash * FUNNY_NUMBER11 + ep->code[2]; + hash = hash * FUNNY_NUMBER11 + ep->info.mfa.arity; hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue); hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue); break; } @@ -3307,13 +3305,15 @@ tailrecur_ne: Export* a_exp = *((Export **) (export_val(a) + 1)); Export* b_exp = *((Export **) (export_val(b) + 1)); - if ((j = erts_cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) { + if ((j = erts_cmp_atoms(a_exp->info.mfa.module, + b_exp->info.mfa.module)) != 0) { RETURN_NEQ(j); } - if ((j = erts_cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) { + if ((j = erts_cmp_atoms(a_exp->info.mfa.function, + b_exp->info.mfa.function)) != 0) { RETURN_NEQ(j); } - ON_CMP_GOTO((Sint) a_exp->code[2] - (Sint) b_exp->code[2]); + ON_CMP_GOTO((Sint) a_exp->info.mfa.arity - (Sint) b_exp->info.mfa.arity); } break; case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE): diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index 3adb8db661..d64f015a6a 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -1937,7 +1937,8 @@ static void free_sendfile(void *data) { MUTEX_LOCK(d->c.sendfile.q_mtx); driver_deq(d->c.sendfile.port,1); MUTEX_UNLOCK(d->c.sendfile.q_mtx); - driver_select(d->c.sendfile.port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 0); + driver_select(d->c.sendfile.port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, + ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 0); } EF_FREE(data); } @@ -2555,7 +2556,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) desc->sendfile_state = sending; desc->d = d; driver_select(desc->port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, - ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 1); + ERL_DRV_USE|ERL_DRV_WRITE, 1); } break; #endif diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c index 440ba956d8..066cf87c9d 100644 --- a/erts/emulator/drivers/common/zlib_drv.c +++ b/erts/emulator/drivers/common/zlib_drv.c @@ -44,30 +44,34 @@ #define INFLATE_INIT 8 #define INFLATE_INIT2 9 #define INFLATE_SETDICT 10 -#define INFLATE_SYNC 11 -#define INFLATE_RESET 12 -#define INFLATE_END 13 -#define INFLATE 14 +#define INFLATE_GETDICT 11 +#define INFLATE_SYNC 12 +#define INFLATE_RESET 13 +#define INFLATE_END 14 +#define INFLATE 15 -#define CRC32_0 15 -#define CRC32_1 16 -#define CRC32_2 17 +#define CRC32_0 16 +#define CRC32_1 17 +#define CRC32_2 18 -#define SET_BUFSZ 18 -#define GET_BUFSZ 19 -#define GET_QSIZE 20 +#define SET_BUFSZ 19 +#define GET_BUFSZ 20 +#define GET_QSIZE 21 -#define ADLER32_1 21 -#define ADLER32_2 22 +#define ADLER32_1 22 +#define ADLER32_2 23 -#define CRC32_COMBINE 23 -#define ADLER32_COMBINE 24 +#define CRC32_COMBINE 24 +#define ADLER32_COMBINE 25 -#define INFLATE_CHUNK 25 +#define INFLATE_CHUNK 26 #define DEFAULT_BUFSZ 4000 +/* According to zlib documentation, it can never exceed this */ +#define INFL_DICT_SZ 32768 + /* This flag is used in the same places, where zlib return codes * (Z_OK, Z_STREAM_END, Z_NEED_DICT) are. So, we need to set it to * relatively large value to avoid possible value clashes in future. @@ -248,6 +252,20 @@ static int zlib_output(ZLibData* d) return zlib_output_init(d); } +#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY +static int zlib_inflate_get_dictionary(ZLibData* d) +{ + ErlDrvBinary* dbin = driver_alloc_binary(INFL_DICT_SZ); + uInt dlen = 0; + int res = inflateGetDictionary(&d->s, (unsigned char*)dbin->orig_bytes, &dlen); + if ((res == Z_OK) && (driver_output_binary(d->port, NULL, 0, dbin, 0, dlen) < 0)) { + res = Z_ERRNO; + } + driver_free_binary(dbin); + return res; +} +#endif + static int zlib_inflate(ZLibData* d, int flush) { int res = Z_OK; @@ -586,6 +604,16 @@ static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *bu res = inflateSetDictionary(&d->s, (unsigned char*)buf, len); return zlib_return(res, rbuf, rlen); + case INFLATE_GETDICT: +#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY + if (d->state != ST_INFLATE) goto badarg; + res = zlib_inflate_get_dictionary(d); + return zlib_return(res, rbuf, rlen); +#else + errno = ENOTSUP; + return zlib_return(Z_ERRNO, rbuf, rlen); +#endif + case INFLATE_SYNC: if (d->state != ST_INFLATE) goto badarg; if (len != 0) goto badarg; diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index 62739d2a78..e3cff4a4ba 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -73,8 +73,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Sint rel32; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + rel32 = (Sint)destAddress - (Sint)callAddress - 4; if ((Sint)(Sint32)rel32 != rel32) return -1; @@ -83,29 +83,6 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - /* * Memory allocator for executable code. * @@ -116,9 +93,6 @@ static void atexit_alloc_code_stats(void) */ static void *alloc_code(unsigned int alloc_bytes) { - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } @@ -130,6 +104,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int bytes) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + /* Make stub for native code calling exported beam function. */ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) @@ -234,6 +213,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h index 6f959815bb..059b8e7f29 100644 --- a/erts/emulator/hipe/hipe_arch.h +++ b/erts/emulator/hipe/hipe_arch.h @@ -30,23 +30,31 @@ extern void hipe_patch_load_fe(Uint *address, Uint value); extern int hipe_patch_insn(void *address, Uint value, Eterm type); extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline); -extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p); -extern void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity); +extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, struct process *p); +extern void hipe_free_code(void*, unsigned int); +extern void *hipe_make_native_stub(void *exp, unsigned int beamArity); +extern void hipe_free_native_stub(void*); + #if defined(__sparc__) #include "hipe_sparc.h" +#include "hipe_sparc_asm.h" #endif #if defined(__i386__) #include "hipe_x86.h" +#include "hipe_x86_asm.h" #endif #if defined(__x86_64__) #include "hipe_amd64.h" +#include "hipe_amd64_asm.h" #endif #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) #include "hipe_ppc.h" +#include "hipe_ppc_asm.h" #endif #if defined(__arm__) #include "hipe_arm.h" +#include "hipe_arm_asm.h" #endif #if !defined(AEXTERN) diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c index f8ef468341..b61939724c 100644 --- a/erts/emulator/hipe/hipe_arm.c +++ b/erts/emulator/hipe/hipe_arm.c @@ -25,7 +25,6 @@ #endif #include "global.h" #include "erl_binary.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -57,30 +56,6 @@ void hipe_flush_icache_word(void *address) hipe_flush_icache_range(address, 4); } -/* - * Management of 32MB code segments for regular code and trampolines. - */ - -#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */ - -static struct segment { - unsigned int *base; /* [base,base+32MB[ */ - unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */ - unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */ - /* On ARM we always allocate a trampoline at base+32MB-8 for - nbif_callemu, so tramp_pos <= base+32MB-8. */ -} curseg; - -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) - -static void *new_code_mapping(void) -{ - return mmap(0, SEGMENT_NRBYTES, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, - -1, 0); -} static int check_callees(Eterm callees) { @@ -107,126 +82,53 @@ static int check_callees(Eterm callees) return arity; } -static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec) -{ - unsigned int *base, *address, *tramp_pos, nrfreewords; - int trampnr; - Eterm mfa, m, f; - unsigned int a, *trampoline; - - m = NIL; f = NIL; a = 0; /* silence stupid compiler warning */ - tramp_pos = curseg.tramp_pos; - address = curseg.code_pos; - nrfreewords = tramp_pos - address; - if (nrwords > nrfreewords) - return NULL; - curseg.code_pos = address + nrwords; - nrfreewords -= nrwords; +#define TRAMPOLINE_WORDS 2 - base = curseg.base; - for (trampnr = 1; trampnr <= nrcallees; ++trampnr) { - mfa = tuple_val(callees)[trampnr]; - if (is_atom(mfa)) - trampoline = hipe_primop_get_trampoline(mfa); - else { - m = tuple_val(mfa)[1]; - f = tuple_val(mfa)[2]; - a = unsigned_val(tuple_val(mfa)[3]); - trampoline = hipe_mfa_get_trampoline(m, f, a); - } - if (!in_area(trampoline, base, SEGMENT_NRBYTES)) { - if (nrfreewords < 2) - return NULL; - nrfreewords -= 2; - tramp_pos = trampoline = tramp_pos - 2; - trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - trampoline[1] = 0; /* callee's address */ - hipe_flush_icache_range(trampoline, 2*sizeof(int)); - if (is_atom(mfa)) - hipe_primop_set_trampoline(mfa, trampoline); - else - hipe_mfa_set_trampoline(m, f, a, trampoline); - } - trampvec[trampnr-1] = trampoline; +static void generate_trampolines(Uint32* address, + int nrcallees, Eterm callees, + Uint32** trampvec) +{ + Uint32* trampoline = address; + int i; + + for (i = 0; i < nrcallees; ++i) { + trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ + trampoline[1] = 0; /* callee's address */ + trampvec[i] = trampoline; + trampoline += TRAMPOLINE_WORDS; } - curseg.tramp_pos = tramp_pos; - return address; + hipe_flush_icache_range(address, nrcallees*2*sizeof(Uint32)); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) { - Uint nrwords; + Uint code_words; int nrcallees; Eterm trampvecbin; - unsigned int **trampvec; - unsigned int *address; - unsigned int *base; - struct segment oldseg; + Uint32 **trampvec; + Uint32 *address; if (nrbytes & 0x3) return NULL; - nrwords = nrbytes >> 2; + code_words = nrbytes / sizeof(Uint32); nrcallees = check_callees(callees); if (nrcallees < 0) return NULL; - trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*)); - trampvec = (unsigned int**)binary_bytes(trampvecbin); + trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*)); + trampvec = (Uint32**)binary_bytes(trampvecbin); - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - curseg.tramp_pos -= 2; - curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - curseg.tramp_pos[1] = (unsigned int)&nbif_callemu; + address = erts_alloc(ERTS_ALC_T_HIPE_EXEC, + (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32)); - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } + generate_trampolines(address + code_words, nrcallees, callees, trampvec); *trampolines = trampvecbin; return address; } -static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu) +void hipe_free_code(void* code, unsigned int bytes) { - unsigned int *address; - unsigned int *base; - struct segment oldseg; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - curseg.tramp_pos -= 2; - curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - curseg.tramp_pos[1] = (unsigned int)&nbif_callemu; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } - *tramp_callemu = (unsigned int*)((char*)curseg.base + SEGMENT_NRBYTES) - 2; - return address; + erts_free(ERTS_ALC_T_HIPE_EXEC, code); } /* @@ -266,8 +168,8 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { unsigned int *code; - unsigned int *tramp_callemu; int callemu_offset; + int is_short_jmp; /* * Native code calls BEAM via a stub looking as follows: @@ -277,36 +179,57 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) * b nbif_callemu * .long callee_exp * + * or if nbif_callemu is too far away: + * + * mov r0, #beamArity + * ldr r8, [pc,#0] // callee_exp + * ldr pc, [pc,#0] // nbif_callemu + * .long callee_exp + * .long nbif_callemu + * * I'm using r0 and r8 since they aren't used for - * parameter passing in native code. The branch to - * nbif_callemu may need to go via a trampoline. - * (Trampolines are allowed to modify r12, but they don't.) + * parameter passing in native code. */ - code = alloc_stub(4, &tramp_callemu); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 5*sizeof(Uint32)); if (!code) return NULL; callemu_offset = ((int)&nbif_callemu - ((int)&code[2] + 8)) >> 2; - if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) { - callemu_offset = ((int)tramp_callemu - ((int)&code[2] + 8)) >> 2; - if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) - abort(); + is_short_jmp = (callemu_offset >= -0x00800000 && + callemu_offset <= 0x007FFFFF); +#ifdef DEBUG + if (is_short_jmp && (callemu_offset % 3)==0) { + is_short_jmp = 0; } +#endif /* mov r0, #beamArity */ code[0] = 0xE3A00000 | (beamArity & 0xFF); /* ldr r8, [pc,#0] // callee_exp */ code[1] = 0xE59F8000; - /* b nbif_callemu */ - code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF); + if (is_short_jmp) { + /* b nbif_callemu */ + code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF); + } + else { + /* ldr pc, [pc,#0] // nbif_callemu */ + code[2] = 0xE59FF000; + /* .long nbif_callemu */ + code[4] = (unsigned int)&nbif_callemu; + } /* .long callee_exp */ code[3] = (unsigned int)callee_exp; - hipe_flush_icache_range(code, 4*sizeof(int)); + hipe_flush_icache_range(code, 5*sizeof(Uint32)); return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + static void patch_b(Uint32 *address, Sint32 offset, Uint32 AA) { Uint32 oldI = *address; diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index dcb6c35bfa..5c29473443 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -44,6 +44,7 @@ #include "hipe_mode_switch.h" #include "hipe_native_bif.h" #include "hipe_bif0.h" +#include "hipe_load.h" /* We need hipe_literals.h for HIPE_SYSTEM_CRC, but it redefines a few constants. #undef them here to avoid warnings. */ #undef F_TIMO @@ -54,6 +55,7 @@ #define BeamOpCode(Op) ((Uint)BeamOp(Op)) + int term_to_Sint32(Eterm term, Sint *sp) { Sint val; @@ -374,15 +376,31 @@ BIF_RETTYPE hipe_bifs_ref_set_2(BIF_ALIST_2) } /* + * BIFs for loading code. + */ + +static HipeLoaderState *get_loader_state(Eterm term) +{ + ProcBin *pb; + + if (!ERTS_TERM_IS_MAGIC_BINARY(term)) return NULL; + + pb = (ProcBin*) binary_val(term); + return hipe_get_loader_state(pb->val); +} + + +/* * Allocate memory and copy machine code to it. */ -BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_enter_code_3(BIF_ALIST_3) { Uint nrbytes; void *bytes; void *address; Eterm trampolines; Eterm *hp; + HipeLoaderState *stp; #ifndef DEBUG ERTS_DECLARE_DUMMY(Uint bitoffs); ERTS_DECLARE_DUMMY(Uint bitsize); @@ -391,7 +409,8 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) Uint bitsize; #endif - if (is_not_binary(BIF_ARG_1)) + if (is_not_binary(BIF_ARG_1) || + (!(stp = get_loader_state(BIF_ARG_3)))) BIF_ERROR(BIF_P, BADARG); nrbytes = binary_size(BIF_ARG_1); ERTS_GET_BINARY_BYTES(BIF_ARG_1, bytes, bitoffs, bitsize); @@ -406,11 +425,15 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) nrcallees = arityval(tuple_val(BIF_ARG_2)[0]); else nrcallees = 0; + // XXX: Is there any reason to not just BIF_ERROR, so that the runtime + // survives? erts_exit(ERTS_ERROR_EXIT, "%s: failed to allocate %lu bytes and %lu trampolines\r\n", __func__, (unsigned long)nrbytes, (unsigned long)nrcallees); } memcpy(address, bytes, nrbytes); hipe_flush_icache_range(address, nrbytes); + stp->text_segment = address; + stp->text_segment_size = nrbytes; hp = HAlloc(BIF_P, 3); hp[0] = make_arityval(2); hp[1] = address_to_term(address, BIF_P); @@ -423,25 +446,34 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) /* * Allocate memory for arbitrary non-Erlang data. */ -BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_alloc_data_3(BIF_ALIST_3) { - Uint align, nrbytes; - void *block; + Uint align; + HipeLoaderState *stp; if (is_not_small(BIF_ARG_1) || is_not_small(BIF_ARG_2) || + (!(stp = get_loader_state(BIF_ARG_3))) || (align = unsigned_val(BIF_ARG_1), !IS_POWER_OF_TWO(align))) BIF_ERROR(BIF_P, BADARG); - nrbytes = unsigned_val(BIF_ARG_2); - if (nrbytes == 0) + + if (stp->data_segment_size || stp->data_segment) + BIF_ERROR(BIF_P, BADARG); + + stp->data_segment_size = unsigned_val(BIF_ARG_2); + if (stp->data_segment_size == 0) BIF_RET(make_small(0)); - block = erts_alloc(ERTS_ALC_T_HIPE, nrbytes); - if ((unsigned long)block & (align-1)) { - fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte aligned\r\n", - __FUNCTION__, (unsigned long)nrbytes, block, (unsigned long)align); - erts_free(ERTS_ALC_T_HIPE, block); + stp->data_segment = erts_alloc(ERTS_ALC_T_HIPE, stp->data_segment_size); + if ((unsigned long)stp->data_segment & (align-1)) { + fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte " + "aligned\r\n", + __FUNCTION__, (unsigned long)stp->data_segment_size, + stp->data_segment, (unsigned long)align); + erts_free(ERTS_ALC_T_HIPE, stp->data_segment); + stp->data_segment = NULL; + stp->data_segment_size = 0; BIF_ERROR(BIF_P, EXC_NOTSUP); } - BIF_RET(address_to_term(block, BIF_P)); + BIF_RET(address_to_term(stp->data_segment, BIF_P)); } /* @@ -537,13 +569,13 @@ BIF_RETTYPE hipe_bifs_merge_term_1(BIF_ALIST_1) BIF_RET(val); } -struct mfa_t { +struct hipe_mfa { Eterm mod; Eterm fun; Uint ari; }; -static int term_to_mfa(Eterm term, struct mfa_t *mfa) +static int term_to_mfa(Eterm term, struct hipe_mfa *mfa) { Eterm mod, fun, a; Uint ari; @@ -601,7 +633,7 @@ static Uint *hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity) Uint *hipe_bifs_find_pc_from_mfa(Eterm term) { - struct mfa_t mfa; + struct hipe_mfa mfa; if (!term_to_mfa(term, &mfa)) return NULL; @@ -616,12 +648,20 @@ BIF_RETTYPE hipe_bifs_fun_to_address_1(BIF_ALIST_1) BIF_RET(address_to_term(pc, BIF_P)); } +BIF_RETTYPE hipe_bifs_commit_patch_load_1(BIF_ALIST_1) +{ + if (!erts_commit_hipe_patch_load(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + BIF_RET(am_ok); +} + BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) { Eterm *pc; void *address; int is_closure; - struct mfa_t mfa; + struct hipe_mfa mfa; switch (BIF_ARG_3) { case am_false: @@ -644,28 +684,22 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) pc = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari); if (pc) { - hipe_mfa_save_orig_beam_op(mfa.mod, mfa.fun, mfa.ari, pc); -#if HIPE -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": planting call trap to %p at BEAM pc %p\r\n", address, pc); -#endif + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", pc, address); hipe_set_call_trap(pc, address, is_closure); BIF_RET(am_true); -#endif } -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": no BEAM pc found\r\n"); -#endif + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "failed set call trap to %p, no beam code found", address); BIF_RET(am_false); } -BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) +BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2) { - struct sdesc *sdesc; + struct hipe_sdesc *sdesc; + HipeLoaderState* stp; + + stp = get_loader_state(BIF_ARG_2); + if (!stp) + BIF_ERROR(BIF_P, BADARG); sdesc = hipe_decode_sdesc(BIF_ARG_1); if (!sdesc) { @@ -676,6 +710,13 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) fprintf(stderr, "%s: duplicate entry!\r\n", __FUNCTION__); BIF_ERROR(BIF_P, BADARG); } + + /* + * Link into list of sdesc's in same module instance + */ + sdesc->next_in_modi = stp->new_hipe_sdesc; + stp->new_hipe_sdesc = sdesc; + BIF_RET(NIL); } @@ -778,9 +819,6 @@ BIF_RETTYPE hipe_bifs_bif_address_3(BIF_ALIST_3) struct primop { HashBucket bucket; /* bucket.hvalue == atom_val(name) */ const void *address; -#if defined(__arm__) - void *trampoline; -#endif }; static struct primop primops[] = { @@ -839,29 +877,6 @@ static struct primop *primop_table_get(Eterm name) return hash_get(&primop_table, &tmpl); } -#if defined(__arm__) -static struct primop *primop_table_put(Eterm name) -{ - struct primop tmpl; - - init_primop_table(); - tmpl.bucket.hvalue = atom_val(name); - return hash_put(&primop_table, &tmpl); -} - -void *hipe_primop_get_trampoline(Eterm name) -{ - struct primop *primop = primop_table_get(name); - return primop ? primop->trampoline : NULL; -} - -void hipe_primop_set_trampoline(Eterm name, void *trampoline) -{ - struct primop *primop = primop_table_put(name); - primop->trampoline = trampoline; -} -#endif - /* * hipe_bifs_primop_address(Atom) -> address or false */ @@ -970,7 +985,7 @@ BIF_RETTYPE hipe_bifs_get_fe_2(BIF_ALIST_2) atom_buf[0] = '\0'; strncat(atom_buf, (char*)atom_tab(i)->name, atom_tab(i)->len); - printf("no fun entry for %s %ld:%ld\n", atom_buf, uniq, index); + printf("no fun entry for %s %ld:%ld\n", atom_buf, (unsigned long)uniq, (unsigned long)index); BIF_ERROR(BIF_P, BADARG); } BIF_RET(address_to_term((void *)fe, BIF_P)); @@ -997,30 +1012,35 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2) BIF_RET(am_true); } +struct hipe_ref_head { + struct hipe_ref_head* next; + struct hipe_ref_head* prev; +}; + /* - * MFA info hash table: + * An exported function called from or implemented by native code * - maps MFA to native code entry point - * - the MFAs it calls (refers_to) - * - the references to it (referred_from) + * - all references to it (callers) * - maps MFA to most recent trampoline [if powerpc or arm] */ struct hipe_mfa_info { + HashBucket mod2mfa; struct { unsigned long hvalue; struct hipe_mfa_info *next; } bucket; Eterm m; /* atom */ Eterm f; /* atom */ - unsigned int a; + unsigned int a : sizeof(int)*8 - 1; + unsigned int is_stub : 1; /* if beam or not (yet) loaded */ void *remote_address; - void *local_address; - Eterm *beam_code; - Uint orig_beam_op; - struct hipe_mfa_info_list *refers_to; - struct ref *referred_from; -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) - void *trampoline; + void *new_address; + struct hipe_ref_head callers; /* sentinel in list of hipe_ref's */ + struct hipe_mfa_info* next_in_mod; +#ifdef DEBUG + Export* dbg_export; #endif + }; static struct { @@ -1038,6 +1058,82 @@ static struct { erts_smp_rwmtx_t lock; } hipe_mfa_info_table; +Hash mod2mfa_tab; /* map from module atom to list of hipe_mfa_info */ + +static HashValue mod2mfa_hash(struct hipe_mfa_info* mfa) +{ + return mfa->mod2mfa.hvalue; +} + +static int mod2mfa_cmp(HashBucket* tmpl, struct hipe_mfa_info* mfa) +{ + return tmpl->hvalue != mfa->mod2mfa.hvalue; +} + +static struct hipe_mfa_info* mod2mfa_alloc(struct hipe_mfa_info* tmpl) +{ + return tmpl; /* hash_put always use mfa itself at template */ +} + +static void mod2mfa_free(struct hipe_mfa_info* mfa) +{ +} + +static void mod2mfa_tab_init(void) +{ + HashFunctions f; + static int init_done = 0; + + if (init_done) + return; + init_done = 1; + + f.hash = (H_FUN) mod2mfa_hash; + f.cmp = (HCMP_FUN) mod2mfa_cmp; + f.alloc = (HALLOC_FUN) mod2mfa_alloc; + f.free = (HFREE_FUN) mod2mfa_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; + + hash_init(ERTS_ALC_T_HIPE, &mod2mfa_tab, "mod2mfa_tab", 50, f); +} + +static struct hipe_mfa_info* mod2mfa_get(Module* modp) +{ + HashBucket tmpl; + tmpl.hvalue = modp->module; + return hash_get(&mod2mfa_tab, &tmpl); +} + +static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa) +{ + mfa->mod2mfa.hvalue = atom_val(mfa->m); + return hash_put(&mod2mfa_tab, mfa); +} + + + +/* + * An external native call site M:F(...) + * to be patched when the callee changes. + */ +struct hipe_ref { + struct hipe_ref_head head; /* list of refs to same calleee */ + void *address; +#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) + void *trampoline; +#endif + unsigned int flags; + struct hipe_ref* next_from_modi; /* list of refs from same module instance */ +#if defined(DEBUG) + struct hipe_mfa_info* callee; + Eterm caller_m, caller_f, caller_a; +#endif +}; +#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ + + static inline void hipe_mfa_info_table_init_lock(void) { erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock"); @@ -1063,6 +1159,16 @@ static inline void hipe_mfa_info_table_rwunlock(void) erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock); } +static ERTS_INLINE +struct hipe_mfa_info* mod2mfa_get_safe(Module* modp) +{ + struct hipe_mfa_info* mfa; + hipe_mfa_info_table_rlock(); + mfa = mod2mfa_get(modp); + hipe_mfa_info_table_runlock(); + return mfa; +} + #define HIPE_MFA_HASH(M,F,A) (atom_val(M) ^ atom_val(F) ^ (A)) static struct hipe_mfa_info **hipe_mfa_info_table_alloc_bucket(unsigned int size) @@ -1108,14 +1214,14 @@ static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigne res->m = m; res->f = f; res->a = arity; + res->is_stub = 0; res->remote_address = NULL; - res->local_address = NULL; - res->beam_code = NULL; - res->orig_beam_op = 0; - res->refers_to = NULL; - res->referred_from = NULL; -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) - res->trampoline = NULL; + res->new_address = NULL; + res->callers.next = &res->callers; + res->callers.prev = &res->callers; + res->next_in_mod = NULL; +#ifdef DEBUG + res->dbg_export = NULL; #endif return res; @@ -1133,6 +1239,8 @@ void hipe_mfa_info_table_init(void) hipe_mfa_info_table.bucket = hipe_mfa_info_table_alloc_bucket(size); hipe_mfa_info_table_init_lock(); + + mod2mfa_tab_init(); } static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eterm f, unsigned int arity) @@ -1154,21 +1262,12 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter return NULL; } -#if 0 /* XXX: unused */ -void *hipe_mfa_find_na(Eterm m, Eterm f, unsigned int arity) -{ - const struct hipe_mfa_info *p; - - p = hipe_mfa_info_table_get(m, f, arity); - return p ? p->address : NULL; -} -#endif - static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned int arity) { unsigned long h; unsigned int i; struct hipe_mfa_info *p; + struct hipe_mfa_info *first_in_mod; unsigned int size; h = HIPE_MFA_HASH(m, f, arity); @@ -1189,216 +1288,137 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, size = 1 << hipe_mfa_info_table.log2size; if (hipe_mfa_info_table.used > (4*size/5)) /* rehash at 80% */ hipe_mfa_info_table_grow(); - return p; -} -static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address, int is_exported) -{ - struct hipe_mfa_info *p; + first_in_mod = mod2mfa_put(p); + if (p != first_in_mod) { + p->next_in_mod = first_in_mod->next_in_mod; + first_in_mod->next_in_mod = p; + } + else { + p->next_in_mod = NULL; + } - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_put_rwlocked(m, f, arity); -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(m, f, arity); - printf(": changing address from %p to %p\r\n", p->local_address, address); -#endif - p->local_address = address; - if (is_exported) - p->remote_address = address; - hipe_mfa_info_table_rwunlock(); + DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p); + + return p; } -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) -void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int arity) +static void remove_mfa_info(struct hipe_mfa_info* rm) { + unsigned int i; struct hipe_mfa_info *p; - void *trampoline; - - hipe_mfa_info_table_rlock(); - p = hipe_mfa_info_table_get_locked(m, f, arity); - trampoline = p ? p->trampoline : NULL; - hipe_mfa_info_table_runlock(); - return trampoline; + struct hipe_mfa_info **prevp; + + i = rm->bucket.hvalue & hipe_mfa_info_table.mask; + prevp = &hipe_mfa_info_table.bucket[i]; + for (;;) { + p = *prevp; + ASSERT(p); + if (p == rm) { + *prevp = p->bucket.next; + ASSERT(hipe_mfa_info_table.used > 0); + hipe_mfa_info_table.used--; + return; + } + prevp = &p->bucket.next; + } } -void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int arity, void *trampoline) +static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address) { struct hipe_mfa_info *p; hipe_mfa_info_table_rwlock(); p = hipe_mfa_info_table_put_rwlocked(m, f, arity); - p->trampoline = trampoline; + DBG_TRACE_MFA(m,f,arity,"set native address in hipe_mfa_info at %p", p); + p->new_address = address; + hipe_mfa_info_table_rwunlock(); } -#endif BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) { - struct mfa_t mfa; + struct hipe_mfa mfa; void *address; - int is_exported; - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - address = term_to_address(BIF_ARG_2); - if (!address) - BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_3 == am_true) - is_exported = 1; - else if (BIF_ARG_3 == am_false) - is_exported = 0; - else + switch (BIF_ARG_3) { + case am_true: /* is_exported */ + if (!term_to_mfa(BIF_ARG_1, &mfa)) + BIF_ERROR(BIF_P, BADARG); + address = term_to_address(BIF_ARG_2); + if (!address) + BIF_ERROR(BIF_P, BADARG); + hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address); + break; + case am_false: + break; /* ignore local functions */ + default: BIF_ERROR(BIF_P, BADARG); - hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address, is_exported); - BIF_RET(NIL); -} - -BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1) -{ - Eterm lst; - struct mfa_t mfa; - struct hipe_mfa_info *p; - - hipe_mfa_info_table_rwlock(); - lst = BIF_ARG_1; - while (is_list(lst)) { - if (!term_to_mfa(CAR(list_val(lst)), &mfa)) - break; - lst = CDR(list_val(lst)); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) { - p->remote_address = NULL; - p->local_address = NULL; - if (p->beam_code) { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": removing call trap from BEAM pc %p (new op %#lx)\r\n", - p->beam_code, p->orig_beam_op); -#endif - p->beam_code[0] = p->orig_beam_op; - p->beam_code = NULL; - p->orig_beam_op = 0; - } else { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": no call trap to remove\r\n"); -#endif - } - } } - hipe_mfa_info_table_rwunlock(); - if (is_not_nil(lst)) - BIF_ERROR(BIF_P, BADARG); BIF_RET(NIL); } -void hipe_mfa_save_orig_beam_op(Eterm mod, Eterm fun, unsigned int ari, Eterm *pc) + +/* Ask if we need to block all threads + * while loading/deleting code for this module? + */ +int hipe_need_blocking(Module* modp) { - Uint orig_beam_op; struct hipe_mfa_info *p; - orig_beam_op = pc[0]; - if (orig_beam_op != BeamOpCode(op_hipe_trap_call_closure) && - orig_beam_op != BeamOpCode(op_hipe_trap_call)) { - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_put_rwlocked(mod, fun, ari); -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mod, fun, ari); - printf(": saving orig op %#lx from BEAM pc %p\r\n", orig_beam_op, pc); -#endif - p->beam_code = pc; - p->orig_beam_op = orig_beam_op; - hipe_mfa_info_table_rwunlock(); - } else { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mod, fun, ari); - printf(": orig op %#lx already saved\r\n", orig_beam_op); -#endif + /* Need to block if we have at least one native caller to this module + * or native code to make unaccessible. + */ + hipe_mfa_info_table_rlock(); + for (p = mod2mfa_get(modp); p; p = p->next_in_mod) { + ASSERT(!p->new_address); + if (p->callers.next != &p->callers || !p->is_stub) { + break; + } } + hipe_mfa_info_table_runlock(); + return (p != NULL); } -static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote) -{ - Export *export_entry; - void *StubAddress; - - ASSERT(is_remote); - - export_entry = erts_export_get_or_make_stub(m, f, arity); - StubAddress = hipe_make_native_stub(export_entry, arity); - if (!StubAddress) - erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); - return StubAddress; -} - -static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info **pp) +static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a) { struct hipe_mfa_info *p; - void *address; p = hipe_mfa_info_table_get_locked(m, f, a); - if (p) { - /* find address, predicting for a runtime apply call */ - address = p->remote_address; - if (!is_remote) - address = p->local_address; - if (address) - return address; - - /* bummer, install stub, checking if one already existed */ - address = p->remote_address; - if (address) - return address; - } - /* Caller must take the slow path with the write lock held, but allow - it to avoid some work if it already holds the write lock. */ - if (pp) - *pp = p; - return NULL; -} - -static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info *p) -{ - void *address; - - if (!p) - p = hipe_mfa_info_table_put_rwlocked(m, f, a); - address = hipe_make_stub(m, f, a, is_remote); - /* XXX: how to tell if a BEAM MFA is exported or not? */ - p->remote_address = address; - return address; + return p ? p->remote_address : NULL; } -static void *hipe_get_na_nofail_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote) +static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a) { - struct hipe_mfa_info *p; - void *address; + struct hipe_mfa_info *p = hipe_mfa_info_table_put_rwlocked(m, f, a); - address = hipe_get_na_try_locked(m, f, a, is_remote, &p); - if (address) - return address; + if (!p->remote_address) { + Export* export_entry = erts_export_get_or_make_stub(m, f, a); + void* stubAddress = hipe_make_native_stub(export_entry, a); + if (!stubAddress) + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); - address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, p); - return address; + p->remote_address = stubAddress; + p->is_stub = 1; +#ifdef DEBUG + p->dbg_export = export_entry; +#endif + } + return p->remote_address; } -static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a, int is_remote) +static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a) { void *address; hipe_mfa_info_table_rlock(); - address = hipe_get_na_try_locked(m, f, a, is_remote, NULL); + address = hipe_get_na_try_locked(m, f, a); hipe_mfa_info_table_runlock(); if (address) return address; hipe_mfa_info_table_rwlock(); - address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, NULL); + address = hipe_get_na_slow_rwlocked(m, f, a); hipe_mfa_info_table_rwunlock(); return address; } @@ -1408,7 +1428,7 @@ void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a) { if (is_not_atom(m) || is_not_atom(f) || a > 255) return NULL; - return hipe_get_na_nofail(m, f, a, 1); + return hipe_get_na_nofail(m, f, a); } /* primop, but called like a BIF for error handling purposes */ @@ -1420,25 +1440,19 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3) if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2)) BIF_ERROR(BIF_P, BADARG); arity = unsigned_val(BIF_ARG_3); /* no error check */ - address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity, 1); + address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity); BIF_RET((Eterm)address); /* semi-Ok */ } -BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_find_na_or_make_stub_1(BIF_ALIST_1) { - struct mfa_t mfa; + struct hipe_mfa mfa; void *address; - int is_remote; if (!term_to_mfa(BIF_ARG_1, &mfa)) BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_2 == am_true) - is_remote = 1; - else if (BIF_ARG_2 == am_false) - is_remote = 0; - else - BIF_ERROR(BIF_P, BADARG); - address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari, is_remote); + + address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari); BIF_RET(address_to_term(address, BIF_P)); } @@ -1453,14 +1467,14 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) hdr = *boxed_val(BIF_ARG_1); if (is_export_header(hdr)) { Export *ep = (Export*)(export_val(BIF_ARG_1)[1]); - unsigned int actual_arity = ep->code[2]; + unsigned int actual_arity = ep->info.mfa.arity; if (actual_arity != BIF_ARG_2) goto badfun; - m = ep->code[0]; - f = ep->code[1]; + m = ep->info.mfa.module; + f = ep->info.mfa.function; } else goto badfun; - address = hipe_get_na_nofail(m, f, BIF_ARG_2, 1); + address = hipe_get_na_nofail(m, f, BIF_ARG_2); BIF_RET((Eterm)address); badfun: @@ -1471,77 +1485,31 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a) { - struct hipe_mfa_info *mfa; - long mfa_offset, ra_offset; - struct hipe_mfa_info **bucket; - unsigned int i, nrbuckets; + const struct hipe_sdesc* sdesc = hipe_find_sdesc((unsigned long)ra); - if (hipe_is_ra_mode_switch(ra)) { + if (!sdesc || sdesc->m_aix == atom_val(am_Empty)) return 0; - } - /* Note about locking: the table is only updated from the - loader, which runs with the rest of the system suspended. */ - /* XXX: alas not true; see comment at hipe_mfa_info_table.lock */ - hipe_mfa_info_table_rlock(); - bucket = hipe_mfa_info_table.bucket; - nrbuckets = 1 << hipe_mfa_info_table.log2size; - mfa = NULL; - mfa_offset = LONG_MAX; - for (i = 0; i < nrbuckets; ++i) { - struct hipe_mfa_info *b = bucket[i]; - while (b != NULL) { - ra_offset = (char*)ra - (char*)b->local_address; - if (ra_offset > 0 && ra_offset < mfa_offset) { - mfa_offset = ra_offset; - mfa = b; - } - b = b->bucket.next; - } - } - if (mfa) { - *m = mfa->m; - *f = mfa->f; - *a = mfa->a; - } - hipe_mfa_info_table_runlock(); - return mfa ? 1 : 0; + *m = make_atom(sdesc->m_aix); + *f = make_atom(sdesc->f_aix); + *a = sdesc->a; + return 1; } -/* - * Patch Reference Handling. - */ -struct hipe_mfa_info_list { - struct hipe_mfa_info *mfa; - struct hipe_mfa_info_list *next; -}; - -struct ref { - struct hipe_mfa_info *caller_mfa; - void *address; - void *trampoline; - unsigned int flags; - struct ref *next; -}; -#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ -#define REF_FLAG_IS_REMOTE 2 /* bit 1: 0 == local, 1 == remote */ -#define REF_FLAG_PENDING_REDIRECT 4 /* bit 2: 1 == pending redirect */ -#define REF_FLAG_PENDING_REMOVE 8 /* bit 3: 1 == pending remove */ -/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,'remote'|'local'}) +/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,LoaderState}) */ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) { - struct mfa_t callee; + struct hipe_mfa callee; Eterm *tuple; - struct mfa_t caller; + struct hipe_mfa caller; void *address; void *trampoline; unsigned int flags; struct hipe_mfa_info *callee_mfa; - struct hipe_mfa_info *caller_mfa; - struct hipe_mfa_info_list *refers_to; - struct ref *ref; + struct hipe_ref *ref; + HipeLoaderState* stp; if (!term_to_mfa(BIF_ARG_1, &callee)) goto badarg; @@ -1572,63 +1540,90 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) if (!trampoline) goto badarg; } - switch (tuple[5]) { - case am_local: - break; - case am_remote: - flags |= REF_FLAG_IS_REMOTE; - break; - default: - goto badarg; - } + stp = get_loader_state(tuple[5]); + if (!stp) + goto badarg; + hipe_mfa_info_table_rwlock(); callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari); - caller_mfa = hipe_mfa_info_table_put_rwlocked(caller.mod, caller.fun, caller.ari); - - refers_to = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*refers_to)); - refers_to->mfa = callee_mfa; - refers_to->next = caller_mfa->refers_to; - caller_mfa->refers_to = refers_to; - ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*ref)); - ref->caller_mfa = caller_mfa; + ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref)); ref->address = address; +#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) ref->trampoline = trampoline; +#endif ref->flags = flags; - ref->next = callee_mfa->referred_from; - callee_mfa->referred_from = ref; + + /* + * Link into list of refs to same callee + */ + ASSERT(callee_mfa->callers.next->prev == &callee_mfa->callers); + ASSERT(callee_mfa->callers.prev->next == &callee_mfa->callers); + ref->head.next = callee_mfa->callers.next; + ref->head.prev = &callee_mfa->callers; + ref->head.next->prev = &ref->head; + ref->head.prev->next = &ref->head; + + /* + * Link into list of refs from same module instance + */ + ref->next_from_modi = stp->new_hipe_refs; + stp->new_hipe_refs = ref; + +#if defined(DEBUG) + ref->callee = callee_mfa; + ref->caller_m = caller.mod; + ref->caller_f = caller.fun; + ref->caller_a = caller.ari; +#endif hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); + DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p)", + ref, callee.mod, callee.fun, callee.ari, ref->address); + DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p)", + ref, caller.mod, caller.fun, caller.ari, ref->address); + BIF_RET(am_ok); badarg: BIF_ERROR(BIF_P, BADARG); } -/* Given a CalleeMFA, mark each ref to it as pending-redirect. - * This ensures that remove_refs_from() won't remove them: any - * removal is instead done at the end of redirect_referred_from(). - */ -BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */ + +static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me) { - struct mfa_t mfa; - const struct hipe_mfa_info *p; - struct ref *ref; + struct hipe_mfa_info* p; - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) - for (ref = p->referred_from; ref != NULL; ref = ref->next) - ref->flags |= REF_FLAG_PENDING_REDIRECT; - hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); + p = hash_get(&mod2mfa_tab, unlink_me); + ASSERT(p); + if (p == unlink_me) { + hash_erase(&mod2mfa_tab, p); + if (p->next_in_mod) + mod2mfa_put(p->next_in_mod); + } + else { + struct hipe_mfa_info** prevp; + + do { + prevp = &p->next_in_mod; + p = *prevp; + ASSERT(p && p->m == unlink_me->m); + } while (p != unlink_me); + + *prevp = p->next_in_mod; + } +} + +static void purge_mfa(struct hipe_mfa_info* p) +{ + ASSERT(p->is_stub); + remove_mfa_info(p); + hipe_free_native_stub(p->remote_address); + erts_free(ERTS_ALC_T_HIPE, p); } /* Called by init:restart after unloading all hipe compiled modules - * to work around bug causing execution of deallocated beam code. - * Can be removed when delete/purge of native modules works better. + * to work around old bug that caused execution of deallocated beam code. + * Can be removed now when delete/purge of native modules works better. * Test: Do init:restart in debug compiled vm with hipe compiled kernel. */ static void hipe_purge_all_refs(void) @@ -1638,126 +1633,249 @@ static void hipe_purge_all_refs(void) hipe_mfa_info_table_rwlock(); + ASSERT(hipe_mfa_info_table.used == 0); bucket = hipe_mfa_info_table.bucket; nrbuckets = 1 << hipe_mfa_info_table.log2size; for (i = 0; i < nrbuckets; ++i) { + ASSERT(bucket[i] == NULL); while (bucket[i] != NULL) { struct hipe_mfa_info* mfa = bucket[i]; bucket[i] = mfa->bucket.next; - while (mfa->refers_to) { - struct hipe_mfa_info_list *to = mfa->refers_to; - mfa->refers_to = to->next; - erts_free(ERTS_ALC_T_HIPE, to); - } - while (mfa->referred_from) { - struct ref* from = mfa->referred_from; - mfa->referred_from = from->next; - erts_free(ERTS_ALC_T_HIPE, from); - } - erts_free(ERTS_ALC_T_HIPE, mfa); + hash_erase(&mod2mfa_tab, mfa); + erts_free(ERTS_ALC_T_HIPE, mfa); } } + hipe_mfa_info_table.used = 0; hipe_mfa_info_table_rwunlock(); } BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) { - struct mfa_t mfa; - struct hipe_mfa_info *caller_mfa, *callee_mfa; - struct hipe_mfa_info_list *refers_to, *tmp_refers_to; - struct ref **prev, *ref; - if (BIF_ARG_1 == am_all) { hipe_purge_all_refs(); BIF_RET(am_ok); } - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - caller_mfa = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (caller_mfa) { - refers_to = caller_mfa->refers_to; - while (refers_to) { - callee_mfa = refers_to->mfa; - prev = &callee_mfa->referred_from; - ref = *prev; - while (ref) { - if (ref->caller_mfa == caller_mfa) { - if (ref->flags & REF_FLAG_PENDING_REDIRECT) { - ref->flags |= REF_FLAG_PENDING_REMOVE; - prev = &ref->next; - ref = ref->next; - } else { - struct ref *tmp = ref; - ref = ref->next; - *prev = ref; - erts_free(ERTS_ALC_T_HIPE, tmp); - } - } else { - prev = &ref->next; - ref = ref->next; - } - } - tmp_refers_to = refers_to; - refers_to = refers_to->next; - erts_free(ERTS_ALC_T_HIPE, tmp_refers_to); - } - caller_mfa->refers_to = NULL; + ASSERT(!"hipe_bifs_remove_refs_from_1() called"); + BIF_ERROR(BIF_P, BADARG); +} + +int hipe_purge_need_blocking(Module* modp) +{ + /* SVERK: Verify if this is really necessary */ + if (modp->old.hipe_code) { + if (modp->old.hipe_code->first_hipe_ref || + modp->old.hipe_code->first_hipe_sdesc) + return 1; + } + if (!modp->curr.code_hdr) { + return mod2mfa_get_safe(modp) != NULL; + } + return 0; +} + +void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module, + int is_blocking) +{ + struct hipe_ref* ref = first_ref; + + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + + while (ref) { + struct hipe_ref* free_ref = ref; + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, + ref->callee->m, ref->callee->f, ref->callee->a); + DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, + ref->caller_m, ref->caller_f, ref->caller_a); + ASSERT(ref->caller_m == caller_module); + + /* + * Unlink from other refs to same callee + */ + ASSERT(ref->head.next->prev == &ref->head); + ASSERT(ref->head.prev->next == &ref->head); + ASSERT(ref->head.next != &ref->head); + ASSERT(ref->head.prev != &ref->head); + ref->head.next->prev = ref->head.prev; + ref->head.prev->next = ref->head.next; + + /* + * Was this the last ref to that callee? + */ + if (ref->head.next == ref->head.prev) { + struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); + if (p->is_stub) { + if (!is_blocking) + hipe_mfa_info_table_rwlock(); + unlink_mfa_from_mod(p); + purge_mfa(p); + if (!is_blocking) + hipe_mfa_info_table_rwunlock(); + } + } + + ref = ref->next_from_modi; + erts_free(ERTS_ALC_T_HIPE, free_ref); } - hipe_mfa_info_table_rwunlock(); - BIF_RET(am_ok); } +void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module, + int is_blocking) +{ + struct hipe_sdesc* sdesc = first_sdesc; + + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + + ERTS_SMP_LC_ASSERT(is_blocking); /*XXX Fix safe sdesc destruction */ + + while (sdesc) { + struct hipe_sdesc* free_sdesc = sdesc; + + DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); + ASSERT(make_atom(sdesc->m_aix) == module); + + sdesc = sdesc->next_in_modi; + hipe_destruct_sdesc(free_sdesc); + } +} + + +void hipe_purge_module(Module* modp, int is_blocking) +{ + ASSERT(modp); + + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module"); + + if (modp->old.hipe_code) { + /* + * Remove all hipe_ref's (external calls) from the old module instance + */ + if (modp->old.hipe_code->first_hipe_ref) { + ERTS_SMP_LC_ASSERT(is_blocking); + + hipe_purge_refs(modp->old.hipe_code->first_hipe_ref, + make_atom(modp->module), is_blocking); + modp->old.hipe_code->first_hipe_ref = NULL; + } -/* redirect_referred_from(CalleeMFA) - * Redirect all pending-redirect refs in CalleeMFA's referred_from. - * Then remove any pending-redirect && pending-remove refs from CalleeMFA's referred_from. + /* + * Remove all hipe_sdesc's for the old module instance + */ + if (modp->old.hipe_code->first_hipe_sdesc) { + ERTS_SMP_LC_ASSERT(is_blocking); + + hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc, + make_atom(modp->module), is_blocking); + modp->old.hipe_code->first_hipe_sdesc = NULL; + } + + hipe_free_module(modp->old.hipe_code); + modp->old.hipe_code = NULL; + } + + + /* + * Remove unreferred hipe_mfa_info's + * when all module instances are removed (like in init:restart) + */ + if (is_blocking && modp->curr.code_hdr == NULL) { + struct hipe_mfa_info* was_first = mod2mfa_get(modp); + struct hipe_mfa_info* is_first = was_first; + struct hipe_mfa_info** prevp = &is_first; + struct hipe_mfa_info *p; + + if (was_first) { + for (p = was_first ; p; p = *prevp) { + if (p->callers.next == &p->callers) { + *prevp = p->next_in_mod; + if (p != was_first) + purge_mfa(p); + } + else + prevp = &p->next_in_mod; + } + if (was_first != is_first) { + hash_erase(&mod2mfa_tab, was_first); + purge_mfa(was_first); + if (is_first) + mod2mfa_put(is_first); + } + } + } +} + + +/* + * Redirect all existing native calls to this module */ -BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1) +void hipe_redirect_to_module(Module* modp) { - struct mfa_t mfa; struct hipe_mfa_info *p; - struct ref **prev, *ref; - int is_remote, res; - void *new_address; + struct hipe_ref_head* refh; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + for (p = mod2mfa_get(modp); p; p = p->next_in_mod) { + if (p->new_address) { + if (p->is_stub) { + hipe_free_native_stub(p->remote_address); + p->is_stub = 0; + } + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit new_address %p", p->new_address); + p->remote_address = p->new_address; + p->new_address = NULL; +#ifdef DEBUG + p->dbg_export = NULL; +#endif + } + else if (!p->is_stub) { + Export* exp = erts_export_get_or_make_stub(p->m, p->f, p->a); + p->remote_address = hipe_make_native_stub(exp, p->a); + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit stub %p", p->remote_address); + if (!p->remote_address) + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); + p->is_stub = 1; +#ifdef DEBUG + p->dbg_export = exp; +#endif + } + else { + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit no-op, already stub"); + ASSERT(p->remote_address && p->dbg_export); + } - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) { - prev = &p->referred_from; - ref = *prev; - while (ref) { - if (ref->flags & REF_FLAG_PENDING_REDIRECT) { - is_remote = ref->flags & REF_FLAG_IS_REMOTE; - new_address = hipe_get_na_nofail_rwlocked(p->m, p->f, p->a, is_remote); - if (ref->flags & REF_FLAG_IS_LOAD_MFA) - res = hipe_patch_insn(ref->address, (Uint)new_address, am_load_mfa); - else - res = hipe_patch_call(ref->address, new_address, ref->trampoline); - if (res) - fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__); - ref->flags &= ~REF_FLAG_PENDING_REDIRECT; - if (ref->flags & REF_FLAG_PENDING_REMOVE) { - struct ref *tmp = ref; - ref = ref->next; - *prev = ref; - erts_free(ERTS_ALC_T_HIPE, tmp); - } else { - prev = &ref->next; - ref = ref->next; - } - } else { - prev = &ref->next; - ref = ref->next; - } + DBG_TRACE_MFA(p->m,p->f,p->a,"START REDIRECT towards hipe_mfa_info at %p", p); + for (refh = p->callers.next; refh != &p->callers; refh = refh->next) { + struct hipe_ref* ref = (struct hipe_ref*) refh; + int res; + + DBG_TRACE_MFA(p->m,p->f,p->a, " REDIRECT ref at %p FROM %T:%T/%u (%p -> %p)", + ref, ref->caller_m, ref->caller_f, ref->caller_a, + ref->address, p->remote_address); + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, + " REDIRECT ref at %p TO %T:%T/%u (%p -> %p)", + ref, p->m,p->f,p->a, ref->address, p->remote_address); + + if (ref->flags & REF_FLAG_IS_LOAD_MFA) + res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa); + else { +#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) + void* trampoline = ref->trampoline; +#else + void* trampoline = NULL; +#endif + res = hipe_patch_call(ref->address, p->remote_address, trampoline); + } + if (res) + fprintf(stderr, "%s: patch failed", __FUNCTION__); } + DBG_TRACE_MFA(p->m,p->f,p->a,"DONE REDIRECT towards hipe_mfa_info at %p", p); } - hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); } BIF_RETTYPE hipe_bifs_check_crc_1(BIF_ALIST_1) @@ -1811,89 +1929,6 @@ void hipe_patch_address(Uint *address, Eterm patchtype, Uint value) } } -struct modinfo { - HashBucket bucket; /* bucket.hvalue == atom_val(the module name) */ - unsigned int code_size; -}; - -static Hash modinfo_table; - -static HashValue modinfo_hash(void *tmpl) -{ - Eterm mod = (Eterm)tmpl; - return atom_val(mod); -} - -static int modinfo_cmp(void *tmpl, void *bucket) -{ - /* bucket->hvalue == modinfo_hash(tmpl), so just return 0 (match) */ - return 0; -} - -static void *modinfo_alloc(void *tmpl) -{ - struct modinfo *p; - - p = (struct modinfo*)erts_alloc(ERTS_ALC_T_HIPE, sizeof(*p)); - p->code_size = 0; - return &p->bucket; -} - -static void init_modinfo_table(void) -{ - HashFunctions f; - static int init_done = 0; - - if (init_done) - return; - init_done = 1; - f.hash = (H_FUN) modinfo_hash; - f.cmp = (HCMP_FUN) modinfo_cmp; - f.alloc = (HALLOC_FUN) modinfo_alloc; - f.free = (HFREE_FUN) NULL; - f.meta_alloc = (HMALLOC_FUN) erts_alloc; - f.meta_free = (HMFREE_FUN) erts_free; - f.meta_print = (HMPRINT_FUN) erts_print; - hash_init(ERTS_ALC_T_HIPE, &modinfo_table, "modinfo_table", 11, f); -} - -BIF_RETTYPE hipe_bifs_update_code_size_3(BIF_ALIST_3) -{ - struct modinfo *p; - Sint code_size; - - init_modinfo_table(); - - if (is_not_atom(BIF_ARG_1) || - is_not_small(BIF_ARG_3) || - (code_size = signed_val(BIF_ARG_3)) < 0) - BIF_ERROR(BIF_P, BADARG); - - p = (struct modinfo*)hash_put(&modinfo_table, (void*)BIF_ARG_1); - - if (is_nil(BIF_ARG_2)) /* some MFAs, not whole module */ - p->code_size += code_size; - else /* whole module */ - p->code_size = code_size; - BIF_RET(NIL); -} - -BIF_RETTYPE hipe_bifs_code_size_1(BIF_ALIST_1) -{ - struct modinfo *p; - unsigned int code_size; - - init_modinfo_table(); - - if (is_not_atom(BIF_ARG_1)) - BIF_ERROR(BIF_P, BADARG); - - p = (struct modinfo*)hash_get(&modinfo_table, (void*)BIF_ARG_1); - - code_size = p ? p->code_size : 0; - BIF_RET(make_small(code_size)); -} - BIF_RETTYPE hipe_bifs_patch_insn_3(BIF_ALIST_3) { Uint *address, value; @@ -1929,3 +1964,23 @@ BIF_RETTYPE hipe_bifs_patch_call_3(BIF_ALIST_3) BIF_ERROR(BIF_P, BADARG); BIF_RET(NIL); } + +BIF_RETTYPE hipe_bifs_alloc_loader_state_1(BIF_ALIST_1) +{ + Binary *magic; + Eterm *hp; + Eterm res; + + if (is_not_atom(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + magic = hipe_alloc_loader_state(BIF_ARG_1); + + if (!magic) + BIF_ERROR(BIF_P, BADARG); + + hp = HAlloc(BIF_P, PROC_BIN_SIZE); + res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic); + erts_refc_dec(&magic->refc, 1); + BIF_RET(res); +} diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index c9a8216368..4a59bacc6e 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -32,17 +32,14 @@ extern void hipe_mfa_info_table_init(void); extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a); extern BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3); extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a); -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) -extern void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int a); -extern void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int a, void *trampoline); -#endif -#if defined(__arm__) -extern void *hipe_primop_get_trampoline(Eterm name); -extern void hipe_primop_set_trampoline(Eterm name, void *trampoline); -#endif /* needed in beam_load.c */ -void hipe_mfa_save_orig_beam_op(Eterm m, Eterm f, unsigned int a, Eterm *pc); +int hipe_need_blocking(Module*); +int hipe_purge_need_blocking(Module*); +void hipe_purge_refs(struct hipe_ref*, Eterm, int is_blocking); +void hipe_purge_sdescs(struct hipe_sdesc*, Eterm, int is_blocking); +void hipe_purge_module(Module*, int is_blocking); +void hipe_redirect_to_module(Module* modp); /* these are also needed in hipe_amd64.c */ extern void *term_to_address(Eterm); diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index 99237aae05..264ea2c34a 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -44,22 +44,20 @@ bif hipe_bifs:ref/1 bif hipe_bifs:ref_get/1 bif hipe_bifs:ref_set/2 -bif hipe_bifs:enter_code/2 -bif hipe_bifs:alloc_data/2 +bif hipe_bifs:enter_code/3 +bif hipe_bifs:alloc_data/3 bif hipe_bifs:constants_size/0 bif hipe_bifs:merge_term/1 bif hipe_bifs:fun_to_address/1 +bif hipe_bifs:commit_patch_load/1 bif hipe_bifs:set_native_address/3 #bif hipe_bifs:address_to_fun/1 bif hipe_bifs:set_funinfo_native_address/3 -bif hipe_bifs:invalidate_funinfo_native_addresses/1 +#bif hipe_bifs:invalidate_funinfo_native_addresses/1 -bif hipe_bifs:update_code_size/3 -bif hipe_bifs:code_size/1 - -bif hipe_bifs:enter_sdesc/1 +bif hipe_bifs:enter_sdesc/2 bif hipe_bifs:bif_address/3 bif hipe_bifs:primop_address/1 @@ -72,7 +70,7 @@ bif hipe_bifs:term_to_word/1 bif hipe_bifs:get_fe/2 bif hipe_bifs:set_native_address_in_fe/2 -bif hipe_bifs:find_na_or_make_stub/2 +bif hipe_bifs:find_na_or_make_stub/1 bif hipe_bifs:check_crc/1 bif hipe_bifs:system_crc/0 @@ -84,9 +82,9 @@ bif hipe_bifs:patch_insn/3 bif hipe_bifs:patch_call/3 bif hipe_bifs:add_ref/2 -bif hipe_bifs:mark_referred_from/1 bif hipe_bifs:remove_refs_from/1 -bif hipe_bifs:redirect_referred_from/1 + +bif hipe_bifs:alloc_loader_state/1 # atoms used by add_ref/2 atom call diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c index 5e127755c6..0c66eb6abe 100644 --- a/erts/emulator/hipe/hipe_bif1.c +++ b/erts/emulator/hipe/hipe_bif1.c @@ -45,7 +45,7 @@ BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1) pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!pc) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI)); if (pc[0] == BeamOpCode(op_hipe_trap_call)) BIF_ERROR(BIF_P, BADARG); if (pc[0] == BeamOpCode(op_hipe_call_count)) @@ -67,7 +67,7 @@ BIF_RETTYPE hipe_bifs_call_count_off_1(BIF_ALIST_1) pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!pc) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI)); if (pc[0] != BeamOpCode(op_hipe_call_count)) BIF_RET(am_false); hcc = (struct hipe_call_count*)pc[-4]; @@ -86,7 +86,7 @@ BIF_RETTYPE hipe_bifs_call_count_get_1(BIF_ALIST_1) pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!pc) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI)); if (pc[0] != BeamOpCode(op_hipe_call_count)) BIF_RET(am_false); hcc = (struct hipe_call_count*)pc[-4]; @@ -102,7 +102,7 @@ BIF_RETTYPE hipe_bifs_call_count_clear_1(BIF_ALIST_1) pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!pc) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI)); if (pc[0] != BeamOpCode(op_hipe_call_count)) BIF_RET(am_false); hcc = (struct hipe_call_count*)pc[-4]; diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4 index dcf3447af9..bb328b5915 100644 --- a/erts/emulator/hipe/hipe_bif_list.m4 +++ b/erts/emulator/hipe/hipe_bif_list.m4 @@ -153,7 +153,7 @@ standard_bif_interface_0(nbif_ports_0, ports_0) * BIFs and primops that may do a GC (change heap limit and walk the native stack). * XXX: erase/1 and put/2 cannot fail */ -gc_bif_interface_2(nbif_erts_internal_check_process_code_2, hipe_erts_internal_check_process_code_2) +gc_bif_interface_1(nbif_erts_internal_check_process_code_1, hipe_erts_internal_check_process_code_1) gc_bif_interface_1(nbif_erase_1, erase_1) gc_bif_interface_0(nbif_garbage_collect_0, garbage_collect_0) gc_nofail_primop_interface_1(nbif_gc_1, hipe_gc) diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c index ace489452f..222a11db3d 100644 --- a/erts/emulator/hipe/hipe_debug.c +++ b/erts/emulator/hipe/hipe_debug.c @@ -62,10 +62,12 @@ static void print_beam_pc(BeamInstr *pc) } else if (pc == &beam_apply[1]) { printf("normal-process-exit"); } else { - BeamInstr *mfa = find_function_from_pc(pc); - if (mfa) + ErtsCodeMFA *cmfa = find_function_from_pc(pc); + if (cmfa) erts_printf("%T:%T/%bpu + 0x%bpx", - mfa[0], mfa[1], mfa[2], pc - &mfa[3]); + cmfa->module, cmfa->function, + cmfa->arity, + pc - erts_codemfa_to_code(cmfa)); else printf("?"); } @@ -214,10 +216,10 @@ void hipe_print_pcb(Process *p) U("seq..clock ", seq_trace_clock); U("seq..astcnt", seq_trace_lastcnt); U("seq..token ", seq_trace_token); - U("intial[0] ", u.initial[0]); - U("intial[1] ", u.initial[1]); - U("intial[2] ", u.initial[2]); - P("current ", current); + U("intial.mod ", u.initial.module); + U("intial.fun ", u.initial.function); + U("intial.ari ", u.initial.arity); + U("current ", current); P("cp ", cp); P("i ", i); U("catches ", catches); diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index 68c65dea27..e6ce7ce628 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -38,7 +38,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int sdesc_size; unsigned long ra; unsigned int i; @@ -123,7 +123,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int sdesc_size; unsigned long ra; unsigned int i; @@ -244,7 +244,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area, /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; /* arch-specific nstack walk state */ struct nstack_walk_state walk_state; @@ -311,7 +311,7 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size) { Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; /* arch-specific nstack walk state */ struct nstack_walk_state walk_state; @@ -356,3 +356,36 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size) } return 0; } + +int +nstack_any_cps_in_segment(Process *p, char* seg_start, Uint seg_size) +{ + Eterm *nsp; + Eterm *nsp_end; + const struct hipe_sdesc *sdesc; + /* arch-specific nstack walk state */ + struct nstack_walk_state walk_state; + + if (!p->hipe.nstack || !nstack_walk_init_check(p)) + return 0; + ASSERT(p->hipe.nsp && p->hipe.nstend); + nsp = nstack_walk_nsp_begin(p); + nsp_end = nstack_walk_nsp_end(p); + sdesc = nstack_walk_init_sdesc_ignore_trap(p, &walk_state); + + /* Check the topmost frame */ + if (ErtsInArea(sdesc->bucket.hvalue, seg_start, seg_size)) + return 1; + + while (!nstack_walk_nsp_reached_end(nsp, nsp_end)) { + unsigned sdesc_size = nstack_walk_frame_size(sdesc); + unsigned long ra = nstack_walk_frame_ra(nsp, sdesc); + if (ra == (unsigned long)nbif_stack_trap_ra) + ra = (unsigned long)p->hipe.ngra; + if (ErtsInArea(ra, seg_start, seg_size)) + return 1; + sdesc = hipe_find_sdesc(ra); + nsp = nstack_walk_next_frame(nsp, sdesc_size); + } + return 0; +} diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c new file mode 100644 index 0000000000..2998ed87a2 --- /dev/null +++ b/erts/emulator/hipe/hipe_load.c @@ -0,0 +1,105 @@ +/* + * %CopyrightBegin% + + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_load.c + * + * HiPE atomic code loader + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "sys.h" +#include "global.h" +#include "erl_binary.h" +#include "hipe_load.h" +#include "hipe_bif0.h" + +void hipe_free_loader_state(HipeLoaderState *stp) +{ + if (stp->module == NIL) return; + + // TODO: Needs to be freed separately. We'd like have a unified executable + // code allocator, so postpone this for now. + /* if (stp->text_segment) */ + /* erts_free(ERTS_ALC_T_HIPE, stp->text_segment); */ + stp->text_segment = NULL; + stp->text_segment_size = 0; + + if (stp->data_segment) + erts_free(ERTS_ALC_T_HIPE, stp->data_segment); + stp->data_segment = NULL; + stp->data_segment_size = 0; + + if (stp->new_hipe_refs) { + hipe_purge_refs(stp->new_hipe_refs, stp->module, 0); + stp->new_hipe_refs = NULL; + } + if (stp->new_hipe_sdesc) { + hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module, 0); + stp->new_hipe_sdesc = NULL; + } + + stp->module = NIL; +} + +static void +hipe_loader_state_dtor(Binary* magic) +{ + HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(magic) == hipe_loader_state_dtor); + + hipe_free_loader_state(stp); +} + +Binary *hipe_alloc_loader_state(Eterm module) +{ + HipeLoaderState *stp; + Binary *magic; + + if (is_not_atom(module)) return NULL; + + magic = erts_create_magic_binary(sizeof(HipeLoaderState), + hipe_loader_state_dtor); + erts_refc_inc(&magic->refc, 1); + stp = ERTS_MAGIC_BIN_DATA(magic); + + stp->module = module; + stp->text_segment = NULL; + stp->text_segment_size = 0; + stp->data_segment = NULL; + stp->data_segment_size = 0; + + stp->new_hipe_refs = NULL; + stp->new_hipe_sdesc = NULL; + + return magic; +} + +HipeLoaderState * +hipe_get_loader_state(Binary *magic) +{ + if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor) + return NULL; + + return (HipeLoaderState*) ERTS_MAGIC_BIN_DATA(magic); +} diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h new file mode 100644 index 0000000000..40c8a8aa2a --- /dev/null +++ b/erts/emulator/hipe/hipe_load.h @@ -0,0 +1,48 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_load.h + * + * HiPE atomic code loader + */ +#ifndef HIPE_LOAD_H +#define HIPE_LOAD_H + +#include "global.h" + +typedef struct hipe_loader_state { + Eterm module; /* Module name, atom */ + + void *text_segment; + Uint text_segment_size; + + void *data_segment; + Uint data_segment_size; + + struct hipe_ref* new_hipe_refs; + struct hipe_sdesc* new_hipe_sdesc; + +} HipeLoaderState; + +extern Binary *hipe_alloc_loader_state(Eterm module); +extern void hipe_free_loader_state(HipeLoaderState*); +extern HipeLoaderState *hipe_get_loader_state(Binary *binary); + +#endif /* HIPE_LOAD_H */ diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index b9d4226705..4573980e1e 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -435,9 +435,6 @@ static const struct rts_param rts_params[] = { presence or absence of struct erl_fun_thing's "next" field. */ { 5, "EFT_CREATOR", 1, offsetof(struct erl_fun_thing, creator) }, { 6, "EFT_FE", 1, offsetof(struct erl_fun_thing, fe) }, -#ifdef HIPE - { 7, "EFT_NATIVE_ADDRESS", 1, offsetof(struct erl_fun_thing, native_address) }, -#endif { 8, "EFT_ARITY", 1, offsetof(struct erl_fun_thing, arity) }, { 9, "EFT_NUM_FREE", 1, offsetof(struct erl_fun_thing, num_free) }, { 10, "EFT_ENV", 1, offsetof(struct erl_fun_thing, env[0]) }, diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index ed95045292..0706f8d2c9 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -718,12 +718,9 @@ void hipe_empty_nstack(Process *p) p->hipe.nstend = NULL; } -void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free) +void hipe_set_closure_stub(ErlFunEntry *fe) { - unsigned arity; - - arity = fe->arity; - fe->native_address = (Eterm*) hipe_closure_stub_address(arity); + fe->native_address = (Eterm*) hipe_closure_stub_address(fe->arity); } Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s) diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h index e54b81cf78..334e978307 100644 --- a/erts/emulator/hipe/hipe_mode_switch.h +++ b/erts/emulator/hipe/hipe_mode_switch.h @@ -59,12 +59,11 @@ void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure); Process *hipe_mode_switch(Process*, unsigned, Eterm*); void hipe_inc_nstack(Process *p); void hipe_empty_nstack(Process *p); -void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free); +void hipe_set_closure_stub(ErlFunEntry *fe); Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s); ERTS_GLB_INLINE void hipe_reserve_beam_trap_frame(Process*, Eterm reg[], unsigned arity); ERTS_GLB_INLINE void hipe_unreserve_beam_trap_frame(Process*); -ERTS_GLB_INLINE int hipe_is_ra_mode_switch(const void* ra); extern Uint hipe_beam_pc_return[]; extern Uint hipe_beam_pc_throw[]; @@ -113,11 +112,6 @@ ERTS_GLB_INLINE void hipe_unreserve_beam_trap_frame(Process *p) p->stop += 2; } -ERTS_GLB_INLINE int hipe_is_ra_mode_switch(const void* ra) -{ - return ra == nbif_return; -} - #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* ASM */ diff --git a/erts/emulator/hipe/hipe_module.c b/erts/emulator/hipe/hipe_module.c new file mode 100644 index 0000000000..469f077dd2 --- /dev/null +++ b/erts/emulator/hipe/hipe_module.c @@ -0,0 +1,35 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "sys.h" +#include "hipe_arch.h" +#include "hipe_module.h" + +void hipe_free_module(HipeModule *mod) +{ + hipe_free_code(mod->text_segment, mod->text_segment_size); + if (mod->data_segment) /* Some modules lack data segments */ + erts_free(ERTS_ALC_T_HIPE, mod->data_segment); + + erts_free(ERTS_ALC_T_HIPE, mod); +} diff --git a/erts/emulator/hipe/hipe_module.h b/erts/emulator/hipe/hipe_module.h new file mode 100644 index 0000000000..b489f567cb --- /dev/null +++ b/erts/emulator/hipe/hipe_module.h @@ -0,0 +1,45 @@ +/* + * %CopyrightBegin% + + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_module.h + * + * + */ +#ifndef HIPE_MODULE_H +#define HIPE_MODULE_H + +/* Forward-declare type to resolve circular dependency with module.h */ +typedef struct hipe_module HipeModule; + +#include "global.h" + +struct hipe_module { + void *text_segment; + Uint text_segment_size; + void *data_segment; + + struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */ + struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */ +}; + +extern void hipe_free_module(HipeModule *mod); + +#endif /* HIPE_MODULE_H */ diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c index 9c03b3811c..9439b823ab 100644 --- a/erts/emulator/hipe/hipe_native_bif.c +++ b/erts/emulator/hipe/hipe_native_bif.c @@ -42,7 +42,7 @@ */ /* for -Wmissing-prototypes :-( */ -extern Eterm hipe_erts_internal_check_process_code_2(BIF_ALIST_2); +extern Eterm hipe_erts_internal_check_process_code_1(BIF_ALIST_1); extern Eterm hipe_show_nstack_1(BIF_ALIST_1); /* Used when a BIF can trigger a stack walk. */ @@ -51,12 +51,12 @@ static __inline__ void hipe_set_narity(Process *p, unsigned int arity) p->hipe.narity = arity; } -Eterm hipe_erts_internal_check_process_code_2(BIF_ALIST_2) +Eterm hipe_erts_internal_check_process_code_1(BIF_ALIST_1) { Eterm ret; - hipe_set_narity(BIF_P, 2); - ret = erts_internal_check_process_code_2(BIF_P, BIF__ARGS); + hipe_set_narity(BIF_P, 1); + ret = erts_internal_check_process_code_1(BIF_P, BIF__ARGS); hipe_set_narity(BIF_P, 0); return ret; } diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c index 9b2048c457..4413748936 100644 --- a/erts/emulator/hipe/hipe_ppc.c +++ b/erts/emulator/hipe/hipe_ppc.c @@ -25,7 +25,6 @@ #endif #include "global.h" #include "erl_binary.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -68,34 +67,6 @@ void hipe_flush_icache_range(void *address, unsigned int nbytes) asm volatile("sync\n\tisync"); } -/* - * Management of 32MB code segments for regular code and trampolines. - */ - -#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */ - -static struct segment { - unsigned int *base; /* [base,base+32MB[ */ - unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */ - unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */ -} curseg; - -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) - -/* Darwin breakage */ -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif - -static void *new_code_mapping(void) -{ - return mmap(0, SEGMENT_NRBYTES, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, - -1, 0); -} - static int check_callees(Eterm callees) { Eterm *tuple; @@ -119,126 +90,71 @@ static int check_callees(Eterm callees) return arity; } -static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec) + +static void generate_trampolines(Uint32* address, + int nrcallees, Eterm callees, + Uint32** trampvec) { - unsigned int *base, *address, *tramp_pos, nrfreewords; - int trampnr; + Uint32* trampoline = address; + int i; - tramp_pos = curseg.tramp_pos; - address = curseg.code_pos; - nrfreewords = tramp_pos - address; - if (nrwords > nrfreewords) - return NULL; - curseg.code_pos = address + nrwords; - nrfreewords -= nrwords; - - base = curseg.base; - for (trampnr = 1; trampnr <= nrcallees; ++trampnr) { - Eterm mfa = tuple_val(callees)[trampnr]; - Eterm m = tuple_val(mfa)[1]; - Eterm f = tuple_val(mfa)[2]; - unsigned int a = unsigned_val(tuple_val(mfa)[3]); - unsigned int *trampoline = hipe_mfa_get_trampoline(m, f, a); - if (!in_area(trampoline, base, SEGMENT_NRBYTES)) { + for (i = 0; i < nrcallees; ++i) { #if defined(__powerpc64__) - if (nrfreewords < 7) - return NULL; - nrfreewords -= 7; - tramp_pos = trampoline = tramp_pos - 7; - trampoline[0] = 0x3D600000; /* addis r11,r0,0 */ - trampoline[1] = 0x616B0000; /* ori r11,r11,0 */ - trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */ - trampoline[3] = 0x656B0000; /* oris r11,r11,0 */ - trampoline[4] = 0x616B0000; /* ori r11,r11,0 */ - trampoline[5] = 0x7D6903A6; /* mtctr r11 */ - trampoline[6] = 0x4E800420; /* bctr */ - hipe_flush_icache_range(trampoline, 7*sizeof(int)); +# define TRAMPOLINE_WORDS 7 + trampoline[0] = 0x3D600000; /* addis r11,r0,0 */ + trampoline[1] = 0x616B0000; /* ori r11,r11,0 */ + trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */ + trampoline[3] = 0x656B0000; /* oris r11,r11,0 */ + trampoline[4] = 0x616B0000; /* ori r11,r11,0 */ + trampoline[5] = 0x7D6903A6; /* mtctr r11 */ + trampoline[6] = 0x4E800420; /* bctr */ #else - if (nrfreewords < 4) - return NULL; - nrfreewords -= 4; - tramp_pos = trampoline = tramp_pos - 4; - trampoline[0] = 0x39600000; /* addi r11,r0,0 */ - trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */ - trampoline[2] = 0x7D6903A6; /* mtctr r11 */ - trampoline[3] = 0x4E800420; /* bctr */ - hipe_flush_icache_range(trampoline, 4*sizeof(int)); +# define TRAMPOLINE_WORDS 4 + trampoline[0] = 0x39600000; /* addi r11,r0,0 */ + trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */ + trampoline[2] = 0x7D6903A6; /* mtctr r11 */ + trampoline[3] = 0x4E800420; /* bctr */ #endif - hipe_mfa_set_trampoline(m, f, a, trampoline); - } - trampvec[trampnr-1] = trampoline; + trampvec[i] = trampoline; + trampoline += TRAMPOLINE_WORDS; } - curseg.tramp_pos = tramp_pos; - return address; + hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_WORDS*sizeof(Uint32)); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) { - Uint nrwords; + Uint code_words; int nrcallees; Eterm trampvecbin; - unsigned int **trampvec; - unsigned int *address; - unsigned int *base; - struct segment oldseg; + Uint32 **trampvec; + Uint32 *address; if (nrbytes & 0x3) return NULL; - nrwords = nrbytes >> 2; + code_words = nrbytes / sizeof(Uint32); nrcallees = check_callees(callees); if (nrcallees < 0) return NULL; - trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*)); - trampvec = (unsigned int**)binary_bytes(trampvecbin); - - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } + trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*)); + trampvec = (Uint32**)binary_bytes(trampvecbin); + + address = erts_alloc(ERTS_ALC_T_HIPE_EXEC, + (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32)); + + generate_trampolines(address + code_words, nrcallees, callees, trampvec); *trampolines = trampvecbin; return address; } -static unsigned int *alloc_stub(Uint nrwords) +void hipe_free_code(void* code, unsigned int bytes) { - unsigned int *address; - unsigned int *base; - struct segment oldseg; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } - return address; + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); } static void patch_imm16(Uint32 *address, unsigned int imm16) @@ -288,12 +204,12 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { - unsigned int *code; + Uint32 *code; if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL) abort(); - code = alloc_stub(7); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 7*sizeof(Uint32)); if (!code) return NULL; @@ -312,7 +228,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) /* ba nbif_callemu */ code[6] = 0x48000002 | (unsigned long)&nbif_callemu; - hipe_flush_icache_range(code, 7*sizeof(int)); + hipe_flush_icache_range(code, 7*sizeof(Uint32)); return code; } @@ -360,7 +276,7 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { - unsigned int *code; + Uint32 *code; /* * Native code calls BEAM via a stub looking as follows: @@ -383,7 +299,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL) abort(); - code = alloc_stub(4); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 4*sizeof(Uint32)); if (!code) return NULL; @@ -396,7 +312,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) /* ba nbif_callemu */ code[3] = 0x48000002 | (unsigned long)&nbif_callemu; - hipe_flush_icache_range(code, 4*sizeof(int)); + hipe_flush_icache_range(code, 4*sizeof(Uint32)); return code; } diff --git a/erts/emulator/hipe/hipe_risc_gc.h b/erts/emulator/hipe/hipe_risc_gc.h index 09568c140e..f019434f67 100644 --- a/erts/emulator/hipe/hipe_risc_gc.h +++ b/erts/emulator/hipe/hipe_risc_gc.h @@ -27,7 +27,7 @@ /* arch wrapper includes hipe_${arch}_asm.h to define NR_ARG_REGS */ struct nstack_walk_state { - const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ + const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ }; static inline int nstack_walk_init_check(const Process *p) @@ -43,20 +43,20 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p) return p->hipe.nsp + nstkarity; } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) { - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra); state->sdesc0 = sdesc; return sdesc; } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc_ignore_trap(const Process *p, struct nstack_walk_state *state) { unsigned long ra = (unsigned long)p->hipe.nra; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; if (ra == (unsigned long)&nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; sdesc = hipe_find_sdesc(ra); @@ -64,7 +64,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p, return sdesc; } -static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0) +static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0) { Eterm *nsp = p->hipe.nsp; p->hipe.nsp = nstack_walk_nsp_begin(p); @@ -103,7 +103,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp return nsp >= nsp_end; } -static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc) +static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc) { return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc); } @@ -114,7 +114,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i) } static inline unsigned long -nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc) +nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc) { return nsp[sdesc_fsize(sdesc)]; } diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h index 0284265307..1369b392fe 100644 --- a/erts/emulator/hipe/hipe_risc_glue.h +++ b/erts/emulator/hipe/hipe_risc_glue.h @@ -66,13 +66,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct sdesc_with_exnra nbif_return_sdesc = { - .exnra = (unsigned long)&nbif_fail, - .sdesc = { - .bucket = { .hvalue = (unsigned long)&nbif_return }, - .summary = (1<<8), - }, - }; + static struct hipe_sdesc_with_exnra nbif_return_sdesc; + + nbif_return_sdesc.exnra = (unsigned long)nbif_fail; + nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return; + nbif_return_sdesc.sdesc.fsize = 0; + nbif_return_sdesc.sdesc.has_exnra = 1; + nbif_return_sdesc.sdesc.stk_nargs = 0; + nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty); + nbif_return_sdesc.sdesc.f_aix = atom_val(am_return); + nbif_return_sdesc.sdesc.a = 0; + hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); } diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c index dc98c96b8f..4001bedeb6 100644 --- a/erts/emulator/hipe/hipe_risc_stack.c +++ b/erts/emulator/hipe/hipe_risc_stack.c @@ -56,8 +56,8 @@ void hipe_print_nstack(Process *p) { Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc1; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc1; + const struct hipe_sdesc *sdesc; unsigned long ra; unsigned long exnra; unsigned int mask; @@ -175,7 +175,7 @@ void hipe_print_nstack(Process *p) #define MINSTACK 128 #define NSKIPFRAMES 4 -void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) +void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc) { Eterm *nsp; Eterm *nsp_end; @@ -216,7 +216,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) void (*hipe_handle_stack_trap(Process *p))(void) { void (*ngra)(void) = p->hipe.ngra; - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); hipe_update_stack_trap(p, sdesc); return ngra; } @@ -237,7 +237,7 @@ void hipe_find_handler(Process *p) unsigned long ra; unsigned long exnra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; nsp = p->hipe.nsp; nsp_end = p->hipe.nstend; @@ -277,7 +277,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) Eterm *nsp_end; unsigned long ra, prev_ra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; int i; if (depth < 1) @@ -292,7 +292,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) ra = (unsigned long)p->hipe.nra; prev_ra = 0; i = 0; - for (;;) { + while (nsp < nsp_end) { if (ra == (unsigned long)nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; if (ra != prev_ra) { @@ -302,8 +302,6 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) break; prev_ra = ra; } - if (nsp >= nsp_end) - break; sdesc = hipe_find_sdesc(ra); nsp += arity + sdesc_fsize(sdesc); arity = sdesc_arity(sdesc); diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c index 23020f34ee..876b20bb15 100644 --- a/erts/emulator/hipe/hipe_sparc.c +++ b/erts/emulator/hipe/hipe_sparc.c @@ -24,7 +24,6 @@ #include "config.h" #endif #include "global.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -88,8 +87,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Uint32 relDest, newI; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + relDest = (Uint32)((Sint32)destAddress - (Sint32)callAddress); newI = (1 << 30) | (relDest >> 2); *(Uint32*)callAddress = newI; @@ -97,105 +96,9 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -/* - * Memory allocator for executable code. - * - * This is required on x86 because some combinations - * of Linux kernels and CPU generations default to - * non-executable memory mappings, causing ordinary - * malloc() memory to be non-executable. - */ -static unsigned int code_bytes; -static char *code_next; - -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - -static int morecore(unsigned int alloc_bytes) -{ - unsigned int map_bytes; - char *map_hint, *map_start; - - /* Page-align the amount to allocate. */ - map_bytes = (alloc_bytes + 4095) & ~4095; - - /* Round up small allocations. */ - if (map_bytes < 1024*1024) - map_bytes = 1024*1024; - else - ALLOC_CODE_STATS(++nr_large); - - /* Create a new memory mapping, ensuring it is executable - and in the low 2GB of the address space. Also attempt - to make it adjacent to the previous mapping. */ - map_hint = code_next + code_bytes; - if ((unsigned long)map_hint & 4095) - abort(); - map_start = mmap(map_hint, map_bytes, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS -#ifdef __x86_64__ - |MAP_32BIT -#endif - , - -1, 0); - if (map_start == MAP_FAILED) - return -1; - - ALLOC_CODE_STATS(total_mapped += map_bytes); - - /* Merge adjacent mappings, so the trailing portion of the previous - mapping isn't lost. In practice this is quite successful. */ - if (map_start == map_hint) { - ALLOC_CODE_STATS(++nr_joins); - code_bytes += map_bytes; - } else { - ALLOC_CODE_STATS(++nr_splits); - ALLOC_CODE_STATS(total_lost += code_bytes); - code_next = map_start; - code_bytes = map_bytes; - } - - ALLOC_CODE_STATS(atexit_alloc_code_stats()); - - return 0; -} - static void *alloc_code(unsigned int alloc_bytes) { - void *res; - - /* Align function entries. */ - alloc_bytes = (alloc_bytes + 3) & ~3; - - if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0) - return NULL; - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - res = code_next; - code_next += alloc_bytes; - code_bytes -= alloc_bytes; - return res; + return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) @@ -206,6 +109,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int nrbytes) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { unsigned int *code; @@ -235,6 +143,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c index e2e6eb74b1..b80e44bc37 100644 --- a/erts/emulator/hipe/hipe_stack.c +++ b/erts/emulator/hipe/hipe_stack.c @@ -43,10 +43,10 @@ */ struct hipe_sdesc_table hipe_sdesc_table; -static struct sdesc **alloc_bucket(unsigned int size) +static struct hipe_sdesc **alloc_bucket(unsigned int size) { - unsigned long nbytes = size * sizeof(struct sdesc*); - struct sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes); + unsigned long nbytes = size * sizeof(struct hipe_sdesc*); + struct hipe_sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes); sys_memzero(bucket, nbytes); return bucket; } @@ -54,7 +54,7 @@ static struct sdesc **alloc_bucket(unsigned int size) static void hipe_grow_sdesc_table(void) { unsigned int old_size, new_size, new_mask; - struct sdesc **old_bucket, **new_bucket; + struct hipe_sdesc **old_bucket, **new_bucket; unsigned int i; old_size = 1 << hipe_sdesc_table.log2size; @@ -66,9 +66,9 @@ static void hipe_grow_sdesc_table(void) new_bucket = alloc_bucket(new_size); hipe_sdesc_table.bucket = new_bucket; for (i = 0; i < old_size; ++i) { - struct sdesc *b = old_bucket[i]; + struct hipe_sdesc *b = old_bucket[i]; while (b != NULL) { - struct sdesc *next = b->bucket.next; + struct hipe_sdesc *next = b->bucket.next; unsigned int j = (b->bucket.hvalue >> HIPE_RA_LSR_COUNT) & new_mask; b->bucket.next = new_bucket[j]; new_bucket[j] = b; @@ -78,11 +78,11 @@ static void hipe_grow_sdesc_table(void) erts_free(ERTS_ALC_T_HIPE, old_bucket); } -struct sdesc *hipe_put_sdesc(struct sdesc *sdesc) +struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc) { unsigned long ra; unsigned int i; - struct sdesc *chain; + struct hipe_sdesc *chain; unsigned int size; ra = sdesc->bucket.hvalue; @@ -102,7 +102,29 @@ struct sdesc *hipe_put_sdesc(struct sdesc *sdesc) return sdesc; } -void hipe_init_sdesc_table(struct sdesc *sdesc) +void hipe_destruct_sdesc(struct hipe_sdesc *sdesc) +{ + unsigned int i; + struct hipe_sdesc** prevp; + void* free_me; + + i = (sdesc->bucket.hvalue >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask; + prevp = &hipe_sdesc_table.bucket[i]; + + for (; *prevp != sdesc; prevp = &(*prevp)->bucket.next) + ASSERT(*prevp); + + *prevp = sdesc->bucket.next; + hipe_sdesc_table.used -= 1; + + if (sdesc->has_exnra) + free_me = ErtsContainerStruct(sdesc, struct hipe_sdesc_with_exnra, sdesc); + else + free_me = sdesc; + erts_free(ERTS_ALC_T_HIPE, free_me); +} + +void hipe_init_sdesc_table(struct hipe_sdesc *sdesc) { unsigned int log2size, size; @@ -121,31 +143,46 @@ void hipe_init_sdesc_table(struct sdesc *sdesc) * representation. If different representations are needed in * the future, this code has to be made target dependent. */ -struct sdesc *hipe_decode_sdesc(Eterm arg) +struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) { Uint ra, exnra; Eterm *live; - Uint fsize, arity, nlive, i, nslots, off; + Uint fsize, nargs, stk_nargs, nlive, i, nslots, off; Uint livebitswords, sdescbytes; void *p; - struct sdesc *sdesc; - - if (is_not_tuple(arg) || - (tuple_val(arg))[0] != make_arityval(6) || - term_to_Uint((tuple_val(arg))[1], &ra) == 0 || - term_to_Uint((tuple_val(arg))[2], &exnra) == 0 || - is_not_small((tuple_val(arg))[3]) || - (fsize = unsigned_val((tuple_val(arg))[3])) > 65535 || - is_not_small((tuple_val(arg))[4]) || - (arity = unsigned_val((tuple_val(arg))[4])) > 255 || - is_not_tuple((tuple_val(arg))[5])) + struct hipe_sdesc *sdesc; + Eterm* mfa_tpl; + Eterm* tp; + + if (is_not_tuple(arg)) + return 0; + + tp = tuple_val(arg); + if (tp[0] != make_arityval(6) || + term_to_Uint(tp[1], &ra) == 0 || + term_to_Uint(tp[2], &exnra) == 0 || + is_not_small(tp[3]) || + (fsize = unsigned_val(tp[3])) > 65535 || + is_not_small(tp[4]) || + (stk_nargs = unsigned_val(tp[4])) > 255 || + is_not_tuple(tp[5]) || + is_not_tuple(tp[6]) || + (mfa_tpl = tuple_val(tp[6]))[0] != make_arityval(3) || + is_not_atom(mfa_tpl[1]) || + is_not_atom(mfa_tpl[2]) || + is_not_small(mfa_tpl[3]) || + (nargs = unsigned_val(mfa_tpl[3])) > 255) return 0; + + if (stk_nargs > nargs) + return 0; + /* Get tuple with live slots */ - live = tuple_val((tuple_val(arg))[5]) + 1; + live = tuple_val(tp[5]) + 1; /* Get number of live slots */ nlive = arityval(live[-1]); - /* Calculate size of frame = locals + ra + arguments */ - nslots = fsize + 1 + arity; + /* Calculate size of frame = locals + ra + stack arguments */ + nslots = fsize + 1 + stk_nargs; /* Check that only valid slots are given. */ for (i = 0; i < nlive; ++i) { if (is_not_small(live[i]) || @@ -155,27 +192,34 @@ struct sdesc *hipe_decode_sdesc(Eterm arg) } /* Calculate number of words for the live bitmap. */ - livebitswords = (fsize + arity + 1 + 31) / 32; + livebitswords = (fsize + stk_nargs + 1 + 31) / 32; /* Calculate number of bytes needed for the stack descriptor. */ sdescbytes = (exnra - ? offsetof(struct sdesc_with_exnra, sdesc.livebits) - : offsetof(struct sdesc, livebits)) + ? offsetof(struct hipe_sdesc_with_exnra, sdesc.livebits) + : offsetof(struct hipe_sdesc, livebits)) + livebitswords * sizeof(int); p = erts_alloc(ERTS_ALC_T_HIPE, sdescbytes); /* If we have an exception handler use the special sdesc_with_exnra structure. */ if (exnra) { - struct sdesc_with_exnra *sdesc_we = p; + struct hipe_sdesc_with_exnra *sdesc_we = p; sdesc_we->exnra = exnra; sdesc = &(sdesc_we->sdesc); } else sdesc = p; + sdesc->m_aix = atom_val(mfa_tpl[1]); + sdesc->f_aix = atom_val(mfa_tpl[2]); + sdesc->a = nargs; + + /* Initialise head of sdesc. */ sdesc->bucket.next = 0; sdesc->bucket.hvalue = ra; - sdesc->summary = (fsize << 9) | (exnra ? (1<<8) : 0) | arity; + sdesc->fsize = fsize; + sdesc->has_exnra = (exnra ? 1 : 0); + sdesc->stk_nargs = stk_nargs; /* Clear all live-bits */ for (i = 0; i < livebitswords; ++i) sdesc->livebits[i] = 0; @@ -184,13 +228,5 @@ struct sdesc *hipe_decode_sdesc(Eterm arg) off = unsigned_val(live[i]); sdesc->livebits[off / 32] |= (1 << (off & 31)); } -#ifdef DEBUG - { - Eterm mfa_tpl = tuple_val(arg)[6]; - sdesc->dbg_M = tuple_val(mfa_tpl)[1]; - sdesc->dbg_F = tuple_val(mfa_tpl)[2]; - sdesc->dbg_A = tuple_val(mfa_tpl)[3]; - } -#endif return sdesc; } diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index afa0ed4256..7e30358767 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -30,40 +30,43 @@ #include <stddef.h> /* offsetof() */ -struct sdesc { +struct hipe_sdesc { struct { unsigned long hvalue; /* return address */ - struct sdesc *next; /* hash collision chain */ + struct hipe_sdesc *next; /* hash collision chain */ } bucket; - unsigned int summary; /* frame size, exn handler presence flag, arity */ -#ifdef DEBUG - Eterm dbg_M, dbg_F; - unsigned dbg_A; -#endif - unsigned int livebits[1]; /* size depends on arch & data in summary field */ + unsigned int fsize : 23; /* frame size */ + unsigned int has_exnra : 1; /* exn handler presence flag */ + unsigned int stk_nargs : 8; /* arguments on stack */ + Uint32 m_aix; + Uint32 f_aix; + Uint32 a; + struct hipe_sdesc* next_in_modi; + Uint32 livebits[1]; /* size depends on arch & data in summary field */ }; -struct sdesc_with_exnra { +struct hipe_sdesc_with_exnra { unsigned long exnra; - struct sdesc sdesc; + struct hipe_sdesc sdesc; }; -static __inline__ unsigned int sdesc_fsize(const struct sdesc *sdesc) +static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc) { - return sdesc->summary >> 9; + return sdesc->fsize; } -static __inline__ unsigned int sdesc_arity(const struct sdesc *sdesc) +/* Nr of arguments pushed on stack */ +static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc) { - return sdesc->summary & 0xFF; + return sdesc->stk_nargs; } -static __inline__ unsigned long sdesc_exnra(const struct sdesc *sdesc) +static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc) { - if ((sdesc->summary & (1<<8))) { + if (sdesc->has_exnra) { const char *tmp; - tmp = (const char*)sdesc - offsetof(struct sdesc_with_exnra, sdesc); - return ((const struct sdesc_with_exnra*)tmp)->exnra; + tmp = (const char*)sdesc - offsetof(struct hipe_sdesc_with_exnra, sdesc); + return ((const struct hipe_sdesc_with_exnra*)tmp)->exnra; } return 0; } @@ -72,13 +75,14 @@ struct hipe_sdesc_table { unsigned int log2size; unsigned int mask; /* INV: mask == (1 << log2size)-1 */ unsigned int used; - struct sdesc **bucket; + struct hipe_sdesc **bucket; }; extern struct hipe_sdesc_table hipe_sdesc_table; -extern struct sdesc *hipe_put_sdesc(struct sdesc*); -extern void hipe_init_sdesc_table(struct sdesc*); -extern struct sdesc *hipe_decode_sdesc(Eterm); +extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*); +extern void hipe_destruct_sdesc(struct hipe_sdesc*); +extern void hipe_init_sdesc_table(struct hipe_sdesc*); +extern struct hipe_sdesc *hipe_decode_sdesc(Eterm); #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) #define __builtin_expect(x, expected_value) (x) @@ -86,10 +90,10 @@ extern struct sdesc *hipe_decode_sdesc(Eterm); #define likely(x) __builtin_expect((x),1) #define unlikely(x) __builtin_expect((x),0) -static __inline__ const struct sdesc *hipe_find_sdesc(unsigned long ra) +static __inline__ const struct hipe_sdesc *hipe_find_sdesc(unsigned long ra) { unsigned int i = (ra >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask; - const struct sdesc *sdesc = hipe_sdesc_table.bucket[i]; + const struct hipe_sdesc *sdesc = hipe_sdesc_table.bucket[i]; if (likely(sdesc->bucket.hvalue == ra)) return sdesc; do { @@ -103,7 +107,7 @@ AEXTERN(void,nbif_stack_trap_ra,(void)); extern void hipe_print_nstack(Process*); extern void hipe_find_handler(Process*); extern void (*hipe_handle_stack_trap(Process*))(void); -extern void hipe_update_stack_trap(Process*, const struct sdesc*); +extern void hipe_update_stack_trap(Process*, const struct hipe_sdesc*); extern int hipe_fill_stacktrace(Process*, int, Eterm**); #if 0 && defined(HIPE_NSTACK_GROWS_UP) @@ -134,5 +138,7 @@ extern void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop extern Eterm *sweep_literals_nstack(Process *p, Eterm *n_htop, char *area, Uint area_size); extern int nstack_any_heap_ref_ptrs(Process *, char* mod_start, Uint mod_size); +extern int nstack_any_cps_in_segment(Process *, char* seg_start, Uint seg_size); + #endif /* HIPE_STACK_H */ diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c index 5f6c8c200e..c7e24673ac 100644 --- a/erts/emulator/hipe/hipe_x86.c +++ b/erts/emulator/hipe/hipe_x86.c @@ -24,7 +24,6 @@ #include "config.h" #endif #include "global.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -62,118 +61,17 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Uint rel32; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + rel32 = (Uint)destAddress - (Uint)callAddress - 4; *(Uint32*)callAddress = rel32; hipe_flush_icache_word(callAddress); return 0; } -/* - * Memory allocator for executable code. - * - * This is required on x86 because some combinations - * of Linux kernels and CPU generations default to - * non-executable memory mappings, causing ordinary - * malloc() memory to be non-executable. - */ -static unsigned int code_bytes; -static char *code_next; - -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - -/* FreeBSD 6.1 and Darwin breakage */ -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif - -static int morecore(unsigned int alloc_bytes) -{ - unsigned int map_bytes; - char *map_hint, *map_start; - - /* Page-align the amount to allocate. */ - map_bytes = (alloc_bytes + 4095) & ~4095; - - /* Round up small allocations. */ - if (map_bytes < 1024*1024) - map_bytes = 1024*1024; - else - ALLOC_CODE_STATS(++nr_large); - - /* Create a new memory mapping, ensuring it is executable - and in the low 2GB of the address space. Also attempt - to make it adjacent to the previous mapping. */ - map_hint = code_next + code_bytes; - if ((unsigned long)map_hint & 4095) - abort(); - map_start = mmap(map_hint, map_bytes, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS -#ifdef __x86_64__ - |MAP_32BIT -#endif - , - -1, 0); - if (map_start == MAP_FAILED) - return -1; - - ALLOC_CODE_STATS(total_mapped += map_bytes); - - /* Merge adjacent mappings, so the trailing portion of the previous - mapping isn't lost. In practice this is quite successful. */ - if (map_start == map_hint) { - ALLOC_CODE_STATS(++nr_joins); - code_bytes += map_bytes; - } else { - ALLOC_CODE_STATS(++nr_splits); - ALLOC_CODE_STATS(total_lost += code_bytes); - code_next = map_start; - code_bytes = map_bytes; - } - - ALLOC_CODE_STATS(atexit_alloc_code_stats()); - - return 0; -} - static void *alloc_code(unsigned int alloc_bytes) { - void *res; - - /* Align function entries. */ - alloc_bytes = (alloc_bytes + 3) & ~3; - - if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0) - return NULL; - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - res = code_next; - code_next += alloc_bytes; - code_bytes -= alloc_bytes; - return res; + return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) @@ -184,6 +82,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int bytes) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { /* @@ -264,6 +167,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h index 00fe03d8f9..a703e24b8c 100644 --- a/erts/emulator/hipe/hipe_x86_gc.h +++ b/erts/emulator/hipe/hipe_x86_gc.h @@ -30,9 +30,9 @@ struct nstack_walk_state { #ifdef SKIP_YOUNGEST_FRAME - const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ + const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ #else - struct sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */ + struct hipe_sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */ #endif }; @@ -57,37 +57,36 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p) #endif } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) { #ifdef SKIP_YOUNGEST_FRAME - const struct sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]); + const struct hipe_sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]); state->sdesc0 = sdesc; return sdesc; #else - unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS; - if ((int)nstkarity < 0) - nstkarity = 0; - state->sdesc0[0].summary = (0 << 9) | (0 << 8) | nstkarity; + state->sdesc0[0].bucket.hvalue = 0; /* for nstack_any_cps_in_segment */ + state->sdesc0[0].fsize = 0; + state->sdesc0[0].has_exnra = 0; + state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : + p->hipe.narity - NR_ARG_REGS); state->sdesc0[0].livebits[0] = 0; -# ifdef DEBUG - state->sdesc0[0].dbg_M = 0; - state->sdesc0[0].dbg_F = am_undefined; - state->sdesc0[0].dbg_A = 0; -# endif + state->sdesc0[0].m_aix = 0; + state->sdesc0[0].f_aix = atom_val(am_undefined); + state->sdesc0[0].a = 0; /* XXX: this appears to prevent a gcc-4.1.1 bug on x86 */ __asm__ __volatile__("" : : "m"(*state) : "memory"); return &state->sdesc0[0]; #endif } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc_ignore_trap(const Process *p, struct nstack_walk_state *state) { #ifdef SKIP_YOUNGEST_FRAME unsigned long ra = p->hipe.nsp[0]; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; if (ra == (unsigned long)nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; sdesc = hipe_find_sdesc(ra); @@ -98,7 +97,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p, #endif } -static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0) +static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0) { #ifdef SKIP_YOUNGEST_FRAME Eterm *nsp = p->hipe.nsp; @@ -137,7 +136,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp return nsp >= nsp_end; } -static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc) +static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc) { return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc); } @@ -148,7 +147,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i) } static inline unsigned long -nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc) +nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc) { return nsp[sdesc_fsize(sdesc)]; } diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h index 818d7444e2..de2b061706 100644 --- a/erts/emulator/hipe/hipe_x86_glue.h +++ b/erts/emulator/hipe/hipe_x86_glue.h @@ -58,16 +58,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct sdesc_with_exnra nbif_return_sdesc = { - .exnra = (unsigned long)nbif_fail, - .sdesc = { - .bucket = { .hvalue = (unsigned long)nbif_return }, - .summary = (1<<8), - #ifdef DEBUG - .dbg_F = am_return, - #endif - }, - }; + static struct hipe_sdesc_with_exnra nbif_return_sdesc; + + nbif_return_sdesc.exnra = (unsigned long)nbif_fail; + nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return; + nbif_return_sdesc.sdesc.fsize = 0; + nbif_return_sdesc.sdesc.has_exnra = 1; + nbif_return_sdesc.sdesc.stk_nargs = 0; + nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty); + nbif_return_sdesc.sdesc.f_aix = atom_val(am_return); + nbif_return_sdesc.sdesc.a = 0; + hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); } diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c index 50d08b96d3..1a34ce786c 100644 --- a/erts/emulator/hipe/hipe_x86_signal.c +++ b/erts/emulator/hipe/hipe_x86_signal.c @@ -51,7 +51,7 @@ #endif #include "hipe_signal.h" -#if __GLIBC__ == 2 && (__GLIBC_MINOR__ >= 3) +#if defined(__GLIBC__) && __GLIBC__ == 2 && (__GLIBC_MINOR__ >= 3) /* * __libc_sigaction() is the core routine. * Without libpthread, sigaction() and __sigaction() are both aliases diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c index f1559b1451..31582b3a2e 100644 --- a/erts/emulator/hipe/hipe_x86_stack.c +++ b/erts/emulator/hipe/hipe_x86_stack.c @@ -52,15 +52,14 @@ void hipe_print_nstack(Process *p) { Eterm *nsp; Eterm *nsp_end; - struct sdesc sdesc0; - const struct sdesc *sdesc1; - const struct sdesc *sdesc; + struct hipe_sdesc sdesc0; + const struct hipe_sdesc *sdesc1; + const struct hipe_sdesc *sdesc; unsigned long ra; unsigned long exnra; unsigned int mask; unsigned int sdesc_size; unsigned int i; - unsigned int nstkarity; static const char dashes[2*sizeof(long)+5] = { [0 ... 2*sizeof(long)+3] = '-' }; @@ -68,10 +67,10 @@ void hipe_print_nstack(Process *p) nsp = p->hipe.nsp; nsp_end = p->hipe.nstend; - nstkarity = p->hipe.narity - NR_ARG_REGS; - if ((int)nstkarity < 0) - nstkarity = 0; - sdesc0.summary = nstkarity; + sdesc0.fsize = 0; + sdesc0.has_exnra = 0; + sdesc0.stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : + p->hipe.narity - NR_ARG_REGS); sdesc0.livebits[0] = ~1; sdesc = &sdesc0; @@ -158,7 +157,7 @@ void hipe_print_nstack(Process *p) #define MINSTACK 128 #define NSKIPFRAMES 4 -void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) +void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc) { Eterm *nsp; Eterm *nsp_end; @@ -199,7 +198,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) void (*hipe_handle_stack_trap(Process *p))(void) { void (*ngra)(void) = p->hipe.ngra; - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); hipe_update_stack_trap(p, sdesc); return ngra; } @@ -220,7 +219,7 @@ void hipe_find_handler(Process *p) unsigned long ra; unsigned long exnra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int nstkarity; nsp = p->hipe.nsp; @@ -262,7 +261,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) Eterm *nsp_end; unsigned long ra, prev_ra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int nstkarity; int i; diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index a5714f8325..bb930ff03b 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -21,6 +21,7 @@ # include "config.h" #endif +#define ERTS_WANT_MEM_MAPPERS #include "sys.h" #include "erl_process.h" #include "erl_smp.h" @@ -358,12 +359,11 @@ char* erts_literals_start; UWord erts_literals_size; #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER ErtsMemMapper erts_exec_mmapper; #endif - #define ERTS_MMAP_SIZE_SC_SA_INC(SZ) \ do { \ mm->size.supercarrier.used.total += (SZ); \ @@ -1878,7 +1878,7 @@ erts_mremap(ErtsMemMapper* mm, return NULL; } -#if ERTS_HAVE_OS_MREMAP || ERTS_HAVE_GENUINE_OS_MMAP +#if defined(ERTS_HAVE_OS_MREMAP) || defined(ERTS_HAVE_GENUINE_OS_MMAP) superaligned = (ERTS_MMAPFLG_SUPERALIGNED & flags); if (superaligned) { @@ -1898,7 +1898,7 @@ erts_mremap(ErtsMemMapper* mm, } } -#if ERTS_HAVE_GENUINE_OS_MMAP +#ifdef ERTS_HAVE_GENUINE_OS_MMAP if (asize < old_size && (!superaligned || ERTS_IS_SUPERALIGNED(ptr))) { @@ -1913,7 +1913,7 @@ erts_mremap(ErtsMemMapper* mm, return ptr; } #endif -#if ERTS_HAVE_OS_MREMAP +#ifdef ERTS_HAVE_OS_MREMAP if (superaligned) return remap_move(mm, flags, new_ptr, old_size, sizep); else { diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index 623b5188ac..2a07d93c8c 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -158,12 +158,23 @@ Eterm erts_mmap_info_options(ErtsMemMapper*, # include "erl_alloc_types.h" extern ErtsMemMapper erts_dflt_mmapper; -# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + +# if defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + +# if defined(ARCH_64) extern ErtsMemMapper erts_literal_mmapper; # endif -# ifdef ERTS_ALC_A_EXEC + +# if defined(ERTS_ALC_A_EXEC) && defined(__x86_64__) + /* + * On x86_64, exec_alloc employs its own super carrier 'erts_exec_mmaper' + * to ensure low memory for HiPE AMD64 small code model. + */ +# define ERTS_HAVE_EXEC_MMAPPER extern ErtsMemMapper erts_exec_mmapper; # endif + +# endif /* ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ #endif /* ERTS_WANT_MEM_MAPPERS */ /*#define HARD_DEBUG_MSEG*/ diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 882c93a83c..4dd986f4c8 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -1414,7 +1414,7 @@ erts_mseg_init(ErtsMsegInit_t *init) erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms"); -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER /* Initialize erts_exec_mapper *FIRST*, to increase probability * of getting low memory for HiPE AMD64's small code model. */ diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h index b64b0d87f6..bd554238b7 100644 --- a/erts/emulator/sys/unix/erl_unix_sys.h +++ b/erts/emulator/sys/unix/erl_unix_sys.h @@ -46,10 +46,10 @@ #include <signal.h> #include <setjmp.h> -#if HAVE_SYS_SOCKETIO_H +#ifdef HAVE_SYS_SOCKETIO_H # include <sys/socketio.h> #endif -#if HAVE_SYS_SOCKIO_H +#ifdef HAVE_SYS_SOCKIO_H # include <sys/sockio.h> #endif diff --git a/erts/emulator/test/alloc_SUITE_data/testcase_driver.h b/erts/emulator/test/alloc_SUITE_data/testcase_driver.h index f0ca91bd06..2b742dd7e3 100644 --- a/erts/emulator/test/alloc_SUITE_data/testcase_driver.h +++ b/erts/emulator/test/alloc_SUITE_data/testcase_driver.h @@ -20,7 +20,7 @@ #ifndef TESTCASE_DRIVER_H__ #define TESTCASE_DRIVER_H__ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdlib.h> typedef struct { diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 3ee14f2d1c..b29520ab9f 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -22,10 +22,9 @@ -export([all/0, suite/0, init_per_suite/1, end_per_suite/1, versions/1,new_binary_types/1, call_purged_fun_code_gone/1, call_purged_fun_code_reload/1, call_purged_fun_code_there/1, - t_check_process_code/1,t_check_old_code/1, - t_check_process_code_ets/1, - external_fun/1,get_chunk/1,module_md5/1,make_stub/1, - make_stub_many_funs/1,constant_pools/1,constant_refc_binaries/1, + multi_proc_purge/1, t_check_old_code/1, + external_fun/1,get_chunk/1,module_md5/1, + constant_pools/1,constant_refc_binaries/1, false_dependency/1,coverage/1,fun_confusion/1, t_copy_literals/1, t_copy_literals_frags/1]). @@ -36,9 +35,9 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [versions, new_binary_types, call_purged_fun_code_gone, - call_purged_fun_code_reload, call_purged_fun_code_there, t_check_process_code, - t_check_process_code_ets, t_check_old_code, external_fun, get_chunk, - module_md5, make_stub, make_stub_many_funs, + call_purged_fun_code_reload, call_purged_fun_code_there, + multi_proc_purge, t_check_old_code, external_fun, get_chunk, + module_md5, constant_pools, constant_refc_binaries, false_dependency, coverage, fun_confusion, t_copy_literals, t_copy_literals_frags]. @@ -154,218 +153,80 @@ call_purged_fun_code_there(Config) when is_list(Config) -> ok. call_purged_fun_test(Priv, Data, Type) -> - File = filename:join(Data, "my_code_test2"), - Code = filename:join(Priv, "my_code_test2"), - - catch erlang:purge_module(my_code_test2), - catch erlang:delete_module(my_code_test2), - catch erlang:purge_module(my_code_test2), - - {ok,my_code_test2} = c:c(File, [{outdir,Priv}]), - - T = ets:new(my_code_test2_fun_table, []), - ets:insert(T, {my_fun,my_code_test2:make_fun(4711)}), - ets:insert(T, {my_fun2,my_code_test2:make_fun2()}), - - spawn(fun () -> - [{my_fun2,F2}] = ets:lookup(T, my_fun2), - F2(fun () -> - receive after infinity -> ok end - end, - fun () -> ok end), - exit(completed) - end), - - PurgeType = case Type of - code_gone -> - ok = file:delete(Code++".beam"), - true; - code_reload -> - true; - code_there -> - false - end, - - true = erlang:delete_module(my_code_test2), - - Purge = start_purge(my_code_test2, PurgeType), - - {P0, M0} = spawn_monitor(fun () -> - [{my_fun,F}] = ets:lookup(T, my_fun), - 4712 = F(1), - exit(completed) - end), - - wait_until(fun () -> - {status, suspended} - == process_info(P0, status) - end), - - ok = continue_purge(Purge), - - {P1, M1} = spawn_monitor(fun () -> - [{my_fun,F}] = ets:lookup(T, my_fun), - 4713 = F(2), - exit(completed) - end), - {P2, M2} = spawn_monitor(fun () -> - [{my_fun,F}] = ets:lookup(T, my_fun), - 4714 = F(3), - exit(completed) - end), - - wait_until(fun () -> - {status, suspended} - == process_info(P1, status) - end), - wait_until(fun () -> - {status, suspended} - == process_info(P2, status) - end), - - {current_function, - {erts_code_purger, - pending_purge_lambda, - 3}} = process_info(P0, current_function), - {current_function, - {erts_code_purger, - pending_purge_lambda, - 3}} = process_info(P1, current_function), - {current_function, - {erts_code_purger, - pending_purge_lambda, - 3}} = process_info(P2, current_function), - - case Type of - code_there -> - false = complete_purge(Purge); - _ -> - {true, true} = complete_purge(Purge) - end, - - case Type of - code_gone -> - receive - {'DOWN', M0, process, P0, Reason0} -> - {undef, _} = Reason0 - end, - receive - {'DOWN', M1, process, P1, Reason1} -> - {undef, _} = Reason1 - end, - receive - {'DOWN', M2, process, P2, Reason2} -> - {undef, _} = Reason2 - end; - _ -> - receive - {'DOWN', M0, process, P0, Reason0} -> - completed = Reason0 - end, - receive - {'DOWN', M1, process, P1, Reason1} -> - completed = Reason1 - end, - receive - {'DOWN', M2, process, P2, Reason2} -> - completed = Reason2 - end, - catch erlang:purge_module(my_code_test2), - catch erlang:delete_module(my_code_test2), - catch erlang:purge_module(my_code_test2) - end, - ok. - -t_check_process_code(Config) when is_list(Config) -> - case check_process_code_handle(indirect_references) of - false -> {skipped, "check_process_code() ignores funs"}; - true -> t_check_process_code_test(Config) - end. - -t_check_process_code_test(Config) -> - Priv = proplists:get_value(priv_dir, Config), - Data = proplists:get_value(data_dir, Config), - File = filename:join(Data, "my_code_test"), - Code = filename:join(Priv, "my_code_test"), + OptsList = case erlang:system_info(hipe_architecture) of + undefined -> [[]]; + _ -> [[], [native,{d,hipe}]] + end, + [call_purged_fun_test_do(Priv, Data, Type, CO, FO) + || CO <- OptsList, FO <- OptsList]. - catch erlang:purge_module(my_code_test), - catch erlang:delete_module(my_code_test), - catch erlang:purge_module(my_code_test), - - {ok,my_code_test} = c:c(File, [{outdir,Priv}]), - - MyFun = fun(X, Y) -> X + Y end, %Confuse things. - F = my_code_test:make_fun(42), - 2 = fun_refc(F), - MyFun2 = fun(X, Y) -> X * Y end, %Confuse things. - 44 = F(2), - %% Delete the module and call the fun again. - true = erlang:delete_module(my_code_test), - 2 = fun_refc(F), - 45 = F(3), - {'EXIT',{undef,_}} = (catch my_code_test:make_fun(33)), - - %% The fun should still be there, preventing purge. - true = erlang:check_process_code(self(), my_code_test), - gc(), - gc(), %Place funs on the old heap. - true = erlang:check_process_code(self(), my_code_test), - - %% Using the funs here guarantees that they will not be prematurely garbed. - 48 = F(6), - 3 = MyFun(1, 2), - 12 = MyFun2(3, 4), - - %% Kill all funs. - t_check_process_code1(Code, []). - -%% The real fun was killed, but we have some fakes which look similar. - -t_check_process_code1(Code, Fakes) -> - MyFun = fun(X, Y) -> X + Y + 1 end, %Confuse things. - false = erlang:check_process_code(self(), my_code_test), - 4 = MyFun(1, 2), - t_check_process_code2(Code, Fakes). - -t_check_process_code2(Code, _) -> - false = erlang:check_process_code(self(), my_code_test), - true = erlang:purge_module(my_code_test), - - %% In the next test we will load the same module twice. - {module,my_code_test} = code:load_abs(Code), - F = my_code_test:make_fun(37), - 2 = fun_refc(F), - false = erlang:check_process_code(self(), my_code_test), - {module,my_code_test} = code:load_abs(Code), - 2 = fun_refc(F), +call_purged_fun_test_do(Priv, Data, Type, CallerOpts, FunOpts) -> + io:format("Compile caller as ~p and funs as ~p\n", [CallerOpts, FunOpts]), + SrcFile = filename:join(Data, "call_purged_fun_tester.erl"), + ObjFile = filename:join(Priv, "call_purged_fun_tester.beam"), + {ok,Mod,Code} = compile:file(SrcFile, [binary, report | CallerOpts]), + {module,Mod} = code:load_binary(Mod, ObjFile, Code), - %% Still false because the fun with the same identify is found - %% in the current code. - false = erlang:check_process_code(self(), my_code_test), + call_purged_fun_tester:do(Priv, Data, Type, FunOpts). - %% Some fake funs in the same module should not do any difference. - false = erlang:check_process_code(self(), my_code_test), - 38 = F(1), - t_check_process_code3(Code, F, []). +multi_proc_purge(Config) when is_list(Config) -> + %% + %% Make sure purge requests aren't lost when + %% purger process is working. + %% + Priv = proplists:get_value(priv_dir, Config), + Data = proplists:get_value(data_dir, Config), + File1 = filename:join(Data, "my_code_test"), + File2 = filename:join(Data, "my_code_test2"), + + {ok,my_code_test} = c:c(File1, [{outdir,Priv}]), + {ok,my_code_test2} = c:c(File2, [{outdir,Priv}]), + erlang:delete_module(my_code_test), + erlang:delete_module(my_code_test2), -t_check_process_code3(Code, F, Fakes) -> - Pid = spawn_link(fun() -> body(F, Fakes) end), - true = erlang:purge_module(my_code_test), - false = erlang:check_process_code(self(), my_code_test), - false = erlang:check_process_code(Pid, my_code_test), + Self = self(), - true = erlang:delete_module(my_code_test), - true = erlang:check_process_code(self(), my_code_test), - true = erlang:check_process_code(Pid, my_code_test), - 39 = F(2), - t_check_process_code4(Code, Pid). - -t_check_process_code4(_Code, Pid) -> - Pid ! drop_funs, - receive after 1 -> ok end, - false = erlang:check_process_code(Pid, my_code_test), + Fun1 = fun () -> + erts_code_purger:purge(my_code_test), + Self ! {self(), done} + end, + Fun2 = fun () -> + erts_code_purger:soft_purge(my_code_test2), + Self ! {self(), done} + end, + Fun3 = fun () -> + erts_code_purger:purge('__nonexisting_module__'), + Self ! {self(), done} + end, + Fun4 = fun () -> + erts_code_purger:soft_purge('__another_nonexisting_module__'), + Self ! {self(), done} + end, + + Pid1 = spawn_link(Fun1), + Pid2 = spawn_link(Fun2), + Pid3 = spawn_link(Fun3), + Pid4 = spawn_link(Fun4), + Pid5 = spawn_link(Fun1), + Pid6 = spawn_link(Fun2), + Pid7 = spawn_link(Fun3), + receive after 50 -> ok end, + Pid8 = spawn_link(Fun4), + Pid9 = spawn_link(Fun1), + Pid10 = spawn_link(Fun2), + Pid11 = spawn_link(Fun3), + Pid12 = spawn_link(Fun4), + Pid13 = spawn_link(Fun1), + receive after 50 -> ok end, + Pid14 = spawn_link(Fun2), + Pid15 = spawn_link(Fun3), + Pid16 = spawn_link(Fun4), + + lists:foreach(fun (P) -> receive {P, done} -> ok end end, + [Pid1, Pid2, Pid3, Pid4, Pid5, Pid6, Pid7, Pid8, + Pid9, Pid10, Pid11, Pid12, Pid13, Pid14, Pid15, Pid16]), ok. body(F, Fakes) -> @@ -388,72 +249,6 @@ gc() -> gc1(). gc1() -> ok. -%% Test check_process_code/2 in combination with a fun obtained from an ets table. -t_check_process_code_ets(Config) when is_list(Config) -> - case check_process_code_handle(indirect_references) of - false -> - {skipped, "check_process_code() ignores funs"}; - true -> - case test_server:is_native(?MODULE) of - true -> - {skip,"Native code"}; - false -> - do_check_process_code_ets(Config) - end - end. - -do_check_process_code_ets(Config) -> - Priv = proplists:get_value(priv_dir, Config), - Data = proplists:get_value(data_dir, Config), - File = filename:join(Data, "my_code_test"), - - catch erlang:purge_module(my_code_test), - catch erlang:delete_module(my_code_test), - catch erlang:purge_module(my_code_test), - {ok,my_code_test} = c:c(File, [{outdir,Priv}]), - - T = ets:new(my_code_test, []), - ets:insert(T, {7,my_code_test:make_fun(107)}), - ets:insert(T, {8,my_code_test:make_fun(108)}), - erlang:delete_module(my_code_test), - false = erlang:check_process_code(self(), my_code_test), - Body = fun() -> - [{7,F1}] = ets:lookup(T, 7), - [{8,F2}] = ets:lookup(T, 8), - IdleLoop = fun() -> receive _X -> ok end end, - RecLoop = fun(Again) -> - receive - call -> 110 = F1(3), - 100 = F2(-8), - Again(Again); - {drop_funs,To} -> - To ! funs_dropped, - IdleLoop() - end - end, - true = erlang:check_process_code(self(), my_code_test), - RecLoop(RecLoop) - end, - Pid = spawn_link(Body), - receive after 1 -> ok end, - true = erlang:check_process_code(Pid, my_code_test), - Pid ! call, - Pid ! {drop_funs,self()}, - - receive - funs_dropped -> ok; - Other -> ct:fail({unexpected,Other}) - after 10000 -> - ct:fail(no_funs_dropped_answer) - end, - - false = erlang:check_process_code(Pid, my_code_test), - ok. - -fun_refc(F) -> - {refc,Count} = erlang:fun_info(F, refc), - Count. - %% Test the erlang:check_old_code/1 BIF. t_check_old_code(Config) when is_list(Config) -> @@ -543,67 +338,6 @@ module_md5_ok(Code) -> end. -make_stub(Config) when is_list(Config) -> - catch erlang:purge_module(my_code_test), - MD5 = erlang:md5(<<>>), - - Data = proplists:get_value(data_dir, Config), - File = filename:join(Data, "my_code_test"), - {ok,my_code_test,Code} = compile:file(File, [binary]), - - my_code_test = code:make_stub_module(my_code_test, Code, {[],[],MD5}), - true = erlang:delete_module(my_code_test), - true = erlang:purge_module(my_code_test), - - my_code_test = code:make_stub_module(my_code_test, - make_unaligned_sub_binary(Code), - {[],[],MD5}), - true = erlang:delete_module(my_code_test), - true = erlang:purge_module(my_code_test), - - my_code_test = code:make_stub_module(my_code_test, zlib:gzip(Code), - {[],[],MD5}), - true = erlang:delete_module(my_code_test), - true = erlang:purge_module(my_code_test), - - %% Should fail. - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(my_code_test, <<"bad">>, {[],[],MD5})), - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(my_code_test, - bit_sized_binary(Code), - {[],[],MD5})), - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(my_code_test_with_wrong_name, - Code, {[],[],MD5})), - ok. - -make_stub_many_funs(Config) when is_list(Config) -> - catch erlang:purge_module(many_funs), - MD5 = erlang:md5(<<>>), - - Data = proplists:get_value(data_dir, Config), - File = filename:join(Data, "many_funs"), - {ok,many_funs,Code} = compile:file(File, [binary]), - - many_funs = code:make_stub_module(many_funs, Code, {[],[],MD5}), - true = erlang:delete_module(many_funs), - true = erlang:purge_module(many_funs), - many_funs = code:make_stub_module(many_funs, - make_unaligned_sub_binary(Code), - {[],[],MD5}), - true = erlang:delete_module(many_funs), - true = erlang:purge_module(many_funs), - - %% Should fail. - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(many_funs, <<"bad">>, {[],[],MD5})), - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(many_funs, - bit_sized_binary(Code), - {[],[],MD5})), - ok. - constant_pools(Config) when is_list(Config) -> Data = proplists:get_value(data_dir, Config), File = filename:join(Data, "literals"), @@ -1137,38 +871,3 @@ flush() -> id(I) -> I. -check_process_code_handle(What) -> - lists:member(What, erlang:system_info(check_process_code)). - -wait_until(Fun) -> - case Fun() of - true -> - ok; - false -> - receive after 100 -> ok end, - wait_until(Fun) - end. - -start_purge(Mod, Type) when is_atom(Mod) - andalso ((Type == true) - orelse (Type == false)) -> - Ref = make_ref(), - erts_code_purger ! {test_purge, Mod, self(), Type, Ref}, - receive - {started, Ref} -> - Ref - end. - -continue_purge(Ref) when is_reference(Ref) -> - erts_code_purger ! {continue, Ref}, - receive - {continued, Ref} -> - ok - end. - -complete_purge(Ref) when is_reference(Ref) -> - erts_code_purger ! {complete, Ref}, - receive - {test_purge, Res, Ref} -> - Res - end. diff --git a/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl b/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl new file mode 100644 index 0000000000..699f0c1161 --- /dev/null +++ b/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl @@ -0,0 +1,186 @@ +-module(call_purged_fun_tester). + +-export([do/4]). + +%% Resurrect line macro when hipe compiled +-ifdef(hipe). +-define(line, put(the_line,?LINE),). +do(Priv, Data, Type, Opts) -> + try do_it(Priv, Data, Type, Opts) + catch + C:E -> + ST = erlang:get_stacktrace(), + io:format("Caught exception from line ~p:\n~p\n", + [get(the_line), ST]), + io:format("Message queue: ~p\n", [process_info(self(), messages)]), + erlang:raise(C, E, ST) + end. +-else. +-define(line,). +do(P,D,T,O) -> + do_it(P,D,T,O). +-endif. + + +do_it(Priv, Data, Type, Opts) -> + File = filename:join(Data, "my_code_test2"), + Code = filename:join(Priv, "my_code_test2"), + + catch erlang:purge_module(my_code_test2), + catch erlang:delete_module(my_code_test2), + catch erlang:purge_module(my_code_test2), + + ?line {ok,my_code_test2} = c:c(File, [{outdir,Priv} | Opts]), + + ?line IsNative = lists:member(native,Opts), + ?line IsNative = code:is_module_native(my_code_test2), + + ?line T = ets:new(my_code_test2_fun_table, []), + ets:insert(T, {my_fun,my_code_test2:make_fun(4711)}), + ets:insert(T, {my_fun2,my_code_test2:make_fun2()}), + + Papa = self(), + {P0,M0} = spawn_monitor(fun () -> + [{my_fun2,F2}] = ets:lookup(T, my_fun2), + F2(fun () -> + Papa ! {self(),"going to sleep"}, + receive {Papa,"wake up"} -> ok end + end, + fun () -> ok end), + exit(completed) + end), + + ?line PurgeType = case Type of + code_gone -> + ok = file:delete(Code++".beam"), + true; + code_reload -> + true; + code_there -> + false + end, + + ?line true = erlang:delete_module(my_code_test2), + + ?line ok = receive {P0, "going to sleep"} -> ok + after 1000 -> timeout + end, + + ?line Purge = start_purge(my_code_test2, PurgeType), + + ?line {P1, M1} = spawn_monitor(fun () -> + ?line [{my_fun,F}] = ets:lookup(T, my_fun), + ?line 4712 = F(1), + exit(completed) + end), + + ?line ok = wait_until(fun () -> + {status, suspended} + == process_info(P1, status) + end), + + ?line ok = continue_purge(Purge), + + ?line {P2, M2} = spawn_monitor(fun () -> + ?line [{my_fun,F}] = ets:lookup(T, my_fun), + ?line 4713 = F(2), + exit(completed) + end), + ?line {P3, M3} = spawn_monitor(fun () -> + ?line [{my_fun,F}] = ets:lookup(T, my_fun), + ?line 4714 = F(3), + exit(completed) + end), + + ?line ok = wait_until(fun () -> + {status, suspended} + == process_info(P2, status) + end), + ?line ok = wait_until(fun () -> + {status, suspended} + == process_info(P3, status) + end), + + ?line {current_function, + {erts_code_purger, + pending_purge_lambda, + 3}} = process_info(P1, current_function), + ?line {current_function, + {erts_code_purger, + pending_purge_lambda, + 3}} = process_info(P2, current_function), + ?line {current_function, + {erts_code_purger, + pending_purge_lambda, + 3}} = process_info(P3, current_function), + + case Type of + code_there -> + ?line false = complete_purge(Purge), + P0 ! {self(), "wake up"}, + ?line completed = wait_for_down(P0,M0); + _ -> + ?line {true, true} = complete_purge(Purge), + ?line killed = wait_for_down(P0,M0) + end, + + case Type of + code_gone -> + ?line {undef, _} = wait_for_down(P1,M1), + ?line {undef, _} = wait_for_down(P2,M2), + ?line {undef, _} = wait_for_down(P3,M3); + _ -> + ?line completed = wait_for_down(P1,M1), + ?line completed = wait_for_down(P2,M2), + ?line completed = wait_for_down(P3,M3), + catch erlang:purge_module(my_code_test2), + catch erlang:delete_module(my_code_test2), + catch erlang:purge_module(my_code_test2) + end, + ok. + +wait_for_down(P,M) -> + receive + {'DOWN', M, process, P, Reason} -> + Reason + after 1000 -> + timeout + end. + +wait_until(Fun) -> + wait_until(Fun, 20). + +wait_until(Fun, N) -> + case {Fun(),N} of + {true, _} -> + ok; + {false, 0} -> + timeout; + {false, _} -> + receive after 100 -> ok end, + wait_until(Fun, N-1) + end. + +start_purge(Mod, Type) when is_atom(Mod) + andalso ((Type == true) + orelse (Type == false)) -> + Ref = make_ref(), + erts_code_purger ! {test_purge, Mod, self(), Type, Ref}, + receive + {started, Ref} -> + Ref + end. + +continue_purge(Ref) when is_reference(Ref) -> + erts_code_purger ! {continue, Ref}, + receive + {continued, Ref} -> + ok + end. + +complete_purge(Ref) when is_reference(Ref) -> + erts_code_purger ! {complete, Ref}, + receive + {test_purge, Res, Ref} -> + Res + end. diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c index 08efa23c81..caf99c952f 100644 --- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c +++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c @@ -17,7 +17,7 @@ * * %CopyrightEnd% */ -#include "erl_nif.h" +#include <erl_nif.h> #include <assert.h> #ifdef __WIN32__ #include <windows.h> diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c index 614b68e865..8e5e81665c 100644 --- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c +++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c @@ -1397,10 +1397,18 @@ static void assert_print(char* str, int line) static void assert_failed(ErlDrvPort port, char* str, int line) { char buf[30]; + size_t bufsz = sizeof(buf); + assert_print(str,line); - snprintf(buf,sizeof(buf),"failed_at_line_%d",line); - driver_failure_atom(port,buf); - /*abort();*/ + + if (erl_drv_getenv("ERL_ABORT_ON_FAILURE", buf, &bufsz) == 0 + && (strcmp("true", buf) == 0 || strcmp("yes", buf) == 0)) { + abort(); + } + else { + snprintf(buf,sizeof(buf),"failed_at_line_%d",line); + driver_failure_atom(port,buf); + } } #define my_driver_select(PORT,FD,MODE,ON) \ diff --git a/erts/emulator/test/gc_SUITE.erl b/erts/emulator/test/gc_SUITE.erl index 8a600b7d9f..35dd147550 100644 --- a/erts/emulator/test/gc_SUITE.erl +++ b/erts/emulator/test/gc_SUITE.erl @@ -23,15 +23,26 @@ -module(gc_SUITE). -include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + -export([all/0, suite/0]). --export([grow_heap/1, grow_stack/1, grow_stack_heap/1, max_heap_size/1]). +-export([ + grow_heap/1, + grow_stack/1, + grow_stack_heap/1, + max_heap_size/1, + minor_major_gc_option_async/1, + minor_major_gc_option_self/1 +]). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [grow_heap, grow_stack, grow_stack_heap, max_heap_size]. + [grow_heap, grow_stack, grow_stack_heap, max_heap_size, + minor_major_gc_option_self, + minor_major_gc_option_async]. %% Produce a growing list of elements, @@ -190,3 +201,91 @@ long_receive() -> after 10000 -> ok end. + +minor_major_gc_option_self(_Config) -> + Endless = fun Endless() -> + receive + {gc, Type} -> erlang:garbage_collect(self(), [{type, Type}]) + after 100 -> ok end, + Endless() + end, + + %% Try as major, a test process will self-trigger GC + P1 = spawn(Endless), + erlang:garbage_collect(P1, []), + erlang:trace(P1, true, [garbage_collection]), + P1 ! {gc, major}, + expect_trace_messages(P1, [gc_major_start, gc_major_end]), + erlang:trace(P1, false, [garbage_collection]), + erlang:exit(P1, kill), + + %% Try as minor, a test process will self-trigger GC + P2 = spawn(Endless), + erlang:garbage_collect(P2, []), + erlang:trace(P2, true, [garbage_collection]), + P2 ! {gc, minor}, + expect_trace_messages(P2, [gc_minor_start, gc_minor_end]), + erlang:trace(P2, false, [garbage_collection]), + erlang:exit(P2, kill). + +minor_major_gc_option_async(_Config) -> + Endless = fun Endless() -> + receive after 100 -> ok end, + Endless() + end, + + %% Try with default option, must be major gc + P1 = spawn(Endless), + erlang:garbage_collect(P1, []), + erlang:trace(P1, true, [garbage_collection]), + erlang:garbage_collect(P1, []), + expect_trace_messages(P1, [gc_major_start, gc_major_end]), + erlang:trace(P1, false, [garbage_collection]), + erlang:exit(P1, kill), + + %% Try with the 'major' type + P2 = spawn(Endless), + erlang:garbage_collect(P2, []), + erlang:trace(P2, true, [garbage_collection]), + erlang:garbage_collect(P2, [{type, major}]), + expect_trace_messages(P2, [gc_major_start, gc_major_end]), + erlang:trace(P2, false, [garbage_collection]), + erlang:exit(P2, kill), + + %% Try with 'minor' option, once + P3 = spawn(Endless), + erlang:garbage_collect(P3, []), + erlang:trace(P3, true, [garbage_collection]), + erlang:garbage_collect(P3, [{type, minor}]), + expect_trace_messages(P3, [gc_minor_start, gc_minor_end]), + erlang:trace(P3, false, [garbage_collection]), + erlang:exit(P3, kill), + + %% Try with 'minor' option, once, async + P4 = spawn(Endless), + Ref = erlang:make_ref(), + erlang:garbage_collect(P4, []), + erlang:trace(P4, true, [garbage_collection]), + ?assertEqual(async, + erlang:garbage_collect(P4, [{type, minor}, {async, Ref}])), + expect_trace_messages(P4, [gc_minor_start, gc_minor_end]), + erlang:trace(P4, false, [garbage_collection]), + receive {garbage_collect, Ref, true} -> ok; + Other4 -> ct:pal("Unexpected message: ~p~n" + ++ "while waiting for async gc result", [Other4]) + after 2000 -> ?assert(false) + end, + erlang:exit(P4, kill). + +%% Given a list of atoms, trace tags - receives messages and checks if they are +%% trace events, and if the tag matches. Else will crash failing the test. +expect_trace_messages(_Pid, []) -> ok; +expect_trace_messages(Pid, [Tag | TraceTags]) -> + receive + {trace, Pid, Tag, _Data} -> ok; + AnythingElse -> + ct:pal("Unexpected message: ~p~nWhile expected {trace, _, ~p, _}", + [AnythingElse, Tag]), + ?assert(false) + end, + expect_trace_messages(Pid, TraceTags). diff --git a/erts/emulator/test/hipe_SUITE.erl b/erts/emulator/test/hipe_SUITE.erl index a556b4ddc0..0b44dd7fb7 100644 --- a/erts/emulator/test/hipe_SUITE.erl +++ b/erts/emulator/test/hipe_SUITE.erl @@ -19,12 +19,17 @@ %% -module(hipe_SUITE). --export([all/0, t_copy_literals/1]). +-export([all/0 + ,t_copy_literals/1 + ,t_purge/1 + ]). all() -> case erlang:system_info(hipe_architecture) of undefined -> {skip, "HiPE is disabled"}; - _ -> [t_copy_literals] + _ -> [t_copy_literals + ,t_purge + ] end. t_copy_literals(doc) -> @@ -65,3 +70,51 @@ t_copy_literals(Config) when is_list(Config) -> true = erlang:delete_module(ref_cell), true = erlang:purge_module(ref_cell), ok. + +t_purge(doc) -> "Checks that native code is properly found and purged"; +t_purge(Config) when is_list(Config) -> + Data = proplists:get_value(data_dir, Config), + Priv = proplists:get_value(priv_dir, Config), + SrcFile = filename:join(Data, "ref_cell"), + BeamFile = filename:join(Priv, "ref_cell"), + {ok,ref_cell} = c:c(SrcFile, [{outdir,Priv},native]), + true = code:is_module_native(ref_cell), + + PA = ref_cell:start_link(), + + %% Unload, PA should still be running + true = erlang:delete_module(ref_cell), + %% Can't use ref_cel:call/2, it's in old code! + call(PA, {put_res_of, fun()-> hej end}), + hej = call(PA, get), + + %% Load same module again + code:load_abs(BeamFile), + true = code:is_module_native(ref_cell), + PB = ref_cell:start_link(), + + %% Purge old code, PA should be killed, PB should survive + unlink(PA), + ARef = monitor(process, PA), + true = erlang:purge_module(ref_cell), + receive {'DOWN', ARef, process, PA, killed} -> ok + after 1 -> ct:fail("PA was not killed") + end, + + %% Unload, PB should still be running + true = erlang:delete_module(ref_cell), + call(PB, {put_res_of, fun()-> svejs end}), + svejs = call(PB, get), + + unlink(PB), + BRef = monitor(process, PB), + true = erlang:purge_module(ref_cell), + receive {'DOWN', BRef, process, PB, killed} -> ok + after 1 -> ct:fail("PB was not killed") + end, + + ok. + +call(Pid, Call) -> + Pid ! {Call, self()}, + receive {Pid, Res} -> Res end. diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl index 6733237b20..34e956bc21 100644 --- a/erts/emulator/test/match_spec_SUITE.erl +++ b/erts/emulator/test/match_spec_SUITE.erl @@ -427,13 +427,13 @@ silent_no_ms(Config) when is_list(Config) -> %% [{trace,Tracee,call,{?MODULE,f1,[start]}}, {trace,Tracee,return_to, - {?MODULE,'-silent_no_ms/1-fun-2-',0}}, + {?MODULE,'-silent_no_ms/1-fun-3-',0}}, {trace,Tracee,call,{?MODULE,f2,[f,g]}}, {trace,Tracee,return_to, - {?MODULE,'-silent_no_ms/1-fun-2-',0}}, + {?MODULE,'-silent_no_ms/1-fun-3-',0}}, {trace,Tracee,call,{erlang,integer_to_list,[2]}}, {trace,Tracee,return_to, - {?MODULE,'-silent_no_ms/1-fun-2-',0}}, + {?MODULE,'-silent_no_ms/1-fun-3-',0}}, {trace,Tracee,call,{?MODULE,f2,[h,i]}}, {trace,Tracee,return_to,{?MODULE,f3,2}}] end). @@ -484,7 +484,7 @@ ms_trace2(Config) when is_list(Config) -> %% %% Expected: (no return_to for global call trace) %% - Origin = {match_spec_SUITE,'-ms_trace2/1-fun-0-',1}, + Origin = {match_spec_SUITE,'-ms_trace2/1-fun-1-',1}, [{trace_ts,Tracee,call, {?MODULE,fn, [[all],[call,return_to,{tracer,Tracer}]]}, @@ -574,7 +574,7 @@ ms_trace3(Config) when is_list(Config) -> %% %% Expected: (no return_to for global call trace) %% - Origin = {match_spec_SUITE,'-ms_trace3/1-fun-1-',2}, + Origin = {match_spec_SUITE,'-ms_trace3/1-fun-2-',2}, [{trace_ts,Controller,call, {?MODULE,fn,[TraceeName,[all], [call,return_to,send,'receive', diff --git a/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c index e011aadce9..46ee8b5540 100644 --- a/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c +++ b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c @@ -24,7 +24,7 @@ * Author: Rickard Green */ -#include "erl_nif.h" +#include <erl_nif.h> #ifdef __WIN32__ # ifndef WIN32_LEAN_AND_MEAN diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 9c1694fa8a..0d3910b2e2 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -21,15 +21,17 @@ -module(nif_SUITE). %%-define(line_trace,true). --define(CHECK(Exp,Got), check(Exp,Got,?LINE)). +-define(CHECK(Exp,Got), Exp = check(Exp,Got,?LINE)). %%-define(CHECK(Exp,Got), Exp = Got). -include_lib("common_test/include/ct.hrl"). --export([all/0, suite/0, +-export([all/0, suite/0, groups/0, + init_per_group/2, end_per_group/2, init_per_testcase/2, end_per_testcase/2, - basic/1, reload/1, upgrade/1, heap_frag/1, + basic/1, reload_error/1, upgrade/1, heap_frag/1, t_on_load/1, + hipe/1, types/1, many_args/1, binaries/1, get_string/1, get_atom/1, maps/1, api_macros/1, @@ -52,27 +54,20 @@ -export([many_args_100/100]). - -%% -export([lib_version/0,call_history/0,hold_nif_mod_priv_data/1,nif_mod_call_history/0, -%% list_seq/1,type_test/0,tuple_2_list/1,is_identical/2,compare/2, -%% clone_bin/1,make_sub_bin/3,string_to_bin/2,atom_to_bin/2,macros/1, -%% tuple_2_list_and_tuple/1,iolist_2_bin/1,get_resource_type/1,alloc_resource/2, -%% make_resource/1,get_resource/2,release_resource/1,last_resource_dtor_call/0, suite/0, -%% make_new_resource/2,make_new_resource_binary/1,send_list_seq/2,send_new_blob/2, -%% alloc_msgenv/0,clear_msgenv/1,grow_blob/2,send_blob/2,send_blob_thread/3, -%% join_send_thread/1]). - - -define(nif_stub,nif_stub_error(?LINE)). suite() -> [{ct_hooks,[ts_install_cth]}]. -all() -> - [basic, reload, upgrade, heap_frag, types, many_args, - t_on_load, +all() -> + [basic] + ++ + [{group, G} || G <- api_groups()] + ++ + [reload_error, heap_frag, types, many_args, + hipe, binaries, get_string, get_atom, maps, api_macros, from_array, iolist_as_binary, resource, resource_binary, - resource_takeover, threading, send, send2, send3, + threading, send, send2, send3, send_threaded, neg, is_checks, get_length, make_atom, make_string,reverse_list_test, otp_9828, @@ -85,9 +80,35 @@ all() -> nif_port_command, nif_snprintf]. +groups() -> + [{G, [], api_repeaters()} || G <- api_groups()]. + +api_groups() -> [api_latest, api_2_4, api_2_0]. + +api_repeaters() -> [upgrade, resource_takeover, t_on_load]. + +init_per_group(api_latest, Config) -> Config; +init_per_group(api_2_4, Config) -> + [{nif_api_version, ".2_4"} | Config]; +init_per_group(api_2_0, Config) -> + case {os:type(),erlang:system_info({wordsize, internal})} of + {{win32,_}, 8} -> + %% ERL_NIF_TERM was declared as 32-bit 'long' until 2.3 + {skip, "API 2.0 buggy on Windows 64-bit"}; + _ -> + [{nif_api_version, ".2_0"} | Config] + end. + +end_per_group(_,_) -> ok. + init_per_testcase(t_on_load, Config) -> ets:new(nif_SUITE, [named_table]), Config; +init_per_testcase(hipe, Config) -> + case erlang:system_info(hipe_architecture) of + undefined -> {skip, "HiPE is disabled"}; + _ -> Config + end; init_per_testcase(_Case, Config) -> Config. @@ -112,8 +133,8 @@ basic(Config) when is_list(Config) -> true = lists:member(?MODULE, erlang:system_info(taints)), ok. -%% Test reload callback in nif lib -reload(Config) when is_list(Config) -> +%% Test old reload feature now always fails +reload_error(Config) when is_list(Config) -> TmpMem = tmpmem(), ensure_lib_loaded(Config), @@ -127,20 +148,20 @@ reload(Config) when is_list(Config) -> hold_nif_mod_priv_data(nif_mod:get_priv_data_ptr()), [{load,1,1,101},{get_priv_data_ptr,1,2,102}] = nif_mod_call_history(), - ok = nif_mod:load_nif_lib(Config, 2), - 2 = nif_mod:lib_version(), - [{reload,2,1,201},{lib_version,2,2,202}] = nif_mod_call_history(), + {error, {reload, _}} = nif_mod:load_nif_lib(Config, 2), + 1 = nif_mod:lib_version(), + [{lib_version,1,3,103}] = nif_mod_call_history(), - ok = nif_mod:load_nif_lib(Config, 1), + {error, {reload, _}} = nif_mod:load_nif_lib(Config, 1), 1 = nif_mod:lib_version(), - [{reload,1,1,101},{lib_version,1,2,102}] = nif_mod_call_history(), + [{lib_version,1,4,104}] = nif_mod_call_history(), true = erlang:delete_module(nif_mod), [] = nif_mod_call_history(), %%false= check_process_code(Pid, nif_mod), true = erlang:purge_module(nif_mod), - [{unload,1,3,103}] = nif_mod_call_history(), + [{unload,1,5,105}] = nif_mod_call_history(), true = lists:member(?MODULE, erlang:system_info(taints)), true = lists:member(nif_mod, erlang:system_info(taints)), @@ -148,7 +169,7 @@ reload(Config) when is_list(Config) -> ok. %% Test upgrade callback in nif lib -upgrade(Config) when is_list(Config) -> +upgrade(Config) when is_list(Config) -> TmpMem = tmpmem(), ensure_lib_loaded(Config), @@ -299,6 +320,8 @@ t_on_load(Config) when is_list(Config) -> %% Use ETS to tell nif_mod:on_load what to do ets:insert(nif_SUITE, {data_dir, Data}), ets:insert(nif_SUITE, {lib_version, 1}), + API = proplists:get_value(nif_api_version, Config, ""), + ets:insert(nif_SUITE, {nif_api_version, API}), {module,nif_mod} = code:load_binary(nif_mod,File,Bin), hold_nif_mod_priv_data(nif_mod:get_priv_data_ptr()), [{load,1,1,101},{get_priv_data_ptr,1,2,102}] = nif_mod_call_history(), @@ -411,6 +434,17 @@ t_on_load(Config) when is_list(Config) -> verify_tmpmem(TmpMem), ok. +hipe(Config) when is_list(Config) -> + Data = proplists:get_value(data_dir, Config), + Priv = proplists:get_value(priv_dir, Config), + Src = filename:join(Data, "hipe_compiled"), + {ok,hipe_compiled} = c:c(Src, [{outdir,Priv},native]), + true = code:is_module_native(hipe_compiled), + {error, {notsup,_}} = hipe_compiled:try_load_nif(), + true = code:delete(hipe_compiled), + false = code:purge(hipe_compiled), + ok. + %% Test NIF building heap fragments heap_frag(Config) when is_list(Config) -> @@ -828,7 +862,7 @@ resource_binary_do() -> -define(RT_CREATE,1). -define(RT_TAKEOVER,2). -%% Test resource takeover by module reload and upgrade +%% Test resource takeover by module upgrade resource_takeover(Config) when is_list(Config) -> TmpMem = tmpmem(), ensure_lib_loaded(Config), @@ -893,6 +927,7 @@ resource_takeover(Config) when is_list(Config) -> ok = forget_resource(NGX1), ?CHECK([], nif_mod_call_history()), % no dtor + {module,nif_mod} = erlang:load_module(nif_mod,ModBin), ok = nif_mod:load_nif_lib(Config, 2, [{resource_type, 0, ?RT_TAKEOVER, "resource_type_A",resource_dtor_A, ?RT_TAKEOVER}, @@ -911,7 +946,9 @@ resource_takeover(Config) when is_list(Config) -> {resource_type, 4, ?RT_CREATE, "resource_type_null_goneY",null, ?RT_CREATE} ]), - ?CHECK([{reload,2,1,201}], nif_mod_call_history()), + ?CHECK([{upgrade,2,1,201}], nif_mod_call_history()), + true = erlang:purge_module(nif_mod), + ?CHECK([], nif_mod_call_history()), % BGX2 keeping lib loaded BinA2 = read_resource(0,A2), ok = forget_resource(A2), @@ -924,8 +961,8 @@ resource_takeover(Config) when is_list(Config) -> ?CHECK([], nif_mod_call_history()), % no dtor ok = forget_resource(BGX2), % calling dtor in orphan library v1 still loaded - ?CHECK([{{resource_dtor_B_v1,BinBGX2},1,6,106}], nif_mod_call_history()), - % How to test that lib v1 is closed here? + ?CHECK([{{resource_dtor_B_v1,BinBGX2},1,6,106}, {unload,1,7,107}], + nif_mod_call_history()), ok = forget_resource(NGX2), ?CHECK([], nif_mod_call_history()), % no dtor @@ -1148,6 +1185,9 @@ resource_takeover(Config) when is_list(Config) -> ok = forget_resource(AN7), [] = nif_mod_call_history(), + true = erlang:delete_module(nif_mod), + true = erlang:purge_module(nif_mod), + true = lists:member(?MODULE, erlang:system_info(taints)), true = lists:member(nif_mod, erlang:system_info(taints)), verify_tmpmem(TmpMem), @@ -1221,11 +1261,19 @@ threading_do(Config) -> ok = tester:load_nif_lib(Config, "basic"), ok = tester:run(), + erlang:load_module(tester,ModBin), + erlang:purge_module(tester), ok = tester:load_nif_lib(Config, "rwlock"), ok = tester:run(), + erlang:load_module(tester,ModBin), + erlang:purge_module(tester), ok = tester:load_nif_lib(Config, "tsd"), - ok = tester:run(). + ok = tester:run(), + + erlang:delete_module(tester), + erlang:purge_module(tester). + %% Test NIF message sending send(Config) when is_list(Config) -> @@ -1513,13 +1561,13 @@ send3_new_state(State, Blob) -> neg(Config) when is_list(Config) -> TmpMem = tmpmem(), {'EXIT',{badarg,_}} = (catch erlang:load_nif(badarg, 0)), - {error,{load_failed,_}} = erlang:load_nif("pink_unicorn", 0), Data = proplists:get_value(data_dir, Config), File = filename:join(Data, "nif_mod"), {ok,nif_mod,Bin} = compile:file(File, [binary,return_errors]), {module,nif_mod} = erlang:load_module(nif_mod,Bin), + {error,{load_failed,_}} = nif_mod:load_nif_lib(Config, 0), {error,{bad_lib,_}} = nif_mod:load_nif_lib(Config, no_init), verify_tmpmem(TmpMem), ok. @@ -1640,6 +1688,7 @@ consume_timeslice(Config) when is_list(Config) -> end. consume_timeslice_test(Config) when is_list(Config) -> + ensure_lib_loaded(Config), CONTEXT_REDS = 2000, Me = self(), Go = make_ref(), @@ -1716,7 +1765,7 @@ consume_timeslice_test(Config) when is_list(Config) -> io:format("Reductions = ~p~n", [Reductions]), ok; {RedDiff, Reductions} -> - ct:fail({unexpected_reduction_count, Reductions}) + ct:fail({unexpected_reduction_count, Reductions, ExpReds}) end, none = next_msg(P), @@ -1870,7 +1919,8 @@ check(Exp,Got,Line) -> case Got of Exp -> Exp; _ -> - io:format("CHECK at ~p: Expected ~p but got ~p\n",[Line,Exp,Got]), + io:format("CHECK at line ~p\nExpected: ~p\nGot : ~p\n", + [Line,Exp,Got]), Got end. diff --git a/erts/emulator/test/nif_SUITE_data/Makefile.src b/erts/emulator/test/nif_SUITE_data/Makefile.src index fbb8978771..de06026780 100644 --- a/erts/emulator/test/nif_SUITE_data/Makefile.src +++ b/erts/emulator/test/nif_SUITE_data/Makefile.src @@ -2,7 +2,13 @@ NIF_LIBS = nif_SUITE.1@dll@ \ nif_mod.1@dll@ \ nif_mod.2@dll@ \ - nif_mod.3@dll@ + nif_mod.3@dll@ \ + nif_mod.1.2_0@dll@ \ + nif_mod.2.2_0@dll@ \ + nif_mod.3.2_0@dll@ \ + nif_mod.1.2_4@dll@ \ + nif_mod.2.2_4@dll@ \ + nif_mod.3.2_4@dll@ all: $(NIF_LIBS) basic@dll@ rwlock@dll@ tsd@dll@ echo_drv@dll@ diff --git a/erts/emulator/test/nif_SUITE_data/hipe_compiled.erl b/erts/emulator/test/nif_SUITE_data/hipe_compiled.erl new file mode 100644 index 0000000000..84ddbc8d63 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/hipe_compiled.erl @@ -0,0 +1,6 @@ +-module(hipe_compiled). + +-export([try_load_nif/0]). + +try_load_nif() -> + erlang:load_nif("doesn't matter", 0). diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index f2b1ef9d24..2c93891852 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -17,7 +17,7 @@ * * %CopyrightEnd% */ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdio.h> #include <string.h> @@ -104,21 +104,24 @@ struct binary_resource { static int get_pointer(ErlNifEnv* env, ERL_NIF_TERM term, void** pp) { - ErlNifUInt64 i64; - int r = enif_get_uint64(env, term, &i64); + ErlNifBinary bin; + int r = enif_inspect_binary(env, term, &bin); if (r) { - *pp = (void*)i64; + *pp = *(void**)bin.data; } return r; } static ERL_NIF_TERM make_pointer(ErlNifEnv* env, void* p) { - ErlNifUInt64 i64 = (ErlNifUInt64) p; - return enif_make_uint64(env, i64); + void** bin_data; + ERL_NIF_TERM res; + ADD_CALL("get_priv_data_ptr"); + bin_data = (void**)enif_make_new_binary(env, sizeof(void*), &res); + *bin_data = p; + return res; } - static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) { PrivData* data = enif_alloc(sizeof(PrivData)); @@ -184,14 +187,6 @@ static void resource_takeover(ErlNifEnv* env, PrivData* priv) msgenv_resource_type = rt; } -static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) -{ - PrivData* priv = (PrivData*) *priv_data; - add_call(env, priv, "reload"); - resource_takeover(env,priv); - return 0; -} - static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) { PrivData* priv = (PrivData*) *old_priv_data; @@ -2094,4 +2089,4 @@ static ErlNifFunc nif_funcs[] = {"format_term_nif", 2, format_term} }; -ERL_NIF_INIT(nif_SUITE,nif_funcs,load,reload,upgrade,unload) +ERL_NIF_INIT(nif_SUITE,nif_funcs,load,NULL,upgrade,unload) diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_0/README b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/README new file mode 100644 index 0000000000..a6ed36f634 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/README @@ -0,0 +1,5 @@ +These are old genuine header files +checked out from tag OTP_R14A c1e94fa9a6fe4ae717d35. + +I choose this API version (2.0) to test as it's +before the addition of vm_variant in ErlNifEntry. diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_drv_nif.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_drv_nif.h new file mode 100644 index 0000000000..ea013a49a3 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_drv_nif.h @@ -0,0 +1,48 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Common structures for both erl_driver.h and erl_nif.h + */ + +#ifndef __ERL_DRV_NIF_H__ +#define __ERL_DRV_NIF_H__ + +typedef struct { + int driver_major_version; + int driver_minor_version; + char *erts_version; + char *otp_release; + int thread_support; + int smp_support; + int async_threads; + int scheduler_threads; + int nif_major_version; + int nif_minor_version; +} ErlDrvSysInfo; + +typedef struct { + int suggested_stack_size; +} ErlDrvThreadOpts; + +#endif /* __ERL_DRV_NIF_H__ */ + + + + diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif.h new file mode 100644 index 0000000000..936f03bce1 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif.h @@ -0,0 +1,206 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* Include file for writers of Native Implemented Functions. +*/ + +#ifndef __ERL_NIF_H__ +#define __ERL_NIF_H__ + + +#include "erl_drv_nif.h" + +/* Version history: +** 0.1: R13B03 +** 1.0: R13B04 +** 2.0: R14A +*/ +#define ERL_NIF_MAJOR_VERSION 2 +#define ERL_NIF_MINOR_VERSION 0 + +#include <stdlib.h> + +#ifdef SIZEOF_CHAR +# define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR +# undef SIZEOF_CHAR +#endif +#ifdef SIZEOF_SHORT +# define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT +# undef SIZEOF_SHORT +#endif +#ifdef SIZEOF_INT +# define SIZEOF_INT_SAVED__ SIZEOF_INT +# undef SIZEOF_INT +#endif +#ifdef SIZEOF_LONG +# define SIZEOF_LONG_SAVED__ SIZEOF_LONG +# undef SIZEOF_LONG +#endif +#ifdef SIZEOF_LONG_LONG +# define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG +# undef SIZEOF_LONG_LONG +#endif +#ifdef HALFWORD_HEAP_EMULATOR +# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR +# undef HALFWORD_HEAP_EMULATOR +#endif +#include "erl_int_sizes_config.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef HALFWORD_HEAP_EMULATOR +typedef unsigned int ERL_NIF_TERM; +#else +typedef unsigned long ERL_NIF_TERM; +#endif + +struct enif_environment_t; +typedef struct enif_environment_t ErlNifEnv; + +typedef struct +{ + const char* name; + unsigned arity; + ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +}ErlNifFunc; + +typedef struct enif_entry_t +{ + int major; + int minor; + const char* name; + int num_of_funcs; + ErlNifFunc* funcs; + int (*load) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info); + int (*reload) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info); + int (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); + void (*unload) (ErlNifEnv*, void* priv_data); +}ErlNifEntry; + + + +typedef struct +{ + size_t size; + unsigned char* data; + + /* Internals (avert your eyes) */ + ERL_NIF_TERM bin_term; + void* ref_bin; +}ErlNifBinary; + +typedef struct enif_resource_type_t ErlNifResourceType; +typedef void ErlNifResourceDtor(ErlNifEnv*, void*); +typedef enum +{ + ERL_NIF_RT_CREATE = 1, + ERL_NIF_RT_TAKEOVER = 2 +}ErlNifResourceFlags; + +typedef enum +{ + ERL_NIF_LATIN1 = 1 +}ErlNifCharEncoding; + +typedef struct +{ + ERL_NIF_TERM pid; /* internal, may change */ +}ErlNifPid; + +typedef ErlDrvSysInfo ErlNifSysInfo; + +typedef struct ErlDrvTid_ *ErlNifTid; +typedef struct ErlDrvMutex_ ErlNifMutex; +typedef struct ErlDrvCond_ ErlNifCond; +typedef struct ErlDrvRWLock_ ErlNifRWLock; +typedef int ErlNifTSDKey; + +typedef ErlDrvThreadOpts ErlNifThreadOpts; + +#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) +# define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS +typedef struct { +# include "erl_nif_api_funcs.h" +} TWinDynNifCallbacks; +extern TWinDynNifCallbacks WinDynNifCallbacks; +# undef ERL_NIF_API_FUNC_DECL +#endif + +#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) && !defined(STATIC_ERLANG_DRIVER) +# define ERL_NIF_API_FUNC_MACRO(NAME) (WinDynNifCallbacks.NAME) +# include "erl_nif_api_funcs.h" +/* note that we have to keep ERL_NIF_API_FUNC_MACRO defined */ + +#else /* non windows or included from emulator itself */ + +# define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) extern RET_TYPE NAME ARGS +# include "erl_nif_api_funcs.h" +# undef ERL_NIF_API_FUNC_DECL +#endif + + +#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) +# define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks; +# define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks) +# define ERL_NIF_INIT_BODY memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks)) +#else +# define ERL_NIF_INIT_GLOB +# define ERL_NIF_INIT_BODY +# if defined(VXWORKS) +# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* MODNAME ## _init(void) +# else +# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* nif_init(void) +# endif +#endif + + +#ifdef __cplusplus +} +# define ERL_NIF_INIT_PROLOGUE extern "C" { +# define ERL_NIF_INIT_EPILOGUE } +#else +# define ERL_NIF_INIT_PROLOGUE +# define ERL_NIF_INIT_EPILOGUE +#endif + + +#define ERL_NIF_INIT(NAME, FUNCS, LOAD, RELOAD, UPGRADE, UNLOAD) \ +ERL_NIF_INIT_PROLOGUE \ +ERL_NIF_INIT_GLOB \ +ERL_NIF_INIT_DECL(NAME) \ +{ \ + static ErlNifEntry entry = \ + { \ + ERL_NIF_MAJOR_VERSION, \ + ERL_NIF_MINOR_VERSION, \ + #NAME, \ + sizeof(FUNCS) / sizeof(*FUNCS), \ + FUNCS, \ + LOAD, RELOAD, UPGRADE, UNLOAD \ + }; \ + ERL_NIF_INIT_BODY; \ + return &entry; \ +} \ +ERL_NIF_INIT_EPILOGUE + + +#endif /* __ERL_NIF_H__ */ + diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif_api_funcs.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif_api_funcs.h new file mode 100644 index 0000000000..ef4e9580b0 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_0/erl_nif_api_funcs.h @@ -0,0 +1,257 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#if !defined(ERL_NIF_API_FUNC_DECL) && !defined(ERL_NIF_API_FUNC_MACRO) +# error This file should not be included directly +#endif + +#ifdef ERL_NIF_API_FUNC_DECL +ERL_NIF_API_FUNC_DECL(void*,enif_priv_data,(ErlNifEnv*)); +ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_free,(void* ptr)); +ERL_NIF_API_FUNC_DECL(int,enif_is_atom,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_is_binary,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_is_ref,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_inspect_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(size_t size, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifBinary* bin, size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(int,enif_get_int,(ErlNifEnv*, ERL_NIF_TERM term, int* ip)); +ERL_NIF_API_FUNC_DECL(int,enif_get_ulong,(ErlNifEnv*, ERL_NIF_TERM term, unsigned long* ip)); +ERL_NIF_API_FUNC_DECL(int,enif_get_double,(ErlNifEnv*, ERL_NIF_TERM term, double* dp)); +ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM* head, ERL_NIF_TERM* tail)); +ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array)); +ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); +ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ulong,(ErlNifEnv* env, unsigned long i)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_double,(ErlNifEnv* env, double d)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_atom,(ErlNifEnv* env, const char* name)); +ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple,(ErlNifEnv* env, unsigned cnt, ...)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list,(ErlNifEnv* env, unsigned cnt, ...)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_cell,(ErlNifEnv* env, ERL_NIF_TERM car, ERL_NIF_TERM cdr)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_string,(ErlNifEnv* env, const char* string, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ref,(ErlNifEnv* env)); + +ERL_NIF_API_FUNC_DECL(ErlNifMutex*,enif_mutex_create,(char *name)); +ERL_NIF_API_FUNC_DECL(void,enif_mutex_destroy,(ErlNifMutex *mtx)); +ERL_NIF_API_FUNC_DECL(int,enif_mutex_trylock,(ErlNifMutex *mtx)); +ERL_NIF_API_FUNC_DECL(void,enif_mutex_lock,(ErlNifMutex *mtx)); +ERL_NIF_API_FUNC_DECL(void,enif_mutex_unlock,(ErlNifMutex *mtx)); +ERL_NIF_API_FUNC_DECL(ErlNifCond*,enif_cond_create,(char *name)); +ERL_NIF_API_FUNC_DECL(void,enif_cond_destroy,(ErlNifCond *cnd)); +ERL_NIF_API_FUNC_DECL(void,enif_cond_signal,(ErlNifCond *cnd)); +ERL_NIF_API_FUNC_DECL(void,enif_cond_broadcast,(ErlNifCond *cnd)); +ERL_NIF_API_FUNC_DECL(void,enif_cond_wait,(ErlNifCond *cnd, ErlNifMutex *mtx)); +ERL_NIF_API_FUNC_DECL(ErlNifRWLock*,enif_rwlock_create,(char *name)); +ERL_NIF_API_FUNC_DECL(void,enif_rwlock_destroy,(ErlNifRWLock *rwlck)); +ERL_NIF_API_FUNC_DECL(int,enif_rwlock_tryrlock,(ErlNifRWLock *rwlck)); +ERL_NIF_API_FUNC_DECL(void,enif_rwlock_rlock,(ErlNifRWLock *rwlck)); +ERL_NIF_API_FUNC_DECL(void,enif_rwlock_runlock,(ErlNifRWLock *rwlck)); +ERL_NIF_API_FUNC_DECL(int,enif_rwlock_tryrwlock,(ErlNifRWLock *rwlck)); +ERL_NIF_API_FUNC_DECL(void,enif_rwlock_rwlock,(ErlNifRWLock *rwlck)); +ERL_NIF_API_FUNC_DECL(void,enif_rwlock_rwunlock,(ErlNifRWLock *rwlck)); +ERL_NIF_API_FUNC_DECL(int,enif_tsd_key_create,(char *name, ErlNifTSDKey *key)); +ERL_NIF_API_FUNC_DECL(void,enif_tsd_key_destroy,(ErlNifTSDKey key)); +ERL_NIF_API_FUNC_DECL(void,enif_tsd_set,(ErlNifTSDKey key, void *data)); +ERL_NIF_API_FUNC_DECL(void*,enif_tsd_get,(ErlNifTSDKey key)); +ERL_NIF_API_FUNC_DECL(ErlNifThreadOpts*,enif_thread_opts_create,(char *name)); +ERL_NIF_API_FUNC_DECL(void,enif_thread_opts_destroy,(ErlNifThreadOpts *opts)); +ERL_NIF_API_FUNC_DECL(int,enif_thread_create,(char *name,ErlNifTid *tid,void * (*func)(void *),void *args,ErlNifThreadOpts *opts)); +ERL_NIF_API_FUNC_DECL(ErlNifTid,enif_thread_self,(void)); +ERL_NIF_API_FUNC_DECL(int,enif_equal_tids,(ErlNifTid tid1, ErlNifTid tid2)); +ERL_NIF_API_FUNC_DECL(void,enif_thread_exit,(void *resp)); +ERL_NIF_API_FUNC_DECL(int,enif_thread_join,(ErlNifTid, void **respp)); + +ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(void* ptr, size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_system_info,(ErlNifSysInfo *sip, size_t si_size)); +ERL_NIF_API_FUNC_DECL(int,enif_fprintf,(void/* FILE* */ *filep, const char *format, ...)); +ERL_NIF_API_FUNC_DECL(int,enif_inspect_iolist_as_binary,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, size_t pos, size_t size)); +ERL_NIF_API_FUNC_DECL(int,enif_get_string,(ErlNifEnv*, ERL_NIF_TERM list, char* buf, unsigned len, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(int,enif_get_atom,(ErlNifEnv*, ERL_NIF_TERM atom, char* buf, unsigned len, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(int,enif_is_fun,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_is_pid,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_is_port,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_get_uint,(ErlNifEnv*, ERL_NIF_TERM term, unsigned* ip)); +ERL_NIF_API_FUNC_DECL(int,enif_get_long,(ErlNifEnv*, ERL_NIF_TERM term, long* ip)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint,(ErlNifEnv*, unsigned i)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_long,(ErlNifEnv*, long i)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt)); +ERL_NIF_API_FUNC_DECL(int,enif_is_empty_list,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type,(ErlNifEnv*, const char* module_str, const char* name_str, void (*dtor)(ErlNifEnv*,void *), ErlNifResourceFlags flags, ErlNifResourceFlags* tried)); +ERL_NIF_API_FUNC_DECL(void*,enif_alloc_resource,(ErlNifResourceType* type, size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_release_resource,(void* obj)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource,(ErlNifEnv*, void* obj)); +ERL_NIF_API_FUNC_DECL(int,enif_get_resource,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp)); +ERL_NIF_API_FUNC_DECL(size_t,enif_sizeof_resource,(void* obj)); +ERL_NIF_API_FUNC_DECL(unsigned char*,enif_make_new_binary,(ErlNifEnv*,size_t size,ERL_NIF_TERM* termp)); +ERL_NIF_API_FUNC_DECL(int,enif_is_list,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_is_tuple,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_get_atom_length,(ErlNifEnv*, ERL_NIF_TERM atom, unsigned* len, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(int,enif_get_list_length,(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_atom_len,(ErlNifEnv* env, const char* name, size_t len)); +ERL_NIF_API_FUNC_DECL(int, enif_make_existing_atom_len,(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_string_len,(ErlNifEnv* env, const char* string, size_t len, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(ErlNifEnv*,enif_alloc_env,(void)); +ERL_NIF_API_FUNC_DECL(void,enif_free_env,(ErlNifEnv* env)); +ERL_NIF_API_FUNC_DECL(void,enif_clear_env,(ErlNifEnv* env)); +ERL_NIF_API_FUNC_DECL(int,enif_send,(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_copy,(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)); +ERL_NIF_API_FUNC_DECL(ErlNifPid*,enif_self,(ErlNifEnv* caller_env, ErlNifPid* pid)); +ERL_NIF_API_FUNC_DECL(int,enif_get_local_pid,(ErlNifEnv* env, ERL_NIF_TERM, ErlNifPid* pid)); +ERL_NIF_API_FUNC_DECL(void,enif_keep_resource,(void* obj)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource_binary,(ErlNifEnv*,void* obj,const void* data, size_t size)); + +/* +** Add last to keep compatibility on Windows!!! +*/ +#endif + +#ifdef ERL_NIF_API_FUNC_MACRO +# define enif_priv_data ERL_NIF_API_FUNC_MACRO(enif_priv_data) +# define enif_alloc ERL_NIF_API_FUNC_MACRO(enif_alloc) +# define enif_free ERL_NIF_API_FUNC_MACRO(enif_free) +# define enif_is_atom ERL_NIF_API_FUNC_MACRO(enif_is_atom) +# define enif_is_binary ERL_NIF_API_FUNC_MACRO(enif_is_binary) +# define enif_is_ref ERL_NIF_API_FUNC_MACRO(enif_is_ref) +# define enif_inspect_binary ERL_NIF_API_FUNC_MACRO(enif_inspect_binary) +# define enif_alloc_binary ERL_NIF_API_FUNC_MACRO(enif_alloc_binary) +# define enif_realloc_binary ERL_NIF_API_FUNC_MACRO(enif_realloc_binary) +# define enif_release_binary ERL_NIF_API_FUNC_MACRO(enif_release_binary) +# define enif_get_int ERL_NIF_API_FUNC_MACRO(enif_get_int) +# define enif_get_ulong ERL_NIF_API_FUNC_MACRO(enif_get_ulong) +# define enif_get_double ERL_NIF_API_FUNC_MACRO(enif_get_double) +# define enif_get_tuple ERL_NIF_API_FUNC_MACRO(enif_get_tuple) +# define enif_get_list_cell ERL_NIF_API_FUNC_MACRO(enif_get_list_cell) +# define enif_is_identical ERL_NIF_API_FUNC_MACRO(enif_is_identical) +# define enif_compare ERL_NIF_API_FUNC_MACRO(enif_compare) + +# define enif_make_binary ERL_NIF_API_FUNC_MACRO(enif_make_binary) +# define enif_make_badarg ERL_NIF_API_FUNC_MACRO(enif_make_badarg) +# define enif_make_int ERL_NIF_API_FUNC_MACRO(enif_make_int) +# define enif_make_ulong ERL_NIF_API_FUNC_MACRO(enif_make_ulong) +# define enif_make_double ERL_NIF_API_FUNC_MACRO(enif_make_double) +# define enif_make_atom ERL_NIF_API_FUNC_MACRO(enif_make_atom) +# define enif_make_existing_atom ERL_NIF_API_FUNC_MACRO(enif_make_existing_atom) +# define enif_make_tuple ERL_NIF_API_FUNC_MACRO(enif_make_tuple) +# define enif_make_list ERL_NIF_API_FUNC_MACRO(enif_make_list) +# define enif_make_list_cell ERL_NIF_API_FUNC_MACRO(enif_make_list_cell) +# define enif_make_string ERL_NIF_API_FUNC_MACRO(enif_make_string) +# define enif_make_ref ERL_NIF_API_FUNC_MACRO(enif_make_ref) + +# define enif_mutex_create ERL_NIF_API_FUNC_MACRO(enif_mutex_create) +# define enif_mutex_destroy ERL_NIF_API_FUNC_MACRO(enif_mutex_destroy) +# define enif_mutex_trylock ERL_NIF_API_FUNC_MACRO(enif_mutex_trylock) +# define enif_mutex_lock ERL_NIF_API_FUNC_MACRO(enif_mutex_lock) +# define enif_mutex_unlock ERL_NIF_API_FUNC_MACRO(enif_mutex_unlock) +# define enif_cond_create ERL_NIF_API_FUNC_MACRO(enif_cond_create) +# define enif_cond_destroy ERL_NIF_API_FUNC_MACRO(enif_cond_destroy) +# define enif_cond_signal ERL_NIF_API_FUNC_MACRO(enif_cond_signal) +# define enif_cond_broadcast ERL_NIF_API_FUNC_MACRO(enif_cond_broadcast) +# define enif_cond_wait ERL_NIF_API_FUNC_MACRO(enif_cond_wait) +# define enif_rwlock_create ERL_NIF_API_FUNC_MACRO(enif_rwlock_create) +# define enif_rwlock_destroy ERL_NIF_API_FUNC_MACRO(enif_rwlock_destroy) +# define enif_rwlock_tryrlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_tryrlock) +# define enif_rwlock_rlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_rlock) +# define enif_rwlock_runlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_runlock) +# define enif_rwlock_tryrwlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_tryrwlock) +# define enif_rwlock_rwlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_rwlock) +# define enif_rwlock_rwunlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_rwunlock) +# define enif_tsd_key_create ERL_NIF_API_FUNC_MACRO(enif_tsd_key_create) +# define enif_tsd_key_destroy ERL_NIF_API_FUNC_MACRO(enif_tsd_key_destroy) +# define enif_tsd_set ERL_NIF_API_FUNC_MACRO(enif_tsd_set) +# define enif_tsd_get ERL_NIF_API_FUNC_MACRO(enif_tsd_get) +# define enif_thread_opts_create ERL_NIF_API_FUNC_MACRO(enif_thread_opts_create) +# define enif_thread_opts_destroy ERL_NIF_API_FUNC_MACRO(enif_thread_opts_destroy) +# define enif_thread_create ERL_NIF_API_FUNC_MACRO(enif_thread_create) +# define enif_thread_self ERL_NIF_API_FUNC_MACRO(enif_thread_self) +# define enif_equal_tids ERL_NIF_API_FUNC_MACRO(enif_equal_tids) +# define enif_thread_exit ERL_NIF_API_FUNC_MACRO(enif_thread_exit) +# define enif_thread_join ERL_NIF_API_FUNC_MACRO(enif_thread_join) + +# define enif_realloc ERL_NIF_API_FUNC_MACRO(enif_realloc) +# define enif_system_info ERL_NIF_API_FUNC_MACRO(enif_system_info) +# define enif_fprintf ERL_NIF_API_FUNC_MACRO(enif_fprintf) +# define enif_inspect_iolist_as_binary ERL_NIF_API_FUNC_MACRO(enif_inspect_iolist_as_binary) +# define enif_make_sub_binary ERL_NIF_API_FUNC_MACRO(enif_make_sub_binary) +# define enif_get_string ERL_NIF_API_FUNC_MACRO(enif_get_string) +# define enif_get_atom ERL_NIF_API_FUNC_MACRO(enif_get_atom) +# define enif_is_fun ERL_NIF_API_FUNC_MACRO(enif_is_fun) +# define enif_is_pid ERL_NIF_API_FUNC_MACRO(enif_is_pid) +# define enif_is_port ERL_NIF_API_FUNC_MACRO(enif_is_port) +# define enif_get_uint ERL_NIF_API_FUNC_MACRO(enif_get_uint) +# define enif_get_long ERL_NIF_API_FUNC_MACRO(enif_get_long) +# define enif_make_uint ERL_NIF_API_FUNC_MACRO(enif_make_uint) +# define enif_make_long ERL_NIF_API_FUNC_MACRO(enif_make_long) +# define enif_make_tuple_from_array ERL_NIF_API_FUNC_MACRO(enif_make_tuple_from_array) +# define enif_make_list_from_array ERL_NIF_API_FUNC_MACRO(enif_make_list_from_array) +# define enif_is_empty_list ERL_NIF_API_FUNC_MACRO(enif_is_empty_list) +# define enif_open_resource_type ERL_NIF_API_FUNC_MACRO(enif_open_resource_type) +# define enif_alloc_resource ERL_NIF_API_FUNC_MACRO(enif_alloc_resource) +# define enif_release_resource ERL_NIF_API_FUNC_MACRO(enif_release_resource) +# define enif_make_resource ERL_NIF_API_FUNC_MACRO(enif_make_resource) +# define enif_get_resource ERL_NIF_API_FUNC_MACRO(enif_get_resource) +# define enif_sizeof_resource ERL_NIF_API_FUNC_MACRO(enif_sizeof_resource) +# define enif_make_new_binary ERL_NIF_API_FUNC_MACRO(enif_make_new_binary) +# define enif_is_list ERL_NIF_API_FUNC_MACRO(enif_is_list) +# define enif_is_tuple ERL_NIF_API_FUNC_MACRO(enif_is_tuple) +# define enif_get_atom_length ERL_NIF_API_FUNC_MACRO(enif_get_atom_length) +# define enif_get_list_length ERL_NIF_API_FUNC_MACRO(enif_get_list_length) +# define enif_make_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_atom_len) +# define enif_make_existing_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_existing_atom_len) +# define enif_make_string_len ERL_NIF_API_FUNC_MACRO(enif_make_string_len) +# define enif_alloc_env ERL_NIF_API_FUNC_MACRO(enif_alloc_env) +# define enif_free_env ERL_NIF_API_FUNC_MACRO(enif_free_env) +# define enif_clear_env ERL_NIF_API_FUNC_MACRO(enif_clear_env) +# define enif_send ERL_NIF_API_FUNC_MACRO(enif_send) +# define enif_make_copy ERL_NIF_API_FUNC_MACRO(enif_make_copy) +# define enif_self ERL_NIF_API_FUNC_MACRO(enif_self) +# define enif_get_local_pid ERL_NIF_API_FUNC_MACRO(enif_get_local_pid) +# define enif_keep_resource ERL_NIF_API_FUNC_MACRO(enif_keep_resource) +# define enif_make_resource_binary ERL_NIF_API_FUNC_MACRO(enif_make_resource_binary) +#endif + +#ifndef enif_make_list1 +# define enif_make_list1(ENV,E1) enif_make_list(ENV,1,E1) +# define enif_make_list2(ENV,E1,E2) enif_make_list(ENV,2,E1,E2) +# define enif_make_list3(ENV,E1,E2,E3) enif_make_list(ENV,3,E1,E2,E3) +# define enif_make_list4(ENV,E1,E2,E3,E4) enif_make_list(ENV,4,E1,E2,E3,E4) +# define enif_make_list5(ENV,E1,E2,E3,E4,E5) enif_make_list(ENV,5,E1,E2,E3,E4,E5) +# define enif_make_list6(ENV,E1,E2,E3,E4,E5,E6) enif_make_list(ENV,6,E1,E2,E3,E4,E5,E6) +# define enif_make_list7(ENV,E1,E2,E3,E4,E5,E6,E7) enif_make_list(ENV,7,E1,E2,E3,E4,E5,E6,E7) +# define enif_make_list8(ENV,E1,E2,E3,E4,E5,E6,E7,E8) enif_make_list(ENV,8,E1,E2,E3,E4,E5,E6,E7,E8) +# define enif_make_list9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_list(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9) +# define enif_make_tuple1(ENV,E1) enif_make_tuple(ENV,1,E1) +# define enif_make_tuple2(ENV,E1,E2) enif_make_tuple(ENV,2,E1,E2) +# define enif_make_tuple3(ENV,E1,E2,E3) enif_make_tuple(ENV,3,E1,E2,E3) +# define enif_make_tuple4(ENV,E1,E2,E3,E4) enif_make_tuple(ENV,4,E1,E2,E3,E4) +# define enif_make_tuple5(ENV,E1,E2,E3,E4,E5) enif_make_tuple(ENV,5,E1,E2,E3,E4,E5) +# define enif_make_tuple6(ENV,E1,E2,E3,E4,E5,E6) enif_make_tuple(ENV,6,E1,E2,E3,E4,E5,E6) +# define enif_make_tuple7(ENV,E1,E2,E3,E4,E5,E6,E7) enif_make_tuple(ENV,7,E1,E2,E3,E4,E5,E6,E7) +# define enif_make_tuple8(ENV,E1,E2,E3,E4,E5,E6,E7,E8) enif_make_tuple(ENV,8,E1,E2,E3,E4,E5,E6,E7,E8) +# define enif_make_tuple9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_tuple(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9) + +# define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid)) +#endif + diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_4/README b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/README new file mode 100644 index 0000000000..7abd0319a6 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/README @@ -0,0 +1,6 @@ +These are old genuine header files +checked out from tag OTP_R16B 05f11890bdfec4bfc3a78e191 + +I choose this API version (2.4) to test, as it's before +the addition of 'options' in ErlNifEntry and 'flags' in ErlNifFunc +and without include of generated erl_native_features_config.h. diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_drv_nif.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_drv_nif.h new file mode 100644 index 0000000000..ea013a49a3 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_drv_nif.h @@ -0,0 +1,48 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Common structures for both erl_driver.h and erl_nif.h + */ + +#ifndef __ERL_DRV_NIF_H__ +#define __ERL_DRV_NIF_H__ + +typedef struct { + int driver_major_version; + int driver_minor_version; + char *erts_version; + char *otp_release; + int thread_support; + int smp_support; + int async_threads; + int scheduler_threads; + int nif_major_version; + int nif_minor_version; +} ErlDrvSysInfo; + +typedef struct { + int suggested_stack_size; +} ErlDrvThreadOpts; + +#endif /* __ERL_DRV_NIF_H__ */ + + + + diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif.h new file mode 100644 index 0000000000..8006741a63 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif.h @@ -0,0 +1,237 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2013. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* Include file for writers of Native Implemented Functions. +*/ + +#ifndef __ERL_NIF_H__ +#define __ERL_NIF_H__ + + +#include "erl_drv_nif.h" + +/* Version history: +** 0.1: R13B03 +** 1.0: R13B04 +** 2.0: R14A +** 2.1: R14B02 "vm_variant" +** 2.2: R14B03 enif_is_exception +** 2.3: R15 enif_make_reverse_list, enif_is_number +** 2.4: R16 enif_consume_timeslice +*/ +#define ERL_NIF_MAJOR_VERSION 2 +#define ERL_NIF_MINOR_VERSION 4 + +#include <stdlib.h> + +#ifdef SIZEOF_CHAR +# define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR +# undef SIZEOF_CHAR +#endif +#ifdef SIZEOF_SHORT +# define SIZEOF_SHORT_SAVED__ SIZEOF_SHORT +# undef SIZEOF_SHORT +#endif +#ifdef SIZEOF_INT +# define SIZEOF_INT_SAVED__ SIZEOF_INT +# undef SIZEOF_INT +#endif +#ifdef SIZEOF_LONG +# define SIZEOF_LONG_SAVED__ SIZEOF_LONG +# undef SIZEOF_LONG +#endif +#ifdef SIZEOF_LONG_LONG +# define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG +# undef SIZEOF_LONG_LONG +#endif +#ifdef HALFWORD_HEAP_EMULATOR +# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR +# undef HALFWORD_HEAP_EMULATOR +#endif +#include "erl_int_sizes_config.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) +typedef unsigned __int64 ErlNifUInt64; +typedef __int64 ErlNifSInt64; +#elif SIZEOF_LONG == 8 +typedef unsigned long ErlNifUInt64; +typedef long ErlNifSInt64; +#elif SIZEOF_LONG_LONG == 8 +typedef unsigned long long ErlNifUInt64; +typedef long long ErlNifSInt64; +#else +#error No 64-bit integer type +#endif + +#ifdef HALFWORD_HEAP_EMULATOR +# define ERL_NIF_VM_VARIANT "beam.halfword" +typedef unsigned int ERL_NIF_TERM; +#else +# define ERL_NIF_VM_VARIANT "beam.vanilla" +# if SIZEOF_LONG == SIZEOF_VOID_P +typedef unsigned long ERL_NIF_TERM; +# elif SIZEOF_LONG_LONG == SIZEOF_VOID_P +typedef unsigned long long ERL_NIF_TERM; +# endif +#endif + +struct enif_environment_t; +typedef struct enif_environment_t ErlNifEnv; + +typedef struct +{ + const char* name; + unsigned arity; + ERL_NIF_TERM (*fptr)(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +}ErlNifFunc; + +typedef struct enif_entry_t +{ + int major; + int minor; + const char* name; + int num_of_funcs; + ErlNifFunc* funcs; + int (*load) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info); + int (*reload) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info); + int (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); + void (*unload) (ErlNifEnv*, void* priv_data); + const char* vm_variant; +}ErlNifEntry; + + + +typedef struct +{ + size_t size; + unsigned char* data; + + /* Internals (avert your eyes) */ + ERL_NIF_TERM bin_term; + void* ref_bin; +}ErlNifBinary; + +typedef struct enif_resource_type_t ErlNifResourceType; +typedef void ErlNifResourceDtor(ErlNifEnv*, void*); +typedef enum +{ + ERL_NIF_RT_CREATE = 1, + ERL_NIF_RT_TAKEOVER = 2 +}ErlNifResourceFlags; + +typedef enum +{ + ERL_NIF_LATIN1 = 1 +}ErlNifCharEncoding; + +typedef struct +{ + ERL_NIF_TERM pid; /* internal, may change */ +}ErlNifPid; + +typedef ErlDrvSysInfo ErlNifSysInfo; + +typedef struct ErlDrvTid_ *ErlNifTid; +typedef struct ErlDrvMutex_ ErlNifMutex; +typedef struct ErlDrvCond_ ErlNifCond; +typedef struct ErlDrvRWLock_ ErlNifRWLock; +typedef int ErlNifTSDKey; + +typedef ErlDrvThreadOpts ErlNifThreadOpts; + +#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) +# define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) RET_TYPE (*NAME) ARGS +typedef struct { +# include "erl_nif_api_funcs.h" +} TWinDynNifCallbacks; +extern TWinDynNifCallbacks WinDynNifCallbacks; +# undef ERL_NIF_API_FUNC_DECL +#endif + +#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) && !defined(STATIC_ERLANG_DRIVER) +# define ERL_NIF_API_FUNC_MACRO(NAME) (WinDynNifCallbacks.NAME) +# include "erl_nif_api_funcs.h" +/* note that we have to keep ERL_NIF_API_FUNC_MACRO defined */ + +#else /* non windows or included from emulator itself */ + +# define ERL_NIF_API_FUNC_DECL(RET_TYPE, NAME, ARGS) extern RET_TYPE NAME ARGS +# include "erl_nif_api_funcs.h" +# undef ERL_NIF_API_FUNC_DECL +#endif + + +#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) +# define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks; +# define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks) +# define ERL_NIF_INIT_BODY memcpy(&WinDynNifCallbacks,callbacks,sizeof(TWinDynNifCallbacks)) +#else +# define ERL_NIF_INIT_GLOB +# define ERL_NIF_INIT_BODY +# define ERL_NIF_INIT_DECL(MODNAME) ErlNifEntry* nif_init(void) +#endif + + +#ifdef __cplusplus +} +# define ERL_NIF_INIT_PROLOGUE extern "C" { +# define ERL_NIF_INIT_EPILOGUE } +#else +# define ERL_NIF_INIT_PROLOGUE +# define ERL_NIF_INIT_EPILOGUE +#endif + + +#define ERL_NIF_INIT(NAME, FUNCS, LOAD, RELOAD, UPGRADE, UNLOAD) \ +ERL_NIF_INIT_PROLOGUE \ +ERL_NIF_INIT_GLOB \ +ERL_NIF_INIT_DECL(NAME); \ +ERL_NIF_INIT_DECL(NAME) \ +{ \ + static ErlNifEntry entry = \ + { \ + ERL_NIF_MAJOR_VERSION, \ + ERL_NIF_MINOR_VERSION, \ + #NAME, \ + sizeof(FUNCS) / sizeof(*FUNCS), \ + FUNCS, \ + LOAD, RELOAD, UPGRADE, UNLOAD, \ + ERL_NIF_VM_VARIANT \ + }; \ + ERL_NIF_INIT_BODY; \ + return &entry; \ +} \ +ERL_NIF_INIT_EPILOGUE + +#if defined(USE_DYNAMIC_TRACE) && (defined(USE_DTRACE) || defined(USE_SYSTEMTAP)) +#define HAVE_USE_DTRACE 1 +#endif + +#ifdef HAVE_USE_DTRACE +ERL_NIF_TERM erl_nif_user_trace_s1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM erl_nif_user_trace_i4s4(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +ERL_NIF_TERM erl_nif_user_trace_n(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +#endif + +#endif /* __ERL_NIF_H__ */ + diff --git a/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif_api_funcs.h b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif_api_funcs.h new file mode 100644 index 0000000000..2f841645e1 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/nif_api_2_4/erl_nif_api_funcs.h @@ -0,0 +1,503 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2013. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#if !defined(ERL_NIF_API_FUNC_DECL) && !defined(ERL_NIF_API_FUNC_MACRO) +# error This file should not be included directly +#endif + +/* +** WARNING: add new ERL_NIF_API_FUNC_DECL entries at the bottom of the list +** to keep compatibility on Windows!!! +** +** And don't forget to increase ERL_NIF_MINOR_VERSION in erl_nif.h +** when adding functions to the API. +*/ +#ifdef ERL_NIF_API_FUNC_DECL +ERL_NIF_API_FUNC_DECL(void*,enif_priv_data,(ErlNifEnv*)); +ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_free,(void* ptr)); +ERL_NIF_API_FUNC_DECL(int,enif_is_atom,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_is_binary,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_is_ref,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_inspect_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(size_t size, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifBinary* bin, size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(int,enif_get_int,(ErlNifEnv*, ERL_NIF_TERM term, int* ip)); +ERL_NIF_API_FUNC_DECL(int,enif_get_ulong,(ErlNifEnv*, ERL_NIF_TERM term, unsigned long* ip)); +ERL_NIF_API_FUNC_DECL(int,enif_get_double,(ErlNifEnv*, ERL_NIF_TERM term, double* dp)); +ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM* head, ERL_NIF_TERM* tail)); +ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array)); +ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); +ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ulong,(ErlNifEnv* env, unsigned long i)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_double,(ErlNifEnv* env, double d)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_atom,(ErlNifEnv* env, const char* name)); +ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple,(ErlNifEnv* env, unsigned cnt, ...)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list,(ErlNifEnv* env, unsigned cnt, ...)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_cell,(ErlNifEnv* env, ERL_NIF_TERM car, ERL_NIF_TERM cdr)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_string,(ErlNifEnv* env, const char* string, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ref,(ErlNifEnv* env)); + +ERL_NIF_API_FUNC_DECL(ErlNifMutex*,enif_mutex_create,(char *name)); +ERL_NIF_API_FUNC_DECL(void,enif_mutex_destroy,(ErlNifMutex *mtx)); +ERL_NIF_API_FUNC_DECL(int,enif_mutex_trylock,(ErlNifMutex *mtx)); +ERL_NIF_API_FUNC_DECL(void,enif_mutex_lock,(ErlNifMutex *mtx)); +ERL_NIF_API_FUNC_DECL(void,enif_mutex_unlock,(ErlNifMutex *mtx)); +ERL_NIF_API_FUNC_DECL(ErlNifCond*,enif_cond_create,(char *name)); +ERL_NIF_API_FUNC_DECL(void,enif_cond_destroy,(ErlNifCond *cnd)); +ERL_NIF_API_FUNC_DECL(void,enif_cond_signal,(ErlNifCond *cnd)); +ERL_NIF_API_FUNC_DECL(void,enif_cond_broadcast,(ErlNifCond *cnd)); +ERL_NIF_API_FUNC_DECL(void,enif_cond_wait,(ErlNifCond *cnd, ErlNifMutex *mtx)); +ERL_NIF_API_FUNC_DECL(ErlNifRWLock*,enif_rwlock_create,(char *name)); +ERL_NIF_API_FUNC_DECL(void,enif_rwlock_destroy,(ErlNifRWLock *rwlck)); +ERL_NIF_API_FUNC_DECL(int,enif_rwlock_tryrlock,(ErlNifRWLock *rwlck)); +ERL_NIF_API_FUNC_DECL(void,enif_rwlock_rlock,(ErlNifRWLock *rwlck)); +ERL_NIF_API_FUNC_DECL(void,enif_rwlock_runlock,(ErlNifRWLock *rwlck)); +ERL_NIF_API_FUNC_DECL(int,enif_rwlock_tryrwlock,(ErlNifRWLock *rwlck)); +ERL_NIF_API_FUNC_DECL(void,enif_rwlock_rwlock,(ErlNifRWLock *rwlck)); +ERL_NIF_API_FUNC_DECL(void,enif_rwlock_rwunlock,(ErlNifRWLock *rwlck)); +ERL_NIF_API_FUNC_DECL(int,enif_tsd_key_create,(char *name, ErlNifTSDKey *key)); +ERL_NIF_API_FUNC_DECL(void,enif_tsd_key_destroy,(ErlNifTSDKey key)); +ERL_NIF_API_FUNC_DECL(void,enif_tsd_set,(ErlNifTSDKey key, void *data)); +ERL_NIF_API_FUNC_DECL(void*,enif_tsd_get,(ErlNifTSDKey key)); +ERL_NIF_API_FUNC_DECL(ErlNifThreadOpts*,enif_thread_opts_create,(char *name)); +ERL_NIF_API_FUNC_DECL(void,enif_thread_opts_destroy,(ErlNifThreadOpts *opts)); +ERL_NIF_API_FUNC_DECL(int,enif_thread_create,(char *name,ErlNifTid *tid,void * (*func)(void *),void *args,ErlNifThreadOpts *opts)); +ERL_NIF_API_FUNC_DECL(ErlNifTid,enif_thread_self,(void)); +ERL_NIF_API_FUNC_DECL(int,enif_equal_tids,(ErlNifTid tid1, ErlNifTid tid2)); +ERL_NIF_API_FUNC_DECL(void,enif_thread_exit,(void *resp)); +ERL_NIF_API_FUNC_DECL(int,enif_thread_join,(ErlNifTid, void **respp)); + +ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(void* ptr, size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_system_info,(ErlNifSysInfo *sip, size_t si_size)); +ERL_NIF_API_FUNC_DECL(int,enif_fprintf,(void/* FILE* */ *filep, const char *format, ...)); +ERL_NIF_API_FUNC_DECL(int,enif_inspect_iolist_as_binary,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, size_t pos, size_t size)); +ERL_NIF_API_FUNC_DECL(int,enif_get_string,(ErlNifEnv*, ERL_NIF_TERM list, char* buf, unsigned len, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(int,enif_get_atom,(ErlNifEnv*, ERL_NIF_TERM atom, char* buf, unsigned len, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(int,enif_is_fun,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_is_pid,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_is_port,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_get_uint,(ErlNifEnv*, ERL_NIF_TERM term, unsigned* ip)); +ERL_NIF_API_FUNC_DECL(int,enif_get_long,(ErlNifEnv*, ERL_NIF_TERM term, long* ip)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint,(ErlNifEnv*, unsigned i)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_long,(ErlNifEnv*, long i)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt)); +ERL_NIF_API_FUNC_DECL(int,enif_is_empty_list,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type,(ErlNifEnv*, const char* module_str, const char* name_str, void (*dtor)(ErlNifEnv*,void *), ErlNifResourceFlags flags, ErlNifResourceFlags* tried)); +ERL_NIF_API_FUNC_DECL(void*,enif_alloc_resource,(ErlNifResourceType* type, size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_release_resource,(void* obj)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource,(ErlNifEnv*, void* obj)); +ERL_NIF_API_FUNC_DECL(int,enif_get_resource,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp)); +ERL_NIF_API_FUNC_DECL(size_t,enif_sizeof_resource,(void* obj)); +ERL_NIF_API_FUNC_DECL(unsigned char*,enif_make_new_binary,(ErlNifEnv*,size_t size,ERL_NIF_TERM* termp)); +ERL_NIF_API_FUNC_DECL(int,enif_is_list,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_is_tuple,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_get_atom_length,(ErlNifEnv*, ERL_NIF_TERM atom, unsigned* len, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(int,enif_get_list_length,(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_atom_len,(ErlNifEnv* env, const char* name, size_t len)); +ERL_NIF_API_FUNC_DECL(int, enif_make_existing_atom_len,(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_string_len,(ErlNifEnv* env, const char* string, size_t len, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(ErlNifEnv*,enif_alloc_env,(void)); +ERL_NIF_API_FUNC_DECL(void,enif_free_env,(ErlNifEnv* env)); +ERL_NIF_API_FUNC_DECL(void,enif_clear_env,(ErlNifEnv* env)); +ERL_NIF_API_FUNC_DECL(int,enif_send,(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_copy,(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)); +ERL_NIF_API_FUNC_DECL(ErlNifPid*,enif_self,(ErlNifEnv* caller_env, ErlNifPid* pid)); +ERL_NIF_API_FUNC_DECL(int,enif_get_local_pid,(ErlNifEnv* env, ERL_NIF_TERM, ErlNifPid* pid)); +ERL_NIF_API_FUNC_DECL(void,enif_keep_resource,(void* obj)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource_binary,(ErlNifEnv*,void* obj,const void* data, size_t size)); +#if SIZEOF_LONG != 8 +ERL_NIF_API_FUNC_DECL(int,enif_get_int64,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifSInt64* ip)); +ERL_NIF_API_FUNC_DECL(int,enif_get_uint64,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifUInt64* ip)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int64,(ErlNifEnv*, ErlNifSInt64)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint64,(ErlNifEnv*, ErlNifUInt64)); +#endif +ERL_NIF_API_FUNC_DECL(int,enif_is_exception,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_make_reverse_list,(ErlNifEnv*, ERL_NIF_TERM term, ERL_NIF_TERM *list)); +ERL_NIF_API_FUNC_DECL(int,enif_is_number,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(void*,enif_dlopen,(const char* lib, void (*err_handler)(void*,const char*), void* err_arg)); +ERL_NIF_API_FUNC_DECL(void*,enif_dlsym,(void* handle, const char* symbol, void (*err_handler)(void*,const char*), void* err_arg)); +ERL_NIF_API_FUNC_DECL(int,enif_consume_timeslice,(ErlNifEnv*, int percent)); + +/* +** Add new entries here to keep compatibility on Windows!!! +*/ +#endif + +/* +** Please keep the ERL_NIF_API_FUNC_MACRO list below in the same order +** as the ERL_NIF_API_FUNC_DECL list above +*/ +#ifdef ERL_NIF_API_FUNC_MACRO +# define enif_priv_data ERL_NIF_API_FUNC_MACRO(enif_priv_data) +# define enif_alloc ERL_NIF_API_FUNC_MACRO(enif_alloc) +# define enif_free ERL_NIF_API_FUNC_MACRO(enif_free) +# define enif_is_atom ERL_NIF_API_FUNC_MACRO(enif_is_atom) +# define enif_is_binary ERL_NIF_API_FUNC_MACRO(enif_is_binary) +# define enif_is_ref ERL_NIF_API_FUNC_MACRO(enif_is_ref) +# define enif_inspect_binary ERL_NIF_API_FUNC_MACRO(enif_inspect_binary) +# define enif_alloc_binary ERL_NIF_API_FUNC_MACRO(enif_alloc_binary) +# define enif_realloc_binary ERL_NIF_API_FUNC_MACRO(enif_realloc_binary) +# define enif_release_binary ERL_NIF_API_FUNC_MACRO(enif_release_binary) +# define enif_get_int ERL_NIF_API_FUNC_MACRO(enif_get_int) +# define enif_get_ulong ERL_NIF_API_FUNC_MACRO(enif_get_ulong) +# define enif_get_double ERL_NIF_API_FUNC_MACRO(enif_get_double) +# define enif_get_tuple ERL_NIF_API_FUNC_MACRO(enif_get_tuple) +# define enif_get_list_cell ERL_NIF_API_FUNC_MACRO(enif_get_list_cell) +# define enif_is_identical ERL_NIF_API_FUNC_MACRO(enif_is_identical) +# define enif_compare ERL_NIF_API_FUNC_MACRO(enif_compare) + +# define enif_make_binary ERL_NIF_API_FUNC_MACRO(enif_make_binary) +# define enif_make_badarg ERL_NIF_API_FUNC_MACRO(enif_make_badarg) +# define enif_make_int ERL_NIF_API_FUNC_MACRO(enif_make_int) +# define enif_make_ulong ERL_NIF_API_FUNC_MACRO(enif_make_ulong) +# define enif_make_double ERL_NIF_API_FUNC_MACRO(enif_make_double) +# define enif_make_atom ERL_NIF_API_FUNC_MACRO(enif_make_atom) +# define enif_make_existing_atom ERL_NIF_API_FUNC_MACRO(enif_make_existing_atom) +# define enif_make_tuple ERL_NIF_API_FUNC_MACRO(enif_make_tuple) +# define enif_make_list ERL_NIF_API_FUNC_MACRO(enif_make_list) +# define enif_make_list_cell ERL_NIF_API_FUNC_MACRO(enif_make_list_cell) +# define enif_make_string ERL_NIF_API_FUNC_MACRO(enif_make_string) +# define enif_make_ref ERL_NIF_API_FUNC_MACRO(enif_make_ref) + +# define enif_mutex_create ERL_NIF_API_FUNC_MACRO(enif_mutex_create) +# define enif_mutex_destroy ERL_NIF_API_FUNC_MACRO(enif_mutex_destroy) +# define enif_mutex_trylock ERL_NIF_API_FUNC_MACRO(enif_mutex_trylock) +# define enif_mutex_lock ERL_NIF_API_FUNC_MACRO(enif_mutex_lock) +# define enif_mutex_unlock ERL_NIF_API_FUNC_MACRO(enif_mutex_unlock) +# define enif_cond_create ERL_NIF_API_FUNC_MACRO(enif_cond_create) +# define enif_cond_destroy ERL_NIF_API_FUNC_MACRO(enif_cond_destroy) +# define enif_cond_signal ERL_NIF_API_FUNC_MACRO(enif_cond_signal) +# define enif_cond_broadcast ERL_NIF_API_FUNC_MACRO(enif_cond_broadcast) +# define enif_cond_wait ERL_NIF_API_FUNC_MACRO(enif_cond_wait) +# define enif_rwlock_create ERL_NIF_API_FUNC_MACRO(enif_rwlock_create) +# define enif_rwlock_destroy ERL_NIF_API_FUNC_MACRO(enif_rwlock_destroy) +# define enif_rwlock_tryrlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_tryrlock) +# define enif_rwlock_rlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_rlock) +# define enif_rwlock_runlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_runlock) +# define enif_rwlock_tryrwlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_tryrwlock) +# define enif_rwlock_rwlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_rwlock) +# define enif_rwlock_rwunlock ERL_NIF_API_FUNC_MACRO(enif_rwlock_rwunlock) +# define enif_tsd_key_create ERL_NIF_API_FUNC_MACRO(enif_tsd_key_create) +# define enif_tsd_key_destroy ERL_NIF_API_FUNC_MACRO(enif_tsd_key_destroy) +# define enif_tsd_set ERL_NIF_API_FUNC_MACRO(enif_tsd_set) +# define enif_tsd_get ERL_NIF_API_FUNC_MACRO(enif_tsd_get) +# define enif_thread_opts_create ERL_NIF_API_FUNC_MACRO(enif_thread_opts_create) +# define enif_thread_opts_destroy ERL_NIF_API_FUNC_MACRO(enif_thread_opts_destroy) +# define enif_thread_create ERL_NIF_API_FUNC_MACRO(enif_thread_create) +# define enif_thread_self ERL_NIF_API_FUNC_MACRO(enif_thread_self) +# define enif_equal_tids ERL_NIF_API_FUNC_MACRO(enif_equal_tids) +# define enif_thread_exit ERL_NIF_API_FUNC_MACRO(enif_thread_exit) +# define enif_thread_join ERL_NIF_API_FUNC_MACRO(enif_thread_join) + +# define enif_realloc ERL_NIF_API_FUNC_MACRO(enif_realloc) +# define enif_system_info ERL_NIF_API_FUNC_MACRO(enif_system_info) +# define enif_fprintf ERL_NIF_API_FUNC_MACRO(enif_fprintf) +# define enif_inspect_iolist_as_binary ERL_NIF_API_FUNC_MACRO(enif_inspect_iolist_as_binary) +# define enif_make_sub_binary ERL_NIF_API_FUNC_MACRO(enif_make_sub_binary) +# define enif_get_string ERL_NIF_API_FUNC_MACRO(enif_get_string) +# define enif_get_atom ERL_NIF_API_FUNC_MACRO(enif_get_atom) +# define enif_is_fun ERL_NIF_API_FUNC_MACRO(enif_is_fun) +# define enif_is_pid ERL_NIF_API_FUNC_MACRO(enif_is_pid) +# define enif_is_port ERL_NIF_API_FUNC_MACRO(enif_is_port) +# define enif_get_uint ERL_NIF_API_FUNC_MACRO(enif_get_uint) +# define enif_get_long ERL_NIF_API_FUNC_MACRO(enif_get_long) +# define enif_make_uint ERL_NIF_API_FUNC_MACRO(enif_make_uint) +# define enif_make_long ERL_NIF_API_FUNC_MACRO(enif_make_long) +# define enif_make_tuple_from_array ERL_NIF_API_FUNC_MACRO(enif_make_tuple_from_array) +# define enif_make_list_from_array ERL_NIF_API_FUNC_MACRO(enif_make_list_from_array) +# define enif_is_empty_list ERL_NIF_API_FUNC_MACRO(enif_is_empty_list) +# define enif_open_resource_type ERL_NIF_API_FUNC_MACRO(enif_open_resource_type) +# define enif_alloc_resource ERL_NIF_API_FUNC_MACRO(enif_alloc_resource) +# define enif_release_resource ERL_NIF_API_FUNC_MACRO(enif_release_resource) +# define enif_make_resource ERL_NIF_API_FUNC_MACRO(enif_make_resource) +# define enif_get_resource ERL_NIF_API_FUNC_MACRO(enif_get_resource) +# define enif_sizeof_resource ERL_NIF_API_FUNC_MACRO(enif_sizeof_resource) +# define enif_make_new_binary ERL_NIF_API_FUNC_MACRO(enif_make_new_binary) +# define enif_is_list ERL_NIF_API_FUNC_MACRO(enif_is_list) +# define enif_is_tuple ERL_NIF_API_FUNC_MACRO(enif_is_tuple) +# define enif_get_atom_length ERL_NIF_API_FUNC_MACRO(enif_get_atom_length) +# define enif_get_list_length ERL_NIF_API_FUNC_MACRO(enif_get_list_length) +# define enif_make_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_atom_len) +# define enif_make_existing_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_existing_atom_len) +# define enif_make_string_len ERL_NIF_API_FUNC_MACRO(enif_make_string_len) +# define enif_alloc_env ERL_NIF_API_FUNC_MACRO(enif_alloc_env) +# define enif_free_env ERL_NIF_API_FUNC_MACRO(enif_free_env) +# define enif_clear_env ERL_NIF_API_FUNC_MACRO(enif_clear_env) +# define enif_send ERL_NIF_API_FUNC_MACRO(enif_send) +# define enif_make_copy ERL_NIF_API_FUNC_MACRO(enif_make_copy) +# define enif_self ERL_NIF_API_FUNC_MACRO(enif_self) +# define enif_get_local_pid ERL_NIF_API_FUNC_MACRO(enif_get_local_pid) +# define enif_keep_resource ERL_NIF_API_FUNC_MACRO(enif_keep_resource) +# define enif_make_resource_binary ERL_NIF_API_FUNC_MACRO(enif_make_resource_binary) +#if SIZEOF_LONG != 8 +# define enif_get_int64 ERL_NIF_API_FUNC_MACRO(enif_get_int64) +# define enif_get_uint64 ERL_NIF_API_FUNC_MACRO(enif_get_uint64) +# define enif_make_int64 ERL_NIF_API_FUNC_MACRO(enif_make_int64) +# define enif_make_uint64 ERL_NIF_API_FUNC_MACRO(enif_make_uint64) +#endif + +# define enif_is_exception ERL_NIF_API_FUNC_MACRO(enif_is_exception) +# define enif_make_reverse_list ERL_NIF_API_FUNC_MACRO(enif_make_reverse_list) +# define enif_is_number ERL_NIF_API_FUNC_MACRO(enif_is_number) +# define enif_dlopen ERL_NIF_API_FUNC_MACRO(enif_dlopen) +# define enif_dlsym ERL_NIF_API_FUNC_MACRO(enif_dlsym) +# define enif_consume_timeslice ERL_NIF_API_FUNC_MACRO(enif_consume_timeslice) + +/* +** Add new entries here +*/ +#endif + + +#if defined(__GNUC__) && !(defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) + +/* Inline functions for compile time type checking of arguments to + variadic functions. +*/ + +# define ERL_NIF_INLINE __inline__ + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple1(ErlNifEnv* env, + ERL_NIF_TERM e1) +{ + return enif_make_tuple(env, 1, e1); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple2(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2) +{ + return enif_make_tuple(env, 2, e1, e2); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple3(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3) +{ + return enif_make_tuple(env, 3, e1, e2, e3); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple4(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4) +{ + return enif_make_tuple(env, 4, e1, e2, e3, e4); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple5(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5) +{ + return enif_make_tuple(env, 5, e1, e2, e3, e4, e5); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple6(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6) +{ + return enif_make_tuple(env, 6, e1, e2, e3, e4, e5, e6); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple7(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7) +{ + return enif_make_tuple(env, 7, e1, e2, e3, e4, e5, e6, e7); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple8(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7, + ERL_NIF_TERM e8) +{ + return enif_make_tuple(env, 8, e1, e2, e3, e4, e5, e6, e7, e8); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_tuple9(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7, + ERL_NIF_TERM e8, + ERL_NIF_TERM e9) +{ + return enif_make_tuple(env, 9, e1, e2, e3, e4, e5, e6, e7, e8, e9); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list1(ErlNifEnv* env, + ERL_NIF_TERM e1) +{ + return enif_make_list(env, 1, e1); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list2(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2) +{ + return enif_make_list(env, 2, e1, e2); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list3(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3) +{ + return enif_make_list(env, 3, e1, e2, e3); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list4(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4) +{ + return enif_make_list(env, 4, e1, e2, e3, e4); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list5(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5) +{ + return enif_make_list(env, 5, e1, e2, e3, e4, e5); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list6(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6) +{ + return enif_make_list(env, 6, e1, e2, e3, e4, e5, e6); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list7(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7) +{ + return enif_make_list(env, 7, e1, e2, e3, e4, e5, e6, e7); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list8(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7, + ERL_NIF_TERM e8) +{ + return enif_make_list(env, 8, e1, e2, e3, e4, e5, e6, e7, e8); +} + +static ERL_NIF_INLINE ERL_NIF_TERM enif_make_list9(ErlNifEnv* env, + ERL_NIF_TERM e1, + ERL_NIF_TERM e2, + ERL_NIF_TERM e3, + ERL_NIF_TERM e4, + ERL_NIF_TERM e5, + ERL_NIF_TERM e6, + ERL_NIF_TERM e7, + ERL_NIF_TERM e8, + ERL_NIF_TERM e9) +{ + return enif_make_list(env, 9, e1, e2, e3, e4, e5, e6, e7, e8, e9); +} + +# undef ERL_NIF_INLINE + +#else /* fallback with macros */ + +#ifndef enif_make_list1 +# define enif_make_list1(ENV,E1) enif_make_list(ENV,1,E1) +# define enif_make_list2(ENV,E1,E2) enif_make_list(ENV,2,E1,E2) +# define enif_make_list3(ENV,E1,E2,E3) enif_make_list(ENV,3,E1,E2,E3) +# define enif_make_list4(ENV,E1,E2,E3,E4) enif_make_list(ENV,4,E1,E2,E3,E4) +# define enif_make_list5(ENV,E1,E2,E3,E4,E5) enif_make_list(ENV,5,E1,E2,E3,E4,E5) +# define enif_make_list6(ENV,E1,E2,E3,E4,E5,E6) enif_make_list(ENV,6,E1,E2,E3,E4,E5,E6) +# define enif_make_list7(ENV,E1,E2,E3,E4,E5,E6,E7) enif_make_list(ENV,7,E1,E2,E3,E4,E5,E6,E7) +# define enif_make_list8(ENV,E1,E2,E3,E4,E5,E6,E7,E8) enif_make_list(ENV,8,E1,E2,E3,E4,E5,E6,E7,E8) +# define enif_make_list9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_list(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9) +# define enif_make_tuple1(ENV,E1) enif_make_tuple(ENV,1,E1) +# define enif_make_tuple2(ENV,E1,E2) enif_make_tuple(ENV,2,E1,E2) +# define enif_make_tuple3(ENV,E1,E2,E3) enif_make_tuple(ENV,3,E1,E2,E3) +# define enif_make_tuple4(ENV,E1,E2,E3,E4) enif_make_tuple(ENV,4,E1,E2,E3,E4) +# define enif_make_tuple5(ENV,E1,E2,E3,E4,E5) enif_make_tuple(ENV,5,E1,E2,E3,E4,E5) +# define enif_make_tuple6(ENV,E1,E2,E3,E4,E5,E6) enif_make_tuple(ENV,6,E1,E2,E3,E4,E5,E6) +# define enif_make_tuple7(ENV,E1,E2,E3,E4,E5,E6,E7) enif_make_tuple(ENV,7,E1,E2,E3,E4,E5,E6,E7) +# define enif_make_tuple8(ENV,E1,E2,E3,E4,E5,E6,E7,E8) enif_make_tuple(ENV,8,E1,E2,E3,E4,E5,E6,E7,E8) +# define enif_make_tuple9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_tuple(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9) +#endif + +#endif /* __GNUC__ && !WIN32 */ + +#ifndef enif_make_pid + +# define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid)) + +#if SIZEOF_LONG == 8 +# define enif_get_int64 enif_get_long +# define enif_get_uint64 enif_get_ulong +# define enif_make_int64 enif_make_long +# define enif_make_uint64 enif_make_ulong +#endif + +#endif + diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.1.2_0.c b/erts/emulator/test/nif_SUITE_data/nif_mod.1.2_0.c new file mode 100644 index 0000000000..a554cc7f25 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.1.2_0.c @@ -0,0 +1,4 @@ +#include "nif_api_2_0/erl_nif.h" + +#define NIF_LIB_VER 1 +#include "nif_mod.c" diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.1.2_4.c b/erts/emulator/test/nif_SUITE_data/nif_mod.1.2_4.c new file mode 100644 index 0000000000..6d28dbb8ba --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.1.2_4.c @@ -0,0 +1,4 @@ +#include "nif_api_2_4/erl_nif.h" + +#define NIF_LIB_VER 1 +#include "nif_mod.c" diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.2.2_0.c b/erts/emulator/test/nif_SUITE_data/nif_mod.2.2_0.c new file mode 100644 index 0000000000..0731e6b5d0 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.2.2_0.c @@ -0,0 +1,4 @@ +#include "nif_api_2_0/erl_nif.h" + +#define NIF_LIB_VER 2 +#include "nif_mod.c" diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.2.2_4.c b/erts/emulator/test/nif_SUITE_data/nif_mod.2.2_4.c new file mode 100644 index 0000000000..628fd42b52 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.2.2_4.c @@ -0,0 +1,4 @@ +#include "nif_api_2_4/erl_nif.h" + +#define NIF_LIB_VER 2 +#include "nif_mod.c" diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.3.2_0.c b/erts/emulator/test/nif_SUITE_data/nif_mod.3.2_0.c new file mode 100644 index 0000000000..d7e676b668 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.3.2_0.c @@ -0,0 +1,4 @@ +#include "nif_api_2_0/erl_nif.h" + +#define NIF_LIB_VER 3 +#include "nif_mod.c" diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.3.2_4.c b/erts/emulator/test/nif_SUITE_data/nif_mod.3.2_4.c new file mode 100644 index 0000000000..bdbe8cf381 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.3.2_4.c @@ -0,0 +1,4 @@ +#include "nif_api_2_4/erl_nif.h" + +#define NIF_LIB_VER 3 +#include "nif_mod.c" diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.c b/erts/emulator/test/nif_SUITE_data/nif_mod.c index fd8a0d0595..e6106b6036 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_mod.c +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.c @@ -17,7 +17,7 @@ * * %CopyrightEnd% */ -#include "erl_nif.h" +#include <erl_nif.h> #include <string.h> #include <stdio.h> @@ -237,10 +237,22 @@ static ERL_NIF_TERM lib_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg return enif_make_int(env, NIF_LIB_VER); } +static ERL_NIF_TERM nif_api_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + /*ADD_CALL("nif_api_version");*/ + return enif_make_tuple2(env, + enif_make_int(env, ERL_NIF_MAJOR_VERSION), + enif_make_int(env, ERL_NIF_MINOR_VERSION)); +} + static ERL_NIF_TERM get_priv_data_ptr(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + NifModPrivData** bin_data; + ERL_NIF_TERM res; ADD_CALL("get_priv_data_ptr"); - return enif_make_uint64(env, (ErlNifUInt64)priv_data(env)); + bin_data = (NifModPrivData**)enif_make_new_binary(env, sizeof(void*), &res); + *bin_data = priv_data(env); + return res; } static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) @@ -279,6 +291,7 @@ static ERL_NIF_TERM get_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar static ErlNifFunc nif_funcs[] = { {"lib_version", 0, lib_version}, + {"nif_api_version", 0, nif_api_version}, {"get_priv_data_ptr", 0, get_priv_data_ptr}, {"make_new_resource", 2, make_new_resource}, {"get_resource", 2, get_resource} diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.erl b/erts/emulator/test/nif_SUITE_data/nif_mod.erl index 1fcc33faa4..8019cfcf82 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_mod.erl +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.erl @@ -22,7 +22,7 @@ -include_lib("common_test/include/ct.hrl"). --export([load_nif_lib/2, load_nif_lib/3, start/0, lib_version/0, call_history/0, +-export([load_nif_lib/2, load_nif_lib/3, start/0, lib_version/0, get_priv_data_ptr/0, make_new_resource/2, get_resource/2]). -export([loop/0, upgrade/1]). @@ -35,21 +35,32 @@ on_load() -> [{data_dir, Path}] = ets:lookup(nif_SUITE, data_dir), [{lib_version, Ver}] = ets:lookup(nif_SUITE, lib_version), - erlang:load_nif(filename:join(Path,libname(Ver)), []). + [{nif_api_version, API}] = ets:lookup(nif_SUITE, nif_api_version), + R = erlang:load_nif(filename:join(Path,libname(Ver,API)), []), + check_api_version(R, API). -endif. +check_api_version(Err, _) when Err =/= ok -> Err; +check_api_version(ok, []) -> ok; +check_api_version(ok, [$., MajC, $_ | MinS]) -> + {Maj, Min} = {list_to_integer([MajC]), list_to_integer(MinS)}, + {Maj, Min} = nif_api_version(), + ok. + load_nif_lib(Config, Ver) -> load_nif_lib(Config, Ver, []). load_nif_lib(Config, Ver, LoadInfo) -> Path = proplists:get_value(data_dir, Config), - erlang:load_nif(filename:join(Path,libname(Ver)), LoadInfo). - -libname(no_init) -> libname(3); -libname(Ver) when is_integer(Ver) -> - "nif_mod." ++ integer_to_list(Ver). + API = proplists:get_value(nif_api_version, Config, ""), + R = erlang:load_nif(filename:join(Path,libname(Ver,API)), LoadInfo), + check_api_version(R, API). +libname(no_init,API) -> libname(3,API); +libname(Ver,API) when is_integer(Ver) -> + "nif_mod." ++ integer_to_list(Ver) ++ API. + start() -> spawn_opt(?MODULE,loop,[], [link, monitor]). @@ -72,7 +83,9 @@ upgrade(Pid) -> lib_version() -> % NIF undefined. -call_history() -> ?nif_stub. +nif_api_version() -> %NIF + {undefined,undefined}. + get_priv_data_ptr() -> ?nif_stub. make_new_resource(_,_) -> ?nif_stub. get_resource(_,_) -> ?nif_stub. diff --git a/erts/emulator/test/nif_SUITE_data/testcase_driver.h b/erts/emulator/test/nif_SUITE_data/testcase_driver.h index e32e63069a..feb10ecaea 100644 --- a/erts/emulator/test/nif_SUITE_data/testcase_driver.h +++ b/erts/emulator/test/nif_SUITE_data/testcase_driver.h @@ -20,7 +20,7 @@ #ifndef TESTCASE_DRIVER_H__ #define TESTCASE_DRIVER_H__ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdlib.h> #include <stdio.h> diff --git a/erts/emulator/test/nif_SUITE_data/tester.c b/erts/emulator/test/nif_SUITE_data/tester.c index 257b116322..ea4afd924d 100644 --- a/erts/emulator/test/nif_SUITE_data/tester.c +++ b/erts/emulator/test/nif_SUITE_data/tester.c @@ -1,4 +1,4 @@ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdio.h> #include <stdarg.h> @@ -53,7 +53,7 @@ void testcase_free(void *ptr) void testcase_run(TestCaseState_t *tcs); -static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) { return 0; } @@ -70,5 +70,5 @@ static ErlNifFunc nif_funcs[] = {"run", 0, run} }; -ERL_NIF_INIT(tester,nif_funcs,NULL,reload,NULL,NULL) +ERL_NIF_INIT(tester,nif_funcs,NULL,NULL,upgrade,NULL) diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl index d1c9648017..1a1ab0e5e0 100644 --- a/erts/emulator/test/num_bif_SUITE.erl +++ b/erts/emulator/test/num_bif_SUITE.erl @@ -32,6 +32,8 @@ %% list_to_integer/1 %% round/1 %% trunc/1 +%% floor/1 +%% ceil/1 %% integer_to_binary/1 %% integer_to_binary/2 %% binary_to_integer/1 @@ -41,7 +43,7 @@ t_float_to_string/1, t_integer_to_string/1, t_string_to_integer/1, t_list_to_integer_edge_cases/1, t_string_to_float_safe/1, t_string_to_float_risky/1, - t_round/1, t_trunc/1 + t_round/1, t_trunc_and_friends/1 ]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -49,7 +51,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [t_abs, t_float, t_float_to_string, t_integer_to_string, {group, t_string_to_float}, t_string_to_integer, t_round, - t_trunc, t_list_to_integer_edge_cases]. + t_trunc_and_friends, t_list_to_integer_edge_cases]. groups() -> [{t_string_to_float, [], @@ -295,30 +297,78 @@ t_round(Config) when is_list(Config) -> -4294967297 = -round(id(4294967296.9)), ok. -t_trunc(Config) when is_list(Config) -> - 0 = trunc(id(0.0)), - 5 = trunc(id(5.3333)), - -10 = trunc(id(-10.978987)), +%% Test trunc/1, floor/1, ceil/1, and round/1. +t_trunc_and_friends(_Config) -> + MinusZero = 0.0 / (-1.0), + 0 = trunc_and_friends(MinusZero), + 0 = trunc_and_friends(0.0), + 5 = trunc_and_friends(5.3333), + -10 = trunc_and_friends(-10.978987), - % The largest smallnum, converted to float (OTP-3722): + %% The largest smallnum, converted to float (OTP-3722): X = id((1 bsl 27) - 1), - F = id(X + 0.0), + F = X + 0.0, io:format("X = ~p/~w/~w, F = ~p/~w/~w, trunc(F) = ~p/~w/~w~n", [X, X, binary_to_list(term_to_binary(X)), F, F, binary_to_list(term_to_binary(F)), - trunc(F), trunc(F), binary_to_list(term_to_binary(trunc(F)))]), - X = trunc(F), - X = trunc(F+1)-1, - X = trunc(F-1)+1, - X = -trunc(-F), - X = -trunc(-F-1)-1, - X = -trunc(-F+1)+1, + trunc_and_friends(F), + trunc_and_friends(F), + binary_to_list(term_to_binary(trunc_and_friends(F)))]), + X = trunc_and_friends(F), + X = trunc_and_friends(F+1)-1, + X = trunc_and_friends(F-1)+1, + X = -trunc_and_friends(-F), + X = -trunc_and_friends(-F-1)-1, + X = -trunc_and_friends(-F+1)+1, %% Bignums. - 4294967305 = trunc(id(4294967305.7)), - -4294967305 = trunc(id(-4294967305.7)), + 4294967305 = trunc_and_friends(4294967305.7), + -4294967305 = trunc_and_friends(-4294967305.7), + 18446744073709551616 = trunc_and_friends(float(1 bsl 64)), + -18446744073709551616 = trunc_and_friends(-float(1 bsl 64)), + + %% Random. + t_trunc_and_friends_rand(100), ok. +t_trunc_and_friends_rand(0) -> + ok; +t_trunc_and_friends_rand(N) -> + F0 = rand:uniform() * math:pow(10, 50*rand:normal()), + F = case rand:uniform() of + U when U < 0.5 -> -F0; + _ -> F0 + end, + _ = trunc_and_friends(F), + t_trunc_and_friends_rand(N-1). + +trunc_and_friends(F) -> + Trunc = trunc(F), + Floor = floor(F), + Ceil = ceil(F), + Round = round(F), + + Trunc = trunc(Trunc), + Floor = floor(Floor), + Ceil = ceil(Ceil), + Round = round(Round), + + Trunc = trunc(float(Trunc)), + Floor = floor(float(Floor)), + Ceil = ceil(float(Ceil)), + Round = round(float(Round)), + + true = Floor =< Trunc andalso Trunc =< Ceil, + true = Ceil - Floor =< 1, + true = Round =:= Floor orelse Round =:= Ceil, + + if + F < 0 -> + Trunc = Ceil; + true -> + Trunc = Floor + end, + Trunc. %% Tests integer_to_binary/1. diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 0a6eb7ffac..5712c9fa74 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -2449,7 +2449,7 @@ no_priority_inversion2(Config) when is_list(Config) -> request_gc(Pid, Prio) -> Ref = make_ref(), - erts_internal:request_system_task(Pid, Prio, {garbage_collect, Ref}), + erts_internal:request_system_task(Pid, Prio, {garbage_collect, Ref, major}), Ref. system_task_blast(Config) when is_list(Config) -> diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl index a4aedb31f6..6a772bf7c9 100644 --- a/erts/emulator/test/system_info_SUITE.erl +++ b/erts/emulator/test/system_info_SUITE.erl @@ -36,7 +36,8 @@ -export([all/0, suite/0]). -export([process_count/1, system_version/1, misc_smoke_tests/1, - heap_size/1, wordsize/1, memory/1, ets_limit/1]). + heap_size/1, wordsize/1, memory/1, ets_limit/1, atom_limit/1, + atom_count/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -44,7 +45,7 @@ suite() -> all() -> [process_count, system_version, misc_smoke_tests, - heap_size, wordsize, memory, ets_limit]. + heap_size, wordsize, memory, ets_limit, atom_limit, atom_count]. %%% %%% The test cases ------------------------------------------------------------- @@ -472,6 +473,17 @@ mapn(_Fun, 0) -> mapn(Fun, N) -> [Fun(N) | mapn(Fun, N-1)]. + +get_node_name(Config) -> + list_to_atom(atom_to_list(?MODULE) + ++ "-" + ++ atom_to_list(proplists:get_value(testcase, Config)) + ++ "-" + ++ integer_to_list(erlang:system_time(second)) + ++ "-" + ++ integer_to_list(erlang:unique_integer([positive]))). + + %% Verify system_info(ets_limit) reflects max ETS table settings. ets_limit(Config0) when is_list(Config0) -> Config = [{testcase,ets_limit}|Config0], @@ -486,7 +498,7 @@ get_ets_limit(Config, EtsMax) -> 0 -> []; _ -> [{"ERL_MAX_ETS_TABLES", integer_to_list(EtsMax)}] end, - {ok, Node} = start_node(Config, Envs), + {ok, Node} = start_node_ets(Config, Envs), Me = self(), Ref = make_ref(), spawn_link(Node, @@ -502,16 +514,50 @@ get_ets_limit(Config, EtsMax) -> stop_node(Node), Res. -start_node(Config, Envs) when is_list(Config) -> +start_node_ets(Config, Envs) when is_list(Config) -> + Pa = filename:dirname(code:which(?MODULE)), + test_server:start_node(get_node_name(Config), peer, + [{args, "-pa "++Pa}, {env, Envs}]). + +start_node_atm(Config, AtomsMax) when is_list(Config) -> Pa = filename:dirname(code:which(?MODULE)), - Name = list_to_atom(atom_to_list(?MODULE) - ++ "-" - ++ atom_to_list(proplists:get_value(testcase, Config)) - ++ "-" - ++ integer_to_list(erlang:system_time(second)) - ++ "-" - ++ integer_to_list(erlang:unique_integer([positive]))), - test_server:start_node(Name, peer, [{args, "-pa "++Pa}, {env, Envs}]). + test_server:start_node(get_node_name(Config), peer, + [{args, "-pa "++ Pa ++ AtomsMax}]). stop_node(Node) -> test_server:stop_node(Node). + + +%% Verify system_info(atom_limit) reflects max atoms settings +%% (using " +t"). +atom_limit(Config0) when is_list(Config0) -> + Config = [{testcase,atom_limit}|Config0], + 2186042 = get_atom_limit(Config, " +t 2186042 "), + ok. + +get_atom_limit(Config, AtomsMax) -> + {ok, Node} = start_node_atm(Config, AtomsMax), + Me = self(), + Ref = make_ref(), + spawn_link(Node, + fun() -> + Res = erlang:system_info(atom_limit), + unlink(Me), + Me ! {Ref, Res} + end), + receive + {Ref, Res} -> + Res + end, + stop_node(Node), + Res. + +%% Verify that system_info(atom_count) works. +atom_count(Config) when is_list(Config) -> + Limit = erlang:system_info(atom_limit), + Count1 = erlang:system_info(atom_count), + list_to_atom(integer_to_list(erlang:unique_integer())), + Count2 = erlang:system_info(atom_count), + true = Limit >= Count2, + true = Count2 > Count1, + ok. diff --git a/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c index 33b346aab7..786be35c9c 100644 --- a/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c +++ b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c @@ -1,4 +1,4 @@ -#include "erl_nif.h" +#include <erl_nif.h> static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) @@ -6,11 +6,6 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) return 0; } -static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) -{ - return 0; -} - static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) { return 0; @@ -34,4 +29,4 @@ static ErlNifFunc nif_funcs[] = {"nif_dec", 1, nif_dec_1} }; -ERL_NIF_INIT(trace_call_time_SUITE,nif_funcs,load,reload,upgrade,unload) +ERL_NIF_INIT(trace_call_time_SUITE,nif_funcs,load,NULL,upgrade,unload) diff --git a/erts/emulator/test/trace_nif_SUITE.erl b/erts/emulator/test/trace_nif_SUITE.erl index 8d5bff2a48..7ac6fce234 100644 --- a/erts/emulator/test/trace_nif_SUITE.erl +++ b/erts/emulator/test/trace_nif_SUITE.erl @@ -265,10 +265,16 @@ nif_process() -> nif_process(). load_nif(Config) -> - Path = proplists:get_value(data_dir, Config), - - ok = erlang:load_nif(filename:join(Path,"trace_nif"), 0). + case is_nif_loaded() of + true -> + ok; + false -> + Path = proplists:get_value(data_dir, Config), + ok = erlang:load_nif(filename:join(Path,"trace_nif"), 0) + end. +is_nif_loaded() -> + false. nif() -> {"Stub0",[]}. %exit("nif/0 stub called"). diff --git a/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c b/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c index 26f2420b8b..1afb5ee919 100644 --- a/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c +++ b/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c @@ -1,4 +1,4 @@ -#include "erl_nif.h" +#include <erl_nif.h> static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) @@ -6,18 +6,18 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) return 0; } -static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) { return 0; } -static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) +static void unload(ErlNifEnv* env, void* priv_data) { - return 0; } -static void unload(ErlNifEnv* env, void* priv_data) +static ERL_NIF_TERM is_nif_loaded(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + return enif_make_atom(env,"true"); } static ERL_NIF_TERM nif_0(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) @@ -38,9 +38,10 @@ static ERL_NIF_TERM nif_1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) static ErlNifFunc nif_funcs[] = { + {"is_nif_loaded", 0, is_nif_loaded}, {"nif", 0, nif_0}, {"nif", 1, nif_1} }; -ERL_NIF_INIT(trace_nif_SUITE,nif_funcs,load,reload,upgrade,unload) +ERL_NIF_INIT(trace_nif_SUITE,nif_funcs,load,NULL,upgrade,unload) diff --git a/erts/emulator/test/tracer_SUITE_data/tracer_test.c b/erts/emulator/test/tracer_SUITE_data/tracer_test.c index d9543b7ab9..1555a95d9a 100644 --- a/erts/emulator/test/tracer_SUITE_data/tracer_test.c +++ b/erts/emulator/test/tracer_SUITE_data/tracer_test.c @@ -18,7 +18,7 @@ * %CopyrightEnd% */ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdio.h> #include <string.h> diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index 4407f7e289..9813142585 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -475,7 +475,7 @@ sub emulator_output { print '#include "beam_load.h"', "\n"; print "\n"; - print "char tag_to_letter[] = {\n "; + print "const char tag_to_letter[] = {\n "; for ($i = 0; $i < length($genop_types); $i++) { print "'$tag_type[$i]', "; } @@ -489,7 +489,7 @@ sub emulator_output { # Generate code for specific ops. # my($spec_opnum) = 0; - print "OpEntry opc[] = {\n"; + print "const OpEntry opc[] = {\n"; foreach $key (sort keys %specific_op) { $gen_to_spec{$key} = $spec_opnum; $num_specific{$key} = @{$specific_op{$key}}; @@ -566,13 +566,22 @@ sub emulator_output { $sep = ","; } $init .= "}"; - init_item($print_name, $init, $involves_r, $size, $pack, $sign, 0); + init_item($print_name, $init, $involves_r, $size, $pack, $sign); $op_to_name[$spec_opnum] = $instr; $spec_opnum++; } } print "};\n\n"; - print "int num_instructions = $spec_opnum;\n\n"; + print "const int num_instructions = $spec_opnum;\n\n"; + + # + # Print the array for instruction counts. + # + + print "#ifdef ERTS_OPCODE_COUNTER_SUPPORT\n"; + print "Uint erts_instr_count[$spec_opnum];\n"; + print "#endif\n"; + print "\n"; # # Generate transformations. @@ -584,7 +593,7 @@ sub emulator_output { # Print the generic instruction table. # - print "GenOpEntry gen_opc[] = {\n"; + print "const GenOpEntry gen_opc[] = {\n"; for ($i = 0; $i < @gen_opname; $i++) { if ($i == $num_file_opcodes) { print "\n/*\n * Internal generic instructions.\n */\n\n"; @@ -678,8 +687,8 @@ sub emulator_output { print "#define TE_MAX_VARS $te_max_vars\n"; print "\n"; - print "extern char tag_to_letter[];\n"; - print "extern Uint op_transform[];\n"; + print "extern const char tag_to_letter[];\n"; + print "extern const Uint op_transform[];\n"; print "\n"; for ($i = 0; $i < @op_to_name; $i++) { @@ -708,7 +717,7 @@ sub emulator_output { print "#define DEFINE_COUNTING_LABELS"; for ($i = 0; $i < @op_to_name; $i++) { my($name) = $op_to_name[$i]; - print " \\\nCountCase($name): opc[$i].count++; goto lb_$name;"; + print " \\\nCountCase($name): erts_instr_count[$i]++; goto lb_$name;"; } print "\n\n"; @@ -1417,7 +1426,7 @@ sub tr_gen { # Print the generated transformation engine. # my($offset) = 0; - print "Uint op_transform[] = {\n"; + print "const Uint op_transform[] = {\n"; foreach $key (sort keys %gen_transform) { $gen_transform_offset{$key} = $offset; my @instr = @{$gen_transform{$key}}; diff --git a/erts/emulator/utils/make_preload b/erts/emulator/utils/make_preload index f489bc2a39..8b629d9517 100755 --- a/erts/emulator/utils/make_preload +++ b/erts/emulator/utils/make_preload @@ -94,8 +94,8 @@ foreach $file (@ARGV) { close(FILE); push(@modules, " {\"$module\", " . length($_) . ", preloaded_$module},\n"); - print "unsigned preloaded_size_$module = ", length($_), ";\n"; - print "unsigned char preloaded_$module", "[] = {\n"; + print "const unsigned preloaded_size_$module = ", length($_), ";\n"; + print "const unsigned char preloaded_$module", "[] = {\n"; for ($i = 0; $i < length($_); $i++) { if ($i % 8 == 0 && $comment ne '') { $comment =~ s@/\*@..@g; # Comment start -- avoid warning. @@ -125,10 +125,10 @@ if ($gen_rc) { print @modules; print "END\n"; } elsif ($gen_old) { - print "struct {\n"; + print "const struct {\n"; print " char* name;\n"; print " int size;\n"; - print " unsigned char* code;\n"; + print " const unsigned char* code;\n"; print "} pre_loaded[] = {\n"; foreach (@modules) { print; diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables index c158778f43..27f9dcc878 100755 --- a/erts/emulator/utils/make_tables +++ b/erts/emulator/utils/make_tables @@ -36,7 +36,6 @@ use File::Basename; # <-src>/erl_am.c # <-src>/erl_bif_table.c # <-src>/erl_bif_wrap.c -# <-src>/erl_pbifs.c # <-include>/erl_atom_table.h # <-include>/erl_bif_table.h # @@ -54,8 +53,7 @@ my %aliases; my $auto_alias_num = 0; my @bif; -my @implementation; -my @pbif; +my @bif_type; while (@ARGV && $ARGV[0] =~ /^-(\w+)/) { my $opt = shift; @@ -79,8 +77,11 @@ while (<>) { my($type, @args) = split; if ($type eq 'atom') { save_atoms(@args); - } elsif ($type eq 'bif' or $type eq 'ubif') { - my($bif,$alias,$alias2) = (@args); + } elsif ($type eq 'bif' or $type eq 'ubif' or $type eq 'gcbif') { + if (@args > 2) { + error("$type only allows two arguments"); + } + my($bif,$alias) = (@args); $bif =~ m@^([a-z_.'0-9]+):(.*)/(\d)$@ or error("invalid BIF"); my($mod,$name,$arity) = ($1,$2,$3); save_atoms($mod, $name); @@ -90,12 +91,14 @@ while (<>) { $alias .= "${name}_$arity"; } my $wrapper; - $wrapper = "wrap_$alias" if $type eq 'bif'; - $wrapper = $alias if $type eq 'ubif'; + if ($type eq 'bif') { + $wrapper = "wrap_$alias"; + } else { + $wrapper = $alias; + } push(@bif, ["am_$atom_alias{$mod}","am_$atom_alias{$name}",$arity, $alias,$wrapper]); - push(@pbif, $bif =~ m/^'/ && $alias =~ m/^ebif_/); - push(@implementation, $alias2); + push(@bif_type, $type); } else { error("invalid line"); } @@ -166,8 +169,14 @@ typedef struct bif_entry { BifFunction traced; } BifEntry; +typedef struct erts_gc_bif { + BifFunction bif; + BifFunction gc_bif; +} ErtsGcBif; + extern BifEntry bif_table[]; extern Export* bif_export[]; +extern const ErtsGcBif erts_gc_bifs[]; #define BIF_SIZE $bif_size @@ -182,8 +191,12 @@ print "\n"; for ($i = 0; $i < @bif; $i++) { my $args = join(', ', 'Process*', 'Eterm*'); - print "Eterm $bif[$i]->[3]($args);\n"; - print "Eterm wrap_$bif[$i]->[3]($args, UWord *I);\n"; + my $name = $bif[$i]->[3]; + print "Eterm $name($args);\n"; + print "Eterm wrap_$name($args, UWord *I);\n"; + print "Eterm erts_gc_$name(Process* p, Eterm* reg, Uint live);\n" + if $bif_type[$i] eq 'gcbif'; + print "\n"; } print "#endif\n"; @@ -225,27 +238,26 @@ for ($i = 0; $i < @bif; $i++) { } # -# Generate the package bif file. +# Generate erl_gc_bifs.c. # -open_file("$src/erl_pbifs.c"); +open_file("$src/erl_gc_bifs.c"); my $i; includes("export.h", "sys.h", "erl_vm.h", "global.h", "erl_process.h", "bif.h", - "erl_bif_table.h", "erl_atom_table.h"); + "erl_bif_table.h"); +print "const ErtsGcBif erts_gc_bifs[] = {\n"; for ($i = 0; $i < @bif; $i++) { + next unless $bif_type[$i] eq 'gcbif'; my $arity = $bif[$i]->[2]; my $func = $bif[$i]->[3]; - my $arg; - next unless $pbif[$i]; - next unless $func =~ m/^ebif_(.*)/; - my $orig_func = $1; - $orig_func = $implementation[$i] if $implementation[$i]; - print "Eterm\n"; - print "$func(Process* p, Eterm* BIF__ARGS)\n"; - print "{\n"; - print " return $orig_func(p, BIF__ARGS);\n"; - print "}\n\n"; + print " {$func, erts_gc_$func},\n"; } +print " {0, 0}\n"; +print "};\n"; + +# +# Utilities follow. +# sub open_file { # or die my($name) = @_; |