diff options
Diffstat (limited to 'erts/emulator/beam')
41 files changed, 2070 insertions, 1399 deletions
diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index fbd0528009..ae60904785 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -21,10 +21,7 @@ #ifndef __ATOM_H__ #define __ATOM_H__ -#ifndef __INDEX_H__ #include "index.h" -#endif - #include "erl_atom_table.h" #define MAX_ATOM_CHARACTERS 255 diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index badd69856e..9dae67cb2d 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -65,6 +65,7 @@ atom undefined_lambda atom DOWN='DOWN' atom UP='UP' atom EXIT='EXIT' +atom abort atom aborted atom abs_path atom absoluteURI @@ -165,6 +166,7 @@ atom commandv atom compact atom compat_rel atom compile +atom complete atom compressed atom config_h atom convert_time_unit @@ -176,6 +178,7 @@ atom const atom context_switches atom control atom copy +atom copy_literals atom counters atom cpu atom cpu_timestamp @@ -195,6 +198,7 @@ atom dgroup_leader atom dictionary atom dirty_cpu atom dirty_cpu_schedulers_online +atom dirty_execution atom dirty_io atom disable_trace atom disabled @@ -235,6 +239,7 @@ atom erlang atom ERROR='ERROR' atom error_handler atom error_logger +atom erts_code_purger atom erts_internal atom ets atom ETS_TRANSFER='ETS-TRANSFER' @@ -384,8 +389,10 @@ atom merge_trap atom meta atom meta_match_spec atom micro_seconds +atom microsecond atom microstate_accounting atom milli_seconds +atom millisecond atom min_heap_size atom min_bin_vheap_size atom minor_version @@ -402,11 +409,13 @@ atom more atom multi_scheduling atom multiline atom nano_seconds +atom nanosecond atom name atom named_table atom namelist atom native atom native_addresses +atom need_gc atom Neq='=/=' atom Neqeq='/=' atom net_kernel @@ -485,6 +494,7 @@ atom pause atom pending atom pending_driver atom pending_process +atom pending_purge_lambda atom pending_reload atom permanent atom pid @@ -494,6 +504,7 @@ atom port_count atom port_limit atom port_op atom positive +atom prepare atom print atom priority atom private @@ -554,6 +565,7 @@ atom schedulers_online atom scheme atom scientific atom scope +atom second atom seconds atom send_to_non_existing_process atom sensitive diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 15e878ba65..dddcfbb77d 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -37,13 +37,90 @@ #include "erl_bits.h" #include "erl_thr_progress.h" +#ifdef HIPE +# include "hipe_stack.h" +#endif + +static struct { + Eterm module; + erts_smp_mtx_t mtx; + Export *pending_purge_lambda; + Eterm *sprocs; + Eterm def_sprocs[10]; + Uint sp_size; + Uint sp_ix; + ErlFunEntry **funs; + ErlFunEntry *def_funs[10]; + Uint fe_size; + Uint fe_ix; +} purge_state; + +Process *erts_code_purger = NULL; + +#ifdef ERTS_DIRTY_SCHEDULERS +Process *erts_dirty_process_code_checker; +#endif +erts_smp_atomic_t erts_copy_literal_area__; +#define ERTS_SET_COPY_LITERAL_AREA(LA) \ + erts_smp_atomic_set_nob(&erts_copy_literal_area__, \ + (erts_aint_t) (LA)) +#ifdef ERTS_NEW_PURGE_STRATEGY +Process *erts_literal_area_collector = NULL; + +typedef struct ErtsLiteralAreaRef_ ErtsLiteralAreaRef; +struct ErtsLiteralAreaRef_ { + ErtsLiteralAreaRef *next; + ErtsLiteralArea *literal_area; +}; + +struct { + erts_smp_mtx_t mtx; + ErtsLiteralAreaRef *first; + ErtsLiteralAreaRef *last; +} release_literal_areas; +#endif + static void set_default_trace_pattern(Eterm module); static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls); static void delete_code(Module* modp); -static void decrement_refc(BeamCodeHeader*); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); +static void +init_purge_state(void) +{ + purge_state.module = THE_NON_VALUE; + + erts_smp_mtx_init(&purge_state.mtx, "purge_state"); + + purge_state.pending_purge_lambda = + erts_export_put(am_erts_code_purger, am_pending_purge_lambda, 3); + + purge_state.sprocs = &purge_state.def_sprocs[0]; + purge_state.sp_size = sizeof(purge_state.def_sprocs); + purge_state.sp_size /= sizeof(purge_state.def_sprocs[0]); + purge_state.sp_ix = 0; + + purge_state.funs = &purge_state.def_funs[0]; + purge_state.fe_size = sizeof(purge_state.def_funs); + purge_state.fe_size /= sizeof(purge_state.def_funs[0]); + purge_state.fe_ix = 0; +} + +void +erts_beam_bif_load_init(void) +{ +#ifdef ERTS_NEW_PURGE_STRATEGY + erts_smp_mtx_init(&release_literal_areas.mtx, "release_literal_areas"); + release_literal_areas.first = NULL; + release_literal_areas.last = NULL; +#endif + erts_smp_atomic_init_nob(&erts_copy_literal_area__, + (erts_aint_t) NULL); + + init_purge_state(); +} + BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) { Module* modp; @@ -516,6 +593,43 @@ badarg: BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2) +{ +#if !defined(ERTS_DIRTY_SCHEDULERS) + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else + Process *rp; + int reds = 0; + Eterm res; + + if (BIF_P != erts_dirty_process_code_checker) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + if (is_not_internal_pid(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + if (is_not_atom(BIF_ARG_2)) + BIF_ERROR(BIF_P, BADARG); + + rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN, + BIF_ARG_1, ERTS_PROC_LOCK_MAIN); + if (rp == ERTS_PROC_LOCK_BUSY) + ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_check_dirty_process_code_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + if (!rp) + BIF_RET(am_false); + + res = erts_check_process_code(rp, BIF_ARG_2, 0, &reds, BIF_P->fcalls); + + if (BIF_P != rp) + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + + ASSERT(is_value(res)); + + BIF_RET2(res, reds); +#endif +} + BIF_RETTYPE delete_module_1(BIF_ALIST_1) { ErtsCodeIndex code_ix; @@ -748,6 +862,8 @@ set_default_trace_pattern(Eterm module) } } +#ifndef ERTS_NEW_PURGE_STRATEGY + static ERTS_INLINE int check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size) { @@ -762,12 +878,249 @@ check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size) return 0; } +#endif + static Uint hfrag_literal_size(Eterm* start, Eterm* end, char* lit_start, Uint lit_size); static void hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, Eterm *start, Eterm *end, char *lit_start, Uint lit_size); +#ifdef ERTS_NEW_PURGE_STRATEGY + +Eterm +erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed) +{ + ErtsLiteralArea *la; + ErtsMessage *msgp; + struct erl_off_heap_header* oh; + char *literals; + Uint lit_bsize; + ErlHeapFragment *hfrag; + + la = ERTS_COPY_LITERAL_AREA(); + if (!la) + return am_ok; + + oh = la->off_heap; + literals = (char *) &la->start[0]; + lit_bsize = (char *) la->end - literals; + + /* + * If a literal is in the message queue we make an explicit copy of + * it and attach it to the heap fragment. Each message needs to be + * self contained, we cannot save the literal in the old_heap or + * any other heap than the message it self. + */ + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + + for (msgp = c_p->msg.first; msgp; msgp = msgp->next) { + ErlHeapFragment *hf; + Uint lit_sz = 0; + + *redsp += 1; + + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) + hfrag = &msgp->hfrag; + else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag) + hfrag = msgp->data.heap_frag; + else + continue; /* Content on heap or in external term format... */ + + for (hf = hfrag; hf; hf = hf->next) { + lit_sz += hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size], + literals, lit_bsize); + *redsp += 1; + } + + *redsp += lit_sz / 16; /* Better value needed... */ + if (lit_sz > 0) { + ErlHeapFragment *bp = new_message_buffer(lit_sz); + Eterm *hp = bp->mem; + + for (hf = hfrag; hf; hf = hf->next) { + hfrag_literal_copy(&hp, &bp->off_heap, + &hf->mem[0], &hf->mem[hf->used_size], + literals, lit_bsize); + hfrag = hf; + } + + /* link new hfrag last */ + ASSERT(hfrag->next == NULL); + hfrag->next = bp; + bp->next = NULL; + } + } + + if (gc_allowed) { + /* + * Current implementation first tests without + * allowing GC, and then restarts the operation + * allowing GC if it is needed. It is therfore + * very likely that we will need the GC (although + * this is not completely certain). We go for + * the GC directly instead of scanning everything + * one more time... + */ + goto literal_gc; + } + + *redsp += 2; + if (any_heap_ref_ptrs(&c_p->fvalue, &c_p->fvalue+1, literals, lit_bsize)) { + c_p->freason = EXC_NULL; + c_p->fvalue = NIL; + c_p->ftrace = NIL; + } + + if (any_heap_ref_ptrs(c_p->stop, c_p->hend, literals, lit_bsize)) + goto literal_gc; + *redsp += 1; +#ifdef HIPE + if (nstack_any_heap_ref_ptrs(c_p, literals, lit_bsize)) + goto literal_gc; + *redsp += 1; +#endif + if (any_heap_refs(c_p->heap, c_p->htop, literals, lit_bsize)) + goto literal_gc; + *redsp += 1; + if (any_heap_refs(c_p->old_heap, c_p->old_htop, literals, lit_bsize)) + goto literal_gc; + + /* Check dictionary */ + *redsp += 1; + if (c_p->dictionary) { + Eterm* start = ERTS_PD_START(c_p->dictionary); + Eterm* end = start + ERTS_PD_SIZE(c_p->dictionary); + + if (any_heap_ref_ptrs(start, end, literals, lit_bsize)) + goto literal_gc; + } + + /* Check heap fragments */ + for (hfrag = c_p->mbuf; hfrag; hfrag = hfrag->next) { + Eterm *hp, *hp_end; + + *redsp += 1; + + hp = &hfrag->mem[0]; + hp_end = &hfrag->mem[hfrag->used_size]; + if (any_heap_refs(hp, hp_end, literals, lit_bsize)) + goto literal_gc; + } + + /* + * Message buffer fragments (matched messages) + * - off heap lists should already have been moved into + * process off heap structure. + * - Check for literals + */ + for (msgp = c_p->msg_frag; msgp; msgp = msgp->next) { + hfrag = erts_message_to_heap_frag(msgp); + for (; hfrag; hfrag = hfrag->next) { + Eterm *hp, *hp_end; + + *redsp += 1; + + hp = &hfrag->mem[0]; + hp_end = &hfrag->mem[hfrag->used_size]; + + if (any_heap_refs(hp, hp_end, literals, lit_bsize)) + goto literal_gc; + } + } + + return am_ok; + +literal_gc: + + if (!gc_allowed) + return am_need_gc; + + if (c_p->flags & F_DISABLE_GC) + return THE_NON_VALUE; + + FLAGS(c_p) |= F_NEED_FULLSWEEP; + + *redsp += erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, c_p->arity, fcalls); + + erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize, oh); + + *redsp += lit_bsize / 64; /* Need, better value... */ + + return am_ok; +} + +static Eterm +check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) +{ + BeamInstr* start; + char* mod_start; + Uint mod_size; + Eterm* sp; + + *redsp += 1; + + /* + * Pick up limits for the module. + */ + start = (BeamInstr*) modp->old.code_hdr; + mod_start = (char *) start; + mod_size = modp->old.code_length; + + /* + * Check if current instruction or continuation pointer points into module. + */ + if (ErtsInArea(rp->i, mod_start, mod_size) + || ErtsInArea(rp->cp, mod_start, mod_size)) { + return am_true; + } + + *redsp += (STACK_START(rp) - rp->stop) / 32; + + /* + * Check all continuation pointers stored on the stack. + */ + for (sp = rp->stop; sp < STACK_START(rp); sp++) { + if (is_CP(*sp) && ErtsInArea(cp_val(*sp), mod_start, mod_size)) { + return am_true; + } + } + + /* + * Check all continuation pointers stored in stackdump + * and clear exception stackdump if there is a pointer + * to the module. + */ + if (rp->ftrace != NIL) { + struct StackTrace *s; + ASSERT(is_list(rp->ftrace)); + s = (struct StackTrace *) big_val(CDR(list_val(rp->ftrace))); + if ((s->pc && ErtsInArea(s->pc, mod_start, mod_size)) || + (s->current && ErtsInArea(s->current, mod_start, mod_size))) { + rp->freason = EXC_NULL; + rp->fvalue = NIL; + rp->ftrace = NIL; + } else { + int i; + for (i = 0; i < s->depth; i++) { + if (ErtsInArea(s->trace[i], mod_start, mod_size)) { + rp->freason = EXC_NULL; + rp->fvalue = NIL; + rp->ftrace = NIL; + break; + } + } + } + } + + return am_false; +} + +#else /* !ERTS_NEW_PURGE_STRATEGY, i.e, old style purge... */ + static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) { @@ -861,8 +1214,14 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); - literals = (char*) modp->old.code_hdr->literals_start; - lit_bsize = (char*) modp->old.code_hdr->literals_end - literals; + if (modp->old.code_hdr->literal_area) { + literals = (char*) modp->old.code_hdr->literal_area->start; + lit_bsize = (char*) modp->old.code_hdr->literal_area->end - literals; + } + else { + literals = NULL; + lit_bsize = 0; + } for (msgp = rp->msg.first; msgp; msgp = msgp->next) { if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) @@ -903,12 +1262,6 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls /* Check heap, stack etc... */ if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size)) goto try_gc; - if (!(flags & ERTS_CPC_COPY_LITERALS)) { - /* Process ok. May contain old literals but we will be called - * again before module is purged. - */ - return am_false; - } if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, literals, lit_bsize)) { rp->freason = EXC_NULL; rp->fvalue = NIL; @@ -916,6 +1269,10 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } if (any_heap_ref_ptrs(rp->stop, rp->hend, literals, lit_bsize)) goto try_literal_gc; +#ifdef HIPE + if (nstack_any_heap_ref_ptrs(rp, literals, lit_bsize)) + goto try_literal_gc; +#endif if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize)) goto try_literal_gc; if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize)) @@ -994,7 +1351,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } if (need_gc & ERTS_LITERAL_GC__) { struct erl_off_heap_header* oh; - oh = modp->old.code_hdr->literals_off_heap; + oh = modp->old.code_hdr->literal_area->off_heap; *redsp += lit_bsize / 64; /* Need, better value... */ erts_garbage_collect_literals(rp, (Eterm*)literals, lit_bsize, oh); done_gc |= ERTS_LITERAL_GC__; @@ -1007,6 +1364,8 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } +#endif /* !ERTS_NEW_PURGE_STRATEGY */ + static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) { @@ -1135,200 +1494,475 @@ hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, } } -#undef in_area +#ifdef ERTS_NEW_PURGE_STRATEGY #ifdef ERTS_SMP -static void copy_literals_commit(void*); -#endif -copy_literals_t erts_clrange = {NULL, 0, THE_NON_VALUE}; +ErtsThrPrgrLaterOp later_literal_area_switch; -/* copy literals - * - * copy_literals.ptr = LitPtr - * copy_literals.sz = LitSz - * ------ THR PROG COMMIT ----- - * - * - check process code - * - check process code - * ... - * copy_literals.ptr = NULL - * copy_literals.sz = 0 - * ------ THR PROG COMMIT ----- - * ... - */ +typedef struct { + ErtsThrPrgrLaterOp lop; + ErtsLiteralArea *la; +} ErtsLaterReleasLiteralArea; +static void +later_release_literal_area(void *vlrlap) +{ + ErtsLaterReleasLiteralArea *lrlap; + lrlap = (ErtsLaterReleasLiteralArea *) vlrlap; + erts_release_literal_area(lrlap->la); + erts_free(ERTS_ALC_T_RELEASE_LAREA, vlrlap); +} -BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2) +static void +complete_literal_area_switch(void *literal_area) { - ErtsCodeIndex code_ix; - Eterm res = am_true; + Process *p = erts_literal_area_collector; + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_resume(p, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + if (literal_area) + erts_release_literal_area((ErtsLiteralArea *) literal_area); +} +#endif - if (is_not_atom(BIF_ARG_1) || (am_true != BIF_ARG_2 && am_false != BIF_ARG_2)) { - BIF_ERROR(BIF_P, BADARG); - } +#endif /* ERTS_NEW_PURGE_STRATEGY */ - if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_copy_literals_2], - BIF_P, BIF_ARG_1, BIF_ARG_2); +BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) +{ +#ifndef ERTS_NEW_PURGE_STRATEGY + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else + ErtsLiteralArea *unused_la; + ErtsLiteralAreaRef *la_ref; + + if (BIF_P != erts_literal_area_collector) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + erts_smp_mtx_lock(&release_literal_areas.mtx); + + la_ref = release_literal_areas.first; + if (la_ref) { + release_literal_areas.first = la_ref->next; + if (!release_literal_areas.first) + release_literal_areas.last = NULL; } - code_ix = erts_active_code_ix(); + erts_smp_mtx_unlock(&release_literal_areas.mtx); - if (BIF_ARG_2 == am_true) { - Module* modp = erts_get_module(BIF_ARG_1, code_ix); - if (!modp || !modp->old.code_hdr) { - res = am_false; - goto done; - } - if (erts_clrange.ptr != NULL - && !(BIF_P->static_flags & ERTS_STC_FLG_SYSTEM_PROC)) { - res = am_aborted; - goto done; - } - erts_clrange.ptr = modp->old.code_hdr->literals_start; - erts_clrange.sz = modp->old.code_hdr->literals_end - erts_clrange.ptr; - erts_clrange.pid = BIF_P->common.id; - } else if (BIF_ARG_2 == am_false) { - if (erts_clrange.pid != BIF_P->common.id) { - res = am_false; - goto done; - } - erts_clrange.ptr = NULL; - erts_clrange.sz = 0; - erts_clrange.pid = THE_NON_VALUE; + unused_la = ERTS_COPY_LITERAL_AREA(); + + if (!la_ref) { + ERTS_SET_COPY_LITERAL_AREA(NULL); + if (unused_la) { +#ifdef ERTS_SMP + ErtsLaterReleasLiteralArea *lrlap; + lrlap = erts_alloc(ERTS_ALC_T_RELEASE_LAREA, + sizeof(ErtsLaterReleasLiteralArea)); + lrlap->la = unused_la; + erts_schedule_thr_prgr_later_cleanup_op( + later_release_literal_area, + (void *) lrlap, + &lrlap->lop, + (sizeof(ErtsLaterReleasLiteralArea) + + sizeof(ErtsLiteralArea) + + ((unused_la->end + - &unused_la->start[0]) + - 1)*(sizeof(Eterm)))); +#else + erts_release_literal_area(unused_la); +#endif + } + BIF_RET(am_false); } + ERTS_SET_COPY_LITERAL_AREA(la_ref->literal_area); + + erts_free(ERTS_ALC_T_LITERAL_REF, la_ref); + #ifdef ERTS_SMP - ASSERT(committer_state.stager == NULL); - committer_state.stager = BIF_P; - erts_schedule_thr_prgr_later_op(copy_literals_commit, NULL, &committer_state.lop); - erts_proc_inc_refc(BIF_P); + erts_schedule_thr_prgr_later_op(complete_literal_area_switch, + unused_la, + &later_literal_area_switch); erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); ERTS_BIF_YIELD_RETURN(BIF_P, am_true); +#else + erts_release_literal_area(unused_la); + BIF_RET(am_true); #endif -done: - erts_release_code_write_permission(); - BIF_RET(res); + +#endif /* ERTS_NEW_PURGE_STRATEGY */ +} + +void +erts_purge_state_add_fun(ErlFunEntry *fe) +{ + ASSERT(is_value(purge_state.module)); + if (purge_state.fe_ix >= purge_state.fe_size) { + ErlFunEntry **funs; + purge_state.fe_size += 100; + funs = erts_alloc(ERTS_ALC_T_PURGE_DATA, + sizeof(ErlFunEntry *)*purge_state.fe_size); + sys_memcpy((void *) funs, + (void *) purge_state.funs, + purge_state.fe_ix*sizeof(ErlFunEntry *)); + if (purge_state.funs != &purge_state.def_funs[0]) + erts_free(ERTS_ALC_T_PURGE_DATA, purge_state.funs); + purge_state.funs = funs; + } + purge_state.funs[purge_state.fe_ix++] = fe; +} + +Export * +erts_suspend_process_on_pending_purge_lambda(Process *c_p) +{ + erts_smp_mtx_lock(&purge_state.mtx); + if (is_value(purge_state.module)) { + /* + * The process c_p is about to call a fun in the code + * that we are trying to purge. Suspend it and call + * erts_code_purger:pending_purge_lambda/3. The process + * will be resumed when the purge completes or aborts, + * and will then try to do the call again. + */ + if (purge_state.sp_ix >= purge_state.sp_size) { + Eterm *sprocs; + purge_state.sp_size += 100; + sprocs = erts_alloc(ERTS_ALC_T_PURGE_DATA, + (sizeof(ErlFunEntry *) + * purge_state.sp_size)); + sys_memcpy((void *) sprocs, + (void *) purge_state.sprocs, + purge_state.sp_ix*sizeof(ErlFunEntry *)); + if (purge_state.sprocs != &purge_state.def_sprocs[0]) + erts_free(ERTS_ALC_T_PURGE_DATA, purge_state.sprocs); + purge_state.sprocs = sprocs; + } + purge_state.sprocs[purge_state.sp_ix++] = c_p->common.id; + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_VBUMP_ALL_REDS(c_p); + } + erts_smp_mtx_unlock(&purge_state.mtx); + return purge_state.pending_purge_lambda; +} + +static void +finalize_purge_operation(Process *c_p, int succeded) +{ + Uint ix; + + if (c_p) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + + erts_smp_mtx_lock(&purge_state.mtx); + + ASSERT(purge_state.module != THE_NON_VALUE); + + purge_state.module = THE_NON_VALUE; + + /* + * Resume all processes that have tried to call + * funs in this code. + */ + for (ix = 0; ix < purge_state.sp_ix; ix++) { + Process *rp = erts_pid2proc(NULL, 0, + purge_state.sprocs[ix], + ERTS_PROC_LOCK_STATUS); + if (rp) { + erts_resume(rp, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + } + } + + erts_smp_mtx_unlock(&purge_state.mtx); + + if (c_p) + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + + if (purge_state.sprocs != &purge_state.def_sprocs[0]) { + erts_free(ERTS_ALC_T_PURGE_DATA, purge_state.sprocs); + purge_state.sprocs = &purge_state.def_sprocs[0]; + purge_state.sp_size = sizeof(purge_state.def_sprocs); + purge_state.sp_size /= sizeof(purge_state.def_sprocs[0]); + } + purge_state.sp_ix = 0; + + if (purge_state.funs != &purge_state.def_funs[0]) { + erts_free(ERTS_ALC_T_PURGE_DATA, purge_state.funs); + purge_state.funs = &purge_state.def_funs[0]; + purge_state.fe_size = sizeof(purge_state.def_funs); + purge_state.fe_size /= sizeof(purge_state.def_funs[0]); + } + purge_state.fe_ix = 0; } #ifdef ERTS_SMP -static void copy_literals_commit(void* null) { - Process* p = committer_state.stager; -#ifdef DEBUG - committer_state.stager = NULL; -#endif - erts_release_code_write_permission(); + +static ErtsThrPrgrLaterOp purger_lop_data; + +static void +resume_purger(void *unused) +{ + Process *p = erts_code_purger; erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - if (!ERTS_PROC_IS_EXITING(p)) { - erts_resume(p, ERTS_PROC_LOCK_STATUS); - } + erts_resume(p, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_proc_dec_refc(p); } -#endif /* ERTS_SMP */ +static void +finalize_purge_abort(void *unused) +{ + erts_fun_purge_abort_finalize(purge_state.funs, purge_state.fe_ix); -/* Do the actualy module purging and return: - * true for success - * false if no such old module - * BADARG if not an atom - */ -BIF_RETTYPE erts_internal_purge_module_1(BIF_ALIST_1) + finalize_purge_operation(NULL, 0); + + resume_purger(NULL); +} + +#endif /* ERTS_SMP */ + +BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) { - ErtsCodeIndex code_ix; - BeamInstr* code; - BeamInstr* end; - Module* modp; - int is_blocking = 0; - Eterm ret; + if (BIF_P != erts_code_purger) + BIF_ERROR(BIF_P, EXC_NOTSUP); - if (is_not_atom(BIF_ARG_1)) { + if (is_not_atom(BIF_ARG_1)) BIF_ERROR(BIF_P, BADARG); - } - if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD1(bif_export[BIF_erts_internal_purge_module_1], - BIF_P, BIF_ARG_1); - } + switch (BIF_ARG_2) { - code_ix = erts_active_code_ix(); + case am_prepare: { + /* + * Prepare for purge by marking all fun + * entries referring to the code to purge + * with "pending purge" markers. + */ + ErtsCodeIndex code_ix; + Module* modp; + Eterm res; - /* - * Correct module? - */ + if (is_value(purge_state.module)) + BIF_ERROR(BIF_P, BADARG); - if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) { - ERTS_BIF_PREP_RET(ret, am_false); - } - else { - erts_rwlock_old_code(code_ix); + code_ix = erts_active_code_ix(); /* - * Any code to purge? + * Correct module? */ - if (!modp->old.code_hdr) { - ERTS_BIF_PREP_RET(ret, am_false); - } + modp = erts_get_module(BIF_ARG_1, code_ix); + if (!modp) + res = am_false; else { /* - * Unload any NIF library + * Any code to purge? */ - if (modp->old.nif != NULL) { - /* ToDo: Do unload nif without blocking */ - erts_rwunlock_old_code(code_ix); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); - is_blocking = 1; - erts_rwlock_old_code(code_ix); - erts_unload_nif(modp->old.nif); - modp->old.nif = NULL; + erts_rlock_old_code(code_ix); + if (!modp->old.code_hdr) + res = am_false; + else { + BeamInstr* code; + BeamInstr* end; + erts_smp_mtx_lock(&purge_state.mtx); + purge_state.module = BIF_ARG_1; + erts_smp_mtx_unlock(&purge_state.mtx); + res = am_true; + code = (BeamInstr*) modp->old.code_hdr; + end = (BeamInstr *)((char *)code + modp->old.code_length); + erts_fun_purge_prepare(code, end); +#if !defined(ERTS_NEW_PURGE_STRATEGY) + ASSERT(!ERTS_COPY_LITERAL_AREA()); + ERTS_SET_COPY_LITERAL_AREA(modp->old.code_hdr->literal_area); +#endif } - + erts_runlock_old_code(code_ix); + } + +#ifndef ERTS_SMP + BIF_RET(res); +#else + if (res != am_true) + BIF_RET(res); + else { /* - * Remove the old code. + * We'll be resumed when all schedulers are guaranteed + * to see the "pending purge" markers that we've made on + * all fun entries of the code that we are about to purge. + * Processes trying to call these funs will be suspended + * before calling the funs. That is we are guaranteed not + * to get any more direct references into the code while + * checking for such references... */ - ASSERT(erts_total_code_size >= modp->old.code_length); - erts_total_code_size -= modp->old.code_length; - code = (BeamInstr*) modp->old.code_hdr; - end = (BeamInstr *)((char *)code + modp->old.code_length); - erts_cleanup_funs_on_purge(code, end); - beam_catches_delmod(modp->old.catches, code, modp->old.code_length, - code_ix); - decrement_refc(modp->old.code_hdr); - if (modp->old.code_hdr->literals_start) { - erts_free(ERTS_ALC_T_LITERAL, modp->old.code_hdr->literals_start); - } - erts_free(ERTS_ALC_T_CODE, (void *) code); - modp->old.code_hdr = NULL; - modp->old.code_length = 0; - modp->old.catches = BEAM_CATCHES_NIL; - erts_remove_from_ranges(code); - ERTS_BIF_PREP_RET(ret, am_true); + erts_schedule_thr_prgr_later_op(resume_purger, + NULL, + &purger_lop_data); + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); } - erts_rwunlock_old_code(code_ix); +#endif } - if (is_blocking) { - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + + case am_abort: { + /* + * Soft purge that detected direct references into the code + * we set out to purge. Abort the purge. + */ + + if (purge_state.module != BIF_ARG_1) + BIF_ERROR(BIF_P, BADARG); + + erts_fun_purge_abort_prepare(purge_state.funs, purge_state.fe_ix); + +#if !defined(ERTS_NEW_PURGE_STRATEGY) + ASSERT(ERTS_COPY_LITERAL_AREA()); + ERTS_SET_COPY_LITERAL_AREA(NULL); +#endif +#ifndef ERTS_SMP + erts_fun_purge_abort_finalize(purge_state.funs, purge_state.fe_ix); + finalize_purge_operation(BIF_P, 0); + BIF_RET(am_false); +#else + /* + * We need to restore the code addresses of the funs in + * two stages in order to ensure that we do not get any + * stale suspended processes due to the purge abort. + * Restore address pointer (erts_fun_purge_abort_prepare); + * wait for thread progress; clear pending purge address + * pointer (erts_fun_purge_abort_finalize), and then + * resume processes that got suspended + * (finalize_purge_operation). + */ + erts_schedule_thr_prgr_later_op(finalize_purge_abort, + NULL, + &purger_lop_data); + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_BIF_YIELD_RETURN(BIF_P, am_false); +#endif } - erts_release_code_write_permission(); - return ret; -} -static void -decrement_refc(BeamCodeHeader* code_hdr) -{ - struct erl_off_heap_header* oh = code_hdr->literals_off_heap; - - while (oh) { - Binary* bptr; - ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG); - bptr = ((ProcBin*)oh)->val; - if (erts_refc_dectest(&bptr->refc, 0) == 0) { - erts_bin_free(bptr); + case am_complete: { + ErtsCodeIndex code_ix; + BeamInstr* code; + Module* modp; + int is_blocking = 0; + Eterm ret; + ErtsLiteralArea *literals = NULL; + + + /* + * We have no direct references into the code. + * Complete to purge. + */ + + if (purge_state.module != BIF_ARG_1) + BIF_ERROR(BIF_P, BADARG); + + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_purge_module_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + } + + code_ix = erts_active_code_ix(); + + /* + * Correct module? + */ + + if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) { + ERTS_BIF_PREP_RET(ret, am_false); + } + else { + + erts_rwlock_old_code(code_ix); + + /* + * Any code to purge? + */ + if (!modp->old.code_hdr) { + ERTS_BIF_PREP_RET(ret, am_false); + } + else { + /* + * Unload any NIF library + */ + if (modp->old.nif != NULL) { + /* ToDo: Do unload nif without blocking */ + erts_rwunlock_old_code(code_ix); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + is_blocking = 1; + erts_rwlock_old_code(code_ix); + erts_unload_nif(modp->old.nif); + modp->old.nif = NULL; + } + + /* + * Remove the old code. + */ + ASSERT(erts_total_code_size >= modp->old.code_length); + erts_total_code_size -= modp->old.code_length; + code = (BeamInstr*) modp->old.code_hdr; + erts_fun_purge_complete(purge_state.funs, purge_state.fe_ix); + beam_catches_delmod(modp->old.catches, code, modp->old.code_length, + code_ix); + literals = modp->old.code_hdr->literal_area; + modp->old.code_hdr->literal_area = NULL; + erts_free(ERTS_ALC_T_CODE, (void *) code); + modp->old.code_hdr = NULL; + modp->old.code_length = 0; + modp->old.catches = BEAM_CATCHES_NIL; + erts_remove_from_ranges(code); + ERTS_BIF_PREP_RET(ret, am_true); + } + erts_rwunlock_old_code(code_ix); + } + if (is_blocking) { + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + } + + erts_release_code_write_permission(); + + finalize_purge_operation(BIF_P, ret == am_true); + +#if !defined(ERTS_NEW_PURGE_STRATEGY) + + ASSERT(ERTS_COPY_LITERAL_AREA() == literals); + ERTS_SET_COPY_LITERAL_AREA(NULL); + erts_release_literal_area(literals); + +#else /* ERTS_NEW_PURGE_STRATEGY */ + + if (literals) { + ErtsLiteralAreaRef *ref; + ref = erts_alloc(ERTS_ALC_T_LITERAL_REF, + sizeof(ErtsLiteralAreaRef)); + ref->literal_area = literals; + ref->next = NULL; + erts_smp_mtx_lock(&release_literal_areas.mtx); + if (release_literal_areas.last) { + release_literal_areas.last->next = ref; + release_literal_areas.last = ref; + } + else { + release_literal_areas.first = ref; + release_literal_areas.last = ref; + } + erts_smp_mtx_unlock(&release_literal_areas.mtx); + erts_queue_message(erts_literal_area_collector, + 0, + erts_alloc_message(0, NULL), + am_copy_literals, + BIF_P->common.id); } - oh = oh->next; + +#endif /* ERTS_NEW_PURGE_STRATEGY */ + + return ret; + } + + default: + BIF_ERROR(BIF_P, BADARG); + } } diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 8489897d3a..920c8b1ed0 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -858,7 +858,8 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) if (flags & MATCH_SET_RX_TRACE) { erts_trace_return(p, ep->code, result, &ERTS_TRACER(p)); } - if (flags & MATCH_SET_RETURN_TO_TRACE) { + if (flags & MATCH_SET_RETURN_TO_TRACE && + IS_TRACED_FL(p, F_TRACE_RETURN_TO)) { /* can only happen if(local)*/ if (applying) { /* Apply of BIF, cp is in calling function */ diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 03aec2c987..ef4cdf9d5a 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1960,6 +1960,8 @@ void process_main(void) ErtsMessage* msgp; PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_CHK_MBUF_SZ(c_p); + PreFetch(0, next); msgp = PEEK_MESSAGE(c_p); @@ -2047,6 +2049,7 @@ void process_main(void) } ERTS_DBG_CHK_REDS(c_p, FCALLS); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2583,7 +2586,9 @@ do { \ c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2614,7 +2619,9 @@ do { \ c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2644,7 +2651,9 @@ do { \ SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2685,7 +2694,9 @@ do { \ SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2728,7 +2739,9 @@ do { \ SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, live); + ERTS_CHK_MBUF_SZ(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2766,7 +2779,9 @@ do { \ c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2793,7 +2808,9 @@ do { \ bf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, tmp_reg); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); @@ -2829,13 +2846,7 @@ do { \ goto context_switch3; } - if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { - if (GET_BIF_MODULE(Arg(0)) == am_ets) { - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS); - } else { - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF); - } - } + ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(Arg(0)), GET_BIF_ADDRESS(Arg(0))); bf = GET_BIF_ADDRESS(Arg(0)); @@ -2849,7 +2860,9 @@ do { \ ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); result = (*bf)(c_p, reg, I); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_HOLE_CHECK(c_p); @@ -3562,11 +3575,13 @@ do { \ ASSERT(c_p->scheduler_data); #endif live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); nif_bif_result = (*fp)(&env, bif_nif_arity, reg); if (env.exception_thrown) nif_bif_result = THE_NON_VALUE; erts_post_nif(&env); + ERTS_CHK_MBUF_SZ(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -3596,13 +3611,7 @@ do { \ goto context_switch; } - if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { - if ((Eterm)I[-3] == am_ets) { - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS); - } else { - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF); - } - } + ERTS_MSACC_SET_BIF_STATE_CACHED_X((Eterm)I[-3], (BifFunction)Arg(0)); c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ c_p->i = I; /* In case we apply check_process_code/2. */ @@ -3624,7 +3633,9 @@ do { \ Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); live_hf_end = c_p->mbuf; + ERTS_CHK_MBUF_SZ(c_p); nif_bif_result = (*bf)(c_p, reg, I); + ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -5264,19 +5275,14 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) else { /* * Dirty CPU scheduler: - * Currently two reductions consumed per - * micro second spent in the dirty NIF. + * Reductions based on time consumed by + * the dirty NIF. */ - ErtsMonotonicTime time; - time = erts_get_monotonic_time(esdp); - time -= start_time; - time = ERTS_MONOTONIC_TO_USEC(time); - time *= (CONTEXT_REDS-1)/1000 + 1; - ASSERT(time >= 0); - if (time == 0) - time = 1; /* At least one reduction */ - time += esdp->virtual_reds; - reds_used = time > INT_MAX ? INT_MAX : (int) time; + Sint64 treds; + treds = erts_time2reds(start_time, + erts_get_monotonic_time(esdp)); + treds += esdp->virtual_reds; + reds_used = treds > INT_MAX ? INT_MAX : (int) treds; } PROCESS_MAIN_CHK_LOCKS(c_p); @@ -5394,7 +5400,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) ASSERT(!c_p->scheduler_data); erts_pre_dirty_nif(esdp, &env, c_p, - (struct erl_module_nif*)I[2], NULL); + (struct erl_module_nif*)I[2]); #ifdef DEBUG result = @@ -5403,7 +5409,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) #endif (*fp)(&env, arity, reg); - erts_post_nif(&env); + erts_post_dirty_nif(&env); ASSERT(!is_value(result)); ASSERT(c_p->freason == TRAP); @@ -6538,34 +6544,49 @@ call_fun(Process* p, /* Current process. */ * representation (the module has never been loaded), * or the module defining the fun has been unloaded. */ + module = fe->module; - if ((modp = erts_get_module(module, code_ix)) != NULL - && modp->curr.code_hdr != NULL) { + + ERTS_SMP_READ_MEMORY_BARRIER; + if (fe->pend_purge_address) { /* - * There is a module loaded, but obviously the fun is not - * defined in it. We must not call the error_handler - * (or we will get into an infinite loop). + * The system is currently trying to purge the + * module containing this fun. Suspend the process + * and let it try again when the purge operation is + * done (may succeed or not). */ - goto badfun; + ep = erts_suspend_process_on_pending_purge_lambda(p); + ASSERT(ep); } + else { + if ((modp = erts_get_module(module, code_ix)) != NULL + && modp->curr.code_hdr != NULL) { + /* + * There is a module loaded, but obviously the fun is not + * defined in it. We must not call the error_handler + * (or we will get into an infinite loop). + */ + goto badfun; + } - /* - * No current code for this module. Call the error_handler module - * to attempt loading the module. - */ + /* + * No current code for this module. Call the error_handler module + * to attempt loading the module. + */ - ep = erts_find_function(erts_proc_get_error_handler(p), - am_undefined_lambda, 3, code_ix); - if (ep == NULL) { /* No error handler */ - p->current = NULL; - p->freason = EXC_UNDEF; - return NULL; + ep = erts_find_function(erts_proc_get_error_handler(p), + am_undefined_lambda, 3, code_ix); + if (ep == NULL) { /* No error handler */ + p->current = NULL; + p->freason = EXC_UNDEF; + return NULL; + } } reg[0] = module; reg[1] = fun; reg[2] = args; reg[3] = NIL; - return ep->addressv[erts_active_code_ix()]; + return ep->addressv[code_ix]; } } } else if (is_export_header(hdr)) { @@ -7020,7 +7041,11 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I) /* The expensive case, need to build a hashmap */ if (n > MAP_SMALL_MAP_LIMIT) { - res = erts_hashmap_from_ks_and_vs(p,flatmap_get_keys(mp),flatmap_get_values(mp),n); + ErtsHeapFactory factory; + erts_factory_proc_init(&factory, p); + res = erts_hashmap_from_ks_and_vs(&factory,flatmap_get_keys(mp), + flatmap_get_values(mp),n); + erts_factory_close(&factory); } return res; } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 0c2743beb2..f63addb309 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -105,7 +105,7 @@ typedef struct { */ typedef struct genop { - int op; /* Opcode. */ + unsigned int op; /* Opcode. */ int arity; /* Number of arguments. */ GenOpArg def_args[MAX_OPARGS]; /* Default buffer for arguments. */ GenOpArg* a; /* The arguments. */ @@ -283,8 +283,8 @@ typedef struct LoaderState { byte* code_start; /* Start of code file. */ unsigned code_size; /* Size of code file. */ int specific_op; /* Specific opcode (-1 if not found). */ - int num_functions; /* Number of functions in module. */ - int num_labels; /* Number of labels. */ + unsigned int num_functions; /* Number of functions in module. */ + unsigned int num_labels; /* Number of labels. */ BeamCodeHeader* hdr; /* Loaded code header */ BeamInstr* codev; /* Loaded code buffer */ int codev_size; /* Size of code buffer in words. */ @@ -303,13 +303,13 @@ typedef struct LoaderState { * Atom table. */ - int num_atoms; /* Number of atoms in atom table. */ + unsigned int num_atoms; /* Number of atoms in atom table. */ Eterm* atom; /* Atom table. */ - int num_exps; /* Number of exports. */ + unsigned int num_exps; /* Number of exports. */ ExportEntry* export; /* Pointer to export table. */ - int num_imports; /* Number of imports. */ + unsigned int num_imports; /* Number of imports. */ ImportEntry* import; /* Import entry (translated information). */ /* @@ -323,8 +323,8 @@ typedef struct LoaderState { * Lambda table. */ - int num_lambdas; /* Number of lambdas in table. */ - int lambdas_allocated; /* Size of allocated lambda table. */ + unsigned int num_lambdas; /* Number of lambdas in table. */ + unsigned int lambdas_allocated; /* Size of allocated lambda table. */ Lambda* lambdas; /* Pointer to lambdas. */ Lambda def_lambdas[16]; /* Default storage for lambda table. */ char* lambda_error; /* Delayed missing 'FunT' error. */ @@ -333,8 +333,8 @@ typedef struct LoaderState { * Literals (constant pool). */ - int num_literals; /* Number of literals in table. */ - int allocated_literals; /* Number of literal entries allocated. */ + unsigned int num_literals; /* Number of literals in table. */ + unsigned int allocated_literals; /* Number of literal entries allocated. */ Literal* literals; /* Array of literals. */ LiteralPatch* literal_patches; /* Operands that need to be patched. */ Uint total_literal_size; /* Total heap size for all literals. */ @@ -343,13 +343,13 @@ typedef struct LoaderState { * Line table. */ BeamInstr* line_item; /* Line items from the BEAM file. */ - int num_line_items; /* Number of line items. */ + unsigned int num_line_items;/* Number of line items. */ LineInstr* line_instr; /* Line instructions */ - int num_line_instrs; /* Maximum number of line instructions */ - int current_li; /* Current line instruction */ - int* func_line; /* Mapping from function to first line instr */ + unsigned int num_line_instrs; /* Maximum number of line instructions */ + unsigned int current_li; /* Current line instruction */ + unsigned int* func_line; /* Mapping from function to first line instr */ Eterm* fname; /* List of file names */ - int num_fnames; /* Number of filenames in fname table */ + unsigned int num_fnames; /* Number of filenames in fname table */ int loc_size; /* Size of location info in bytes (2/4) */ } LoaderState; @@ -663,7 +663,7 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader, stp->hdr->compile_ptr = NULL; stp->hdr->compile_size = 0; stp->hdr->compile_size_on_heap = 0; - stp->hdr->literals_start = NULL; + stp->hdr->literal_area = NULL; stp->hdr->md5_ptr = NULL; /* @@ -1005,8 +1005,9 @@ loader_state_dtor(Binary* magic) stp->bin = 0; } if (stp->hdr != 0) { - if (stp->hdr->literals_start) { - erts_free(ERTS_ALC_T_LITERAL, stp->hdr->literals_start); + if (stp->hdr->literal_area) { + erts_release_literal_area(stp->hdr->literal_area); + stp->hdr->literal_area = NULL; } erts_free(ERTS_ALC_T_CODE, stp->hdr); stp->hdr = 0; @@ -1330,7 +1331,7 @@ verify_chunks(LoaderState* stp) static int load_atom_table(LoaderState* stp) { - int i; + unsigned int i; GetInt(stp, 4, stp->num_atoms); stp->num_atoms++; @@ -1375,13 +1376,13 @@ load_atom_table(LoaderState* stp) static int load_import_table(LoaderState* stp) { - int i; + unsigned int i; GetInt(stp, 4, stp->num_imports); stp->import = erts_alloc(ERTS_ALC_T_PREPARED_CODE, stp->num_imports * sizeof(ImportEntry)); for (i = 0; i < stp->num_imports; i++) { - int n; + unsigned int n; Eterm mod; Eterm func; Uint arity; @@ -1389,17 +1390,17 @@ load_import_table(LoaderState* stp) GetInt(stp, 4, n); if (n >= stp->num_atoms) { - LoadError2(stp, "import entry %d: invalid atom number %d", i, n); + LoadError2(stp, "import entry %u: invalid atom number %u", i, n); } mod = stp->import[i].module = stp->atom[n]; GetInt(stp, 4, n); if (n >= stp->num_atoms) { - LoadError2(stp, "import entry %d: invalid atom number %d", i, n); + LoadError2(stp, "import entry %u: invalid atom number %u", i, n); } func = stp->import[i].function = stp->atom[n]; GetInt(stp, 4, arity); if (arity > MAX_REG) { - LoadError2(stp, "import entry %d: invalid arity %d", i, arity); + LoadError2(stp, "import entry %u: invalid arity %d", i, arity); } stp->import[i].arity = arity; stp->import[i].patches = 0; @@ -1427,12 +1428,12 @@ load_import_table(LoaderState* stp) static int read_export_table(LoaderState* stp) { - int i; + unsigned int i; BeamInstr* address; GetInt(stp, 4, stp->num_exps); if (stp->num_exps > stp->num_functions) { - LoadError2(stp, "%d functions exported; only %d functions defined", + LoadError2(stp, "%u functions exported; only %u functions defined", stp->num_exps, stp->num_functions); } stp->export @@ -1450,16 +1451,16 @@ read_export_table(LoaderState* stp) stp->export[i].function = func; GetInt(stp, 4, arity); if (arity > MAX_REG) { - LoadError2(stp, "export table entry %d: absurdly high arity %d", i, arity); + LoadError2(stp, "export table entry %u: absurdly high arity %u", i, arity); } stp->export[i].arity = arity; GetInt(stp, 4, n); if (n >= stp->num_labels) { - LoadError3(stp, "export table entry %d: invalid label %d (highest defined label is %d)", i, n, stp->num_labels); + LoadError3(stp, "export table entry %u: invalid label %u (highest defined label is %u)", i, n, stp->num_labels); } value = stp->labels[n].value; if (value == 0) { - LoadError2(stp, "export table entry %d: label %d not resolved", i, n); + LoadError2(stp, "export table entry %u: label %u not resolved", i, n); } stp->export[i].address = address = stp->codev + value; @@ -1520,7 +1521,7 @@ is_bif(Eterm mod, Eterm func, unsigned arity) static int read_lambda_table(LoaderState* stp) { - int i; + unsigned int i; GetInt(stp, 4, stp->num_lambdas); if (stp->num_lambdas > stp->lambdas_allocated) { @@ -1540,12 +1541,12 @@ read_lambda_table(LoaderState* stp) GetAtom(stp, n, stp->lambdas[i].function); GetInt(stp, 4, arity); if (arity > MAX_REG) { - LoadError2(stp, "lambda entry %d: absurdly high arity %d", i, arity); + LoadError2(stp, "lambda entry %u: absurdly high arity %u", i, arity); } stp->lambdas[i].arity = arity; GetInt(stp, 4, n); if (n >= stp->num_labels) { - LoadError3(stp, "lambda entry %d: invalid label %d (highest defined label is %d)", + LoadError3(stp, "lambda entry %u: invalid label %u (highest defined label is %u)", i, n, stp->num_labels); } stp->lambdas[i].label = n; @@ -1566,7 +1567,7 @@ read_lambda_table(LoaderState* stp) static int read_literal_table(LoaderState* stp) { - int i; + unsigned int i; uLongf uncompressed_sz; byte* uncompressed = 0; @@ -1588,7 +1589,7 @@ read_literal_table(LoaderState* stp) } for (i = 0; i < stp->num_literals; i++) { - int sz; + Uint sz; Sint heap_size; byte* p; Eterm val; @@ -1597,7 +1598,7 @@ read_literal_table(LoaderState* stp) GetInt(stp, 4, sz); /* Size of external term format. */ GetString(stp, p, sz); if ((heap_size = erts_decode_ext_size(p, sz)) < 0) { - LoadError1(stp, "literal %d: bad external format", i); + LoadError1(stp, "literal %u: bad external format", i); } if (heap_size > 0) { @@ -1607,7 +1608,7 @@ read_literal_table(LoaderState* stp) val = erts_decode_ext(&factory, &p, 0); if (is_non_value(val)) { - LoadError1(stp, "literal %d: bad external format", i); + LoadError1(stp, "literal %u: bad external format", i); } erts_factory_close(&factory); stp->literals[i].heap_frags = factory.heap_frags; @@ -1617,7 +1618,7 @@ read_literal_table(LoaderState* stp) erts_factory_dummy_init(&factory); val = erts_decode_ext(&factory, &p, 0); if (is_non_value(val)) { - LoadError1(stp, "literal %d: bad external format", i); + LoadError1(stp, "literal %u: bad external format", i); } ASSERT(is_immed(val)); stp->literals[i].heap_frags = NULL; @@ -1640,9 +1641,9 @@ read_line_table(LoaderState* stp) { unsigned version; ERTS_DECLARE_DUMMY(unsigned flags); - int num_line_items; + unsigned int num_line_items; BeamInstr* lp; - int i; + unsigned int i; BeamInstr fname_index; BeamInstr tag; @@ -1721,7 +1722,7 @@ read_line_table(LoaderState* stp) } } else if (tag == TAG_a) { if (val > stp->num_fnames) { - LoadError2(stp, "file index overflow (%d/%d)", + LoadError2(stp, "file index overflow (%u/%u)", val, stp->num_fnames); } fname_index = val; @@ -1757,9 +1758,9 @@ read_line_table(LoaderState* stp) stp->num_line_instrs * sizeof(LineInstr)); stp->current_li = 0; - stp->func_line = (int *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, - stp->num_functions * - sizeof(int)); + stp->func_line = (unsigned int *) erts_alloc(ERTS_ALC_T_PREPARED_CODE, + stp->num_functions * + sizeof(int)); return 1; @@ -1783,6 +1784,10 @@ read_code_header(LoaderState* stp) */ GetInt(stp, 4, head_size); + if (head_size > stp->file_left) { + LoadError2(stp, "invalid code header size %u; bytes left %u", + head_size, stp->file_left); + } stp->code_start = stp->file_p + head_size; stp->code_size = stp->file_left - head_size; stp->file_left = head_size; @@ -1887,7 +1892,7 @@ load_code(LoaderState* stp) ci = stp->ci; for (;;) { - int new_op; + unsigned int new_op; GenOp* tmp_op; ASSERT(ci <= codev_size); @@ -1895,10 +1900,10 @@ load_code(LoaderState* stp) get_next_instr: GetByte(stp, new_op); if (new_op >= NUM_GENERIC_OPS) { - LoadError1(stp, "invalid opcode %d", new_op); + LoadError1(stp, "invalid opcode %u", new_op); } if (gen_opc[new_op].name[0] == '\0') { - LoadError1(stp, "invalid opcode %d", new_op); + LoadError1(stp, "invalid opcode %u", new_op); } @@ -2368,7 +2373,7 @@ load_code(LoaderState* stp) VerifyTag(stp, tag, TAG_u); last_label = tmp_op->a[arg].val; if (!(0 < last_label && last_label < stp->num_labels)) { - LoadError2(stp, "invalid label num %d (0 < label < %d)", + LoadError2(stp, "invalid label num %u (0 < label < %u)", tmp_op->a[arg].val, stp->num_labels); } if (stp->labels[last_label].value != 0) { @@ -2512,7 +2517,7 @@ load_code(LoaderState* stp) { Sint offset; if (function_number >= stp->num_functions) { - LoadError1(stp, "too many functions in module (header said %d)", + LoadError1(stp, "too many functions in module (header said %u)", stp->num_functions); } @@ -2591,14 +2596,14 @@ load_code(LoaderState* stp) if (stp->line_item) { BeamInstr item = code[ci-1]; BeamInstr loc; - int li; + unsigned int li; if (item >= stp->num_line_items) { - LoadError2(stp, "line instruction index overflow (%d/%d)", + LoadError2(stp, "line instruction index overflow (%u/%u)", item, stp->num_line_items); } li = stp->current_li; if (li >= stp->num_line_instrs) { - LoadError2(stp, "line instruction table overflow (%d/%d)", + LoadError2(stp, "line instruction table overflow (%u/%u)", li, stp->num_line_instrs); } loc = stp->line_item[item]; @@ -2630,6 +2635,10 @@ load_code(LoaderState* stp) * End of code found. */ case op_int_code_end: + if (function_number != stp->num_functions) { + LoadError2(stp, "too few functions (%u) in module (header said %u)", + function_number, stp->num_functions); + } stp->codev_size = codev_size; stp->ci = ci; stp->function = THE_NON_VALUE; @@ -4560,13 +4569,16 @@ freeze_code(LoaderState* stp) Eterm* ptr; LiteralPatch* lp; ErlOffHeap code_off_heap; + ErtsLiteralArea *literal_area; + Uint lit_asize; ERTS_INIT_OFF_HEAP(&code_off_heap); - ptr = (Eterm*)erts_alloc(ERTS_ALC_T_LITERAL, - stp->total_literal_size*sizeof(Eterm)); - code_hdr->literals_start = ptr; - code_hdr->literals_end = ptr + stp->total_literal_size; + lit_asize = ERTS_LITERAL_AREA_ALLOC_SIZE(stp->total_literal_size); + literal_area = erts_alloc(ERTS_ALC_T_LITERAL, lit_asize); + ptr = &literal_area->start[0]; + literal_area->end = ptr + stp->total_literal_size; + for (i = 0; i < stp->num_literals; i++) { if (is_not_immed(stp->literals[i].term)) { erts_move_multi_frags(&ptr, &code_off_heap, @@ -4576,7 +4588,7 @@ freeze_code(LoaderState* stp) ptr_val(stp->literals[i].term))); } } - code_hdr->literals_off_heap = code_off_heap.first; + literal_area->off_heap = code_off_heap.first; lp = stp->literal_patches; while (lp != 0) { BeamInstr* op_ptr; @@ -4587,6 +4599,7 @@ freeze_code(LoaderState* stp) op_ptr[0] = lit->term; lp = lp->next; } + code_hdr->literal_area = literal_area; } CHKBLK(ERTS_ALC_T_CODE,code); @@ -4598,8 +4611,8 @@ freeze_code(LoaderState* stp) str_table = (byte *) (codev + stp->ci); } else { BeamCodeLineTab* const line_tab = (BeamCodeLineTab *) (codev+stp->ci); - const int ftab_size = stp->num_functions; - const int num_instrs = stp->current_li; + const unsigned int ftab_size = stp->num_functions; + const unsigned int num_instrs = stp->current_li; const BeamInstr** const line_items = (const BeamInstr**) &line_tab->func_tab[ftab_size + 1]; @@ -4759,7 +4772,7 @@ freeze_code(LoaderState* stp) static void final_touch(LoaderState* stp, struct erl_module_instance* inst_p) { - int i; + unsigned int i; int on_load = stp->on_load; unsigned catches; Uint index; @@ -5431,7 +5444,7 @@ new_genop(LoaderState* stp) static int new_label(LoaderState* stp) { - int num = stp->num_labels; + unsigned int num = stp->num_labels; stp->num_labels++; stp->labels = (Label *) erts_realloc(ERTS_ALC_T_PREPARED_CODE, @@ -5642,6 +5655,28 @@ has_native(BeamCodeHeader *code_hdr) return result; } +void +erts_release_literal_area(ErtsLiteralArea* literal_area) +{ + struct erl_off_heap_header* oh; + + if (!literal_area) + return; + + oh = literal_area->off_heap; + + while (oh) { + Binary* bptr; + ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG); + bptr = ((ProcBin*)oh)->val; + if (erts_refc_dectest(&bptr->refc, 0) == 0) { + erts_bin_free(bptr); + } + oh = oh->next; + } + erts_free(ERTS_ALC_T_LITERAL, literal_area); +} + int erts_is_module_native(BeamCodeHeader* code_hdr) { @@ -6029,11 +6064,11 @@ stub_copy_info(LoaderState* stp, static int stub_read_export_table(LoaderState* stp) { - int i; + unsigned int i; GetInt(stp, 4, stp->num_exps); if (stp->num_exps > stp->num_functions) { - LoadError2(stp, "%d functions exported; only %d functions defined", + LoadError2(stp, "%u functions exported; only %u functions defined", stp->num_exps, stp->num_functions); } stp->export @@ -6047,7 +6082,7 @@ stub_read_export_table(LoaderState* stp) GetAtom(stp, n, stp->export[i].function); GetInt(stp, 4, n); if (n > MAX_REG) { - LoadError2(stp, "export table entry %d: absurdly high arity %d", i, n); + LoadError2(stp, "export table entry %u: absurdly high arity %u", i, n); } stp->export[i].arity = n; GetInt(stp, 4, n); /* Ignore label */ @@ -6061,8 +6096,8 @@ stub_read_export_table(LoaderState* stp) static void stub_final_touch(LoaderState* stp, BeamInstr* fp) { - int i; - int n = stp->num_exps; + unsigned int i; + unsigned int n = stp->num_exps; Eterm mod = fp[2]; Eterm function = fp[3]; int arity = fp[4]; @@ -6373,9 +6408,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) code_hdr->compile_ptr = NULL; code_hdr->compile_size = 0; code_hdr->compile_size_on_heap = 0; - code_hdr->literals_start = NULL; - code_hdr->literals_end = NULL; - code_hdr->literals_off_heap = 0; + code_hdr->literal_area = NULL; code_hdr->on_load_function_ptr = NULL; code_hdr->line_table = NULL; code_hdr->md5_ptr = NULL; diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index fd2dd97fee..1200bb9c6f 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -50,6 +50,7 @@ extern BeamInstr* em_call_error_handler; extern BeamInstr* em_apply_bif; extern BeamInstr* em_call_nif; +struct ErtsLiteralArea_; /* * The following variables keep a sorted list of address ranges for @@ -89,9 +90,7 @@ typedef struct beam_code_header { /* * Literal area (constant pool). */ - Eterm* literals_start; - Eterm* literals_end; - struct erl_off_heap_header* literals_off_heap; + struct ErtsLiteralArea_ *literal_area; /* * Pointer to the on_load function (or NULL if none). @@ -120,7 +119,12 @@ typedef struct beam_code_header { }BeamCodeHeader; +void erts_release_literal_area(struct ErtsLiteralArea_* literal_area); int erts_is_module_native(BeamCodeHeader* code); +void erts_beam_bif_load_init(void); +struct erl_fun_entry; +void erts_purge_state_add_fun(struct erl_fun_entry *fe); +Export *erts_suspend_process_on_pending_purge_lambda(Process *c_p); /* * Layout of the line table. diff --git a/erts/emulator/beam/benchmark.c b/erts/emulator/beam/benchmark.c deleted file mode 100644 index c8409784ef..0000000000 --- a/erts/emulator/beam/benchmark.c +++ /dev/null @@ -1,301 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2016. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#include "global.h" -#include "benchmark.h" - -#ifdef BM_COUNTERS -unsigned long long processes_busy; -unsigned long long processes_spawned; -unsigned long long messages_sent; -unsigned long long messages_copied; -unsigned long long messages_ego; -unsigned long long minor_gc; -unsigned long long major_gc; -#endif /* BM_COUNTERS */ - -#ifdef BM_TIMERS - -/* assuming Solaris */ -#include <time.h> -BM_TIMER_T system_clock; - -unsigned long local_pause_times[MAX_PAUSE_TIME]; -unsigned long pause_times[MAX_PAUSE_TIME]; -unsigned long pause_times_old[MAX_PAUSE_TIME]; - -BM_TIMER_T mmu; -BM_TIMER_T mmu_counter; - -BM_NEW_TIMER(timer); -BM_NEW_TIMER(system); -BM_NEW_TIMER(gc); -BM_NEW_TIMER(minor_gc); -BM_NEW_TIMER(major_gc); -BM_NEW_TIMER(minor_global_gc); -BM_NEW_TIMER(major_global_gc); -BM_NEW_TIMER(send); -BM_NEW_TIMER(copy); -BM_NEW_TIMER(size); -BM_NEW_TIMER(max_minor); -BM_NEW_TIMER(max_major); -BM_NEW_TIMER(max_global_minor); -BM_NEW_TIMER(max_global_major); -BM_NEW_TIMER(misc0); -BM_NEW_TIMER(misc1); -BM_NEW_TIMER(misc2); -#endif /* BM_TIMERS */ - -#ifdef BM_HEAP_SIZES -unsigned long long max_used_heap; -unsigned long long max_allocated_heap; -unsigned long long max_used_global_heap; -unsigned long long max_allocated_global_heap; -#endif /* BM_HEAP_SIZES */ - -#ifdef BM_MESSAGE_SIZES -unsigned long long words_sent; -unsigned long long words_copied; -unsigned long long words_prealloc; -unsigned long long message_sizes[1000]; -#endif /* BM_MESSAGE_SIZES */ - -/***** - * The following functions have to be defined, but they only have contents - * if certain keywords are defined. - */ - -void init_benchmarking() -{ -#ifdef BM_TIMERS - int i; - for (i = 0; i < 1000; i++) - { - BM_START_TIMER(system); - BM_STOP_TIMER(system); - } - timer_time = system_time / 1000; - - for (i = 0; i < MAX_PAUSE_TIME; i++) { - local_pause_times[i] = 0; - pause_times[i] = 0; - pause_times_old[i] = 0; - } - - mmu = 0; - mmu_counter = 0; - - BM_MMU_INIT(); -#endif /* BM_TIMERS */ - -#ifdef BM_COUNTERS - processes_busy = 0; - processes_spawned = 0; - messages_sent = 0; - messages_copied = 0; - messages_ego = 0; - minor_gc = 0; - major_gc = 0; -#endif /* BM_COUNTERS */ - -#ifdef BM_HEAP_SIZES - max_used_heap = 0; - max_allocated_heap = 0; - max_used_global_heap = 0; - max_allocated_global_heap = 0; -#endif /* BM_HEAP_SIZES */ - -#ifdef BM_MESSAGE_SIZES - words_sent = 0; - words_copied = 0; - words_prealloc = 0; - { - int i; - for (i = 0; i < 1000; i++) - message_sizes[i] = 0; - } -#endif /* BM_MESSAGE_SIZES */ -} - -void save_statistics() -{ -#ifdef BM_STATISTICS - FILE *file = fopen(BM_STATISTICS_FILE,"a"); - long i = 0; - - if (file) - { - erts_fprintf(file,"-------------------------------------------------------------------------\n"); - erts_fprintf(file,"The counters are reset at system start and are sums over the entire node.\n"); - erts_fprintf(file,"You may reset them manually using the BIFs in the module hipe_bifs.\n"); - erts_fprintf(file,"All times are given in milliseconds.\n"); - erts_fprintf(file,"-------------------------------------------------------------------------\n"); - - erts_fprintf(file,"Node: %T\n",erts_this_node->sysname); - -#ifdef BM_COUNTERS - erts_fprintf(file,"Number of processes spawned: %lld\n",processes_spawned); - erts_fprintf(file,"Number of local minor GCs: %lld\n",minor_gc); - erts_fprintf(file,"Number of local major GCs: %lld\n",major_gc); - erts_fprintf(file,"Number of messages sent: %lld\n",messages_sent); - erts_fprintf(file,"Number of messages copied: %lld\n",messages_copied); - erts_fprintf(file,"Number of messages sent to self: %lld\n",messages_ego); -#endif /* BM_COUNTERS */ - -#ifdef BM_MESSAGE_SIZES - erts_fprintf(file,"Number of words sent: %lld\n",words_sent); - erts_fprintf(file,"Number of words copied: %lld\n",words_copied); - erts_fprintf(file,"Number of words preallocated: %lld\n",words_prealloc); -#endif /* BM_MESSAGE_SIZES */ - -#ifdef BM_HEAP_SIZES - erts_fprintf(file,"Biggest local heap used (in words): %lld\n",max_used_heap); - erts_fprintf(file,"Biggest local heap allocated (in words): %lld\n",max_allocated_heap); - erts_fprintf(file,"Biggest global heap used (in words): %lld\n",max_used_global_heap); - erts_fprintf(file,"Biggest global heap allocated (in words): %lld\n",max_allocated_global_heap); -#endif /* BM_HEAP_SIZES */ - -#ifdef BM_TIMERS - erts_fprintf(file,"--- The total active system time is the sum of all times below ---\n"); - BM_TIME_PRINTER("Mutator time",system_time); - BM_TIME_PRINTER("Time spent in send (excluding size & copy)",send_time); - BM_TIME_PRINTER("Time spent in size",size_time); - BM_TIME_PRINTER("Time spent in copy",copy_time); - BM_TIME_PRINTER("Time spent in local minor GC",minor_gc_time); - BM_TIME_PRINTER("Time spent in local major GC",major_gc_time); - BM_TIME_PRINTER("Time spent in global minor GC",minor_global_gc_time); - BM_TIME_PRINTER("Time spent in global major GC",major_global_gc_time); - erts_fprintf(file,"---\n"); - BM_TIME_PRINTER("Maximum time spent in one separate local minor GC",max_minor_time); - BM_TIME_PRINTER("Maximum time spent in one separate local major GC",max_major_time); - BM_TIME_PRINTER("Maximum time spent in one separate global minor GC",max_global_minor_time); - BM_TIME_PRINTER("Maximum time spent in one separate global major GC",max_global_major_time); -#endif /* BM_TIMERS */ - -#if 0 - /* Save a log file for import into excel */ - - long long total_time, n; - long left, right, mid; - -#ifdef BM_COUNTERS - erts_fprintf(file,"Spawns\tLocalGC\tMAGC\tMessages\tMutator_t\tLocalGC_t\tMAGC_t\tLocMaxP\tLocMeanP\tLocGeoMP\tMAMaxP\tMAMeanP\tMAGeoMP\t\tCMAGC\tCMAGC_t\n"); - erts_fprintf(file,"%lld\t%lld\t%lld\t%lld\t", - processes_spawned, - minor_garbage_cols + major_garbage_cols, - minor_global_garbage_cols + major_global_garbage_cols, - messages_sent); -#endif /* BM_COUNTERS */ - -#ifdef BM_TIMERS - erts_fprintf(file,"%lld\t%lld\t%lld\t", - (long long)(system_time + send_time + size_time + copy_time), - (long long)(minor_gc_time + major_gc_time), - (long long)(minor_global_gc_time + major_global_gc_time)); - - total_time = 0; n = 0; - left = 0; right = 0; mid = 0; - for (i = 0; i < MAX_PAUSE_TIME; i++) { - total_time += local_pause_times[i] * i; - n += local_pause_times[i]; - if (i > mid) - right += local_pause_times[i]; - while(right > left) { - left += local_pause_times[mid++]; - right -= local_pause_times[mid]; - } - } - erts_fprintf(file,"%lld\t%lld\t%ld\t", - (long long)((max_minor_time > max_major_time ? - max_minor_time : - max_major_time)*1000), - total_time / n, - mid); - - total_time = 0; n = 0; - left = 0; right = 0; mid = 0; - for (i = 0; i < MAX_PAUSE_TIME; i++) { - if (pause_times[i] > 0) { - total_time += pause_times[i] * i; - n += pause_times[i]; - if (i > mid) - right += pause_times[i]; - while(right > left) { - left += pause_times[mid++]; - right -= pause_times[mid]; - } - } - } - erts_fprintf(file,"%lld\t%lld\t%ld\t", - (long long)((max_global_minor_time > max_global_major_time ? - max_global_minor_time : - max_global_major_time)*1000), - (n > 0 ? total_time / n : 0), - mid); - - erts_fprintf(file,"\t%lld\t%lld\n",n,total_time); - - erts_fprintf(file,"\nMinor:\n"); - for (i = 0; i < MAX_PAUSE_TIME; i++) { - if (i < 1000 || pause_times[i] > 0) { - erts_fprintf(file,"%d\t%ld\n",i,pause_times[i]); - } - } - - fprintf(file,"Major:\n"); - for (i = 0; i < MAX_PAUSE_TIME; i++) { - if (pause_times_old[i] > 0) { - fprintf(file,"%d\t%ld\n",i,pause_times_old[i]); - } - } -#endif /* BM_TIMERS */ - -#ifdef BM_TIMERS - total_time = 0; n = 0; - left = 0; right = 0; mid = 0; - fprintf(file,"\nLocal:\n"); - for (i = 0; i < MAX_PAUSE_TIME; i++) { - if (local_pause_times[i] > 0) { - erts_fprintf(file,"%d\t%ld\n",i,local_pause_times[i]); - total_time += local_pause_times[i] * i; - n += local_pause_times[i]; - if (i > mid) - right += local_pause_times[i]; - while(right > left) { - left += local_pause_times[mid++]; - right -= local_pause_times[mid]; - } - } - } - erts_fprintf(file,"Mid: %ld Mean: %ld\n",(long)mid, - (long)(n > 0 ? total_time / n : 0)); -#endif -#endif /* 0 */ - fclose(file); - } - else - fprintf(stderr,"Sorry... Can not write to %s!\n\r",BM_STATISTICS_FILE); -#endif /* BM_STATISTICS */ -} diff --git a/erts/emulator/beam/benchmark.h b/erts/emulator/beam/benchmark.h deleted file mode 100644 index 0272896f4f..0000000000 --- a/erts/emulator/beam/benchmark.h +++ /dev/null @@ -1,295 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2016. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * %CopyrightEnd% - */ - -#ifndef __BENCHMARK_H__ -#define __BENCHMARK_H__ - -/* The define __BENCHMARK__ is the master switch to turn on and off - * benchmarking. This will enable the benchmark-BIFs in hipe_bif1.c. - * Documentation for the BIFs is in hipe_bif1.c, and that is where you - * will find the information about how to accually get some data out - * from these timers and counters. - */ -/* #define __BENCHMARK__ */ - -#ifdef __BENCHMARK__ -/* - * The defines below enables different parts of the benchmaring. - * Counters and timers that are disabled, always report zero in - * the BIFs. - */ - -/* BM_TIMERS keeps track of the time spent in diferent parts of the - * system. It only measures accual active time, not time spent in idle - * mode. Currently, the Solaris hrtime_t will be used. - * To add new timers look below. - */ -#define BM_TIMERS - -/* BM_COUNTERS count all kinds of events that occurs in the system. - * Among other things it counts the number of messages, then number of - * garbage collections, the number of processes spawned etc. - * To add new counters look below. - */ -#define BM_COUNTERS - -/* BM_MESSAGE_SIZES keeps a log of the size of all messages sent in - * the system. This introduce an overhead in time for the shared heap - * system since all message sizes have to be calculated at send. - */ -/* #define BM_MESSAGE_SIZES */ - -/* BM_HEAP_SIZES goes through all processes at garbage collection time - * to sum their allocated and used heap sizes. In anything else than a - * shared heap system, this will cost. - */ -/* #define BM_HEAP_SIZES */ - -/* BM_STATISTICS saves an entry in the file BM_STATISTICS_FILE. This - * is done for each erlang node at exit time. - */ -/* #define BM_STATISTICS */ - -#endif /* __BENCHMARK__ */ - - -#ifdef BM_STATISTICS -# define BM_STATISTICS_FILE "/tmp/erlang_statistics.joppe.log" -#endif /* BM_STATISTICS */ - - -/************ There are no more settings below this line *************/ - -/* - * Maintenance and how to add new stuff is documented by the code - * below ;-) - */ - -#ifdef BM_COUNTERS -/********************************************************************* - * To add new counters: - * - * Add the variable here AND in benchmark.c. Use the macro - * BM_COUNT(var) in the code where you want to increase it. - * - */ -extern unsigned long long processes_busy; -extern unsigned long long processes_spawned; -extern unsigned long long messages_sent; -extern unsigned long long messages_copied; -extern unsigned long long messages_ego; -extern unsigned long long minor_gc; -extern unsigned long long major_gc; - -#define BM_COUNT(var) (var)++; - -#define BM_EGO_COUNT(send,rec) { \ - if ((send) == (rec)) \ - BM_COUNT(messages_ego); } - -#define BM_LAZY_COPY_START long long gcs = minor_global_gc + major_global_gc; -#define BM_LAZY_COPY_STOP { gcs = (minor_global_gc + major_global_gc) - gcs; \ - if (gcs > gc_in_copy) gc_in_copy = gcs; } - -#else /* !BM_COUNTERS */ -# define BM_COUNT(var) -# define BM_EGO_COUNT(send,rec) -# define BM_LAZY_COPY_START -# define BM_LAZY_COPY_STOP -#endif /* BM_COUNTERS */ - - -#ifdef BM_TIMERS -/********************************************************************* - * To add new timers: - * - * Add the variable below using the form extern BM_TIMER_T blah_time. - * Also add them in benchmark.c using the macro NEW_TIMER(blah). Use - * the macro BM_SWAP_TIMER(from,blah) ... BM_SWAP_TIMER(blah,to) to - * start and stop the new timer. Note, that you have to know what - * timer is running at the place where you want to insert your new - * timer to be able to stop and start (from,to) it. - * - * You can use the macros BM_STOP_TIMER(blah) and BM_START_TIMER(blah) - * around code that should not be timed at all. As above, you have to - * know what timer to start and stop. The system timer is running at - * most places in the emulator. Only the garbage collector and the - * message sending has its own timers at the moment. - * - * The timer_time used when stopping timers is the time it takes to - * start and stop the timers, calculated in init_benchmarking(). If it - * is not there, the time it takes to do this will accually be - * substantial compared to some small times in the system we want to - * meassure (send time in shared heap for instance). - */ - -/* (Assuming Solaris) */ - -#define BM_TIMER_T ErtsMonotonicTime -#define BM_START_TIMER(t) system_clock = ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time()) -#define BM_STOP_TIMER(t) do { \ - BM_TIMER_T tmp = (ERTS_MONOTONIC_TO_NSEC(erts_os_monotonic_time()) - system_clock) - timer_time; \ - t##_time += (tmp > 0 ? tmp : 0); \ -} while(0) - -#define BM_TIME_PRINTER(str,time) do { \ - int min,sec,milli,micro; \ - BM_TIMER_T tmp; \ - tmp = (time) / 1000; \ - micro = tmp % 1000; \ - tmp /= 1000; \ - milli = tmp % 1000; \ - tmp /= 1000; \ - sec = tmp % 60; \ - min = tmp / 60; \ - erts_fprintf(file,str": %d:%02d.%03d %03d\n",min,sec,milli,micro); \ -} while(0) - -extern BM_TIMER_T system_clock; - -extern BM_TIMER_T timer_time; -extern BM_TIMER_T system_time; -extern BM_TIMER_T gc_time; -extern BM_TIMER_T minor_gc_time; -extern BM_TIMER_T major_gc_time; -extern BM_TIMER_T minor_global_gc_time; -extern BM_TIMER_T major_global_gc_time; -extern BM_TIMER_T send_time; -extern BM_TIMER_T copy_time; -extern BM_TIMER_T size_time; -extern BM_TIMER_T max_minor_time; -extern BM_TIMER_T max_major_time; -extern BM_TIMER_T max_global_minor_time; -extern BM_TIMER_T max_global_major_time; -extern BM_TIMER_T misc0_time; -extern BM_TIMER_T misc1_time; -extern BM_TIMER_T misc2_time; - -#define MAX_PAUSE_TIME 500000 -extern unsigned long local_pause_times[MAX_PAUSE_TIME]; -extern unsigned long pause_times[MAX_PAUSE_TIME]; -extern unsigned long pause_times_old[MAX_PAUSE_TIME]; - -#define MMU_INTERVAL 5 /* milli seconds */ -extern BM_TIMER_T mmu_counter; -extern BM_TIMER_T mmu; - -#define BM_NEW_TIMER(t) BM_TIMER_T t##_time = 0; -#define BM_RESET_TIMER(t) t##_time = 0; -#define BM_SWAP_TIMER(t1,t2) do { BM_STOP_TIMER(t1); BM_START_TIMER(t2); } while(0) -#define BM_MMU_INIT() do { \ - BM_TIMER_T gc = gc_time; \ - while (gc > 0) { \ - if (gc > MMU_INTERVAL) { \ - gc -= MMU_INTERVAL - mmu_counter; \ - erts_printf("%d\n",(int)((mmu / MMU_INTERVAL) * 100)); \ - mmu_counter = 0; mmu = 0; \ - } else { \ - mmu_counter += gc; \ - if (mmu_counter >= MMU_INTERVAL) { \ - mmu_counter -= MMU_INTERVAL; \ - erts_printf("%d\n",(int)((mmu / MMU_INTERVAL) * 100)); \ - mmu = 0; \ - } \ - gc = 0; \ - } \ - } \ - BM_RESET_TIMER(system); \ - BM_RESET_TIMER(send); \ - BM_RESET_TIMER(copy); \ - BM_RESET_TIMER(size); \ -} while(0) - -#define BM_MMU_READ() do { \ - BM_TIMER_T mut = system_time + send_time + copy_time + size_time; \ - while (mut > 0) { \ - if (mut > MMU_INTERVAL) { \ - BM_TIMER_T tmp = MMU_INTERVAL - mmu_counter; \ - mmu += tmp; mut -= tmp; \ - erts_printf("%d\n",(int)((mmu / MMU_INTERVAL) * 100)); \ - mmu_counter = 0; mmu = 0; \ - } else { \ - mmu_counter += mut; mmu += mut; \ - if (mmu_counter >= MMU_INTERVAL) { \ - mmu_counter -= MMU_INTERVAL; \ - mmu -= mmu_counter; \ - erts_printf("%d\n",(int)((mmu / MMU_INTERVAL) * 100)); \ - mmu = mmu_counter; \ - } \ - mut = 0; \ - } \ - } \ -} while(0) - -#else /* !BM_TIMERS */ -# define BM_NEW_TIMER(t) -# define BM_START_TIMER(t) -# define BM_STOP_TIMER(t) -# define BM_RESET_TIMER(t) -# define BM_SWAP_TIMER(t1,t2) -# define BM_TIME_PRINTER(str,time) -# define BM_MMU_INIT() -# define BM_MMU_READ() -#endif /* BM_TIMERS */ - -#ifdef BM_HEAP_SIZES -extern unsigned long long max_used_heap; -extern unsigned long long max_allocated_heap; -extern unsigned long long max_used_global_heap; -extern unsigned long long max_allocated_global_heap; -#endif /* BM_HEAP_SIZES */ - -#ifdef BM_MESSAGE_SIZES -extern unsigned long long words_sent; -extern unsigned long long words_copied; -extern unsigned long long words_prealloc; -extern unsigned long long message_sizes[1000]; - -#define BM_MESSAGE_COPIED(size) { \ - words_copied += size; \ - BM_COUNT(messages_copied); } - -#define BM_PREALLOC_DATA(size) { \ - words_prealloc += size; } - -#define BM_MESSAGE(mess,send,rec) { \ - Uint msize = size_object(mess); \ - words_sent += msize; \ - if (msize < 1000) \ - message_sizes[msize]++; \ - else \ - message_sizes[999]++; \ - BM_EGO_COUNT(send,rec); \ - BM_COUNT(messages_sent); } - -#else /* !BM_MESSAGE_SIZES */ - -#define BM_MESSAGE_COPIED(size) BM_COUNT(messages_copied); -#define BM_PREALLOC_DATA(size) -#define BM_MESSAGE(mess,send,rec) { \ - BM_EGO_COUNT(send,rec); \ - BM_COUNT(messages_sent); } - -#endif /* BM_MESSAGE_SIZES */ - -void init_benchmarking(void); -void save_statistics(void); - -#endif /* _BENCHMARK_H_ */ diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 065018514a..80db4eb6ff 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -161,6 +161,7 @@ bif erts_internal:port_close/1 bif erts_internal:port_connect/2 bif erts_internal:request_system_task/3 +bif erts_internal:request_system_task/4 bif erts_internal:check_process_code/2 bif erts_internal:map_to_tuple_keys/1 @@ -174,6 +175,8 @@ bif erts_internal:is_system_process/1 bif erts_internal:system_check/1 +bif erts_internal:release_literal_area_switch/0 + # inet_db support bif erlang:port_set_data/2 bif erlang:port_get_data/1 @@ -642,8 +645,9 @@ bif erts_debug:map_info/1 # New in 19.0 # -bif erts_internal:copy_literals/2 -bif erts_internal:purge_module/1 +bif erts_internal:is_process_executing_dirty/1 +bif erts_internal:check_dirty_process_code/2 +bif erts_internal:purge_module/2 bif binary:split/2 bif binary:split/3 bif erts_debug:size_shared/1 diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 464acd67f6..4a96d971c3 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -21,17 +21,8 @@ #ifndef __BIG_H__ #define __BIG_H__ -#ifndef __SYS_H__ #include "sys.h" -#endif - -#ifndef __CONFIG_H__ -#include "erl_vm.h" -#endif - -#ifndef __GLOBAL_H__ #include "global.h" -#endif typedef Uint ErtsDigit; diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 227fedfb69..6e8710eb8a 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -279,6 +279,7 @@ type TRACER_NIF LONG_LIVED SYSTEM tracer_nif type TRACE_MSG_QUEUE SHORT_LIVED SYSTEM trace_message_queue type SCHED_ASYNC_JOB SHORT_LIVED SYSTEM async_calls type DIRTY_START STANDARD PROCESSES dirty_start +type DIRTY_SL SHORT_LIVED SYSTEM dirty_short_lived +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; @@ -313,6 +314,7 @@ type RUNQ_BLNS LONG_LIVED SYSTEM run_queue_balancing type THR_PRGR_IDATA LONG_LIVED SYSTEM thr_prgr_internal_data type THR_PRGR_DATA LONG_LIVED SYSTEM thr_prgr_data type T_THR_PRGR_DATA SHORT_LIVED SYSTEM temp_thr_prgr_data +type RELEASE_LAREA SHORT_LIVED SYSTEM release_literal_area +endif # @@ -367,6 +369,8 @@ type MONITOR_LH STANDARD PROCESSES monitor_lh type NLINK_LH STANDARD PROCESSES nlink_lh type CODE LONG_LIVED CODE code type LITERAL LITERAL CODE literal +type LITERAL_REF SHORT_LIVED CODE literal_area_ref +type PURGE_DATA SHORT_LIVED CODE purge_data type DB_HEIR_DATA STANDARD ETS db_heir_data type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 3fb866733c..29ba12dfdb 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2284,9 +2284,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("dist_ctrl", BIF_ARG_1)) { DistEntry *dep; i = 0; - /* Need to be the only thread running... */ - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); for (dep = erts_visible_dist_entries; dep; dep = dep->next) ++i; for (dep = erts_hidden_dist_entries; dep; dep = dep->next) @@ -2309,8 +2307,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = CONS(hp, tpl, res); hp += 2; } - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); BIF_RET(res); } else if (BIF_ARG_1 == am_system_version) { erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); @@ -2886,6 +2883,27 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(AM_tag); #endif } + else if (ERTS_IS_ATOM_STR("check_process_code",BIF_ARG_1)) { + Eterm terms[3]; + Sint length = 1; + Uint sz = 0; + Eterm *hp, res; + DECL_AM(direct_references); + + terms[0] = AM_direct_references; +#if !defined(ERTS_NEW_PURGE_STRATEGY) + { + DECL_AM(indirect_references); + terms[1] = AM_indirect_references; + terms[2] = am_copy_literals; + length = 3; + } +#endif + erts_bld_list(NULL, &sz, length, terms); + hp = HAlloc(BIF_P, sz); + res = erts_bld_list(&hp, NULL, length, terms); + BIF_RET(res); + } BIF_ERROR(BIF_P, BADARG); } diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index bad34211a5..128a7b3865 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1094,7 +1094,7 @@ BIF_RETTYPE ets_insert_2(BIF_ALIST_2) CHECK_TABLES(); - /* Write lock table if more than one object to keep atomicy */ + /* Write lock table if more than one object to keep atomicity */ kind = ((is_list(BIF_ARG_2) && CDR(list_val(BIF_ARG_2)) != NIL) ? LCK_WRITE : LCK_WRITE_REC); @@ -1164,7 +1164,7 @@ BIF_RETTYPE ets_insert_new_2(BIF_ALIST_2) Eterm lookup_ret; DbTableMethod* meth; - /* More than one object, use LCK_WRITE to keep atomicy */ + /* More than one object, use LCK_WRITE to keep atomicity */ kind = LCK_WRITE; tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, kind); if (tb == NULL) { diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 12ae086b31..5e6fe4f460 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -40,7 +40,7 @@ ** DB_FINE_LOCKED set. The table variable is_thread_safe will then indicate ** if operations need to obtain fine grained locks or not. Some operations ** will for example always use exclusive table lock to guarantee -** a higher level of atomicy. +** a higher level of atomicity. */ /* FIXATION: diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 6ce1376c81..c639ba623f 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -199,14 +199,13 @@ erts_erase_fun_entry(ErlFunEntry* fe) } void -erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end) +erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end) { int limit; HashBucket** bucket; - ErlFunEntry* to_delete = NULL; int i; - erts_fun_write_lock(); + erts_fun_read_lock(); limit = erts_fun_table.size; bucket = erts_fun_table.bucket; for (i = 0; i < limit; i++) { @@ -217,22 +216,51 @@ erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end) BeamInstr* addr = fe->address; if (start <= addr && addr < end) { + fe->pend_purge_address = addr; + ERTS_SMP_WRITE_MEMORY_BARRIER; fe->address = unloaded_fun; - if (erts_refc_dectest(&fe->refc, 0) == 0) { - fe->address = (void *) to_delete; - to_delete = fe; - } + erts_purge_state_add_fun(fe); } b = b->next; } } + erts_fun_read_unlock(); +} + +void +erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no) +{ + Uint ix; - while (to_delete != NULL) { - ErlFunEntry* next = (ErlFunEntry *) to_delete->address; - erts_erase_fun_entry_unlocked(to_delete); - to_delete = next; + for (ix = 0; ix < no; ix++) { + ErlFunEntry *fe = funs[ix]; + if (fe->address == unloaded_fun) + fe->address = fe->pend_purge_address; + fe->pend_purge_address = NULL; } - erts_fun_write_unlock(); +} + +void +erts_fun_purge_abort_finalize(ErlFunEntry **funs, Uint no) +{ + Uint ix; + + for (ix = 0; ix < no; ix++) + funs[ix]->pend_purge_address = NULL; +} + +void +erts_fun_purge_complete(ErlFunEntry **funs, Uint no) +{ + Uint ix; + + for (ix = 0; ix < no; ix++) { + ErlFunEntry *fe = funs[ix]; + fe->pend_purge_address = NULL; + if (erts_refc_dectest(&fe->refc, 0) == 0) + erts_erase_fun_entry(fe); + } + ERTS_SMP_WRITE_MEMORY_BARRIER; } void @@ -294,6 +322,7 @@ fun_alloc(ErlFunEntry* template) obj->module = template->module; erts_refc_init(&obj->refc, -1); obj->address = unloaded_fun; + obj->pend_purge_address = NULL; #ifdef HIPE obj->native_address = NULL; #endif diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 8c4deea7a0..73c3e19c1c 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -44,6 +44,7 @@ typedef struct erl_fun_entry { Eterm module; /* Tagged atom for module. */ erts_refc_t refc; /* Reference count: One for code + one for each fun object in each process. */ + BeamInstr *pend_purge_address; /* address stored during a pending purge */ } ErlFunEntry; /* @@ -81,7 +82,10 @@ ErlFunEntry* erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, void erts_erase_fun_entry(ErlFunEntry* fe); void erts_cleanup_funs(ErlFunThing* funp); -void erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end); +void erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end); +void erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no); +void erts_fun_purge_abort_finalize(ErlFunEntry **funs, Uint no); +void erts_fun_purge_complete(ErlFunEntry **funs, Uint no); void erts_dump_fun_entries(int, void *); #endif diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 9e0919cc9c..1495d06459 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -517,6 +517,8 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int erts_proc_sched_data((p))->virtual_reds += vreds; } + ERTS_CHK_MBUF_SZ(p); + ASSERT(CONTEXT_REDS >= erts_proc_sched_data(p)->virtual_reds); return reds_left; } @@ -527,6 +529,8 @@ young_gen_usage(Process *p) Uint hsz; Eterm *aheap; + ERTS_CHK_MBUF_SZ(p); + hsz = p->mbuf_sz; if (p->flags & F_ON_HEAP_MSGQ) { @@ -600,6 +604,8 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); #endif + ERTS_CHK_MBUF_SZ(p); + ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) >= erts_proc_sched_data(p)->virtual_reds); @@ -682,6 +688,7 @@ do_major_collection: killed before a GC could be done. */ if (reds == -2) { ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL; + int res; erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); erts_send_exit_signal(p, p->common.id, p, &locks, @@ -693,7 +700,9 @@ do_major_collection: erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); /* We have to make sure that we have space for need on the heap */ - return delay_garbage_collection(p, live_hf_end, need, fcalls); + res = delay_garbage_collection(p, live_hf_end, need, fcalls); + ERTS_MSACC_POP_STATE_M(); + return res; } erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); @@ -892,6 +901,58 @@ erts_garbage_collect_hibernate(Process* p) } +/* + * HiPE native code stack scanning procedures: + * - fullsweep_nstack() + * - gensweep_nstack() + * - offset_nstack() + * - sweep_literals_nstack() + */ +#if defined(HIPE) + +#define GENSWEEP_NSTACK(p,old_htop,n_htop) \ + do { \ + Eterm *tmp_old_htop = old_htop; \ + Eterm *tmp_n_htop = n_htop; \ + gensweep_nstack((p), &tmp_old_htop, &tmp_n_htop); \ + old_htop = tmp_old_htop; \ + n_htop = tmp_n_htop; \ + } while(0) + +/* + * offset_nstack() can ignore the descriptor-based traversal the other + * nstack procedures use and simply call offset_heap_ptr() instead. + * This relies on two facts: + * 1. The only live non-Erlang terms on an nstack are return addresses, + * and they will be skipped thanks to the low/high range check. + * 2. Dead values, even if mistaken for pointers into the low/high area, + * can be offset safely since they won't be dereferenced. + * + * XXX: WARNING: If HiPE starts storing other non-Erlang values on the + * nstack, such as floats, then this will have to be changed. + */ +static ERTS_INLINE void offset_nstack(Process* p, Sint offs, + char* area, Uint area_size) +{ + if (p->hipe.nstack) { + ASSERT(p->hipe.nsp && p->hipe.nstend); + offset_heap_ptr(hipe_nstack_start(p), hipe_nstack_used(p), + offs, area, area_size); + } + else { + ASSERT(!p->hipe.nsp && !p->hipe.nstend); + } +} + +#else /* !HIPE */ + +#define fullsweep_nstack(p,n_htop) (n_htop) +#define GENSWEEP_NSTACK(p,old_htop,n_htop) do{}while(0) +#define offset_nstack(p,offs,area,area_size) do{}while(0) +#define sweep_literals_nstack(p,old_htop,area,area_size) (old_htop) + +#endif /* HIPE */ + void erts_garbage_collect_literals(Process* p, Eterm* literals, Uint byte_lit_size, @@ -954,7 +1015,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, area_size = byte_lit_size; n = setup_rootset(p, p->arg_reg, p->arity, &rootset); roots = rootset.roots; - old_htop = p->old_htop; + old_htop = sweep_literals_nstack(p, p->old_htop, area, area_size); while (n--) { Eterm* g_ptr = roots->v; Uint g_sz = roots->sz; @@ -1221,56 +1282,6 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, return -1; } -/* - * HiPE native code stack scanning procedures: - * - fullsweep_nstack() - * - gensweep_nstack() - * - offset_nstack() - */ -#if defined(HIPE) - -#define GENSWEEP_NSTACK(p,old_htop,n_htop) \ - do { \ - Eterm *tmp_old_htop = old_htop; \ - Eterm *tmp_n_htop = n_htop; \ - gensweep_nstack((p), &tmp_old_htop, &tmp_n_htop); \ - old_htop = tmp_old_htop; \ - n_htop = tmp_n_htop; \ - } while(0) - -/* - * offset_nstack() can ignore the descriptor-based traversal the other - * nstack procedures use and simply call offset_heap_ptr() instead. - * This relies on two facts: - * 1. The only live non-Erlang terms on an nstack are return addresses, - * and they will be skipped thanks to the low/high range check. - * 2. Dead values, even if mistaken for pointers into the low/high area, - * can be offset safely since they won't be dereferenced. - * - * XXX: WARNING: If HiPE starts storing other non-Erlang values on the - * nstack, such as floats, then this will have to be changed. - */ -static ERTS_INLINE void offset_nstack(Process* p, Sint offs, - char* area, Uint area_size) -{ - if (p->hipe.nstack) { - ASSERT(p->hipe.nsp && p->hipe.nstend); - offset_heap_ptr(hipe_nstack_start(p), hipe_nstack_used(p), - offs, area, area_size); - } - else { - ASSERT(!p->hipe.nsp && !p->hipe.nstend); - } -} - -#else /* !HIPE */ - -#define fullsweep_nstack(p,n_htop) (n_htop) -#define GENSWEEP_NSTACK(p,old_htop,n_htop) do{}while(0) -#define offset_nstack(p,offs,area,area_size) do{}while(0) - -#endif /* HIPE */ - static void do_minor(Process *p, ErlHeapFragment *live_hf_end, char *mature, Uint mature_size, diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 0a396bb069..781bf024dd 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -332,8 +332,6 @@ erl_init(int ncpu, int node_tab_delete_delay, ErtsDbSpinCount db_spin_count) { - init_benchmarking(); - erts_bif_unique_init(); erts_init_monitors(); erts_init_time(time_correction, time_warp_mode); @@ -384,6 +382,7 @@ erl_init(int ncpu, erts_init_unicode(); /* after RE to get access to PCRE unicode */ erts_init_external(); erts_init_map(); + erts_beam_bif_load_init(); erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2); erts_late_init_process(); #if HAVE_ERTS_MSEG @@ -2250,7 +2249,42 @@ erl_start(int argc, char **argv) otp_ring0_pid = erl_first_process_otp("otp_ring0", NULL, 0, boot_argc, boot_argv); - (void) erl_system_process_otp(otp_ring0_pid, "erts_code_purger"); + { + /* + * The erts_code_purger and the erts_literal_area_collector + * system processes are *always* alive. If they terminate + * they bring the whole VM down. + */ + Eterm pid; + + pid = erl_system_process_otp(otp_ring0_pid, "erts_code_purger"); + erts_code_purger + = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, + internal_pid_index(pid)); + ASSERT(erts_code_purger && erts_code_purger->common.id == pid); + erts_proc_inc_refc(erts_code_purger); + +#ifdef ERTS_NEW_PURGE_STRATEGY + pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector"); + erts_literal_area_collector + = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, + internal_pid_index(pid)); + ASSERT(erts_literal_area_collector + && erts_literal_area_collector->common.id == pid); + erts_proc_inc_refc(erts_literal_area_collector); +#endif + +#ifdef ERTS_DIRTY_SCHEDULERS + pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker"); + erts_dirty_process_code_checker + = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, + internal_pid_index(pid)); + ASSERT(erts_dirty_process_code_checker + && erts_dirty_process_code_checker->common.id == pid); + erts_proc_inc_refc(erts_dirty_process_code_checker); +#endif + + } #ifdef ERTS_SMP erts_start_schedulers(); @@ -2345,8 +2379,6 @@ erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) { system_cleanup(flush_async); - save_statistics(); - if (erts_mtrace_enabled) erts_mtrace_exit((Uint32) n); diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 39c0617143..06266363b5 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -96,6 +96,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "dist_entry", "address" }, { "dist_entry_links", "address" }, { "code_write_permission", NULL }, + { "purge_state", NULL }, { "proc_status", "pid" }, { "proc_trace", "pid" }, { "ports_snapshot", NULL }, @@ -112,6 +113,9 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "export_tab", NULL }, { "fun_tab", NULL }, { "environ", NULL }, +#ifdef ERTS_NEW_PURGE_STRATEGY + { "release_literal_areas", NULL }, +#endif #endif { "efile_drv", "address" }, { "drv_ev_state_grow", NULL, }, diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index 8efc983f04..979a0040b0 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -497,18 +497,50 @@ Eterm erts_hashmap_from_array(ErtsHeapFactory* factory, Eterm *leafs, Uint n, return res; } +Eterm erts_map_from_ks_and_vs(ErtsHeapFactory *factory, Eterm *ks0, Eterm *vs0, Uint n) +{ + if (n < MAP_SMALL_MAP_LIMIT) { + Eterm *ks, *vs, *hp; + flatmap_t *mp; + Eterm keys; -Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n, + hp = erts_produce_heap(factory, 3 + 1 + (2 * n), 0); + keys = make_tuple(hp); + *hp++ = make_arityval(n); + ks = hp; + hp += n; + mp = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + vs = hp; + + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = n; + mp->keys = keys; + + sys_memcpy(ks, ks0, n * sizeof(Eterm)); + sys_memcpy(vs, vs0, n * sizeof(Eterm)); + + erts_validate_and_sort_flatmap(mp); + + return make_flatmap(mp); + } else { + return erts_hashmap_from_ks_and_vs(factory, ks0, vs0, n); + } + return THE_NON_VALUE; +} + + +Eterm erts_hashmap_from_ks_and_vs_extra(ErtsHeapFactory *factory, + Eterm *ks, Eterm *vs, Uint n, Eterm key, Eterm value) { Uint32 sw, hx; Uint i,sz; hxnode_t *hxns; - ErtsHeapFactory factory; Eterm *hp, res; sz = (key == THE_NON_VALUE) ? n : (n + 1); ASSERT(sz > MAP_SMALL_MAP_LIMIT); - hp = HAlloc(p, 2 * sz); + hp = erts_produce_heap(factory, 2 * sz, 0); /* create tmp hx values and leaf ptrs */ hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, sz * sizeof(hxnode_t)); @@ -531,12 +563,9 @@ Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n hxns[i].i = i; } - erts_factory_proc_init(&factory, p); - res = hashmap_from_unsorted_array(&factory, hxns, sz, 0); - erts_factory_close(&factory); + res = hashmap_from_unsorted_array(factory, hxns, sz, 0); erts_free(ERTS_ALC_T_TMP, (void *) hxns); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); return res; } @@ -1780,11 +1809,14 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) { /* the map will grow */ if (n >= MAP_SMALL_MAP_LIMIT) { + ErtsHeapFactory factory; HRelease(p, shp + MAP_HEADER_FLATMAP_SZ + n, shp); ks = flatmap_get_keys(mp); vs = flatmap_get_values(mp); - res = erts_hashmap_from_ks_and_vs_extra(p,ks,vs,n,key,value); + erts_factory_proc_init(&factory, p); + res = erts_hashmap_from_ks_and_vs_extra(&factory,ks,vs,n,key,value); + erts_factory_close(&factory); return res; } diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h index 8b5c9582ba..61a841f7f0 100644 --- a/erts/emulator/beam/erl_map.h +++ b/erts/emulator/beam/erl_map.h @@ -98,10 +98,12 @@ Eterm* hashmap_iterator_prev(struct ErtsWStack_* s); int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp); Eterm erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n, int reject_dupkeys); -#define erts_hashmap_from_ks_and_vs(P, KS, VS, N) \ - erts_hashmap_from_ks_and_vs_extra((P), (KS), (VS), (N), THE_NON_VALUE, THE_NON_VALUE); +#define erts_hashmap_from_ks_and_vs(F, KS, VS, N) \ + erts_hashmap_from_ks_and_vs_extra((F), (KS), (VS), (N), THE_NON_VALUE, THE_NON_VALUE); -Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n, +Eterm erts_map_from_ks_and_vs(ErtsHeapFactory *factory, Eterm *ks, Eterm *vs, Uint n); +Eterm erts_hashmap_from_ks_and_vs_extra(ErtsHeapFactory *factory, + Eterm *ks, Eterm *vs, Uint n, Eterm k, Eterm v); const Eterm *erts_maps_get(Eterm key, Eterm map); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 71ab92937d..e4c696ae3b 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -697,9 +697,6 @@ erts_send_message(Process* sender, #ifdef SHCOPY_SEND erts_shcopy_t info; #endif - BM_STOP_TIMER(system); - BM_MESSAGE(message,sender,receiver); - BM_START_TIMER(send); #ifdef USE_VM_PROBES *sender_name = *receiver_name = '\0'; @@ -720,7 +717,6 @@ erts_send_message(Process* sender, #ifdef USE_VM_PROBES Uint dt_utag_size = 0; #endif - BM_SWAP_TIMER(send,size); /* SHCOPY corrupts the heap between * copy_shared_calculate, and @@ -747,8 +743,6 @@ erts_send_message(Process* sender, #else msize = size_object(message); #endif - BM_SWAP_TIMER(size,send); - mp = erts_alloc_message_heap_state(receiver, &receiver_state, receiver_locks, @@ -760,8 +754,6 @@ erts_send_message(Process* sender, &hp, &ohp); - BM_SWAP_TIMER(send,copy); - #ifdef SHCOPY_SEND if (is_not_immed(message)) message = copy_shared_perform(message, msize, &info, &hp, ohp); @@ -792,9 +784,6 @@ erts_send_message(Process* sender, msize, tok_label, tok_lastcnt, tok_serial); } #endif - BM_MESSAGE_COPIED(msize); - BM_SWAP_TIMER(copy,send); - } else { Eterm *hp; @@ -803,22 +792,18 @@ erts_send_message(Process* sender, msize = 0; } else { - BM_SWAP_TIMER(send,size); #ifdef SHCOPY_SEND INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else msize = size_object(message); #endif - BM_SWAP_TIMER(size,send); - mp = erts_alloc_message_heap_state(receiver, &receiver_state, receiver_locks, msize, &hp, &ohp); - BM_SWAP_TIMER(send,copy); #ifdef SHCOPY_SEND if (is_not_immed(message)) message = copy_shared_perform(message, msize, &info, &hp, ohp); @@ -827,8 +812,6 @@ erts_send_message(Process* sender, if (is_not_immed(message)) message = copy_struct(message, msize, &hp, ohp); #endif - BM_MESSAGE_COPIED(msz); - BM_SWAP_TIMER(copy,send); } #ifdef USE_VM_PROBES DTRACE6(message_send, sender_name, receiver_name, @@ -846,8 +829,6 @@ erts_send_message(Process* sender, mp, message, sender->common.id); - BM_SWAP_TIMER(send,system); - return res; } @@ -1861,3 +1842,20 @@ void erts_factory_undo(ErtsHeapFactory* factory) factory->heap_frags = NULL; #endif } + +Uint +erts_mbuf_size(Process *p) +{ + Uint sz = 0; + ErlHeapFragment* bp; + ErtsMessage* mp; + + for (bp = p->mbuf; bp; bp = bp->next) + sz += bp->used_size; + + for (mp = p->msg_frag; mp; mp = mp->next) + for (bp = erts_message_to_heap_frag(mp); bp; bp = bp->next) + sz += bp->used_size; + + return sz; +} diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 6df969367b..42ed14e69c 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -499,4 +499,15 @@ erts_msgq_replace_msg_ref(ErlMessageQueue *msgq, ErtsMessage *newp, ErtsMessage #endif +Uint erts_mbuf_size(Process *p); +#if defined(DEBUG) || 0 +# define ERTS_CHK_MBUF_SZ(P) \ + do { \ + Uint actual_mbuf_sz__ = erts_mbuf_size((P)); \ + ERTS_ASSERT((P)->mbuf_sz >= actual_mbuf_sz__); \ + } while (0) +#else +# define ERTS_CHK_MBUF_SZ(P) ((void) 1) +#endif + #endif diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c index 544bc8b983..421445fbad 100644 --- a/erts/emulator/beam/erl_msacc.c +++ b/erts/emulator/beam/erl_msacc.c @@ -40,10 +40,11 @@ #include "erl_bif_unique.h" #include "erl_map.h" #include "erl_msacc.h" +#include "erl_bif_table.h" #if ERTS_ENABLE_MSACC -static Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, Eterm **hpp, Uint *szp); +static Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, ErtsHeapFactory *factory); static void erts_msacc_reset(ErtsMsAcc *msacc); static ErtsMsAcc* get_msacc(void); @@ -52,7 +53,9 @@ erts_tsd_key_t ERTS_WRITE_UNLIKELY(erts_msacc_key); #else ErtsMsAcc *ERTS_WRITE_UNLIKELY(erts_msacc) = NULL; #endif +#ifndef ERTS_MSACC_ALWAYS_ON int ERTS_WRITE_UNLIKELY(erts_msacc_enabled); +#endif static Eterm *erts_msacc_state_atoms = NULL; static erts_rwmtx_t msacc_mutex; @@ -62,6 +65,12 @@ static ErtsMsAcc *msacc_unmanaged = NULL; static Uint msacc_unmanaged_count = 0; #endif +#if ERTS_MSACC_STATE_COUNT < MAP_SMALL_MAP_LIMIT +#define DEFAULT_MSACC_MSG_SIZE (3 + 1 + ERTS_MSACC_STATE_COUNT * 2 + 3 + REF_THING_SIZE) +#else +#define DEFAULT_MSACC_MSG_SIZE (3 + ERTS_MSACC_STATE_COUNT * 3 + 3 + REF_THING_SIZE) +#endif + /* we have to split initiation as atoms are not inited in early init */ void erts_msacc_early_init(void) { #ifndef ERTS_MSACC_ALWAYS_ON @@ -88,7 +97,8 @@ void erts_msacc_init(void) { void erts_msacc_init_thread(char *type, int id, int managed) { ErtsMsAcc *msacc; - msacc = erts_alloc(ERTS_ALC_T_MSACC, sizeof(ErtsMsAcc)); + msacc = erts_alloc(ERTS_ALC_T_MSACC, sizeof(ErtsMsAcc) + + sizeof(ErtsMsAccPerfCntr) * ERTS_MSACC_STATE_COUNT); msacc->type = strdup(type); msacc->id = make_small(id); @@ -122,79 +132,80 @@ void erts_msacc_init_thread(char *type, int id, int managed) { #endif } +#ifdef ERTS_MSACC_EXTENDED_STATES + +void erts_msacc_set_bif_state(ErtsMsAcc *__erts_msacc_cache, Eterm mod, void *fn) { + +#ifdef ERTS_MSACC_EXTENDED_BIFS +#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num) \ + if (fn == &FuncAddr) { \ + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATIC_STATE_COUNT + Num); \ + } else +#include "erl_bif_list.h" +#undef BIF_LIST + { /* The last else in the macro expansion, + this happens for internal bifs, i.e. traps etc */ + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF); + } +#else + if (mod == am_ets) { + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS); + } else { + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF); + } +#endif +} + +#endif + /* * Creates a structure looking like this * #{ type => scheduler, id => 1, counters => #{ State1 => Counter1 ... StateN => CounterN}} */ static -Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, Eterm **hpp, Uint *szp) { +Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, ErtsHeapFactory *factory) { + Uint sz = 0; + Eterm *hp, cvs[ERTS_MSACC_STATE_COUNT]; + Eterm key, state_map; int i; - Eterm *hp; - Eterm key, state_key, state_map; - Eterm res = THE_NON_VALUE; - flatmap_t *map; - - if (szp) { - *szp += MAP_HEADER_FLATMAP_SZ + 1 + 2*(3); - *szp += MAP_HEADER_FLATMAP_SZ + 1 + 2*(ERTS_MSACC_STATE_COUNT); - for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { - (void)erts_bld_sint64(NULL,szp,(Sint64)msacc->perf_counters[i]); + flatmap_t *map; + + hp = erts_produce_heap(factory, 4, 0); + key = TUPLE3(hp,am_counters,am_id,am_type); + + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { + cvs[i] = erts_bld_sint64(NULL, &sz,(Sint64)msacc->counters[i].pc); #ifdef ERTS_MSACC_STATE_COUNTERS - (void)erts_bld_uint64(NULL,szp,msacc->state_counters[i]); - *szp += 3; /* tuple to put state+perf counter in */ + erts_bld_uint64(NULL,&sz,msacc->counters[i].sc); + sz += 3; #endif - } } - if (hpp) { - Eterm counters[ERTS_MSACC_STATE_COUNT]; - hp = *hpp; - for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { - Eterm counter = erts_bld_sint64(&hp,NULL,(Sint64)msacc->perf_counters[i]); + hp = erts_produce_heap(factory, sz, 0); + + for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { + cvs[i] = erts_bld_sint64(&hp,NULL,(Sint64)msacc->counters[i].pc); #ifdef ERTS_MSACC_STATE_COUNTERS - Eterm counter__ = erts_bld_uint64(&hp,NULL,msacc->state_counters[i]); - counters[i] = TUPLE2(hp,counter,counter__); - hp += 3; -#else - counters[i] = counter; + Eterm counter__ = erts_bld_uint64(&hp,NULL,msacc->counters[i].sc); + cvs[i] = TUPLE2(hp,cvs[i],counter__); + hp += 3; #endif - } - - key = TUPLE3(hp,am_counters,am_id,am_type); - hp += 4; - - state_key = make_tuple(hp); - hp[0] = make_arityval(ERTS_MSACC_STATE_COUNT); - - for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) - hp[1+i] = erts_msacc_state_atoms[i]; - hp += 1 + ERTS_MSACC_STATE_COUNT; - - map = (flatmap_t*)hp; - hp += MAP_HEADER_FLATMAP_SZ; - map->thing_word = MAP_HEADER_FLATMAP; - map->size = ERTS_MSACC_STATE_COUNT; - map->keys = state_key; - for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) - hp[i] = counters[i]; - hp += ERTS_MSACC_STATE_COUNT; - state_map = make_flatmap(map); - - map = (flatmap_t*)hp; - hp += MAP_HEADER_FLATMAP_SZ; - map->thing_word = MAP_HEADER_FLATMAP; - map->size = 3; - map->keys = key; - hp[0] = state_map; - hp[1] = msacc->id; - hp[2] = am_atom_put(msacc->type,strlen(msacc->type)); - hp += 3; - - *hpp = hp; - res = make_flatmap(map); } - return res; + state_map = erts_map_from_ks_and_vs(factory, erts_msacc_state_atoms, cvs, + ERTS_MSACC_STATE_COUNT); + + hp = erts_produce_heap(factory, MAP_HEADER_FLATMAP_SZ + 3, 0); + map = (flatmap_t*)hp; + hp += MAP_HEADER_FLATMAP_SZ; + map->thing_word = MAP_HEADER_FLATMAP; + map->size = 3; + map->keys = key; + hp[0] = state_map; + hp[1] = msacc->id; + hp[2] = am_atom_put(msacc->type,strlen(msacc->type)); + + return make_flatmap(map); } typedef struct { @@ -222,40 +233,31 @@ static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); Process *rp = msaccrp->proc; ErtsMessage *msgp = NULL; - Eterm **hpp, *hp; + Eterm *hp; Eterm ref_copy = NIL, msg; - Uint sz, *szp; - ErlOffHeap *ohp = NULL; ErtsProcLocks rp_locks = (esdp && msaccrp->req_sched == esdp->no ? ERTS_PROC_LOCK_MAIN : 0); + ErtsHeapFactory factory; - sz = 0; - hpp = NULL; - szp = &sz; - - if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx); - - while (1) { - if (hpp) - ref_copy = STORE_NC(hpp, ohp, msaccrp->ref); - else - *szp += REF_THING_SIZE; - - if (msaccrp->action != ERTS_MSACC_GATHER) - msg = ref_copy; - else { - msg = erts_msacc_gather_stats(msacc, hpp, szp); - msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); - } - if (hpp) - break; - - msgp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); - hpp = &hp; - szp = NULL; - } + if (msaccrp->action == ERTS_MSACC_GATHER) { + + msgp = erts_factory_message_create(&factory, rp, &rp_locks, DEFAULT_MSACC_MSG_SIZE); + + if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx); - if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx); + hp = erts_produce_heap(&factory, REF_THING_SIZE + 3 /* tuple */, 0); + ref_copy = STORE_NC(&hp, &msgp->hfrag.off_heap, msaccrp->ref); + msg = erts_msacc_gather_stats(msacc, &factory); + msg = TUPLE2(hp, ref_copy, msg); + + if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx); + + erts_factory_close(&factory); + } else { + ErlOffHeap *ohp = NULL; + msgp = erts_alloc_message_heap(rp, &rp_locks, REF_THING_SIZE, &hp, &ohp); + msg = STORE_NC(&hp, &msgp->hfrag.off_heap, msaccrp->ref); + } erts_queue_message(rp, rp_locks, msgp, msg, am_system); @@ -308,9 +310,9 @@ static void erts_msacc_reset(ErtsMsAcc *msacc) { if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx); for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) { - msacc->perf_counters[i] = 0; + msacc->counters[i].pc = 0; #ifdef ERTS_MSACC_STATE_COUNTERS - msacc->state_counters[i] = 0; + msacc->counters[i].sc = 0; #endif } @@ -415,7 +417,7 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads) ErtsSysPerfCounter perf_counter; /* if enabled update stats */ perf_counter = erts_sys_perf_counter(); - unmanaged[i]->perf_counters[unmanaged[i]->state] += + unmanaged[i]->counters[unmanaged[i]->state].pc += perf_counter - unmanaged[i]->perf_counter; unmanaged[i]->perf_counter = perf_counter; } @@ -454,7 +456,7 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads) for (msacc = msacc_unmanaged; msacc != NULL; msacc = msacc->next) { erts_mtx_lock(&msacc->mtx); perf_counter = erts_sys_perf_counter(); - msacc->perf_counters[msacc->state] += perf_counter - msacc->perf_counter; + msacc->counters[msacc->state].pc += perf_counter - msacc->perf_counter; msacc->perf_counter = 0; erts_mtx_unlock(&msacc->mtx); } diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index 284388f7aa..ad7c8c5eee 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -35,6 +35,10 @@ this reduces overhead a little bit when profiling */ /* #define ERTS_MSACC_ALWAYS_ON 1 */ +/* Uncomment this to keep individual stats for all + of the bifs when extended states is enabled */ +/* #define ERTS_MSACC_EXTENDED_BIFS 1 */ + #define ERTS_MSACC_DISABLE 0 #define ERTS_MSACC_ENABLE 1 #define ERTS_MSACC_RESET 2 @@ -92,7 +96,13 @@ static char *erts_msacc_states[] = { #define ERTS_MSACC_STATE_SLEEP 13 #define ERTS_MSACC_STATE_TIMERS 14 -#define ERTS_MSACC_STATE_COUNT 15 +#define ERTS_MSACC_STATIC_STATE_COUNT 15 + +#ifdef ERTS_MSACC_EXTENDED_BIFS +#define ERTS_MSACC_STATE_COUNT (ERTS_MSACC_STATIC_STATE_COUNT + BIF_SIZE) +#else +#define ERTS_MSACC_STATE_COUNT ERTS_MSACC_STATIC_STATE_COUNT +#endif #if ERTS_MSACC_STATE_STRINGS static char *erts_msacc_states[] = { @@ -111,22 +121,26 @@ static char *erts_msacc_states[] = { "send", "sleep", "timers" +#ifdef ERTS_MSACC_EXTENDED_BIFS +#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num) \ + ,"bif_" #Mod "_" #Func "_" #Arity +#include "erl_bif_list.h" +#undef BIF_LIST +#endif }; #endif #endif typedef struct erl_msacc_t_ ErtsMsAcc; - -struct erl_msacc_t_ { - - /* the the values below are protected by mtx iff unmanaged = 1 */ - ErtsSysPerfCounter perf_counter; - ErtsSysPerfCounter perf_counters[ERTS_MSACC_STATE_COUNT]; +typedef struct erl_msacc_p_cnt_t_ { + ErtsSysPerfCounter pc; #ifdef ERTS_MSACC_STATE_COUNTERS - Uint64 state_counters[ERTS_MSACC_STATE_COUNT]; + Uint64 sc; #endif - Uint state; +} ErtsMsAccPerfCntr; + +struct erl_msacc_t_ { /* protected by msacc_mutex in erl_msacc.c, and should be constant */ int unmanaged; @@ -135,12 +149,16 @@ struct erl_msacc_t_ { erts_tid_t tid; Eterm id; char *type; + + /* the the values below are protected by mtx iff unmanaged = 1 */ + ErtsSysPerfCounter perf_counter; + Uint state; + ErtsMsAccPerfCntr counters[]; + }; #if ERTS_ENABLE_MSACC -#define ERTS_MSACC_INLINE ERTS_GLB_INLINE - #ifdef USE_THREADS extern erts_tsd_key_t erts_msacc_key; #else @@ -276,20 +294,20 @@ void erts_msacc_init_thread(char *type, int id, int liberty); #define ERTS_MSACC_PUSH_AND_SET_STATE_M(state) \ ERTS_MSACC_PUSH_STATE_M(); ERTS_MSACC_SET_STATE_CACHED_M(state) -ERTS_MSACC_INLINE +ERTS_GLB_INLINE void erts_msacc_set_state_um__(ErtsMsAcc *msacc,Uint state,int increment); -ERTS_MSACC_INLINE +ERTS_GLB_INLINE void erts_msacc_set_state_m__(ErtsMsAcc *msacc,Uint state,int increment); -ERTS_MSACC_INLINE +ERTS_GLB_INLINE Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc); -ERTS_MSACC_INLINE +ERTS_GLB_INLINE Uint erts_msacc_get_state_m__(ErtsMsAcc *msacc); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -ERTS_MSACC_INLINE +ERTS_GLB_INLINE Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc) { Uint state; if (msacc->unmanaged) @@ -300,12 +318,12 @@ Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc) { return state; } -ERTS_MSACC_INLINE +ERTS_GLB_INLINE Uint erts_msacc_get_state_m__(ErtsMsAcc *msacc) { return msacc->state; } -ERTS_MSACC_INLINE +ERTS_GLB_INLINE void erts_msacc_set_state_um__(ErtsMsAcc *msacc, Uint new_state, int increment) { if (ERTS_UNLIKELY(msacc->unmanaged)) { erts_mtx_lock(&msacc->mtx); @@ -322,7 +340,7 @@ void erts_msacc_set_state_um__(ErtsMsAcc *msacc, Uint new_state, int increment) erts_mtx_unlock(&msacc->mtx); } -ERTS_MSACC_INLINE +ERTS_GLB_INLINE void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) { ErtsSysPerfCounter prev_perf_counter; Sint64 diff; @@ -334,9 +352,9 @@ void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) { msacc->perf_counter = erts_sys_perf_counter(); diff = msacc->perf_counter - prev_perf_counter; ASSERT(diff >= 0); - msacc->perf_counters[msacc->state] += diff; + msacc->counters[msacc->state].pc += diff; #ifdef ERTS_MSACC_STATE_COUNTERS - msacc->state_counters[new_state] += increment; + msacc->counters[new_state].sc += increment; #endif msacc->state = new_state; } @@ -364,7 +382,7 @@ void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) { #define ERTS_MSACC_SET_STATE_CACHED_M(state) #define ERTS_MSACC_POP_STATE_M() #define ERTS_MSACC_PUSH_AND_SET_STATE_M(state) - +#define ERTS_MSACC_SET_BIF_STATE_CACHED_X(Mod,Addr) #endif /* ERTS_ENABLE_MSACC */ @@ -385,9 +403,13 @@ void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) { #define ERTS_MSACC_SET_STATE_CACHED_M_X(state) #define ERTS_MSACC_POP_STATE_M_X() #define ERTS_MSACC_PUSH_AND_SET_STATE_M_X(state) +#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_M_X(state) +#define ERTS_MSACC_SET_BIF_STATE_CACHED_X(Mod,Addr) #else +void erts_msacc_set_bif_state(ErtsMsAcc *msacc, Eterm mod, void *addr); + #define ERTS_MSACC_PUSH_STATE_X() ERTS_MSACC_PUSH_STATE() #define ERTS_MSACC_POP_STATE_X() ERTS_MSACC_POP_STATE() #define ERTS_MSACC_SET_STATE_X(state) ERTS_MSACC_SET_STATE(state) @@ -403,6 +425,9 @@ void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) { #define ERTS_MSACC_SET_STATE_CACHED_M_X(state) ERTS_MSACC_SET_STATE_CACHED_M(state) #define ERTS_MSACC_POP_STATE_M_X() ERTS_MSACC_POP_STATE_M() #define ERTS_MSACC_PUSH_AND_SET_STATE_M_X(state) ERTS_MSACC_PUSH_AND_SET_STATE_M(state) +#define ERTS_MSACC_SET_BIF_STATE_CACHED_X(Mod,Addr) \ + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) \ + erts_msacc_set_bif_state(__erts_msacc_cache, Mod, Addr) #endif /* !ERTS_MSACC_EXTENDED_STATES */ diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 23931f0e54..559b4017e7 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -120,8 +120,10 @@ execution_state(ErlNifEnv *env, Process **c_pp, int *schedp) else { Process *c_p = env->proc; - if (!(c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC)) - ASSERT(is_scheduler() > 0); + if (!(c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC)) { + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + } else { c_p = env->proc->next; ASSERT(is_scheduler() < 0); @@ -154,7 +156,9 @@ static Eterm* alloc_heap_heavy(ErlNifEnv* env, size_t need, Eterm* hp) HEAP_TOP(env->proc) = env->hp; } else { - env->heap_frag->used_size = hp - env->heap_frag->mem; + Uint usz = env->hp - env->heap_frag->mem; + env->proc->mbuf_sz += usz - env->heap_frag->used_size; + env->heap_frag->used_size = usz; ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } hp = erts_heap_alloc(env->proc, need, MIN_HEAP_FRAG_SZ); @@ -208,11 +212,16 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, #endif } +static void full_cache_env(ErlNifEnv *env); +static void cache_env(ErlNifEnv* env); +static void full_flush_env(ErlNifEnv *env); +static void flush_env(ErlNifEnv* env); + +#ifdef ERTS_DIRTY_SCHEDULERS void erts_pre_dirty_nif(ErtsSchedulerData *esdp, - ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, - Process* tracee) + ErlNifEnv* env, Process* p, + struct erl_module_nif* mod_nif) { -#ifdef ERTS_DIRTY_SCHEDULERS Process *sproc; #ifdef DEBUG erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); @@ -223,7 +232,7 @@ void erts_pre_dirty_nif(ErtsSchedulerData *esdp, ASSERT(esdp); #endif - erts_pre_nif(env, p, mod_nif, tracee); + erts_pre_nif(env, p, mod_nif, NULL); sproc = esdp->dirty_shadow_process; ASSERT(sproc); @@ -235,22 +244,10 @@ void erts_pre_dirty_nif(ErtsSchedulerData *esdp, sproc->next = p; sproc->common.id = p->common.id; - sproc->htop = p->htop; - sproc->stop = p->stop; - sproc->hend = p->hend; - sproc->heap = p->heap; - sproc->abandoned_heap = p->abandoned_heap; - sproc->heap_sz = p->heap_sz; - sproc->high_water = p->high_water; - sproc->old_hend = p->old_hend; - sproc->old_htop = p->old_htop; - sproc->old_heap = p->old_heap; - sproc->mbuf = NULL; - sproc->mbuf_sz = 0; - ERTS_INIT_OFF_HEAP(&sproc->off_heap); env->proc = sproc; -#endif + full_cache_env(env); } +#endif /* Temporary object header, auto-deallocated when NIF returns * or when independent environment is cleared. @@ -274,32 +271,37 @@ static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env) void erts_post_nif(ErlNifEnv* env) { erts_unblock_fpe(env->fpe_was_unmasked); + full_flush_env(env); + free_tmp_objs(env); + env->exiting = ERTS_PROC_IS_EXITING(env->proc); +} #ifdef ERTS_DIRTY_SCHEDULERS - if (!(env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)) +void erts_post_dirty_nif(ErlNifEnv* env) +{ + Process *c_p; + ASSERT(env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(env->proc->next); + erts_unblock_fpe(env->fpe_was_unmasked); + full_flush_env(env); + free_tmp_objs(env); + c_p = env->proc->next; + env->exiting = ERTS_PROC_IS_EXITING(c_p); + ERTS_VBUMP_ALL_REDS(c_p); +} #endif - { - ASSERT(is_scheduler() > 0); - if (env->heap_frag == NULL) { - ASSERT(env->hp_end == HEAP_LIMIT(env->proc)); - ASSERT(env->hp >= HEAP_TOP(env->proc)); - ASSERT(env->hp <= HEAP_LIMIT(env->proc)); - HEAP_TOP(env->proc) = env->hp; - } - else { - ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); - env->heap_frag->used_size = env->hp - env->heap_frag->mem; - ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); - } - env->exiting = ERTS_PROC_IS_EXITING(env->proc); - } + +static void full_flush_env(ErlNifEnv* env) +{ #ifdef ERTS_DIRTY_SCHEDULERS - else { /* Dirty nif call using shadow process struct */ + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + /* Dirty nif call using shadow process struct */ Process *c_p = env->proc->next; ASSERT(is_scheduler() < 0); ASSERT(env->proc->common.id == c_p->common.id); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); if (!env->heap_frag) { ASSERT(env->hp_end == HEAP_LIMIT(c_p)); @@ -308,11 +310,14 @@ void erts_post_nif(ErlNifEnv* env) HEAP_TOP(c_p) = env->hp; } else { + Uint usz; ASSERT(env->hp_end != HEAP_LIMIT(c_p)); ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); HEAP_TOP(c_p) = HEAP_TOP(env->proc); - env->heap_frag->used_size = env->hp - env->heap_frag->mem; + usz = env->hp - env->heap_frag->mem; + env->proc->mbuf_sz += usz - env->heap_frag->used_size; + env->heap_frag->used_size = usz; ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); @@ -339,11 +344,48 @@ void erts_post_nif(ErlNifEnv* env) } c_p->off_heap.overhead += env->proc->off_heap.overhead; - env->exiting = ERTS_PROC_IS_EXITING(c_p); - BUMP_ALL_REDS(c_p); + return; } #endif - free_tmp_objs(env); + + flush_env(env); +} + +static void full_cache_env(ErlNifEnv* env) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + /* Dirty nif call using shadow process struct */ + Process *sproc = env->proc; + Process *c_p = sproc->next; + ASSERT(c_p); + ASSERT(is_scheduler() < 0); + ASSERT(env->proc->common.id == c_p->common.id); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + sproc->htop = c_p->htop; + sproc->stop = c_p->stop; + sproc->hend = c_p->hend; + sproc->heap = c_p->heap; + sproc->abandoned_heap = c_p->abandoned_heap; + sproc->heap_sz = c_p->heap_sz; + sproc->high_water = c_p->high_water; + sproc->old_hend = c_p->old_hend; + sproc->old_htop = c_p->old_htop; + sproc->old_heap = c_p->old_heap; + sproc->mbuf = NULL; + sproc->mbuf_sz = 0; + ERTS_INIT_OFF_HEAP(&sproc->off_heap); + + env->hp_end = HEAP_LIMIT(c_p); + env->hp = HEAP_TOP(c_p); + env->heap_frag = NULL; + return; + } +#endif + + cache_env(env); } /* Flush out our cached heap pointers to allow an ordinary HAlloc @@ -357,9 +399,12 @@ static void flush_env(ErlNifEnv* env) HEAP_TOP(env->proc) = env->hp; } else { + Uint usz; ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); - env->heap_frag->used_size = env->hp - env->heap_frag->mem; + usz = env->hp - env->heap_frag->mem; + env->proc->mbuf_sz += usz - env->heap_frag->used_size; + env->heap_frag->used_size = usz; ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } } @@ -600,17 +645,32 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, if (scheduler > 0) { /* Normal scheduler */ rp = erts_proc_lookup(receiver); - if (c_p == rp) - rp_locks = ERTS_PROC_LOCK_MAIN; + if (!rp) + return 0; } else { - if (c_p && ERTS_PROC_IS_EXITING(c_p)) - return 0; - rp = erts_pid2proc_opt(c_p, 0, receiver, rp_locks, + if (c_p) { + ASSERT(scheduler < 0); /* Dirty scheduler */ + if (ERTS_PROC_IS_EXITING(c_p)) + return 0; + + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + } + } + + rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + receiver, rp_locks, ERTS_P2P_FLG_INC_REFC); + if (!rp) { + if (c_p && (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + return 0; + } } - if (rp == NULL) - return 0; + + if (c_p == rp) + rp_locks = ERTS_PROC_LOCK_MAIN; if (menv) { flush_env(msg_env); @@ -631,9 +691,9 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ErlOffHeap *ohp; Eterm *hp; if (env && !env->tracee) { - flush_env(env); + full_flush_env(env); mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); - cache_env(env); + full_cache_env(env); } else { erts_aint_t state = erts_smp_atomic32_read_nob(&rp->state); @@ -656,8 +716,11 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, if (!env || !env->tracee) { - if (c_p && IS_TRACED_FL(c_p, F_TRACE_SEND)) + if (c_p && IS_TRACED_FL(c_p, F_TRACE_SEND)) { + full_flush_env(env); trace_send(c_p, receiver, msg); + full_cache_env(env); + } } #ifdef ERTS_SMP else { @@ -690,10 +753,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, erts_smp_proc_trylock(rp, ERTS_PROC_LOCK_MSGQ) == EBUSY) { if (!msgq) { -#ifdef ERTS_SMP - ErtsThrPrgrDelayHandle dhndl; -#endif - msgq = erts_alloc(ERTS_ALC_T_TRACE_MSG_QUEUE, sizeof(ErlTraceMessageQueue)); msgq->receiver = receiver; @@ -707,15 +766,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE); -#ifdef ERTS_SMP - if (!scheduler) - dhndl = erts_thr_progress_unmanaged_delay(); -#endif - erts_schedule_flush_trace_messages(t_p->common.id); -#ifdef ERTS_SMP - if (!scheduler) - erts_thr_progress_unmanaged_continue(dhndl); -#endif + erts_schedule_flush_trace_messages(t_p, 0); } else { msgq->len++; *msgq->last = mp; @@ -740,6 +791,8 @@ done: rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks & ~lc_locks) erts_smp_proc_unlock(rp, rp_locks & ~lc_locks); + if (c_p && (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); #endif if (scheduler <= 0) erts_proc_dec_refc(rp); @@ -2215,7 +2268,7 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent) * NIF exports need a few more items than the Export struct provides, * including the erl_module_nif* and a NIF function pointer, so the * NifExport below adds those. The Export member must be first in the - * struct. The saved_mfa, exception_thrown, saved_argc, rootset_extra, and + * struct. The saved_current, exception_thrown, saved_argc, rootset_extra, and * rootset members are used to track the MFA, any pending exception, and * arguments of the top NIF in case a chain of one or more * enif_schedule_nif() calls results in an exception, since in that case @@ -2229,7 +2282,7 @@ typedef struct { Export exp; struct erl_module_nif* m; NativeFunPtr fp; - Eterm saved_mfa[3]; + BeamInstr *saved_current; int exception_thrown; int saved_argc; int rootset_extra; @@ -2340,9 +2393,8 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec reg[i] = (Eterm) argv[i]; } if (need_save) { - ep->saved_mfa[0] = proc->current[0]; - ep->saved_mfa[1] = proc->current[1]; - ep->saved_mfa[2] = proc->current[2]; + ASSERT(proc->current); + ep->saved_current = proc->current; ep->saved_argc = argc; } proc->i = (BeamInstr*) ep->exp.addressv[0]; @@ -2365,21 +2417,21 @@ static void restore_nif_mfa(Process* proc, NifExport* ep, int exception) { int i; - Eterm* reg = erts_proc_sched_data(proc)->x_reg_array; ERTS_SMP_LC_ASSERT(!(proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)); ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc) & ERTS_PROC_LOCK_MAIN); - proc->current[0] = ep->saved_mfa[0]; - proc->current[1] = ep->saved_mfa[1]; - proc->current[2] = ep->saved_mfa[2]; - if (exception) + ASSERT(ep->saved_current != &ep->exp.code[0]); + proc->current = ep->saved_current; + ep->saved_current = NULL; + if (exception) { + Eterm* reg = erts_proc_sched_data(proc)->x_reg_array; for (i = 0; i < ep->saved_argc; i++) reg[i] = ep->rootset[i+1]; + } ep->saved_argc = 0; - ep->saved_mfa[0] = THE_NON_VALUE; } #ifdef ERTS_DIRTY_SCHEDULERS @@ -2454,6 +2506,7 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) */ ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep && fp); + ep->fp = NULL; erts_smp_atomic32_read_band_mb(&proc->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC | ERTS_PSFLG_DIRTY_IO_PROC)); @@ -2538,7 +2591,7 @@ schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[ } ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - need_save = (ep == NULL || is_non_value(ep->saved_mfa[0])); + need_save = (ep == NULL || !ep->saved_current); result = init_nif_sched_data(env, execute_dirty_nif, fp, need_save, argc, argv); if (scheduler <= 0) erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); @@ -2616,7 +2669,7 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, } ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - need_save = (ep == NULL || is_non_value(ep->saved_mfa[0])); + need_save = (ep == NULL || !ep->saved_current); if (flags) { #ifdef ERTS_DIRTY_SCHEDULERS @@ -3502,6 +3555,7 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, struct enif_environment_t env; ErlHeapFragment *orig_hf = MBUF(p); ErlOffHeap orig_oh = MSO(p); + Eterm *orig_htop = HEAP_TOP(p); ASSERT(is_internal_pid(p->common.id)); MBUF(p) = NULL; clear_offheap(&MSO(p)); @@ -3523,6 +3577,7 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, /* restore original heap fragment list */ MBUF(p) = orig_hf; MSO(p) = orig_oh; + HEAP_TOP(p) = orig_htop; } else { /* Nif call was done without a process context, so we create a phony one. */ diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 66f22979ad..bc59147c6c 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -448,6 +448,9 @@ int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP; typedef enum { ERTS_PSTT_GC, /* Garbage Collect */ ERTS_PSTT_CPC, /* Check Process Code */ +#ifdef ERTS_NEW_PURGE_STRATEGY + ERTS_PSTT_CLA, /* Copy Literal Area */ +#endif ERTS_PSTT_COHMQ, /* Change off heap message queue */ ERTS_PSTT_FTMQ /* Flush trace msg queue */ } ErtsProcSysTaskType; @@ -6671,14 +6674,24 @@ erts_schedule_process(Process *p, erts_aint32_t state, ErtsProcLocks locks) } static int -schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) +schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, + erts_aint32_t *fail_state_p) { int res; int locked; ErtsProcSysTaskQs *stqs, *free_stqs; - erts_aint32_t state, a, n, enq_prio; + erts_aint32_t fail_state, state, a, n, enq_prio; int enqueue; /* < 0 -> use proxy */ unsigned int prof_runnable_procs; + int strict_fail_state; + + fail_state = *fail_state_p; + /* + * If fail state something other than just exiting process, + * ensure that the task wont be scheduled when the + * receiver is in the failure state. + */ + strict_fail_state = fail_state != ERTS_PSFLG_EXITING; res = 1; /* prepare for success */ st->next = st->prev = st; /* Prep for empty prio queue */ @@ -6705,7 +6718,8 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); state = erts_smp_atomic32_read_nob(&p->state); - if (state & ERTS_PSFLG_EXITING) { + if (state & fail_state) { + *fail_state_p = (state & fail_state); free_stqs = stqs; res = 0; goto cleanup; @@ -6759,7 +6773,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) /* Status lock prevents out of order "runnable proc" trace msgs */ ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - if (!prof_runnable_procs) { + if (!prof_runnable_procs && !strict_fail_state) { erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); locked = 0; } @@ -6770,6 +6784,11 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) erts_aint32_t e; n = e = a; + if (strict_fail_state && (a & fail_state)) { + *fail_state_p = (a & fail_state); + goto cleanup; + } + if (a & ERTS_PSFLG_FREE) goto cleanup; /* We don't want to schedule free processes... */ @@ -8177,6 +8196,8 @@ sched_dirty_cpu_thread_func(void *vesdp) esdp->thr_id += erts_no_schedulers; + erts_msacc_init_thread("dirty_cpu_scheduler", no, 0); + erts_thr_progress_register_unmanaged_thread(&callbacks); #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -8222,6 +8243,8 @@ sched_dirty_io_thread_func(void *vesdp) esdp->thr_id += erts_no_schedulers + erts_no_dirty_cpu_schedulers; + erts_msacc_init_thread("dirty_io_scheduler", no, 0); + erts_thr_progress_register_unmanaged_thread(&callbacks); #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -9088,6 +9111,27 @@ resume_process_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE +erts_internal_is_process_executing_dirty_1(BIF_ALIST_1) +{ + if (is_not_internal_pid(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); +#ifdef ERTS_DIRTY_SCHEDULERS + else { + Process *rp = erts_proc_lookup(BIF_ARG_1); + if (rp) { + erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state); + if (state & (ERTS_PSFLG_DIRTY_RUNNING + |ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + BIF_RET(am_true); + } + } + } +#endif + BIF_RET(am_false); +} + + Uint erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched) { @@ -9364,6 +9408,14 @@ erts_set_process_priority(Process *p, Eterm value) } } +#ifdef __WIN32__ +Sint64 +erts_time2reds(ErtsMonotonicTime start, ErtsMonotonicTime end) +{ + return ERTS_TIME2REDS_IMPL__(start, end); +} +#endif + static int scheduler_gc_proc(Process *c_p, int reds_left) { @@ -9494,15 +9546,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) p->reds += actual_reds; -#ifdef ERTS_SMP - erts_smp_proc_lock(p, ERTS_PROC_LOCK_TRACE); - if (p->trace_msg_q) { - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_TRACE); - erts_schedule_flush_trace_messages(p->common.id); - } else - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_TRACE); -#endif - state = erts_smp_atomic32_read_nob(&p->state); if (IS_TRACED(p)) { @@ -9522,7 +9565,15 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } } - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + +#ifdef ERTS_SMP + if (p->trace_msg_q) { + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_schedule_flush_trace_messages(p, 1); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + } +#endif /* have to re-read state after taking lock */ state = erts_smp_atomic32_read_nob(&p->state); @@ -9530,9 +9581,11 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) #ifdef ERTS_SMP if (is_normal_sched && (state & ERTS_PSFLG_PENDING_EXIT)) erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_TRACE | ERTS_PROC_LOCK_STATUS)); if (p->pending_suspenders) handle_pending_suspend(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_TRACE | ERTS_PROC_LOCK_STATUS)); #endif @@ -9552,7 +9605,9 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) p->scheduler_data = NULL; #endif - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_STATUS + | ERTS_PROC_LOCK_TRACE)); ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); @@ -9588,8 +9643,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) erts_smp_runq_lock(rq); } } - BM_STOP_TIMER(system); - } ERTS_SMP_LC_ASSERT(!is_normal_sched || !erts_thr_progress_is_blocking()); @@ -9825,10 +9878,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) goto check_activities_to_run; } - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR); - - BM_START_TIMER(system); - /* * Take the chosen process out of the queue. */ @@ -9934,6 +9983,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR); + #ifdef ERTS_SMP if (flags & ERTS_RUNQ_FLG_PROTECTED) @@ -9984,6 +10035,11 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (state & (ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_PENDING_EXIT | ERTS_PSFLG_EXITING)) { + /* + * IMPORTANT! We need to take care of + * scheduled check-process-code requests + * before continuing with dirty execution! + */ /* Migrate to normal scheduler... */ goto sunlock_sched_out_proc; } @@ -10393,6 +10449,27 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } break; } +#ifdef ERTS_NEW_PURGE_STRATEGY + case ERTS_PSTT_CLA: { + int fcalls; + int cla_reds = 0; + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + fcalls = reds; + else + fcalls = reds - CONTEXT_REDS; + st_res = erts_proc_copy_literal_area(c_p, + &cla_reds, + fcalls, + st->arg[0] == am_true); + reds -= cla_reds; + if (is_non_value(st_res)) { + /* Needed gc, but gc was disabled */ + save_gc_task(c_p, st, st_prio); + st = NULL; + } + break; + } +#endif case ERTS_PSTT_COHMQ: reds -= erts_complete_off_heap_message_queue_change(c_p); st_res = am_true; @@ -10443,14 +10520,15 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) switch (st->type) { case ERTS_PSTT_GC: - st_res = am_false; - break; case ERTS_PSTT_CPC: - st_res = am_false; - break; case ERTS_PSTT_COHMQ: st_res = am_false; break; +#ifdef ERTS_NEW_PURGE_STRATEGY + case ERTS_PSTT_CLA: + st_res = am_ok; + break; +#endif #ifdef ERTS_SMP case ERTS_PSTT_FTMQ: reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN); @@ -10471,22 +10549,83 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) return reds; } -BIF_RETTYPE -erts_internal_request_system_task_3(BIF_ALIST_3) +#ifdef ERTS_DIRTY_SCHEDULERS + +static BIF_RETTYPE +dispatch_system_task(Process *c_p, erts_aint_t fail_state, + ErtsProcSysTask *st, Eterm target, + Eterm prio, Eterm operation) { - Process *rp = erts_proc_lookup(BIF_ARG_1); + Process *rp; + ErtsProcLocks rp_locks = 0; + ErlOffHeap *ohp; + ErtsMessage *mp; + Eterm *hp, msg; + Uint hsz, osz; + BIF_RETTYPE ret; + + switch (st->type) { + case ERTS_PSTT_CPC: + rp = erts_dirty_process_code_checker; + ASSERT(fail_state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)); + if (c_p == rp) { + ERTS_BIF_PREP_RET(ret, am_dirty_execution); + return ret; + } + break; + default: + rp = NULL; + ERTS_INTERNAL_ERROR("Non-dispatchable system task"); + break; + } + + ERTS_BIF_PREP_RET(ret, am_ok); + + /* + * Send message on the form: {Requester, Target, Operation} + */ + + ASSERT(is_immed(st->requester)); + ASSERT(is_immed(target)); + ASSERT(is_immed(prio)); + + osz = size_object(operation); + hsz = 5 /* 4-tuple */ + osz; + + mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp); + + msg = copy_struct(operation, osz, &hp, ohp); + msg = TUPLE4(hp, st->requester, target, prio, msg); + + erts_queue_message(rp, rp_locks, mp, msg, st->requester); + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + return ret; +} + +#endif + +static BIF_RETTYPE +request_system_task(Process *c_p, Eterm requester, Eterm target, + Eterm priority, Eterm operation) +{ + BIF_RETTYPE ret; + Process *rp = erts_proc_lookup(target); ErtsProcSysTask *st = NULL; - erts_aint32_t prio; + erts_aint32_t prio, fail_state = ERTS_PSFLG_EXITING; Eterm noproc_res, req_type; - if (!rp && !is_internal_pid(BIF_ARG_1)) { - if (!is_external_pid(BIF_ARG_1)) + if (!rp && !is_internal_pid(target)) { + if (!is_external_pid(target)) goto badarg; - if (external_pid_dist_entry(BIF_ARG_1) != erts_this_dist_entry) + if (external_pid_dist_entry(target) != erts_this_dist_entry) goto badarg; } - switch (BIF_ARG_2) { + switch (priority) { case am_max: prio = PRIORITY_MAX; break; case am_high: prio = PRIORITY_HIGH; break; case am_normal: prio = PRIORITY_NORMAL; break; @@ -10494,11 +10633,11 @@ erts_internal_request_system_task_3(BIF_ALIST_3) default: goto badarg; } - if (is_not_tuple(BIF_ARG_3)) + if (is_not_tuple(operation)) goto badarg; else { int i; - Eterm *tp = tuple_val(BIF_ARG_3); + Eterm *tp = tuple_val(operation); Uint arity = arityval(*tp); Eterm req_id; Uint req_id_sz; @@ -10536,7 +10675,7 @@ erts_internal_request_system_task_3(BIF_ALIST_3) ERTS_INIT_OFF_HEAP(&st->off_heap); hp = &st->heap[0]; - st->requester = BIF_P->common.id; + st->requester = requester; st->reply_tag = req_type; st->req_id_sz = req_id_sz; st->req_id = req_id_sz == 0 ? req_id : copy_struct(req_id, @@ -10570,26 +10709,85 @@ erts_internal_request_system_task_3(BIF_ALIST_3) st->type = ERTS_PSTT_CPC; if (!rp) goto noproc; +#ifdef ERTS_DIRTY_SCHEDULERS + /* + * If the process should start executing dirty + * code it is important that this task is + * aborted. Therefore this strict fail state... + */ + fail_state |= (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS); +#endif + break; + +#ifdef ERTS_NEW_PURGE_STRATEGY + case am_copy_literals: + if (st->arg[0] != am_true && st->arg[0] != am_false) + goto badarg; + st->type = ERTS_PSTT_CLA; + noproc_res = am_ok; + if (!rp) + goto noproc; break; +#endif default: goto badarg; } - if (!schedule_process_sys_task(rp, prio, st)) { - noproc: - notify_sys_task_executed(BIF_P, st, noproc_res); + if (!schedule_process_sys_task(rp, prio, st, &fail_state)) { + Eterm failure; + if (fail_state & ERTS_PSFLG_EXITING) { + noproc: + failure = noproc_res; + } +#ifdef ERTS_DIRTY_SCHEDULERS + else if (fail_state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + ret = dispatch_system_task(c_p, fail_state, st, + target, priority, operation); + goto cleanup_return; + } +#endif + else { + ERTS_INTERNAL_ERROR("Unknown failure schedule_process_sys_task()"); + failure = am_internal_error; + } + notify_sys_task_executed(c_p, st, failure); } - BIF_RET(am_ok); + ERTS_BIF_PREP_RET(ret, am_ok); + + return ret; badarg: + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + +#ifdef ERTS_DIRTY_SCHEDULERS +cleanup_return: +#endif + if (st) { erts_cleanup_offheap(&st->off_heap); erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); } - BIF_ERROR(BIF_P, BADARG); + + return ret; +} + +BIF_RETTYPE +erts_internal_request_system_task_3(BIF_ALIST_3) +{ + return request_system_task(BIF_P, BIF_P->common.id, + BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +BIF_RETTYPE +erts_internal_request_system_task_4(BIF_ALIST_4) +{ + return request_system_task(BIF_P, BIF_ARG_1, + BIF_ARG_2, BIF_ARG_3, BIF_ARG_4); } static void @@ -10598,7 +10796,7 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type) Process *rp = erts_proc_lookup(pid); if (rp) { ErtsProcSysTask *st; - erts_aint32_t state; + erts_aint32_t state, fail_state; int i; st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK, @@ -10613,21 +10811,103 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type) ERTS_INIT_OFF_HEAP(&st->off_heap); state = erts_smp_atomic32_read_nob(&rp->state); - if (!schedule_process_sys_task(rp, ERTS_PSFLGS_GET_USR_PRIO(state), st)) + fail_state = ERTS_PSFLG_EXITING; + + if (!schedule_process_sys_task(rp, ERTS_PSFLGS_GET_USR_PRIO(state), + st, &fail_state)) erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); } } + void erts_schedule_complete_off_heap_message_queue_change(Eterm pid) { erts_schedule_generic_sys_task(pid, ERTS_PSTT_COHMQ); } +#ifdef ERTS_DIRTY_SCHEDULERS + +static void +flush_dirty_trace_messages(void *vpid) +{ + Process *proc; + Eterm pid; +#ifdef ARCH_64 + pid = (Eterm) vpid; +#else + pid = *((Eterm *) vpid); + erts_free(ERTS_ALC_T_DIRTY_SL, vpid); +#endif + + proc = erts_proc_lookup(pid); + if (proc) + (void) erts_flush_trace_messages(proc, 0); +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ + void -erts_schedule_flush_trace_messages(Eterm pid) +erts_schedule_flush_trace_messages(Process *proc, int force_on_proc) { +#ifdef ERTS_SMP + ErtsThrPrgrDelayHandle dhndl; +#endif + Eterm pid = proc->common.id; + +#ifdef ERTS_DIRTY_SCHEDULERS + erts_aint32_t state; + + if (!force_on_proc) { + state = erts_smp_atomic32_read_nob(&proc->state); + if (state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + goto sched_flush_dirty; + } + } +#endif + +#ifdef ERTS_SMP + dhndl = erts_thr_progress_unmanaged_delay(); +#endif + erts_schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ); + +#ifdef ERTS_SMP + erts_thr_progress_unmanaged_continue(dhndl); +#endif + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!force_on_proc) { + state = erts_smp_atomic32_read_mb(&proc->state); + if (state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + void *vargp; + + sched_flush_dirty: + /* + * We traced 'proc' from another thread than + * it is executing on, and it is executing + * on a dirty scheduler. It might take a + * significant amount of time before it is + * scheduled out (where it gets opportunity + * to flush messages). We therefore schedule + * the flush on the first ordinary scheduler. + */ + +#ifdef ARCH_64 + vargp = (void *) pid; +#else + { + Eterm *argp = erts_alloc(ERTS_ALC_T_DIRTY_SL, sizeof(Eterm)); + *argp = pid; + vargp = (void *) argp; + } +#endif + erts_schedule_misc_aux_work(1, flush_dirty_trace_messages, vargp); + } + } +#endif } static void @@ -11154,18 +11434,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). || (erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_OFF_HEAP_MSGQ)); -#ifdef BM_COUNTERS - processes_busy++; -#endif - BM_COUNT(processes_spawned); - - BM_SWAP_TIMER(system,size); #ifdef SHCOPY_SPAWN arg_size = copy_shared_calculate(args, &info); #else arg_size = size_object(args); #endif - BM_SWAP_TIMER(size,system); heap_need = arg_size; p->flags = flags; @@ -11242,18 +11515,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->max_arg_reg = sizeof(p->def_arg_reg)/sizeof(p->def_arg_reg[0]); p->arg_reg[0] = mod; p->arg_reg[1] = func; - BM_STOP_TIMER(system); - BM_MESSAGE(args,p,parent); - BM_START_TIMER(system); - BM_SWAP_TIMER(system,copy); #ifdef SHCOPY_SPAWN p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap); DESTROY_SHCOPY(info); #else p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap); #endif - BM_MESSAGE_COPIED(arg_size); - BM_SWAP_TIMER(copy,system); p->arity = 3; p->fvalue = NIL; @@ -12289,7 +12556,6 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) erts_port_demonitor(pcontext->p, ERTS_PORT_DEMONITOR_ORIGIN_ON_DEATHBED, prt, mon->ref, NULL); - return; /* let erts_port_demonitor do the deletion */ } else { /* remote by pid */ ASSERT(is_external_pid(mon->pid)); dep = external_pid_dist_entry(mon->pid); @@ -12826,9 +13092,6 @@ erts_continue_exit_process(Process *p) dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL; erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); -#ifdef BM_COUNTERS - processes_busy--; -#endif if (dep) { erts_do_net_exits(dep, reason); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 7c98b60647..3347a7a60e 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -989,8 +989,7 @@ struct process { Uint32 rcount; /* suspend count */ int schedule_count; /* Times left to reschedule a low prio process */ Uint reds; /* No of reductions for this process */ - Eterm group_leader; /* Pid in charge - (can be boxed) */ + Eterm group_leader; /* Pid in charge (can be boxed) */ Uint flags; /* Trap exit, etc (no trace flags anymore) */ Eterm fvalue; /* Exit & Throw value (failure reason) */ Uint freason; /* Reason for detected failure */ @@ -1329,10 +1328,13 @@ ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp); ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp) { ErlHeapFragment* hf = MBUF(p); + Uint sz; ASSERT(hf!=NULL && (hp - hf->mem < hf->alloc_size)); - hf->used_size = hp - hf->mem; + sz = hp - hf->mem; + p->mbuf_sz -= hf->used_size - sz; + hf->used_size = sz; } #endif /* inline */ @@ -1550,9 +1552,9 @@ extern int erts_system_profile_ts_type; #define ERTS_SCHEDULER_IS_DIRTY(ESDP) \ ((ESDP)->dirty_no.s.num != 0) #define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) \ - ((ESDP)->dirty_no.s.type == 0) + (ERTS_SCHEDULER_IS_DIRTY((ESDP)) & ((ESDP)->dirty_no.s.type == 0)) #define ERTS_SCHEDULER_IS_DIRTY_IO(ESDP) \ - ((ESDP)->dirty_no.s.type == 1) + (ERTS_SCHEDULER_IS_DIRTY((ESDP)) & ((ESDP)->dirty_no.s.type == 1)) #else #define ERTS_SCHEDULER_IS_DIRTY(ESDP) 0 #define ERTS_SCHEDULER_IS_DIRTY_CPU(ESDP) 0 @@ -1768,7 +1770,7 @@ void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *), ErtsThrPrgrLaterOp *, UWord); void erts_schedule_complete_off_heap_message_queue_change(Eterm pid); -void erts_schedule_flush_trace_messages(Eterm pid); +void erts_schedule_flush_trace_messages(Process *proc, int force_on_proc); int erts_flush_trace_messages(Process *c_p, ErtsProcLocks locks); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) @@ -2474,6 +2476,35 @@ erts_get_atom_cache_map(Process *c_p) } #endif +#ifdef __WIN32__ +/* + * Don't want erts_time2reds() inlined in beam_emu.c on windows since + * it is compiled with gcc which fails on it. Implementation is in + * erl_process.c on windows. + */ +# define ERTS_TIME2REDS_IMPL__ erts_time2reds__ +#else +# define ERTS_TIME2REDS_IMPL__ erts_time2reds +#endif + +ERTS_GLB_INLINE Sint64 ERTS_TIME2REDS_IMPL__(ErtsMonotonicTime start, + ErtsMonotonicTime end); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE Sint64 +ERTS_TIME2REDS_IMPL__(ErtsMonotonicTime start, ErtsMonotonicTime end) +{ + ErtsMonotonicTime time = end - start; + ASSERT(time >= 0); + time = ERTS_MONOTONIC_TO_USEC(time); + if (time == 0) + return (Sint64) 1; /* At least one reduction */ + /* Currently two reductions per micro second */ + time *= (CONTEXT_REDS-1)/1000 + 1; + return (Sint64) time; +} +#endif + Process *erts_pid2proc_suspend(Process *, ErtsProcLocks, Eterm, diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 21938e7684..700ed90def 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -700,6 +700,7 @@ leader_update(ErtsThrPrgrData *tpd) tpd->leader_state.chk_next_ix = no_managed; erts_atomic32_set_nob(&intrnl->misc.data.umrefc_ix.current, (erts_aint32_t) new_umrefc_ix); + tpd->leader_state.umrefc_ix.current = new_umrefc_ix; ETHR_MEMBAR(ETHR_StoreLoad); refc = erts_atomic_read_nob(&intrnl->umrefc[umrefc_ix].refc); ASSERT(refc >= 0); @@ -969,8 +970,10 @@ erts_thr_progress_unmanaged_continue__(ErtsThrPrgrDelayHandle handle) #ifdef ERTS_ENABLE_LOCK_CHECK ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); ERTS_LC_ASSERT(tpd && tpd->is_delaying); - tpd->is_delaying = 0; - return_tmp_thr_prgr_data(tpd); + tpd->is_delaying--; + ASSERT(tpd->is_delaying >= 0); + if (!tpd->is_delaying) + return_tmp_thr_prgr_data(tpd); #endif ASSERT(!erts_thr_progress_is_managed_thread()); @@ -995,7 +998,7 @@ erts_thr_progress_unmanaged_delay__(void) #ifdef ERTS_ENABLE_LOCK_CHECK { ErtsThrPrgrData *tpd = tmp_thr_prgr_data(NULL); - tpd->is_delaying = 1; + tpd->is_delaying++; } #endif return (ErtsThrPrgrDelayHandle) umrefc_ix; diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 9e37106b88..6aa2a7500f 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -2133,22 +2133,26 @@ time_unit_conversion(Process *c_p, Eterm term, ErtsMonotonicTime val, ErtsMonoto /* Convert to common user specified time units */ switch (term) { + case am_second: case am_seconds: case make_small(1): result = ERTS_MONOTONIC_TO_SEC(val) + muloff*ERTS_MONOTONIC_OFFSET_SEC; ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); break; + case am_millisecond: case am_milli_seconds: case make_small(1000): result = ERTS_MONOTONIC_TO_MSEC(val) + muloff*ERTS_MONOTONIC_OFFSET_MSEC; ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); break; + case am_microsecond: case am_micro_seconds: case make_small(1000*1000): result = ERTS_MONOTONIC_TO_USEC(val) + muloff*ERTS_MONOTONIC_OFFSET_USEC; ERTS_BIF_PREP_RET(ret, make_time_val(c_p, result)); break; #ifdef ARCH_64 + case am_nanosecond: case am_nano_seconds: case make_small(1000*1000*1000): result = ERTS_MONOTONIC_TO_NSEC(val) + muloff*ERTS_MONOTONIC_OFFSET_NSEC; @@ -2159,7 +2163,7 @@ time_unit_conversion(Process *c_p, Eterm term, ErtsMonotonicTime val, ErtsMonoto Eterm value, native_res; #ifndef ARCH_64 Sint user_res; - if (term == am_nano_seconds) + if (term == am_nanosecond || term == am_nano_seconds) goto to_nano_seconds; if (term_to_Sint(term, &user_res)) { if (user_res == 1000*1000*1000) { diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 4cf38bf894..8c84303997 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -493,8 +493,8 @@ erts_get_system_seq_tracer(void) if (st != erts_tracer_nil && call_enabled_tracer(st, NULL, TRACE_FUN_ENABLED, am_trace_status, am_undefined) == am_remove) { - erts_set_system_seq_tracer(NULL, 0, erts_tracer_nil); - st = erts_tracer_nil; + st = erts_set_system_seq_tracer(NULL, 0, erts_tracer_nil); + ERTS_TRACER_CLEAR(&st); } return st; @@ -813,6 +813,9 @@ trace_send(Process *p, Eterm to, Eterm msg) ErtsTracerNif *tnif = NULL; ErtsTracingEvent* te; Eterm pam_result; +#ifdef ERTS_SMP + ErtsThrPrgrDelayHandle dhndl; +#endif ASSERT(ARE_TRACE_FLAGS_ON(p, F_TRACE_SEND)); @@ -837,6 +840,10 @@ trace_send(Process *p, Eterm to, Eterm msg) } else pam_result = am_true; +#ifdef ERTS_SMP + dhndl = erts_thr_progress_unmanaged_delay(); +#endif + if (is_internal_pid(to)) { if (!erts_proc_lookup(to)) goto send_to_non_existing_process; @@ -852,6 +859,11 @@ trace_send(Process *p, Eterm to, Eterm msg) send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_SEND, operation, msg, to, pam_result); } + +#ifdef ERTS_SMP + erts_thr_progress_unmanaged_continue(dhndl); +#endif + erts_match_set_release_result_trace(p, pam_result); } @@ -1167,6 +1179,8 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, Eterm transformed_args[MAX_ARG]; ErtsTracer pre_ms_tracer = erts_tracer_nil; + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN); + ASSERT(tracer); if (ERTS_TRACER_COMPARE(*tracer, erts_tracer_true)) { /* Breakpoint trace enabled without specifying tracer => @@ -3108,6 +3122,7 @@ erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer) if (is_not_nil(*tracer)) { Uint offs = 2; UWord size = 2 * sizeof(Eterm) + sizeof(ErtsThrPrgrLaterOp); + ErtsThrPrgrLaterOp *lop; ASSERT(is_list(*tracer)); if (is_not_immed(ERTS_TRACER_STATE(*tracer))) { hf = (void*)(((char*)(ptr_val(*tracer)) - offsetof(ErlHeapFragment, mem))); @@ -3115,6 +3130,16 @@ erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer) size = hf->alloc_size * sizeof(Eterm) + sizeof(ErlHeapFragment); ASSERT(offs == size_object(*tracer)); } + + /* sparc assumes that all structs are double word aligned, so we + have to align the ErtsThrPrgrLaterOp struct otherwise it may + segfault.*/ + if ((UWord)(ptr_val(*tracer) + offs) % (sizeof(UWord)*2) == sizeof(UWord)) + offs += 1; + + lop = (ErtsThrPrgrLaterOp*)(ptr_val(*tracer) + offs); + ASSERT((UWord)lop % (sizeof(UWord)*2) == 0); + /* We schedule the free:ing of the tracer until after a thread progress has been made so that we know that no schedulers have any references to it. Because we do this, it is possible to release all locks of a @@ -3122,9 +3147,7 @@ erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer) without having to worry if it is free'd. */ erts_schedule_thr_prgr_later_cleanup_op( - free_tracer, (void*)(*tracer), - (ErtsThrPrgrLaterOp*)(ptr_val(*tracer) + offs), - size); + free_tracer, (void*)(*tracer), lop, size); } if (is_nil(new_tracer)) { @@ -3134,16 +3157,17 @@ erts_tracer_update(ErtsTracer *tracer, const ErtsTracer new_tracer) Not sure if it is worth it, we save 4 words (sizeof(ErlHeapFragment)) per tracer. */ Eterm *hp = erts_alloc(ERTS_ALC_T_HEAP_FRAG, - 2*sizeof(Eterm) + sizeof(ErtsThrPrgrLaterOp)); + 3*sizeof(Eterm) + sizeof(ErtsThrPrgrLaterOp)); *tracer = CONS(hp, ERTS_TRACER_MODULE(new_tracer), ERTS_TRACER_STATE(new_tracer)); } else { Eterm *hp, tracer_state = ERTS_TRACER_STATE(new_tracer), tracer_module = ERTS_TRACER_MODULE(new_tracer); Uint sz = size_object(tracer_state); - hf = new_message_buffer(sz + 2 /* cons cell */ + (sizeof(ErtsThrPrgrLaterOp)+sizeof(Eterm)-1)/sizeof(Eterm)); + hf = new_message_buffer(sz + 2 /* cons cell */ + + (sizeof(ErtsThrPrgrLaterOp)+sizeof(Eterm)-1)/sizeof(Eterm) + 1); hp = hf->mem + 2; - hf->used_size -= (sizeof(ErtsThrPrgrLaterOp)+sizeof(Eterm)-1)/sizeof(Eterm); + hf->used_size -= (sizeof(ErtsThrPrgrLaterOp)+sizeof(Eterm)-1)/sizeof(Eterm) + 1; *tracer = copy_struct(tracer_state, sz, &hp, &hf->off_heap); *tracer = CONS(hf->mem, tracer_module, *tracer); ASSERT((void*)(((char*)(ptr_val(*tracer)) - offsetof(ErlHeapFragment, mem))) == hf); diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index 8c81cbd410..1e7bb8514b 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -21,14 +21,8 @@ #ifndef __EXPORT_H__ #define __EXPORT_H__ -#ifndef __SYS_H__ #include "sys.h" -#endif - -#ifndef __INDEX_H__ #include "index.h" -#endif - #include "code_ix.h" /* diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index f3d4ac56cd..b2c76aa605 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -35,7 +35,6 @@ #include "register.h" #include "erl_fun.h" #include "erl_node_tables.h" -#include "benchmark.h" #include "erl_process.h" #include "erl_sys_driver.h" #include "erl_debug.h" @@ -62,9 +61,12 @@ struct enif_environment_t /* ErlNifEnv */ extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); +#ifdef ERTS_DIRTY_SCHEDULERS extern void erts_pre_dirty_nif(ErtsSchedulerData *, struct enif_environment_t*, Process*, - struct erl_module_nif*, Process* tracee); + struct erl_module_nif*); +extern void erts_post_dirty_nif(struct enif_environment_t* env); +#endif extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(int to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); @@ -999,17 +1001,33 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); /* beam_bif_load.c */ #define ERTS_CPC_ALLOW_GC (1 << 0) -#define ERTS_CPC_COPY_LITERALS (1 << 1) -#define ERTS_CPC_ALL (ERTS_CPC_ALLOW_GC | ERTS_CPC_COPY_LITERALS) +#define ERTS_CPC_ALL ERTS_CPC_ALLOW_GC Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls); +#ifdef ERTS_NEW_PURGE_STRATEGY +Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed); +#endif -typedef struct { - Eterm *ptr; - Uint sz; - Eterm pid; -} copy_literals_t; +typedef struct ErtsLiteralArea_ { + struct erl_off_heap_header *off_heap; + Eterm *end; + Eterm start[1]; /* beginning of area */ +} ErtsLiteralArea; + +#define ERTS_LITERAL_AREA_ALLOC_SIZE(N) \ + (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1)) + +extern erts_smp_atomic_t erts_copy_literal_area__; +#define ERTS_COPY_LITERAL_AREA() \ + ((ErtsLiteralArea *) erts_smp_atomic_read_nob(&erts_copy_literal_area__)) -extern copy_literals_t erts_clrange; +#ifdef ERTS_NEW_PURGE_STRATEGY +extern Process *erts_literal_area_collector; +#endif +#ifdef ERTS_DIRTY_SCHEDULERS +extern Process *erts_dirty_process_code_checker; +#endif + +extern Process *erts_code_purger; /* beam_load.c */ typedef struct { @@ -1092,12 +1110,19 @@ typedef struct { #define INITIALIZE_SHCOPY(info) \ do { \ + ErtsLiteralArea *larea__ = ERTS_COPY_LITERAL_AREA();\ info.queue_start = info.queue_default; \ info.bitstore_start = info.bitstore_default; \ info.shtable_start = info.shtable_default; \ info.literal_size = 0; \ - info.range_ptr = erts_clrange.ptr; \ - info.range_sz = erts_clrange.sz; \ + if (larea__) { \ + info.range_ptr = &larea__->start[0]; \ + info.range_sz = larea__->end - info.range_ptr; \ + } \ + else { \ + info.range_ptr = NULL; \ + info.range_sz = 0; \ + } \ } while(0) #define DESTROY_SHCOPY(info) \ diff --git a/erts/emulator/beam/hash.c b/erts/emulator/beam/hash.c index e255b961f1..cd038d100b 100644 --- a/erts/emulator/beam/hash.c +++ b/erts/emulator/beam/hash.c @@ -35,9 +35,9 @@ static const int h_size_table[] = { 2, 5, 11, 23, 47, 97, 197, 397, 797, /* double upto here */ 1201, 1597, - 2411, 3203, + 2411, 3203, 4813, 6421, - 9643, 12853, + 9643, 12853, 19289, 25717, 51437, 102877, @@ -49,8 +49,8 @@ static const int h_size_table[] = { 6584983, 13169977, 26339969, - 52679969, - -1 + 52679969, + -1 }; /* @@ -69,7 +69,7 @@ void hash_get_info(HashInfo *hi, Hash *h) for (i = 0; i < size; i++) { int depth = 0; HashBucket* b = h->bucket[i]; - + while (b != (HashBucket*) 0) { objects++; depth++; @@ -112,7 +112,7 @@ void hash_info(int to, void *arg, Hash* h) /* * Returns size of table in bytes. Stored objects not included. */ -int +int hash_table_sz(Hash *h) { int i; @@ -190,7 +190,7 @@ void hash_delete(Hash* h) HashBucket* b = h->bucket[i]; while (b != (HashBucket*) 0) { HashBucket* b_next = b->next; - + h->fun.free((void*) b); b = b_next; } @@ -250,7 +250,7 @@ void* hash_get(Hash* h, void* tmpl) HashValue hval = h->fun.hash(tmpl); int ix = hval % h->size; HashBucket* b = h->bucket[ix]; - + while(b != (HashBucket*) 0) { if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) return (void*) b; @@ -294,7 +294,7 @@ void* hash_erase(Hash* h, void* tmpl) int ix = hval % h->size; HashBucket* b = h->bucket[ix]; HashBucket* prev = 0; - + while(b != 0) { if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) { if (prev != 0) @@ -326,7 +326,7 @@ hash_remove(Hash *h, void *tmpl) int ix = hval % h->size; HashBucket *b = h->bucket[ix]; HashBucket *prev = NULL; - + while (b) { if ((b->hvalue == hval) && (h->fun.cmp(tmpl, (void*)b) == 0)) { if (prev) @@ -355,4 +355,3 @@ void hash_foreach(Hash* h, void (*func)(void *, void *), void *func_arg2) } } } - diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h index 9f773d8faa..4e769c0119 100644 --- a/erts/emulator/beam/hash.h +++ b/erts/emulator/beam/hash.h @@ -25,9 +25,7 @@ #ifndef __HASH_H__ #define __HASH_H__ -#ifndef __SYS_H__ #include "sys.h" -#endif typedef unsigned long HashValue; typedef struct hash Hash; diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index 218779c33b..0a109d8699 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -26,13 +26,8 @@ #ifndef __INDEX_H__ #define __INDEX_H__ -#ifndef __HASH_H__ #include "hash.h" -#endif - -#ifndef ERL_ALLOC_H__ #include "erl_alloc.h" -#endif typedef struct index_slot { diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index cb8792dffa..77dbe92241 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -6591,16 +6591,20 @@ deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p, ErtsThrPrgrDelayHandle dhndl = erts_thr_progress_unmanaged_delay(); #endif erts_aint32_t state; + int res = 1; Port *prt = erts_port_lookup_raw((Eterm) port_id); - if (!prt) - return -1; + if (!prt) { + res = -1; + goto done; + } state = erts_atomic32_read_nob(&prt->state); if (state & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP | ERTS_PORT_SFLG_CLOSING)) { if (state & ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP) - return -1; + res = -1; else - return 0; + res = 0; + goto done; } if (connected_p) { #ifdef ERTS_SMP @@ -6609,25 +6613,27 @@ deliver_term_check_port(ErlDrvTermData port_id, Eterm *connected_p, #endif *connected_p = ERTS_PORT_GET_CONNECTED(prt); } + +done: + #ifdef ERTS_SMP if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) { + ERTS_SMP_LC_ASSERT(!prt || !erts_lc_is_port_locked(prt)); erts_thr_progress_unmanaged_continue(dhndl); ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); } else #endif - { + if (res == 1) { + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); *trace_prt = prt; } - ERTS_SMP_LC_ASSERT(dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED - ? erts_lc_is_port_locked(prt) - : !erts_lc_is_port_locked(prt)); - return 1; + return res; } int erl_drv_output_term(ErlDrvTermData port_id, ErlDrvTermData* data, int len) { /* May be called from arbitrary thread */ - Eterm connected; + Eterm connected = NIL; /* Shut up faulty warning... */ Port *prt = NULL; int res = deliver_term_check_port(port_id, &connected, &prt); if (res <= 0) diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index 4e12731d85..5a60bc90d9 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -21,9 +21,7 @@ #ifndef __MODULE_H__ #define __MODULE_H__ -#ifndef __INDEX_H__ #include "index.h" -#endif struct erl_module_instance { BeamCodeHeader* code_hdr; diff --git a/erts/emulator/beam/safe_hash.h b/erts/emulator/beam/safe_hash.h index 6910b33004..285103cb17 100644 --- a/erts/emulator/beam/safe_hash.h +++ b/erts/emulator/beam/safe_hash.h @@ -26,14 +26,9 @@ #ifndef __SAFE_HASH_H__ #define __SAFE_HASH_H__ - -#ifndef __SYS_H__ #include "sys.h" -#endif - #include "erl_alloc.h" - typedef unsigned long SafeHashValue; typedef int (*SHCMP_FUN)(void*, void*); diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 675fafa726..6786657faf 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -85,7 +85,7 @@ erts_heap_alloc(Process* p, Uint need, Uint xtra) && HEAP_TOP(p) >= p->space_verified_from && HEAP_TOP(p) + need <= p->space_verified_from + p->space_verified && HEAP_LIMIT(p) - HEAP_TOP(p) >= need) { - + Uint consumed = need + (HEAP_TOP(p) - p->space_verified_from); ASSERT(consumed <= p->space_verified); p->space_verified -= consumed; @@ -102,6 +102,7 @@ erts_heap_alloc(Process* p, Uint need, Uint xtra) if (bp != NULL && need <= (bp->alloc_size - bp->used_size)) { Eterm* ret = bp->mem + bp->used_size; bp->used_size += need; + p->mbuf_sz += need; return ret; } #ifdef DEBUG @@ -124,7 +125,7 @@ erts_heap_alloc(Process* p, Uint need, Uint xtra) MBUF(p) = bp; bp->alloc_size = n; bp->used_size = need; - MBUF_SIZE(p) += n; + MBUF_SIZE(p) += need; bp->off_heap.first = NULL; bp->off_heap.overhead = 0; return bp->mem; @@ -638,7 +639,7 @@ erts_bld_atom_uword_2tup_list(Uint **hpp, Uint *szp, ui = uint_to_big(uints[i], *hpp); *hpp += BIG_UINT_HEAP_SIZE; } - + res = CONS(*hpp+3, TUPLE2(*hpp, atoms[i], ui), res); *hpp += 5; } @@ -676,14 +677,14 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, ui1 = uint_to_big(uints1[i], *hpp); *hpp += BIG_UINT_HEAP_SIZE; } - + if (IS_USMALL(0, uints2[i])) ui2 = make_small(uints2[i]); else { ui2 = uint_to_big(uints2[i], *hpp); *hpp += BIG_UINT_HEAP_SIZE; } - + res = CONS(*hpp+4, TUPLE3(*hpp, atoms[i], ui1, ui2), res); *hpp += 6; } @@ -794,7 +795,7 @@ hash_binary_bytes(Eterm bin, Uint sz, Uint32 hash) Uint b; Uint lshift = bitoffs; Uint rshift = 8 - lshift; - + while (sz--) { b = (previous << lshift) & 0xFF; previous = *ptr++; @@ -805,7 +806,7 @@ hash_binary_bytes(Eterm bin, Uint sz, Uint32 hash) b = (previous << lshift) & 0xFF; previous = *ptr++; b |= previous >> rshift; - + b >>= 8 - bitsize; hash = (hash*FUNNY_NUMBER1 + b) * FUNNY_NUMBER12 + bitsize; } @@ -835,21 +836,21 @@ Uint32 make_hash(Eterm term_arg) do { \ Uint32 x = (Uint32) (Expr); \ hash = \ - (((((hash)*(Prime1) + (x & 0xFF)) * (Prime1) + \ - ((x >> 8) & 0xFF)) * (Prime1) + \ - ((x >> 16) & 0xFF)) * (Prime1) + \ + (((((hash)*(Prime1) + (x & 0xFF)) * (Prime1) + \ + ((x >> 8) & 0xFF)) * (Prime1) + \ + ((x >> 16) & 0xFF)) * (Prime1) + \ (x >> 24)); \ } while(0) -#define UINT32_HASH_RET(Expr, Prime1, Prime2) \ +#define UINT32_HASH_RET(Expr, Prime1, Prime2) \ UINT32_HASH_STEP(Expr, Prime1); \ hash = hash * (Prime2); \ - break - - + break + + /* * Significant additions needed for real 64 bit port with larger fixnums. - */ + */ /* * Note, for the simple 64bit port, not utilizing the @@ -864,7 +865,7 @@ tail_recur: hash = hash*FUNNY_NUMBER3 + 1; break; case ATOM_DEF: - hash = hash*FUNNY_NUMBER1 + + hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(term))->slot.bucket.hvalue); break; case SMALL_DEF: @@ -893,9 +894,9 @@ tail_recur: Export* ep = *((Export **) (export_val(term) + 1)); hash = hash * FUNNY_NUMBER11 + ep->code[2]; - hash = hash*FUNNY_NUMBER1 + + hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue); - hash = hash*FUNNY_NUMBER1 + + hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue); break; } @@ -906,7 +907,7 @@ tail_recur: Uint num_free = funp->num_free; hash = hash * FUNNY_NUMBER10 + num_free; - hash = hash*FUNNY_NUMBER1 + + hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue); hash = hash*FUNNY_NUMBER2 + funp->fe->old_index; hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq; @@ -931,7 +932,7 @@ tail_recur: UINT32_HASH_RET(internal_ref_numbers(term)[0],FUNNY_NUMBER9,FUNNY_NUMBER10); case EXTERNAL_REF_DEF: UINT32_HASH_RET(external_ref_numbers(term)[0],FUNNY_NUMBER9,FUNNY_NUMBER10); - case FLOAT_DEF: + case FLOAT_DEF: { FloatDef ff; GET_DOUBLE(term, ff); @@ -958,12 +959,12 @@ tail_recur: ** as multiplications on a Sparc is so slow. */ hash = hash*FUNNY_NUMBER2 + unsigned_val(*list); - + if (is_not_list(CDR(list))) { WSTACK_PUSH(stack, MAKE_HASH_CDR_POST_OP); term = CDR(list); goto tail_recur; - } + } list = list_val(CDR(list)); } WSTACK_PUSH2(stack, CDR(list), MAKE_HASH_CDR_PRE_OP); @@ -1004,17 +1005,17 @@ tail_recur: } hash *= is_neg ? FUNNY_NUMBER4 : FUNNY_NUMBER3; break; - } + } case MAP_DEF: hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); break; - case TUPLE_DEF: + case TUPLE_DEF: { Eterm* ptr = tuple_val(term); Uint arity = arityval(*ptr); WSTACK_PUSH3(stack, (UWord) arity, (UWord)(ptr+1), (UWord) arity); - op = MAKE_HASH_TUPLE_OP; + op = MAKE_HASH_TUPLE_OP; }/*fall through*/ case MAKE_HASH_TUPLE_OP: case MAKE_HASH_TERM_ARRAY_OP: @@ -1031,8 +1032,8 @@ tail_recur: hash = hash*FUNNY_NUMBER9 + arity; } break; - } - + } + default: erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op); return 0; @@ -1159,8 +1160,8 @@ make_hash2(Eterm term) if (y < 0) { \ UINT32_HASH(-y, AConst); \ /* Negative numbers are unnecessarily mixed twice. */ \ - } \ - UINT32_HASH(y, AConst); \ + } \ + UINT32_HASH(y, AConst); \ } while(0) #define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) @@ -1242,7 +1243,7 @@ make_hash2(Eterm term) int arity = header_arity(hdr); Eterm* elem = tuple_val(term); UINT32_HASH(arity, HCONST_9); - if (arity == 0) /* Empty tuple */ + if (arity == 0) /* Empty tuple */ goto hash2_common; for (i = arity; ; i--) { term = elem[i]; @@ -1329,7 +1330,7 @@ make_hash2(Eterm term) { Export* ep = *((Export **) (export_val(term) + 1)); UINT32_HASH_2 - (ep->code[2], + (ep->code[2], atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue, HCONST); UINT32_HASH @@ -1343,7 +1344,7 @@ make_hash2(Eterm term) ErlFunThing* funp = (ErlFunThing *) fun_val(term); Uint num_free = funp->num_free; UINT32_HASH_2 - (num_free, + (num_free, atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue, HCONST); UINT32_HASH_2 @@ -1468,7 +1469,7 @@ make_hash2(Eterm term) goto hash2_common; } break; - + default: erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); } @@ -1541,7 +1542,7 @@ make_hash2(Eterm term) } case HASH_MAP_PAIR: hash_xor_pairs ^= hash; - hash = 0; + hash = 0; goto hash2_common; default: break; @@ -1678,17 +1679,22 @@ make_internal_hash(Eterm term) * the order in which keys and values are encountered. * We therefore calculate context independent hashes for all . * key-value pairs and then xor them together. + * + * We *do* need to use an initial seed for each pair, i.e. the + * hash value, so the hash value is reset for each pair with 'hash'. + * If we don't, no additional entropy is given to the system and the + * hash collision resolution in maps:from_list/1 would fail. */ ESTACK_PUSH(s, hash_xor_pairs); ESTACK_PUSH(s, hash); ESTACK_PUSH(s, HASH_MAP_TAIL); - hash = 0; - hash_xor_pairs = 0; for (i = size - 1; i >= 0; i--) { + ESTACK_PUSH(s, hash); /* initial seed for all pairs */ ESTACK_PUSH(s, HASH_MAP_PAIR); ESTACK_PUSH(s, vs[i]); ESTACK_PUSH(s, ks[i]); } + hash_xor_pairs = 0; goto pop_next; } case HAMT_SUBTAG_HEAD_ARRAY: @@ -1700,7 +1706,6 @@ make_internal_hash(Eterm term) ESTACK_PUSH(s, hash_xor_pairs); ESTACK_PUSH(s, hash); ESTACK_PUSH(s, HASH_MAP_TAIL); - hash = 0; hash_xor_pairs = 0; } switch (hdr & _HEADER_MAP_SUBTAG_MASK) { @@ -1717,6 +1722,7 @@ make_internal_hash(Eterm term) while (i) { if (is_list(*ptr)) { Eterm* cons = list_val(*ptr); + ESTACK_PUSH(s, hash); /* initial seed for all pairs */ ESTACK_PUSH(s, HASH_MAP_PAIR); ESTACK_PUSH(s, CDR(cons)); ESTACK_PUSH(s, CAR(cons)); @@ -1906,6 +1912,7 @@ make_internal_hash(Eterm term) pop_next: if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); + return hash; } @@ -1920,7 +1927,7 @@ make_internal_hash(Eterm term) } case HASH_MAP_PAIR: hash_xor_pairs ^= hash; - hash = 0; + hash = (Uint32) ESTACK_POP(s); /* initial seed for all pairs */ goto pop_next; case HASH_CDR: @@ -1953,8 +1960,8 @@ Uint32 make_broken_hash(Eterm term) DECLARE_WSTACK(stack); unsigned op; tail_recur: - op = tag_val_def(term); - for (;;) { + op = tag_val_def(term); + for (;;) { switch (op) { case NIL_DEF: hash = hash*FUNNY_NUMBER3 + 1; @@ -1976,8 +1983,7 @@ tail_recur: { /* like a bignum */ Uint32 y4 = (Uint32) y2; hash = hash*FUNNY_NUMBER2 + ((y4 << 16) | (y4 >> 16)); - if (y3) - { + if (y3) { hash = hash*FUNNY_NUMBER2 + ((y3 << 16) | (y3 >> 16)); arity++; } @@ -2020,9 +2026,9 @@ tail_recur: Export* ep = *((Export **) (export_val(term) + 1)); hash = hash * FUNNY_NUMBER11 + ep->code[2]; - hash = hash*FUNNY_NUMBER1 + + hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue); - hash = hash*FUNNY_NUMBER1 + + hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue); break; } @@ -2033,7 +2039,7 @@ tail_recur: Uint num_free = funp->num_free; hash = hash * FUNNY_NUMBER10 + num_free; - hash = hash*FUNNY_NUMBER1 + + hash = hash*FUNNY_NUMBER1 + (atom_tab(atom_val(funp->fe->module))->slot.bucket.hvalue); hash = hash*FUNNY_NUMBER2 + funp->fe->old_index; hash = hash*FUNNY_NUMBER2 + funp->fe->old_uniq; @@ -2065,7 +2071,7 @@ tail_recur: case EXTERNAL_REF_DEF: hash = hash*FUNNY_NUMBER9 + external_ref_numbers(term)[0]; break; - case FLOAT_DEF: + case FLOAT_DEF: { FloatDef ff; GET_DOUBLE(term, ff); @@ -2149,7 +2155,7 @@ tail_recur: } #else -#error "unsupported D_EXP size" +#error "unsupported D_EXP size" #endif hash = hash * (is_neg ? FUNNY_NUMBER3 : FUNNY_NUMBER2) + arity; } @@ -2158,14 +2164,14 @@ tail_recur: case MAP_DEF: hash = hash*FUNNY_NUMBER13 + FUNNY_NUMBER14 + make_hash2(term); break; - case TUPLE_DEF: + case TUPLE_DEF: { Eterm* ptr = tuple_val(term); Uint arity = arityval(*ptr); WSTACK_PUSH3(stack, (UWord) arity, (UWord) (ptr+1), (UWord) arity); op = MAKE_HASH_TUPLE_OP; - }/*fall through*/ + }/*fall through*/ case MAKE_HASH_TUPLE_OP: case MAKE_HASH_TERM_ARRAY_OP: { @@ -2193,7 +2199,7 @@ tail_recur: DESTROY_WSTACK(stack); return hash; - + #undef MAKE_HASH_TUPLE_OP #undef MAKE_HASH_TERM_ARRAY_OP #undef MAKE_HASH_CDR_PRE_OP @@ -2353,13 +2359,13 @@ static int do_send_term_to_logger(Eterm tag, Eterm gleader, } static ERTS_INLINE int -send_info_to_logger(Eterm gleader, char *buf, int len) +send_info_to_logger(Eterm gleader, char *buf, int len) { return do_send_to_logger(am_info_msg, gleader, buf, len); } static ERTS_INLINE int -send_warning_to_logger(Eterm gleader, char *buf, int len) +send_warning_to_logger(Eterm gleader, char *buf, int len) { Eterm tag; switch (erts_error_logger_warnings) { @@ -2371,7 +2377,7 @@ send_warning_to_logger(Eterm gleader, char *buf, int len) } static ERTS_INLINE int -send_error_to_logger(Eterm gleader, char *buf, int len) +send_error_to_logger(Eterm gleader, char *buf, int len) { return do_send_to_logger(am_error, gleader, buf, len); } @@ -2613,7 +2619,7 @@ tailrecur_ne: break; /* not equal */ case TAG_PRIMARY_BOXED: - { + { Eterm hdr = *boxed_val(a); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: @@ -2639,7 +2645,7 @@ tailrecur_ne: Uint b_bitsize; Uint a_bitoffs; Uint b_bitoffs; - + if (!is_binary(b)) { goto not_equal; } @@ -2671,7 +2677,7 @@ tailrecur_ne: { ErlFunThing* f1; ErlFunThing* f2; - + if (!is_fun(b)) goto not_equal; f1 = (ErlFunThing *) fun_val(a); @@ -2702,7 +2708,7 @@ tailrecur_ne: if(ap->header == bp->header && ap->node == bp->node) { ASSERT(1 == external_data_words(a)); ASSERT(1 == external_data_words(b)); - + if (ap->data.ui[0] == bp->data.ui[0]) goto pop_next; } break; /* not equal */ @@ -2759,7 +2765,7 @@ tailrecur_ne: if (alen == 3 && blen == 3) { /* Most refs are of length 3 */ if (anum[1] == bnum[1] && anum[2] == bnum[2]) { - goto pop_next; + goto pop_next; } else { goto not_equal; } @@ -2784,7 +2790,7 @@ tailrecur_ne: for (i = common_len; i < blen; i++) if (bnum[i] != 0) goto not_equal; - } + } } goto pop_next; } @@ -2792,7 +2798,7 @@ tailrecur_ne: case NEG_BIG_SUBTAG: { int i; - + if (!is_big(b)) goto not_equal; aa = big_val(a); @@ -2810,7 +2816,7 @@ tailrecur_ne: { FloatDef af; FloatDef bf; - + if (is_float(b)) { GET_DOUBLE(a, af); GET_DOUBLE(b, bf); @@ -2889,7 +2895,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */ } goto tailrecur_ne; } - + pop_next: if (!WSTACK_ISEMPTY(stack)) { UWord something = WSTACK_POP(stack); @@ -3093,7 +3099,7 @@ tailrecur_ne: } anode = erts_this_node; adata = internal_port_data(a); - + port_common: CMP_NODES(anode, bnode); ON_CMP_GOTO((Sint)(adata - bdata)); @@ -3111,7 +3117,7 @@ tailrecur_ne: } anode = erts_this_node; adata = internal_pid_data(a); - + pid_common: if (adata != bdata) { RETURN_NEQ(adata < bdata ? -1 : 1); @@ -3336,7 +3342,7 @@ tailrecur_ne: diff = f1->num_free - f2->num_free; if (diff != 0) { RETURN_NEQ(diff); - } + } i = f1->num_free; if (i == 0) goto pop_next; aa = f1->env; @@ -3397,10 +3403,10 @@ tailrecur_ne: anum = internal_thing_ref_numbers(athing); alen = internal_thing_ref_no_of_numbers(athing); } - + ref_common: CMP_NODES(anode, bnode); - + ASSERT(alen > 0 && blen > 0); if (alen != blen) { if (alen > blen) { @@ -3418,7 +3424,7 @@ tailrecur_ne: } while (alen < blen); } } - + ASSERT(alen == blen); for (i = (Sint) alen - 1; i >= 0; i--) if (anum[i] != bnum[i]) @@ -3633,8 +3639,8 @@ term_array: /* arrays in 'aa' and 'bb', length in 'i' */ } a = *aa; b = *bb; - goto tailrecur; - + goto tailrecur; + pop_next: if (!WSTACK_ISEMPTY(stack)) { UWord something = WSTACK_POP(stack); @@ -3897,19 +3903,19 @@ intlist_to_buf(Eterm list, char *buf, Sint len) Eterm* listptr; Sint sz = 0; - if (is_nil(list)) + if (is_nil(list)) return 0; if (is_not_list(list)) return -1; listptr = list_val(list); while (sz < len) { - if (!is_byte(*listptr)) + if (!is_byte(*listptr)) return -1; buf[sz++] = unsigned_val(*listptr); if (is_nil(*(listptr + 1))) return(sz); - if (is_not_list(*(listptr + 1))) + if (is_not_list(*(listptr + 1))) return -1; listptr = list_val(*(listptr + 1)); } @@ -4157,10 +4163,10 @@ do { \ } else if (yield_support && --yield_count <= 0) goto L_yield; } - + res = len; - L_return: + L_return: DESTROY_ESTACK(s); @@ -5053,7 +5059,7 @@ Process *p; if(p) print_process_info(ERTS_PRINT_STDERR, NULL, p); } - + void ppi(Eterm pid) { pp(erts_proc_lookup(pid)); @@ -5079,5 +5085,3 @@ ps(Process* p, Eterm* stop) } } #endif - - |