diff options
Diffstat (limited to 'erts')
39 files changed, 2225 insertions, 422 deletions
diff --git a/erts/configure.in b/erts/configure.in index 9f3d267791..c0326beac2 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -121,6 +121,7 @@ AS_HELP_STRING([--enable-bootstrap-only], enable_hipe=no enable_sctp=no enable_dirty_schedulers=no + enable_new_purge=no fi ]) @@ -139,6 +140,13 @@ AS_HELP_STRING([--enable-dirty-schedulers], [enable dirty scheduler support]), *) enable_dirty_schedulers=yes ;; esac ], enable_dirty_schedulers=default) +AC_ARG_ENABLE(new-purge-strategy, +AS_HELP_STRING([--enable-new-purge-strategy], [enable new code purge strategy]), +[ case "$enableval" in + no) enable_new_purge=no ;; + *) enable_new_purge=yes ;; + esac ], enable_new_purge=default) + AC_ARG_ENABLE(smp-support, AS_HELP_STRING([--enable-smp-support], [enable smp support]) AS_HELP_STRING([--disable-smp-support], [disable smp support]), @@ -1028,6 +1036,27 @@ esac AC_MSG_RESULT($DIRTY_SCHEDULER_SUPPORT) AC_SUBST(DIRTY_SCHEDULER_SUPPORT) +AC_MSG_CHECKING(whether the new code purge strategy should be enabled) +case $enable_new_purge-$enable_dirty_schedulers in + yes-*) + AC_MSG_RESULT(yes) + enable_new_purge=yes;; + default-yes) + AC_MSG_RESULT(yes; forced by dirty scheduler support) + enable_new_purge=yes;; + no-yes) + AC_MSG_ERROR([Dirty schedulers enabled, but new code purge strategy disabled]);; + *) + AC_MSG_RESULT(no) + enable_new_purge=no;; +esac + +if test $enable_new_purge = yes; then + AC_DEFINE(ERTS_NEW_PURGE_STRATEGY, 1, [Define if you want to use the new code purge strategy]) +fi +NEW_PURGE_STRATEGY=$enable_new_purge +AC_SUBST(NEW_PURGE_STRATEGY) + if test $ERTS_BUILD_SMP_EMU = yes; then if test $found_threads = no; then diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index 2b627281fe..4ec5ab78d8 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -488,18 +488,6 @@ return term;</code> sending process is not alive. Deallocation of certain internal resources, such as process heap and process control block, is delayed until the dirty NIF has completed.</p> - <p>Known issue that are planned to be fixed:</p> - <list type="bulleted"> - <item> - <p>As purging of a module might need to garbage - collect a process to determine if it has - references to the module, a process executing a dirty - NIF can delay purging for a very long time. Delaying - a purge operation implies delaying <em>all</em> code - loading operations, which can cause severe problems for - the system as a whole.</p> - </item> - </list> </item> </taglist> </item> diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 2b7a3b85e5..51ebddc155 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -776,6 +776,29 @@ Z = erlang:adler32_combine(X,Y,iolist_size(Data2)).</code> by passing option <c>{allow_gc, false}</c>.</p> </item> </taglist> + <note> + <p> + Up until ERTS version 8.*, the check process code operation + checks for all types of references to the old code. That is, + direct references (e.g. return addresses on the process + stack), indirect references (<c>fun</c>s in process + context), and references to literals in the code. + </p> + <p> + As of ERTS version 9.0, the check process code operation + only checks for direct references to the code. Indirect + references via <c>fun</c>s will be ignored. If such + <c>fun</c>s exist and are used after a purge of the old + code, an exception will be raised upon usage (same as + the case when the <c>fun</c> is received by the process + after the purge). Literals will be taken care of (copied) + at a later stage. This behavior can as of ERTS version + 8.1 be enabled when + <seealso marker="doc/installation_guide:INSTALL#Advanced-configuration-and-build-of-ErlangOTP_Configuring">building OTP</seealso>, + and will automatically be enabled if dirty scheduler + support is enabled. + </p> + </note> <p>See also <seealso marker="kernel:code"> <c>kernel:code(3)</c></seealso>.</p> <p>Failures:</p> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 3430ea5f7b..43d1914b42 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -51,6 +51,7 @@ ARFLAGS=rc OMIT_OMIT_FP=no DIRTY_SCHEDULER_SUPPORT=@DIRTY_SCHEDULER_SUPPORT@ +NEW_PURGE_STRATEGY=@NEW_PURGE_STRATEGY@ ifeq ($(TYPE),debug) PURIFY = @@ -591,11 +592,8 @@ GENERATE += $(TTF_DIR)/driver_tab.c # # This list must be consistent with PRE_LOADED_MODULES in # erts/preloaded/src/Makefile. -ifeq ($(TARGET),win32) -# On windows the preloaded objects are in a resource object. -PRELOAD_OBJ = $(OBJDIR)/beams.$(RES_EXT) -PRELOAD_SRC = $(TARGET)/beams.rc -$(PRELOAD_SRC): $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ + +PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ $(ERL_TOP)/erts/preloaded/ebin/init.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \ @@ -606,23 +604,20 @@ $(PRELOAD_SRC): $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/erl_prim_loader.beam \ $(ERL_TOP)/erts/preloaded/ebin/erlang.beam \ $(ERL_TOP)/erts/preloaded/ebin/erts_internal.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam + $(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_code_checker.beam + +ifeq ($(TARGET),win32) +# On windows the preloaded objects are in a resource object. +PRELOAD_OBJ = $(OBJDIR)/beams.$(RES_EXT) +PRELOAD_SRC = $(TARGET)/beams.rc +$(PRELOAD_SRC): $(PRELOAD_BEAM) $(gen_verbose)LANG=C $(PERL) utils/make_preload $(MAKE_PRELOAD_EXTRA) -rc $^ > $@ else PRELOAD_OBJ = $(OBJDIR)/preload.o PRELOAD_SRC = $(TARGET)/preload.c -$(PRELOAD_SRC): $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ - $(ERL_TOP)/erts/preloaded/ebin/init.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ - $(ERL_TOP)/erts/preloaded/ebin/zlib.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_zip.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erl_prim_loader.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erlang.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erts_internal.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam +$(PRELOAD_SRC): $(PRELOAD_BEAM) $(gen_verbose)LANG=C $(PERL) utils/make_preload -old $^ > $@ endif diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index badd69856e..89c42f4700 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -65,6 +65,7 @@ atom undefined_lambda atom DOWN='DOWN' atom UP='UP' atom EXIT='EXIT' +atom abort atom aborted atom abs_path atom absoluteURI @@ -165,6 +166,7 @@ atom commandv atom compact atom compat_rel atom compile +atom complete atom compressed atom config_h atom convert_time_unit @@ -176,6 +178,7 @@ atom const atom context_switches atom control atom copy +atom copy_literals atom counters atom cpu atom cpu_timestamp @@ -195,6 +198,7 @@ atom dgroup_leader atom dictionary atom dirty_cpu atom dirty_cpu_schedulers_online +atom dirty_execution atom dirty_io atom disable_trace atom disabled @@ -235,6 +239,7 @@ atom erlang atom ERROR='ERROR' atom error_handler atom error_logger +atom erts_code_purger atom erts_internal atom ets atom ETS_TRANSFER='ETS-TRANSFER' @@ -407,6 +412,7 @@ atom named_table atom namelist atom native atom native_addresses +atom need_gc atom Neq='=/=' atom Neqeq='/=' atom net_kernel @@ -485,6 +491,7 @@ atom pause atom pending atom pending_driver atom pending_process +atom pending_purge_lambda atom pending_reload atom permanent atom pid @@ -494,6 +501,7 @@ atom port_count atom port_limit atom port_op atom positive +atom prepare atom print atom priority atom private diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 37e74400bc..5c28fb1a98 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -41,13 +41,81 @@ # include "hipe_stack.h" #endif +static struct { + Eterm module; + erts_smp_mtx_t mtx; + Export *pending_purge_lambda; + Eterm *sprocs; + Eterm def_sprocs[10]; + Uint sp_size; + Uint sp_ix; + ErlFunEntry **funs; + ErlFunEntry *def_funs[10]; + Uint fe_size; + Uint fe_ix; +} purge_state; + +Process *erts_code_purger = NULL; + +ErtsLiteralArea *erts_copy_literal_area = NULL; +#ifdef ERTS_DIRTY_SCHEDULERS +Process *erts_dirty_process_code_checker; +#endif +#ifdef ERTS_NEW_PURGE_STRATEGY +Process *erts_literal_area_collector = NULL; + +typedef struct ErtsLiteralAreaRef_ ErtsLiteralAreaRef; +struct ErtsLiteralAreaRef_ { + ErtsLiteralAreaRef *next; + ErtsLiteralArea *literal_area; +}; + +struct { + erts_smp_mtx_t mtx; + ErtsLiteralAreaRef *first; + ErtsLiteralAreaRef *last; +} release_literal_areas; +#endif + static void set_default_trace_pattern(Eterm module); static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls); static void delete_code(Module* modp); -static void decrement_refc(BeamCodeHeader*); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); +static void +init_purge_state(void) +{ + purge_state.module = THE_NON_VALUE; + + erts_smp_mtx_init(&purge_state.mtx, "purge_state"); + + purge_state.pending_purge_lambda = + erts_export_put(am_erts_code_purger, am_pending_purge_lambda, 3); + + purge_state.sprocs = &purge_state.def_sprocs[0]; + purge_state.sp_size = sizeof(purge_state.def_sprocs); + purge_state.sp_size /= sizeof(purge_state.def_sprocs[0]); + purge_state.sp_ix = 0; + + purge_state.funs = &purge_state.def_funs[0]; + purge_state.fe_size = sizeof(purge_state.def_funs); + purge_state.fe_size /= sizeof(purge_state.def_funs[0]); + purge_state.fe_ix = 0; +} + +void +erts_beam_bif_load_init(void) +{ +#ifdef ERTS_NEW_PURGE_STRATEGY + erts_smp_mtx_init(&release_literal_areas.mtx, "release_literal_areas"); + release_literal_areas.first = NULL; + release_literal_areas.last = NULL; +#endif + + init_purge_state(); +} + BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) { Module* modp; @@ -520,6 +588,43 @@ badarg: BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2) +{ +#if !defined(ERTS_DIRTY_SCHEDULERS) + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else + Process *rp; + int reds = 0; + Eterm res; + + if (BIF_P != erts_dirty_process_code_checker) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + if (is_not_internal_pid(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + if (is_not_atom(BIF_ARG_2)) + BIF_ERROR(BIF_P, BADARG); + + rp = erts_pid2proc_not_running(BIF_P, ERTS_PROC_LOCK_MAIN, + BIF_ARG_1, ERTS_PROC_LOCK_MAIN); + if (rp == ERTS_PROC_LOCK_BUSY) + ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_check_dirty_process_code_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + if (!rp) + BIF_RET(am_false); + + res = erts_check_process_code(rp, BIF_ARG_2, 0, &reds, BIF_P->fcalls); + + if (BIF_P != rp) + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); + + ASSERT(is_value(res)); + + BIF_RET2(res, reds); +#endif +} + BIF_RETTYPE delete_module_1(BIF_ALIST_1) { ErtsCodeIndex code_ix; @@ -752,6 +857,8 @@ set_default_trace_pattern(Eterm module) } } +#ifndef ERTS_NEW_PURGE_STRATEGY + static ERTS_INLINE int check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size) { @@ -766,12 +873,244 @@ check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size) return 0; } +#endif + static Uint hfrag_literal_size(Eterm* start, Eterm* end, char* lit_start, Uint lit_size); static void hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, Eterm *start, Eterm *end, char *lit_start, Uint lit_size); +#ifdef ERTS_NEW_PURGE_STRATEGY + +Eterm +erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed) +{ + ErtsLiteralArea *la; + ErtsMessage *msgp; + struct erl_off_heap_header* oh; + char *literals; + Uint lit_bsize; + ErlHeapFragment *hfrag; + + la = erts_copy_literal_area; + if (!la) + return am_ok; + + oh = la->off_heap; + literals = (char *) &la->start[0]; + lit_bsize = (char *) la->end - literals; + + /* + * If a literal is in the message queue we make an explicit copy of + * it and attach it to the heap fragment. Each message needs to be + * self contained, we cannot save the literal in the old_heap or + * any other heap than the message it self. + */ + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + + for (msgp = c_p->msg.first; msgp; msgp = msgp->next) { + ErlHeapFragment *hf; + Uint lit_sz = 0; + + *redsp += 1; + + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) + hfrag = &msgp->hfrag; + else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag) + hfrag = msgp->data.heap_frag; + else + continue; /* Content on heap or in external term format... */ + + for (hf = hfrag; hf; hf = hf->next) { + lit_sz += hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size], + literals, lit_bsize); + *redsp += 1; + } + + *redsp += lit_sz / 16; /* Better value needed... */ + if (lit_sz > 0) { + ErlHeapFragment *bp = new_message_buffer(lit_sz); + Eterm *hp = bp->mem; + + for (hf = hfrag; hf; hf = hf->next) { + hfrag_literal_copy(&hp, &bp->off_heap, + &hf->mem[0], &hf->mem[hf->used_size], + literals, lit_bsize); + hfrag = hf; + } + + /* link new hfrag last */ + ASSERT(hfrag->next == NULL); + hfrag->next = bp; + bp->next = NULL; + } + } + + if (gc_allowed) { + /* + * Current implementation first tests without + * allowing GC, and then restarts the operation + * allowing GC if it is needed. It is therfore + * very likely that we will need the GC (although + * this is not completely certain). We go for + * the GC directly instead of scanning everything + * one more time... + */ + goto literal_gc; + } + + *redsp += 2; + if (any_heap_ref_ptrs(&c_p->fvalue, &c_p->fvalue+1, literals, lit_bsize)) { + c_p->freason = EXC_NULL; + c_p->fvalue = NIL; + c_p->ftrace = NIL; + } + + if (any_heap_ref_ptrs(c_p->stop, c_p->hend, literals, lit_bsize)) + goto literal_gc; + *redsp += 1; + if (any_heap_refs(c_p->heap, c_p->htop, literals, lit_bsize)) + goto literal_gc; + *redsp += 1; + if (any_heap_refs(c_p->old_heap, c_p->old_htop, literals, lit_bsize)) + goto literal_gc; + + /* Check dictionary */ + *redsp += 1; + if (c_p->dictionary) { + Eterm* start = ERTS_PD_START(c_p->dictionary); + Eterm* end = start + ERTS_PD_SIZE(c_p->dictionary); + + if (any_heap_ref_ptrs(start, end, literals, lit_bsize)) + goto literal_gc; + } + + /* Check heap fragments */ + for (hfrag = c_p->mbuf; hfrag; hfrag = hfrag->next) { + Eterm *hp, *hp_end; + + *redsp += 1; + + hp = &hfrag->mem[0]; + hp_end = &hfrag->mem[hfrag->used_size]; + if (any_heap_refs(hp, hp_end, literals, lit_bsize)) + goto literal_gc; + } + + /* + * Message buffer fragments (matched messages) + * - off heap lists should already have been moved into + * process off heap structure. + * - Check for literals + */ + for (msgp = c_p->msg_frag; msgp; msgp = msgp->next) { + hfrag = erts_message_to_heap_frag(msgp); + for (; hfrag; hfrag = hfrag->next) { + Eterm *hp, *hp_end; + + *redsp += 1; + + hp = &hfrag->mem[0]; + hp_end = &hfrag->mem[hfrag->used_size]; + + if (any_heap_refs(hp, hp_end, literals, lit_bsize)) + goto literal_gc; + } + } + + return am_ok; + +literal_gc: + + if (!gc_allowed) + return am_need_gc; + + if (c_p->flags & F_DISABLE_GC) + return THE_NON_VALUE; + + FLAGS(c_p) |= F_NEED_FULLSWEEP; + + *redsp += erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, c_p->arity, fcalls); + + erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize, oh); + + *redsp += lit_bsize / 64; /* Need, better value... */ + + return am_ok; +} + +static Eterm +check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) +{ + BeamInstr* start; + char* mod_start; + Uint mod_size; + Eterm* sp; + + *redsp += 1; + + /* + * Pick up limits for the module. + */ + start = (BeamInstr*) modp->old.code_hdr; + mod_start = (char *) start; + mod_size = modp->old.code_length; + + /* + * Check if current instruction or continuation pointer points into module. + */ + if (ErtsInArea(rp->i, mod_start, mod_size) + || ErtsInArea(rp->cp, mod_start, mod_size)) { + return am_true; + } + + *redsp += (STACK_START(rp) - rp->stop) / 32; + + /* + * Check all continuation pointers stored on the stack. + */ + for (sp = rp->stop; sp < STACK_START(rp); sp++) { + if (is_CP(*sp) && ErtsInArea(cp_val(*sp), mod_start, mod_size)) { + return am_true; + } + } + + /* + * Check all continuation pointers stored in stackdump + * and clear exception stackdump if there is a pointer + * to the module. + */ + if (rp->ftrace != NIL) { + struct StackTrace *s; + ASSERT(is_list(rp->ftrace)); + s = (struct StackTrace *) big_val(CDR(list_val(rp->ftrace))); + if ((s->pc && ErtsInArea(s->pc, mod_start, mod_size)) || + (s->current && ErtsInArea(s->current, mod_start, mod_size))) { + rp->freason = EXC_NULL; + rp->fvalue = NIL; + rp->ftrace = NIL; + } else { + int i; + for (i = 0; i < s->depth; i++) { + if (ErtsInArea(s->trace[i], mod_start, mod_size)) { + rp->freason = EXC_NULL; + rp->fvalue = NIL; + rp->ftrace = NIL; + break; + } + } + } + } + + return am_false; +} + +#else /* !ERTS_NEW_PURGE_STRATEGY, i.e, old style purge... */ + static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) { @@ -865,8 +1204,14 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); - literals = (char*) modp->old.code_hdr->literals_start; - lit_bsize = (char*) modp->old.code_hdr->literals_end - literals; + if (modp->old.code_hdr->literal_area) { + literals = (char*) modp->old.code_hdr->literal_area->start; + lit_bsize = (char*) modp->old.code_hdr->literal_area->end - literals; + } + else { + literals = NULL; + lit_bsize = 0; + } for (msgp = rp->msg.first; msgp; msgp = msgp->next) { if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) @@ -907,12 +1252,6 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls /* Check heap, stack etc... */ if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size)) goto try_gc; - if (!(flags & ERTS_CPC_COPY_LITERALS)) { - /* Process ok. May contain old literals but we will be called - * again before module is purged. - */ - return am_false; - } if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, literals, lit_bsize)) { rp->freason = EXC_NULL; rp->fvalue = NIL; @@ -1002,7 +1341,7 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } if (need_gc & ERTS_LITERAL_GC__) { struct erl_off_heap_header* oh; - oh = modp->old.code_hdr->literals_off_heap; + oh = modp->old.code_hdr->literal_area->off_heap; *redsp += lit_bsize / 64; /* Need, better value... */ erts_garbage_collect_literals(rp, (Eterm*)literals, lit_bsize, oh); done_gc |= ERTS_LITERAL_GC__; @@ -1015,6 +1354,8 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } +#endif /* !ERTS_NEW_PURGE_STRATEGY */ + static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) { @@ -1143,200 +1484,438 @@ hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, } } -#undef in_area +#ifdef ERTS_NEW_PURGE_STRATEGY + +ErtsThrPrgrLaterOp later_literal_area_switch; #ifdef ERTS_SMP -static void copy_literals_commit(void*); +static void +complete_literal_area_switch(void *unused) +{ + Process *p = erts_literal_area_collector; + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_resume(p, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); +} #endif -copy_literals_t erts_clrange = {NULL, 0, THE_NON_VALUE}; +#endif /* ERTS_NEW_PURGE_STRATEGY */ -/* copy literals - * - * copy_literals.ptr = LitPtr - * copy_literals.sz = LitSz - * ------ THR PROG COMMIT ----- - * - * - check process code - * - check process code - * ... - * copy_literals.ptr = NULL - * copy_literals.sz = 0 - * ------ THR PROG COMMIT ----- - * ... - */ +BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) +{ +#ifndef ERTS_NEW_PURGE_STRATEGY + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else + ErtsLiteralAreaRef *la_ref; + if (BIF_P != erts_literal_area_collector) + BIF_ERROR(BIF_P, EXC_NOTSUP); -BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2) -{ - ErtsCodeIndex code_ix; - Eterm res = am_true; + erts_smp_mtx_lock(&release_literal_areas.mtx); - if (is_not_atom(BIF_ARG_1) || (am_true != BIF_ARG_2 && am_false != BIF_ARG_2)) { - BIF_ERROR(BIF_P, BADARG); + la_ref = release_literal_areas.first; + if (la_ref) { + release_literal_areas.first = la_ref->next; + if (!release_literal_areas.first) + release_literal_areas.last = NULL; } - if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_copy_literals_2], - BIF_P, BIF_ARG_1, BIF_ARG_2); - } + erts_smp_mtx_unlock(&release_literal_areas.mtx); - code_ix = erts_active_code_ix(); + if (erts_copy_literal_area) + erts_release_literal_area(erts_copy_literal_area); - if (BIF_ARG_2 == am_true) { - Module* modp = erts_get_module(BIF_ARG_1, code_ix); - if (!modp || !modp->old.code_hdr) { - res = am_false; - goto done; - } - if (erts_clrange.ptr != NULL - && !(BIF_P->static_flags & ERTS_STC_FLG_SYSTEM_PROC)) { - res = am_aborted; - goto done; - } - erts_clrange.ptr = modp->old.code_hdr->literals_start; - erts_clrange.sz = modp->old.code_hdr->literals_end - erts_clrange.ptr; - erts_clrange.pid = BIF_P->common.id; - } else if (BIF_ARG_2 == am_false) { - if (erts_clrange.pid != BIF_P->common.id) { - res = am_false; - goto done; - } - erts_clrange.ptr = NULL; - erts_clrange.sz = 0; - erts_clrange.pid = THE_NON_VALUE; + if (!la_ref) { + erts_copy_literal_area = NULL; + BIF_RET(am_false); } -#ifdef ERTS_SMP - ASSERT(committer_state.stager == NULL); - committer_state.stager = BIF_P; - erts_schedule_thr_prgr_later_op(copy_literals_commit, NULL, &committer_state.lop); - erts_proc_inc_refc(BIF_P); + erts_copy_literal_area = la_ref->literal_area; + + erts_free(ERTS_ALC_T_LITERAL_REF, la_ref); + +#ifndef ERTS_SMP + BIF_RET(am_true); +#else + erts_schedule_thr_prgr_later_op(complete_literal_area_switch, + NULL, + &later_literal_area_switch); erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); ERTS_BIF_YIELD_RETURN(BIF_P, am_true); #endif -done: - erts_release_code_write_permission(); - BIF_RET(res); + +#endif /* ERTS_NEW_PURGE_STRATEGY */ +} + +void +erts_purge_state_add_fun(ErlFunEntry *fe) +{ + ASSERT(is_value(purge_state.module)); + if (purge_state.fe_ix >= purge_state.fe_size) { + ErlFunEntry **funs; + purge_state.fe_size += 100; + funs = erts_alloc(ERTS_ALC_T_PURGE_DATA, + sizeof(ErlFunEntry *)*purge_state.fe_size); + sys_memcpy((void *) funs, + (void *) purge_state.funs, + purge_state.fe_ix*sizeof(ErlFunEntry *)); + if (purge_state.funs != &purge_state.def_funs[0]) + erts_free(ERTS_ALC_T_PURGE_DATA, purge_state.funs); + purge_state.funs = funs; + } + purge_state.funs[purge_state.fe_ix++] = fe; +} + +Export * +erts_suspend_process_on_pending_purge_lambda(Process *c_p) +{ + erts_smp_mtx_lock(&purge_state.mtx); + if (is_value(purge_state.module)) { + /* + * The process c_p is about to call a fun in the code + * that we are trying to purge. Suspend it and call + * erts_code_purger:pending_purge_lambda/3. The process + * will be resumed when the purge completes or aborts, + * and will then try to do the call again. + */ + if (purge_state.sp_ix >= purge_state.sp_size) { + Eterm *sprocs; + purge_state.sp_size += 100; + sprocs = erts_alloc(ERTS_ALC_T_PURGE_DATA, + (sizeof(ErlFunEntry *) + * purge_state.sp_size)); + sys_memcpy((void *) sprocs, + (void *) purge_state.sprocs, + purge_state.sp_ix*sizeof(ErlFunEntry *)); + if (purge_state.sprocs != &purge_state.def_sprocs[0]) + erts_free(ERTS_ALC_T_PURGE_DATA, purge_state.sprocs); + purge_state.sprocs = sprocs; + } + purge_state.sprocs[purge_state.sp_ix++] = c_p->common.id; + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_VBUMP_ALL_REDS(c_p); + } + erts_smp_mtx_unlock(&purge_state.mtx); + return purge_state.pending_purge_lambda; +} + +static void +finalize_purge_operation(Process *c_p, int succeded) +{ + Uint ix; + + if (c_p) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + + erts_smp_mtx_lock(&purge_state.mtx); + + ASSERT(purge_state.module != THE_NON_VALUE); + + purge_state.module = THE_NON_VALUE; + + /* + * Resume all processes that have tried to call + * funs in this code. + */ + for (ix = 0; ix < purge_state.sp_ix; ix++) { + Process *rp = erts_pid2proc(NULL, 0, + purge_state.sprocs[ix], + ERTS_PROC_LOCK_STATUS); + if (rp) { + erts_resume(rp, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + } + } + + erts_smp_mtx_unlock(&purge_state.mtx); + + if (c_p) + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + + if (purge_state.sprocs != &purge_state.def_sprocs[0]) { + erts_free(ERTS_ALC_T_PURGE_DATA, purge_state.sprocs); + purge_state.sprocs = &purge_state.def_sprocs[0]; + purge_state.sp_size = sizeof(purge_state.def_sprocs); + purge_state.sp_size /= sizeof(purge_state.def_sprocs[0]); + } + purge_state.sp_ix = 0; + + if (purge_state.funs != &purge_state.def_funs[0]) { + erts_free(ERTS_ALC_T_PURGE_DATA, purge_state.funs); + purge_state.funs = &purge_state.def_funs[0]; + purge_state.fe_size = sizeof(purge_state.def_funs); + purge_state.fe_size /= sizeof(purge_state.def_funs[0]); + } + purge_state.fe_ix = 0; } #ifdef ERTS_SMP -static void copy_literals_commit(void* null) { - Process* p = committer_state.stager; -#ifdef DEBUG - committer_state.stager = NULL; -#endif - erts_release_code_write_permission(); + +static ErtsThrPrgrLaterOp purger_lop_data; + +static void +resume_purger(void *unused) +{ + Process *p = erts_code_purger; erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); - if (!ERTS_PROC_IS_EXITING(p)) { - erts_resume(p, ERTS_PROC_LOCK_STATUS); - } + erts_resume(p, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - erts_proc_dec_refc(p); } -#endif /* ERTS_SMP */ +static void +finalize_purge_abort(void *unused) +{ + erts_fun_purge_abort_finalize(purge_state.funs, purge_state.fe_ix); + + finalize_purge_operation(NULL, 0); -/* Do the actualy module purging and return: - * true for success - * false if no such old module - * BADARG if not an atom - */ -BIF_RETTYPE erts_internal_purge_module_1(BIF_ALIST_1) + resume_purger(NULL); +} + +#endif /* ERTS_SMP */ + +BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) { - ErtsCodeIndex code_ix; - BeamInstr* code; - BeamInstr* end; - Module* modp; - int is_blocking = 0; - Eterm ret; + if (BIF_P != erts_code_purger) + BIF_ERROR(BIF_P, EXC_NOTSUP); - if (is_not_atom(BIF_ARG_1)) { + if (is_not_atom(BIF_ARG_1)) BIF_ERROR(BIF_P, BADARG); - } - if (!erts_try_seize_code_write_permission(BIF_P)) { - ERTS_BIF_YIELD1(bif_export[BIF_erts_internal_purge_module_1], - BIF_P, BIF_ARG_1); - } + switch (BIF_ARG_2) { - code_ix = erts_active_code_ix(); + case am_prepare: { + /* + * Prepare for purge by marking all fun + * entries referring to the code to purge + * with "pending purge" markers. + */ + ErtsCodeIndex code_ix; + Module* modp; + Eterm res; - /* - * Correct module? - */ + if (is_value(purge_state.module)) + BIF_ERROR(BIF_P, BADARG); - if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) { - ERTS_BIF_PREP_RET(ret, am_false); - } - else { - erts_rwlock_old_code(code_ix); + code_ix = erts_active_code_ix(); /* - * Any code to purge? + * Correct module? */ - if (!modp->old.code_hdr) { - ERTS_BIF_PREP_RET(ret, am_false); - } + modp = erts_get_module(BIF_ARG_1, code_ix); + if (!modp) + res = am_false; else { /* - * Unload any NIF library + * Any code to purge? */ - if (modp->old.nif != NULL) { - /* ToDo: Do unload nif without blocking */ - erts_rwunlock_old_code(code_ix); - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); - is_blocking = 1; - erts_rwlock_old_code(code_ix); - erts_unload_nif(modp->old.nif); - modp->old.nif = NULL; + erts_rlock_old_code(code_ix); + if (!modp->old.code_hdr) + res = am_false; + else { + BeamInstr* code; + BeamInstr* end; + erts_smp_mtx_lock(&purge_state.mtx); + purge_state.module = BIF_ARG_1; + erts_smp_mtx_unlock(&purge_state.mtx); + res = am_true; + code = (BeamInstr*) modp->old.code_hdr; + end = (BeamInstr *)((char *)code + modp->old.code_length); + erts_fun_purge_prepare(code, end); +#if !defined(ERTS_NEW_PURGE_STRATEGY) + ASSERT(!erts_copy_literal_area); + erts_copy_literal_area = modp->old.code_hdr->literal_area; +#endif } - + erts_runlock_old_code(code_ix); + } + +#ifndef ERTS_SMP + BIF_RET(res); +#else + if (res != am_true) + BIF_RET(res); + else { /* - * Remove the old code. + * We'll be resumed when all schedulers are guaranteed + * to see the "pending purge" markers that we've made on + * all fun entries of the code that we are about to purge. + * Processes trying to call these funs will be suspended + * before calling the funs. That is we are guaranteed not + * to get any more direct references into the code while + * checking for such references... */ - ASSERT(erts_total_code_size >= modp->old.code_length); - erts_total_code_size -= modp->old.code_length; - code = (BeamInstr*) modp->old.code_hdr; - end = (BeamInstr *)((char *)code + modp->old.code_length); - erts_cleanup_funs_on_purge(code, end); - beam_catches_delmod(modp->old.catches, code, modp->old.code_length, - code_ix); - decrement_refc(modp->old.code_hdr); - if (modp->old.code_hdr->literals_start) { - erts_free(ERTS_ALC_T_LITERAL, modp->old.code_hdr->literals_start); - } - erts_free(ERTS_ALC_T_CODE, (void *) code); - modp->old.code_hdr = NULL; - modp->old.code_length = 0; - modp->old.catches = BEAM_CATCHES_NIL; - erts_remove_from_ranges(code); - ERTS_BIF_PREP_RET(ret, am_true); + erts_schedule_thr_prgr_later_op(resume_purger, + NULL, + &purger_lop_data); + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); } - erts_rwunlock_old_code(code_ix); +#endif } - if (is_blocking) { - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + + case am_abort: { + /* + * Soft purge that detected direct references into the code + * we set out to purge. Abort the purge. + */ + + if (purge_state.module != BIF_ARG_1) + BIF_ERROR(BIF_P, BADARG); + + erts_fun_purge_abort_prepare(purge_state.funs, purge_state.fe_ix); + +#if !defined(ERTS_NEW_PURGE_STRATEGY) + ASSERT(erts_copy_literal_area); + erts_copy_literal_area = NULL; +#endif +#ifndef ERTS_SMP + erts_fun_purge_abort_finalize(purge_state.funs, purge_state.fe_ix); + finalize_purge_operation(BIF_P, 0); + BIF_RET(am_false); +#else + /* + * We need to restore the code addresses of the funs in + * two stages in order to ensure that we do not get any + * stale suspended processes due to the purge abort. + * Restore address pointer (erts_fun_purge_abort_prepare); + * wait for thread progress; clear pending purge address + * pointer (erts_fun_purge_abort_finalize), and then + * resume processes that got suspended + * (finalize_purge_operation). + */ + erts_schedule_thr_prgr_later_op(finalize_purge_abort, + NULL, + &purger_lop_data); + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_BIF_YIELD_RETURN(BIF_P, am_false); +#endif } - erts_release_code_write_permission(); - return ret; -} -static void -decrement_refc(BeamCodeHeader* code_hdr) -{ - struct erl_off_heap_header* oh = code_hdr->literals_off_heap; - - while (oh) { - Binary* bptr; - ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG); - bptr = ((ProcBin*)oh)->val; - if (erts_refc_dectest(&bptr->refc, 0) == 0) { - erts_bin_free(bptr); + case am_complete: { + ErtsCodeIndex code_ix; + BeamInstr* code; + Module* modp; + int is_blocking = 0; + Eterm ret; + ErtsLiteralArea *literals = NULL; + + + /* + * We have no direct references into the code. + * Complete to purge. + */ + + if (purge_state.module != BIF_ARG_1) + BIF_ERROR(BIF_P, BADARG); + + if (!erts_try_seize_code_write_permission(BIF_P)) { + ERTS_BIF_YIELD2(bif_export[BIF_erts_internal_purge_module_2], + BIF_P, BIF_ARG_1, BIF_ARG_2); + } + + code_ix = erts_active_code_ix(); + + /* + * Correct module? + */ + + if ((modp = erts_get_module(BIF_ARG_1, code_ix)) == NULL) { + ERTS_BIF_PREP_RET(ret, am_false); } - oh = oh->next; + else { + + erts_rwlock_old_code(code_ix); + + /* + * Any code to purge? + */ + if (!modp->old.code_hdr) { + ERTS_BIF_PREP_RET(ret, am_false); + } + else { + /* + * Unload any NIF library + */ + if (modp->old.nif != NULL) { + /* ToDo: Do unload nif without blocking */ + erts_rwunlock_old_code(code_ix); + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + is_blocking = 1; + erts_rwlock_old_code(code_ix); + erts_unload_nif(modp->old.nif); + modp->old.nif = NULL; + } + + /* + * Remove the old code. + */ + ASSERT(erts_total_code_size >= modp->old.code_length); + erts_total_code_size -= modp->old.code_length; + code = (BeamInstr*) modp->old.code_hdr; + erts_fun_purge_complete(purge_state.funs, purge_state.fe_ix); + beam_catches_delmod(modp->old.catches, code, modp->old.code_length, + code_ix); + literals = modp->old.code_hdr->literal_area; + modp->old.code_hdr->literal_area = NULL; + erts_free(ERTS_ALC_T_CODE, (void *) code); + modp->old.code_hdr = NULL; + modp->old.code_length = 0; + modp->old.catches = BEAM_CATCHES_NIL; + erts_remove_from_ranges(code); + ERTS_BIF_PREP_RET(ret, am_true); + } + erts_rwunlock_old_code(code_ix); + } + if (is_blocking) { + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + } + + erts_release_code_write_permission(); + + finalize_purge_operation(BIF_P, ret == am_true); + +#if !defined(ERTS_NEW_PURGE_STRATEGY) + + ASSERT(erts_copy_literal_area == literals); + erts_copy_literal_area = NULL; + erts_release_literal_area(literals); + +#else /* ERTS_NEW_PURGE_STRATEGY */ + + if (literals) { + ErtsLiteralAreaRef *ref; + ref = erts_alloc(ERTS_ALC_T_LITERAL_REF, + sizeof(ErtsLiteralAreaRef)); + ref->literal_area = literals; + ref->next = NULL; + erts_smp_mtx_lock(&release_literal_areas.mtx); + if (release_literal_areas.last) { + release_literal_areas.last->next = ref; + release_literal_areas.last = ref; + } + else { + release_literal_areas.first = ref; + release_literal_areas.last = ref; + } + erts_smp_mtx_unlock(&release_literal_areas.mtx); + erts_queue_message(erts_literal_area_collector, + 0, + erts_alloc_message(0, NULL), + am_copy_literals, + BIF_P->common.id); + } + +#endif /* ERTS_NEW_PURGE_STRATEGY */ + + return ret; + } + + default: + BIF_ERROR(BIF_P, BADARG); + } } diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 6100c6c923..3b12f78ced 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -5382,7 +5382,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) ASSERT(!c_p->scheduler_data); erts_pre_dirty_nif(esdp, &env, c_p, - (struct erl_module_nif*)I[2], NULL); + (struct erl_module_nif*)I[2]); #ifdef DEBUG result = @@ -5391,7 +5391,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) #endif (*fp)(&env, arity, reg); - erts_post_nif(&env); + erts_post_dirty_nif(&env); ASSERT(!is_value(result)); ASSERT(c_p->freason == TRAP); @@ -6509,34 +6509,49 @@ call_fun(Process* p, /* Current process. */ * representation (the module has never been loaded), * or the module defining the fun has been unloaded. */ + module = fe->module; - if ((modp = erts_get_module(module, code_ix)) != NULL - && modp->curr.code_hdr != NULL) { + + ERTS_SMP_READ_MEMORY_BARRIER; + if (fe->pend_purge_address) { /* - * There is a module loaded, but obviously the fun is not - * defined in it. We must not call the error_handler - * (or we will get into an infinite loop). + * The system is currently trying to purge the + * module containing this fun. Suspend the process + * and let it try again when the purge operation is + * done (may succeed or not). */ - goto badfun; + ep = erts_suspend_process_on_pending_purge_lambda(p); + ASSERT(ep); } + else { + if ((modp = erts_get_module(module, code_ix)) != NULL + && modp->curr.code_hdr != NULL) { + /* + * There is a module loaded, but obviously the fun is not + * defined in it. We must not call the error_handler + * (or we will get into an infinite loop). + */ + goto badfun; + } - /* - * No current code for this module. Call the error_handler module - * to attempt loading the module. - */ + /* + * No current code for this module. Call the error_handler module + * to attempt loading the module. + */ - ep = erts_find_function(erts_proc_get_error_handler(p), - am_undefined_lambda, 3, code_ix); - if (ep == NULL) { /* No error handler */ - p->current = NULL; - p->freason = EXC_UNDEF; - return NULL; + ep = erts_find_function(erts_proc_get_error_handler(p), + am_undefined_lambda, 3, code_ix); + if (ep == NULL) { /* No error handler */ + p->current = NULL; + p->freason = EXC_UNDEF; + return NULL; + } } reg[0] = module; reg[1] = fun; reg[2] = args; reg[3] = NIL; - return ep->addressv[erts_active_code_ix()]; + return ep->addressv[code_ix]; } } } else if (is_export_header(hdr)) { diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 517004d3a0..f5aa9e1c0c 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -663,7 +663,7 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader, stp->hdr->compile_ptr = NULL; stp->hdr->compile_size = 0; stp->hdr->compile_size_on_heap = 0; - stp->hdr->literals_start = NULL; + stp->hdr->literal_area = NULL; stp->hdr->md5_ptr = NULL; /* @@ -1005,8 +1005,9 @@ loader_state_dtor(Binary* magic) stp->bin = 0; } if (stp->hdr != 0) { - if (stp->hdr->literals_start) { - erts_free(ERTS_ALC_T_LITERAL, stp->hdr->literals_start); + if (stp->hdr->literal_area) { + erts_release_literal_area(stp->hdr->literal_area); + stp->hdr->literal_area = NULL; } erts_free(ERTS_ALC_T_CODE, stp->hdr); stp->hdr = 0; @@ -4517,13 +4518,16 @@ freeze_code(LoaderState* stp) Eterm* ptr; LiteralPatch* lp; ErlOffHeap code_off_heap; + ErtsLiteralArea *literal_area; + Uint lit_asize; ERTS_INIT_OFF_HEAP(&code_off_heap); - ptr = (Eterm*)erts_alloc(ERTS_ALC_T_LITERAL, - stp->total_literal_size*sizeof(Eterm)); - code_hdr->literals_start = ptr; - code_hdr->literals_end = ptr + stp->total_literal_size; + lit_asize = ERTS_LITERAL_AREA_ALLOC_SIZE(stp->total_literal_size); + literal_area = erts_alloc(ERTS_ALC_T_LITERAL, lit_asize); + ptr = &literal_area->start[0]; + literal_area->end = ptr + stp->total_literal_size; + for (i = 0; i < stp->num_literals; i++) { if (is_not_immed(stp->literals[i].term)) { erts_move_multi_frags(&ptr, &code_off_heap, @@ -4533,7 +4537,7 @@ freeze_code(LoaderState* stp) ptr_val(stp->literals[i].term))); } } - code_hdr->literals_off_heap = code_off_heap.first; + literal_area->off_heap = code_off_heap.first; lp = stp->literal_patches; while (lp != 0) { BeamInstr* op_ptr; @@ -4544,6 +4548,7 @@ freeze_code(LoaderState* stp) op_ptr[0] = lit->term; lp = lp->next; } + code_hdr->literal_area = literal_area; } CHKBLK(ERTS_ALC_T_CODE,code); @@ -5599,6 +5604,28 @@ has_native(BeamCodeHeader *code_hdr) return result; } +void +erts_release_literal_area(ErtsLiteralArea* literal_area) +{ + struct erl_off_heap_header* oh; + + if (!literal_area) + return; + + oh = literal_area->off_heap; + + while (oh) { + Binary* bptr; + ASSERT(thing_subtag(oh->thing_word) == REFC_BINARY_SUBTAG); + bptr = ((ProcBin*)oh)->val; + if (erts_refc_dectest(&bptr->refc, 0) == 0) { + erts_bin_free(bptr); + } + oh = oh->next; + } + erts_free(ERTS_ALC_T_LITERAL, literal_area); +} + int erts_is_module_native(BeamCodeHeader* code_hdr) { @@ -6330,9 +6357,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) code_hdr->compile_ptr = NULL; code_hdr->compile_size = 0; code_hdr->compile_size_on_heap = 0; - code_hdr->literals_start = NULL; - code_hdr->literals_end = NULL; - code_hdr->literals_off_heap = 0; + code_hdr->literal_area = NULL; code_hdr->on_load_function_ptr = NULL; code_hdr->line_table = NULL; code_hdr->md5_ptr = NULL; diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 92a9abddbf..6be4031822 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -50,6 +50,7 @@ extern BeamInstr* em_call_error_handler; extern BeamInstr* em_apply_bif; extern BeamInstr* em_call_nif; +struct ErtsLiteralArea_; /* * The following variables keep a sorted list of address ranges for @@ -89,9 +90,7 @@ typedef struct beam_code_header { /* * Literal area (constant pool). */ - Eterm* literals_start; - Eterm* literals_end; - struct erl_off_heap_header* literals_off_heap; + struct ErtsLiteralArea_ *literal_area; /* * Pointer to the on_load function (or NULL if none). @@ -120,7 +119,12 @@ typedef struct beam_code_header { }BeamCodeHeader; +void erts_release_literal_area(struct ErtsLiteralArea_* literal_area); int erts_is_module_native(BeamCodeHeader* code); +void erts_beam_bif_load_init(void); +struct erl_fun_entry; +void erts_purge_state_add_fun(struct erl_fun_entry *fe); +Export *erts_suspend_process_on_pending_purge_lambda(Process *c_p); /* * Layout of the line table. diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 978bcd4bba..ea66f165a0 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -163,6 +163,7 @@ bif erts_internal:port_close/1 bif erts_internal:port_connect/2 bif erts_internal:request_system_task/3 +bif erts_internal:request_system_task/4 bif erts_internal:check_process_code/2 bif erts_internal:map_to_tuple_keys/1 @@ -176,6 +177,8 @@ bif erts_internal:is_system_process/1 bif erts_internal:system_check/1 +bif erts_internal:release_literal_area_switch/0 + # inet_db support bif erlang:port_set_data/2 bif erlang:port_get_data/1 @@ -644,8 +647,9 @@ bif erts_debug:map_info/1 # New in 19.0 # -bif erts_internal:copy_literals/2 -bif erts_internal:purge_module/1 +bif erts_internal:is_process_executing_dirty/1 +bif erts_internal:check_dirty_process_code/2 +bif erts_internal:purge_module/2 bif binary:split/2 bif binary:split/3 bif erts_debug:size_shared/1 diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 227fedfb69..971052b947 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -279,6 +279,7 @@ type TRACER_NIF LONG_LIVED SYSTEM tracer_nif type TRACE_MSG_QUEUE SHORT_LIVED SYSTEM trace_message_queue type SCHED_ASYNC_JOB SHORT_LIVED SYSTEM async_calls type DIRTY_START STANDARD PROCESSES dirty_start +type DIRTY_SL SHORT_LIVED SYSTEM dirty_short_lived +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; @@ -367,6 +368,8 @@ type MONITOR_LH STANDARD PROCESSES monitor_lh type NLINK_LH STANDARD PROCESSES nlink_lh type CODE LONG_LIVED CODE code type LITERAL LITERAL CODE literal +type LITERAL_REF SHORT_LIVED CODE literal_area_ref +type PURGE_DATA SHORT_LIVED CODE purge_data type DB_HEIR_DATA STANDARD ETS db_heir_data type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index cb7278696f..107c0c01d9 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2883,6 +2883,27 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(AM_tag); #endif } + else if (ERTS_IS_ATOM_STR("check_process_code",BIF_ARG_1)) { + Eterm terms[3]; + Sint length = 1; + Uint sz = 0; + Eterm *hp, res; + DECL_AM(direct_references); + + terms[0] = AM_direct_references; +#if !defined(ERTS_NEW_PURGE_STRATEGY) + { + DECL_AM(indirect_references); + terms[1] = AM_indirect_references; + terms[2] = am_copy_literals; + length = 3; + } +#endif + erts_bld_list(NULL, &sz, length, terms); + hp = HAlloc(BIF_P, sz); + res = erts_bld_list(&hp, NULL, length, terms); + BIF_RET(res); + } BIF_ERROR(BIF_P, BADARG); } diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 6ce1376c81..c639ba623f 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -199,14 +199,13 @@ erts_erase_fun_entry(ErlFunEntry* fe) } void -erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end) +erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end) { int limit; HashBucket** bucket; - ErlFunEntry* to_delete = NULL; int i; - erts_fun_write_lock(); + erts_fun_read_lock(); limit = erts_fun_table.size; bucket = erts_fun_table.bucket; for (i = 0; i < limit; i++) { @@ -217,22 +216,51 @@ erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end) BeamInstr* addr = fe->address; if (start <= addr && addr < end) { + fe->pend_purge_address = addr; + ERTS_SMP_WRITE_MEMORY_BARRIER; fe->address = unloaded_fun; - if (erts_refc_dectest(&fe->refc, 0) == 0) { - fe->address = (void *) to_delete; - to_delete = fe; - } + erts_purge_state_add_fun(fe); } b = b->next; } } + erts_fun_read_unlock(); +} + +void +erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no) +{ + Uint ix; - while (to_delete != NULL) { - ErlFunEntry* next = (ErlFunEntry *) to_delete->address; - erts_erase_fun_entry_unlocked(to_delete); - to_delete = next; + for (ix = 0; ix < no; ix++) { + ErlFunEntry *fe = funs[ix]; + if (fe->address == unloaded_fun) + fe->address = fe->pend_purge_address; + fe->pend_purge_address = NULL; } - erts_fun_write_unlock(); +} + +void +erts_fun_purge_abort_finalize(ErlFunEntry **funs, Uint no) +{ + Uint ix; + + for (ix = 0; ix < no; ix++) + funs[ix]->pend_purge_address = NULL; +} + +void +erts_fun_purge_complete(ErlFunEntry **funs, Uint no) +{ + Uint ix; + + for (ix = 0; ix < no; ix++) { + ErlFunEntry *fe = funs[ix]; + fe->pend_purge_address = NULL; + if (erts_refc_dectest(&fe->refc, 0) == 0) + erts_erase_fun_entry(fe); + } + ERTS_SMP_WRITE_MEMORY_BARRIER; } void @@ -294,6 +322,7 @@ fun_alloc(ErlFunEntry* template) obj->module = template->module; erts_refc_init(&obj->refc, -1); obj->address = unloaded_fun; + obj->pend_purge_address = NULL; #ifdef HIPE obj->native_address = NULL; #endif diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 8c4deea7a0..73c3e19c1c 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -44,6 +44,7 @@ typedef struct erl_fun_entry { Eterm module; /* Tagged atom for module. */ erts_refc_t refc; /* Reference count: One for code + one for each fun object in each process. */ + BeamInstr *pend_purge_address; /* address stored during a pending purge */ } ErlFunEntry; /* @@ -81,7 +82,10 @@ ErlFunEntry* erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, void erts_erase_fun_entry(ErlFunEntry* fe); void erts_cleanup_funs(ErlFunThing* funp); -void erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end); +void erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end); +void erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no); +void erts_fun_purge_abort_finalize(ErlFunEntry **funs, Uint no); +void erts_fun_purge_complete(ErlFunEntry **funs, Uint no); void erts_dump_fun_entries(int, void *); #endif diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 07cfacf14a..781bf024dd 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -382,6 +382,7 @@ erl_init(int ncpu, erts_init_unicode(); /* after RE to get access to PCRE unicode */ erts_init_external(); erts_init_map(); + erts_beam_bif_load_init(); erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2); erts_late_init_process(); #if HAVE_ERTS_MSEG @@ -2248,7 +2249,42 @@ erl_start(int argc, char **argv) otp_ring0_pid = erl_first_process_otp("otp_ring0", NULL, 0, boot_argc, boot_argv); - (void) erl_system_process_otp(otp_ring0_pid, "erts_code_purger"); + { + /* + * The erts_code_purger and the erts_literal_area_collector + * system processes are *always* alive. If they terminate + * they bring the whole VM down. + */ + Eterm pid; + + pid = erl_system_process_otp(otp_ring0_pid, "erts_code_purger"); + erts_code_purger + = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, + internal_pid_index(pid)); + ASSERT(erts_code_purger && erts_code_purger->common.id == pid); + erts_proc_inc_refc(erts_code_purger); + +#ifdef ERTS_NEW_PURGE_STRATEGY + pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector"); + erts_literal_area_collector + = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, + internal_pid_index(pid)); + ASSERT(erts_literal_area_collector + && erts_literal_area_collector->common.id == pid); + erts_proc_inc_refc(erts_literal_area_collector); +#endif + +#ifdef ERTS_DIRTY_SCHEDULERS + pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker"); + erts_dirty_process_code_checker + = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, + internal_pid_index(pid)); + ASSERT(erts_dirty_process_code_checker + && erts_dirty_process_code_checker->common.id == pid); + erts_proc_inc_refc(erts_dirty_process_code_checker); +#endif + + } #ifdef ERTS_SMP erts_start_schedulers(); diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 39c0617143..06266363b5 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -96,6 +96,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "dist_entry", "address" }, { "dist_entry_links", "address" }, { "code_write_permission", NULL }, + { "purge_state", NULL }, { "proc_status", "pid" }, { "proc_trace", "pid" }, { "ports_snapshot", NULL }, @@ -112,6 +113,9 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "export_tab", NULL }, { "fun_tab", NULL }, { "environ", NULL }, +#ifdef ERTS_NEW_PURGE_STRATEGY + { "release_literal_areas", NULL }, +#endif #endif { "efile_drv", "address" }, { "drv_ev_state_grow", NULL, }, diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 6bd27d11f4..f572577bde 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -120,8 +120,10 @@ execution_state(ErlNifEnv *env, Process **c_pp, int *schedp) else { Process *c_p = env->proc; - if (!(c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC)) - ASSERT(is_scheduler() > 0); + if (!(c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC)) { + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + } else { c_p = env->proc->next; ASSERT(is_scheduler() < 0); @@ -208,11 +210,16 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, #endif } +static void full_cache_env(ErlNifEnv *env); +static void cache_env(ErlNifEnv* env); +static void full_flush_env(ErlNifEnv *env); +static void flush_env(ErlNifEnv* env); + +#ifdef ERTS_DIRTY_SCHEDULERS void erts_pre_dirty_nif(ErtsSchedulerData *esdp, - ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, - Process* tracee) + ErlNifEnv* env, Process* p, + struct erl_module_nif* mod_nif) { -#ifdef ERTS_DIRTY_SCHEDULERS Process *sproc; #ifdef DEBUG erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); @@ -223,7 +230,7 @@ void erts_pre_dirty_nif(ErtsSchedulerData *esdp, ASSERT(esdp); #endif - erts_pre_nif(env, p, mod_nif, tracee); + erts_pre_nif(env, p, mod_nif, NULL); sproc = esdp->dirty_shadow_process; ASSERT(sproc); @@ -235,22 +242,10 @@ void erts_pre_dirty_nif(ErtsSchedulerData *esdp, sproc->next = p; sproc->common.id = p->common.id; - sproc->htop = p->htop; - sproc->stop = p->stop; - sproc->hend = p->hend; - sproc->heap = p->heap; - sproc->abandoned_heap = p->abandoned_heap; - sproc->heap_sz = p->heap_sz; - sproc->high_water = p->high_water; - sproc->old_hend = p->old_hend; - sproc->old_htop = p->old_htop; - sproc->old_heap = p->old_heap; - sproc->mbuf = NULL; - sproc->mbuf_sz = 0; - ERTS_INIT_OFF_HEAP(&sproc->off_heap); env->proc = sproc; -#endif + full_cache_env(env); } +#endif /* Temporary object header, auto-deallocated when NIF returns * or when independent environment is cleared. @@ -274,32 +269,37 @@ static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env) void erts_post_nif(ErlNifEnv* env) { erts_unblock_fpe(env->fpe_was_unmasked); + full_flush_env(env); + free_tmp_objs(env); + env->exiting = ERTS_PROC_IS_EXITING(env->proc); +} #ifdef ERTS_DIRTY_SCHEDULERS - if (!(env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)) +void erts_post_dirty_nif(ErlNifEnv* env) +{ + Process *c_p; + ASSERT(env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(env->proc->next); + erts_unblock_fpe(env->fpe_was_unmasked); + full_flush_env(env); + free_tmp_objs(env); + c_p = env->proc->next; + env->exiting = ERTS_PROC_IS_EXITING(c_p); + ERTS_VBUMP_ALL_REDS(c_p); +} #endif - { - ASSERT(is_scheduler() > 0); - if (env->heap_frag == NULL) { - ASSERT(env->hp_end == HEAP_LIMIT(env->proc)); - ASSERT(env->hp >= HEAP_TOP(env->proc)); - ASSERT(env->hp <= HEAP_LIMIT(env->proc)); - HEAP_TOP(env->proc) = env->hp; - } - else { - ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); - env->heap_frag->used_size = env->hp - env->heap_frag->mem; - ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); - } - env->exiting = ERTS_PROC_IS_EXITING(env->proc); - } + +static void full_flush_env(ErlNifEnv* env) +{ #ifdef ERTS_DIRTY_SCHEDULERS - else { /* Dirty nif call using shadow process struct */ + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + /* Dirty nif call using shadow process struct */ Process *c_p = env->proc->next; ASSERT(is_scheduler() < 0); ASSERT(env->proc->common.id == c_p->common.id); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); if (!env->heap_frag) { ASSERT(env->hp_end == HEAP_LIMIT(c_p)); @@ -339,11 +339,48 @@ void erts_post_nif(ErlNifEnv* env) } c_p->off_heap.overhead += env->proc->off_heap.overhead; - env->exiting = ERTS_PROC_IS_EXITING(c_p); - BUMP_ALL_REDS(c_p); + return; } #endif - free_tmp_objs(env); + + flush_env(env); +} + +static void full_cache_env(ErlNifEnv* env) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + /* Dirty nif call using shadow process struct */ + Process *sproc = env->proc; + Process *c_p = sproc->next; + ASSERT(c_p); + ASSERT(is_scheduler() < 0); + ASSERT(env->proc->common.id == c_p->common.id); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + sproc->htop = c_p->htop; + sproc->stop = c_p->stop; + sproc->hend = c_p->hend; + sproc->heap = c_p->heap; + sproc->abandoned_heap = c_p->abandoned_heap; + sproc->heap_sz = c_p->heap_sz; + sproc->high_water = c_p->high_water; + sproc->old_hend = c_p->old_hend; + sproc->old_htop = c_p->old_htop; + sproc->old_heap = c_p->old_heap; + sproc->mbuf = NULL; + sproc->mbuf_sz = 0; + ERTS_INIT_OFF_HEAP(&sproc->off_heap); + + env->hp_end = HEAP_LIMIT(c_p); + env->hp = HEAP_TOP(c_p); + env->heap_frag = NULL; + return; + } +#endif + + cache_env(env); } /* Flush out our cached heap pointers to allow an ordinary HAlloc @@ -600,17 +637,32 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, if (scheduler > 0) { /* Normal scheduler */ rp = erts_proc_lookup(receiver); - if (c_p == rp) - rp_locks = ERTS_PROC_LOCK_MAIN; + if (!rp) + return 0; } else { - if (c_p && ERTS_PROC_IS_EXITING(c_p)) - return 0; - rp = erts_pid2proc_opt(c_p, 0, receiver, rp_locks, + if (c_p) { + ASSERT(scheduler < 0); /* Dirty scheduler */ + if (ERTS_PROC_IS_EXITING(c_p)) + return 0; + + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + } + } + + rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + receiver, rp_locks, ERTS_P2P_FLG_INC_REFC); + if (!rp) { + if (c_p && (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + return 0; + } } - if (rp == NULL) - return 0; + + if (c_p == rp) + rp_locks = ERTS_PROC_LOCK_MAIN; if (menv) { flush_env(msg_env); @@ -634,9 +686,9 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, INITIALIZE_LITERAL_PURGE_AREA(litarea); sz = size_object_litopt(msg, &litarea); if (env && !env->tracee) { - flush_env(env); + full_flush_env(env); mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); - cache_env(env); + full_cache_env(env); } else { erts_aint_t state = erts_smp_atomic32_read_nob(&rp->state); @@ -659,8 +711,11 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, if (!env || !env->tracee) { - if (c_p && IS_TRACED_FL(c_p, F_TRACE_SEND)) + if (c_p && IS_TRACED_FL(c_p, F_TRACE_SEND)) { + full_flush_env(env); trace_send(c_p, receiver, msg); + full_cache_env(env); + } } #ifdef ERTS_SMP else { @@ -693,10 +748,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, erts_smp_proc_trylock(rp, ERTS_PROC_LOCK_MSGQ) == EBUSY) { if (!msgq) { -#ifdef ERTS_SMP - ErtsThrPrgrDelayHandle dhndl; -#endif - msgq = erts_alloc(ERTS_ALC_T_TRACE_MSG_QUEUE, sizeof(ErlTraceMessageQueue)); msgq->receiver = receiver; @@ -710,15 +761,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, erts_smp_proc_unlock(t_p, ERTS_PROC_LOCK_TRACE); -#ifdef ERTS_SMP - if (!scheduler) - dhndl = erts_thr_progress_unmanaged_delay(); -#endif - erts_schedule_flush_trace_messages(t_p->common.id); -#ifdef ERTS_SMP - if (!scheduler) - erts_thr_progress_unmanaged_continue(dhndl); -#endif + erts_schedule_flush_trace_messages(t_p, 0); } else { msgq->len++; *msgq->last = mp; @@ -743,6 +786,8 @@ done: rp_locks &= ~ERTS_PROC_LOCK_MAIN; if (rp_locks & ~lc_locks) erts_smp_proc_unlock(rp, rp_locks & ~lc_locks); + if (c_p && (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); #endif if (scheduler <= 0) erts_proc_dec_refc(rp); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 3b246f5def..fa8290028a 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -448,6 +448,9 @@ int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP; typedef enum { ERTS_PSTT_GC, /* Garbage Collect */ ERTS_PSTT_CPC, /* Check Process Code */ +#ifdef ERTS_NEW_PURGE_STRATEGY + ERTS_PSTT_CLA, /* Copy Literal Area */ +#endif ERTS_PSTT_COHMQ, /* Change off heap message queue */ ERTS_PSTT_FTMQ /* Flush trace msg queue */ } ErtsProcSysTaskType; @@ -6671,14 +6674,24 @@ erts_schedule_process(Process *p, erts_aint32_t state, ErtsProcLocks locks) } static int -schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) +schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, + erts_aint32_t *fail_state_p) { int res; int locked; ErtsProcSysTaskQs *stqs, *free_stqs; - erts_aint32_t state, a, n, enq_prio; + erts_aint32_t fail_state, state, a, n, enq_prio; int enqueue; /* < 0 -> use proxy */ unsigned int prof_runnable_procs; + int strict_fail_state; + + fail_state = *fail_state_p; + /* + * If fail state something other than just exiting process, + * ensure that the task wont be scheduled when the + * receiver is in the failure state. + */ + strict_fail_state = fail_state != ERTS_PSFLG_EXITING; res = 1; /* prepare for success */ st->next = st->prev = st; /* Prep for empty prio queue */ @@ -6705,7 +6718,8 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); state = erts_smp_atomic32_read_nob(&p->state); - if (state & ERTS_PSFLG_EXITING) { + if (state & fail_state) { + *fail_state_p = (state & fail_state); free_stqs = stqs; res = 0; goto cleanup; @@ -6759,7 +6773,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) /* Status lock prevents out of order "runnable proc" trace msgs */ ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - if (!prof_runnable_procs) { + if (!prof_runnable_procs && !strict_fail_state) { erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); locked = 0; } @@ -6770,6 +6784,11 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) erts_aint32_t e; n = e = a; + if (strict_fail_state && (a & fail_state)) { + *fail_state_p = (a & fail_state); + goto cleanup; + } + if (a & ERTS_PSFLG_FREE) goto cleanup; /* We don't want to schedule free processes... */ @@ -9092,6 +9111,27 @@ resume_process_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE +erts_internal_is_process_executing_dirty_1(BIF_ALIST_1) +{ + if (is_not_internal_pid(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); +#ifdef ERTS_DIRTY_SCHEDULERS + else { + Process *rp = erts_proc_lookup(BIF_ARG_1); + if (rp) { + erts_aint32_t state = erts_smp_atomic32_read_nob(&rp->state); + if (state & (ERTS_PSFLG_DIRTY_RUNNING + |ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + BIF_RET(am_true); + } + } + } +#endif + BIF_RET(am_false); +} + + Uint erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched) { @@ -9498,15 +9538,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) p->reds += actual_reds; -#ifdef ERTS_SMP - erts_smp_proc_lock(p, ERTS_PROC_LOCK_TRACE); - if (p->trace_msg_q) { - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_TRACE); - erts_schedule_flush_trace_messages(p->common.id); - } else - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_TRACE); -#endif - state = erts_smp_atomic32_read_nob(&p->state); if (IS_TRACED(p)) { @@ -9526,7 +9557,15 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } } - erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + +#ifdef ERTS_SMP + if (p->trace_msg_q) { + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + erts_schedule_flush_trace_messages(p, 1); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE); + } +#endif /* have to re-read state after taking lock */ state = erts_smp_atomic32_read_nob(&p->state); @@ -9534,9 +9573,11 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) #ifdef ERTS_SMP if (is_normal_sched && (state & ERTS_PSFLG_PENDING_EXIT)) erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_TRACE | ERTS_PROC_LOCK_STATUS)); if (p->pending_suspenders) handle_pending_suspend(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_TRACE | ERTS_PROC_LOCK_STATUS)); #endif @@ -9556,7 +9597,9 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) p->scheduler_data = NULL; #endif - erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_STATUS + | ERTS_PROC_LOCK_TRACE)); ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); @@ -9984,6 +10027,11 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (state & (ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_PENDING_EXIT | ERTS_PSFLG_EXITING)) { + /* + * IMPORTANT! We need to take care of + * scheduled check-process-code requests + * before continuing with dirty execution! + */ /* Migrate to normal scheduler... */ goto sunlock_sched_out_proc; } @@ -10393,6 +10441,27 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } break; } +#ifdef ERTS_NEW_PURGE_STRATEGY + case ERTS_PSTT_CLA: { + int fcalls; + int cla_reds = 0; + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + fcalls = reds; + else + fcalls = reds - CONTEXT_REDS; + st_res = erts_proc_copy_literal_area(c_p, + &cla_reds, + fcalls, + st->arg[0] == am_true); + reds -= cla_reds; + if (is_non_value(st_res)) { + /* Needed gc, but gc was disabled */ + save_gc_task(c_p, st, st_prio); + st = NULL; + } + break; + } +#endif case ERTS_PSTT_COHMQ: reds -= erts_complete_off_heap_message_queue_change(c_p); st_res = am_true; @@ -10443,14 +10512,15 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) switch (st->type) { case ERTS_PSTT_GC: - st_res = am_false; - break; case ERTS_PSTT_CPC: - st_res = am_false; - break; case ERTS_PSTT_COHMQ: st_res = am_false; break; +#ifdef ERTS_NEW_PURGE_STRATEGY + case ERTS_PSTT_CLA: + st_res = am_ok; + break; +#endif #ifdef ERTS_SMP case ERTS_PSTT_FTMQ: reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN); @@ -10471,22 +10541,83 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) return reds; } -BIF_RETTYPE -erts_internal_request_system_task_3(BIF_ALIST_3) +#ifdef ERTS_DIRTY_SCHEDULERS + +static BIF_RETTYPE +dispatch_system_task(Process *c_p, erts_aint_t fail_state, + ErtsProcSysTask *st, Eterm target, + Eterm prio, Eterm operation) +{ + Process *rp; + ErtsProcLocks rp_locks = 0; + ErlOffHeap *ohp; + ErtsMessage *mp; + Eterm *hp, msg; + Uint hsz, osz; + BIF_RETTYPE ret; + + switch (st->type) { + case ERTS_PSTT_CPC: + rp = erts_dirty_process_code_checker; + ASSERT(fail_state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)); + if (c_p == rp) { + ERTS_BIF_PREP_RET(ret, am_dirty_execution); + return ret; + } + break; + default: + rp = NULL; + ERTS_INTERNAL_ERROR("Non-dispatchable system task"); + break; + } + + ERTS_BIF_PREP_RET(ret, am_ok); + + /* + * Send message on the form: {Requester, Target, Operation} + */ + + ASSERT(is_immed(st->requester)); + ASSERT(is_immed(target)); + ASSERT(is_immed(prio)); + + osz = size_object(operation); + hsz = 5 /* 4-tuple */ + osz; + + mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp); + + msg = copy_struct(operation, osz, &hp, ohp); + msg = TUPLE4(hp, st->requester, target, prio, msg); + + erts_queue_message(rp, rp_locks, mp, msg, st->requester); + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + return ret; +} + +#endif + +static BIF_RETTYPE +request_system_task(Process *c_p, Eterm requester, Eterm target, + Eterm priority, Eterm operation) { - Process *rp = erts_proc_lookup(BIF_ARG_1); + BIF_RETTYPE ret; + Process *rp = erts_proc_lookup(target); ErtsProcSysTask *st = NULL; - erts_aint32_t prio; + erts_aint32_t prio, fail_state = ERTS_PSFLG_EXITING; Eterm noproc_res, req_type; - if (!rp && !is_internal_pid(BIF_ARG_1)) { - if (!is_external_pid(BIF_ARG_1)) + if (!rp && !is_internal_pid(target)) { + if (!is_external_pid(target)) goto badarg; - if (external_pid_dist_entry(BIF_ARG_1) != erts_this_dist_entry) + if (external_pid_dist_entry(target) != erts_this_dist_entry) goto badarg; } - switch (BIF_ARG_2) { + switch (priority) { case am_max: prio = PRIORITY_MAX; break; case am_high: prio = PRIORITY_HIGH; break; case am_normal: prio = PRIORITY_NORMAL; break; @@ -10494,11 +10625,11 @@ erts_internal_request_system_task_3(BIF_ALIST_3) default: goto badarg; } - if (is_not_tuple(BIF_ARG_3)) + if (is_not_tuple(operation)) goto badarg; else { int i; - Eterm *tp = tuple_val(BIF_ARG_3); + Eterm *tp = tuple_val(operation); Uint arity = arityval(*tp); Eterm req_id; Uint req_id_sz; @@ -10536,7 +10667,7 @@ erts_internal_request_system_task_3(BIF_ALIST_3) ERTS_INIT_OFF_HEAP(&st->off_heap); hp = &st->heap[0]; - st->requester = BIF_P->common.id; + st->requester = requester; st->reply_tag = req_type; st->req_id_sz = req_id_sz; st->req_id = req_id_sz == 0 ? req_id : copy_struct(req_id, @@ -10570,26 +10701,85 @@ erts_internal_request_system_task_3(BIF_ALIST_3) st->type = ERTS_PSTT_CPC; if (!rp) goto noproc; +#ifdef ERTS_DIRTY_SCHEDULERS + /* + * If the process should start executing dirty + * code it is important that this task is + * aborted. Therefore this strict fail state... + */ + fail_state |= (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS); +#endif + break; + +#ifdef ERTS_NEW_PURGE_STRATEGY + case am_copy_literals: + if (st->arg[0] != am_true && st->arg[0] != am_false) + goto badarg; + st->type = ERTS_PSTT_CLA; + noproc_res = am_ok; + if (!rp) + goto noproc; break; +#endif default: goto badarg; } - if (!schedule_process_sys_task(rp, prio, st)) { - noproc: - notify_sys_task_executed(BIF_P, st, noproc_res); + if (!schedule_process_sys_task(rp, prio, st, &fail_state)) { + Eterm failure; + if (fail_state & ERTS_PSFLG_EXITING) { + noproc: + failure = noproc_res; + } +#ifdef ERTS_DIRTY_SCHEDULERS + else if (fail_state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + ret = dispatch_system_task(c_p, fail_state, st, + target, priority, operation); + goto cleanup_return; + } +#endif + else { + ERTS_INTERNAL_ERROR("Unknown failure schedule_process_sys_task()"); + failure = am_internal_error; + } + notify_sys_task_executed(c_p, st, failure); } - BIF_RET(am_ok); + ERTS_BIF_PREP_RET(ret, am_ok); + + return ret; badarg: + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + +#ifdef ERTS_DIRTY_SCHEDULERS +cleanup_return: +#endif + if (st) { erts_cleanup_offheap(&st->off_heap); erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); } - BIF_ERROR(BIF_P, BADARG); + + return ret; +} + +BIF_RETTYPE +erts_internal_request_system_task_3(BIF_ALIST_3) +{ + return request_system_task(BIF_P, BIF_P->common.id, + BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); +} + +BIF_RETTYPE +erts_internal_request_system_task_4(BIF_ALIST_4) +{ + return request_system_task(BIF_P, BIF_ARG_1, + BIF_ARG_2, BIF_ARG_3, BIF_ARG_4); } static void @@ -10598,7 +10788,7 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type) Process *rp = erts_proc_lookup(pid); if (rp) { ErtsProcSysTask *st; - erts_aint32_t state; + erts_aint32_t state, fail_state; int i; st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK, @@ -10613,21 +10803,103 @@ erts_schedule_generic_sys_task(Eterm pid, ErtsProcSysTaskType type) ERTS_INIT_OFF_HEAP(&st->off_heap); state = erts_smp_atomic32_read_nob(&rp->state); - if (!schedule_process_sys_task(rp, ERTS_PSFLGS_GET_USR_PRIO(state), st)) + fail_state = ERTS_PSFLG_EXITING; + + if (!schedule_process_sys_task(rp, ERTS_PSFLGS_GET_USR_PRIO(state), + st, &fail_state)) erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); } } + void erts_schedule_complete_off_heap_message_queue_change(Eterm pid) { erts_schedule_generic_sys_task(pid, ERTS_PSTT_COHMQ); } +#ifdef ERTS_DIRTY_SCHEDULERS + +static void +flush_dirty_trace_messages(void *vpid) +{ + Process *proc; + Eterm pid; +#ifdef ARCH_64 + pid = (Eterm) vpid; +#else + pid = *((Eterm *) vpid); + erts_free(ERTS_ALC_T_DIRTY_SL, vpid); +#endif + + proc = erts_proc_lookup(pid); + if (proc) + (void) erts_flush_trace_messages(proc, 0); +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ + void -erts_schedule_flush_trace_messages(Eterm pid) +erts_schedule_flush_trace_messages(Process *proc, int force_on_proc) { +#ifdef ERTS_SMP + ErtsThrPrgrDelayHandle dhndl; +#endif + Eterm pid = proc->common.id; + +#ifdef ERTS_DIRTY_SCHEDULERS + erts_aint32_t state; + + if (!force_on_proc) { + state = erts_smp_atomic32_read_nob(&proc->state); + if (state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + goto sched_flush_dirty; + } + } +#endif + +#ifdef ERTS_SMP + dhndl = erts_thr_progress_unmanaged_delay(); +#endif + erts_schedule_generic_sys_task(pid, ERTS_PSTT_FTMQ); + +#ifdef ERTS_SMP + erts_thr_progress_unmanaged_continue(dhndl); +#endif + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!force_on_proc) { + state = erts_smp_atomic32_read_mb(&proc->state); + if (state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + void *vargp; + + sched_flush_dirty: + /* + * We traced 'proc' from another thread than + * it is executing on, and it is executing + * on a dirty scheduler. It might take a + * significant amount of time before it is + * scheduled out (where it gets opportunity + * to flush messages). We therefore schedule + * the flush on the first ordinary scheduler. + */ + +#ifdef ARCH_64 + vargp = (void *) pid; +#else + { + Eterm *argp = erts_alloc(ERTS_ALC_T_DIRTY_SL, sizeof(Eterm)); + *argp = pid; + vargp = (void *) argp; + } +#endif + erts_schedule_misc_aux_work(1, flush_dirty_trace_messages, vargp); + } + } +#endif } static void diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 714e63e986..f1788c3f71 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1767,7 +1767,7 @@ void erts_schedule_thr_prgr_later_cleanup_op(void (*)(void *), ErtsThrPrgrLaterOp *, UWord); void erts_schedule_complete_off_heap_message_queue_change(Eterm pid); -void erts_schedule_flush_trace_messages(Eterm pid); +void erts_schedule_flush_trace_messages(Process *proc, int force_on_proc); int erts_flush_trace_messages(Process *c_p, ErtsProcLocks locks); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 21938e7684..26d71f573f 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -969,8 +969,10 @@ erts_thr_progress_unmanaged_continue__(ErtsThrPrgrDelayHandle handle) #ifdef ERTS_ENABLE_LOCK_CHECK ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); ERTS_LC_ASSERT(tpd && tpd->is_delaying); - tpd->is_delaying = 0; - return_tmp_thr_prgr_data(tpd); + tpd->is_delaying--; + ASSERT(tpd->is_delaying >= 0); + if (!tpd->is_delaying) + return_tmp_thr_prgr_data(tpd); #endif ASSERT(!erts_thr_progress_is_managed_thread()); @@ -995,7 +997,7 @@ erts_thr_progress_unmanaged_delay__(void) #ifdef ERTS_ENABLE_LOCK_CHECK { ErtsThrPrgrData *tpd = tmp_thr_prgr_data(NULL); - tpd->is_delaying = 1; + tpd->is_delaying++; } #endif return (ErtsThrPrgrDelayHandle) umrefc_ix; diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index c4396488f5..f4d92564c1 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -813,6 +813,9 @@ trace_send(Process *p, Eterm to, Eterm msg) ErtsTracerNif *tnif = NULL; ErtsTracingEvent* te; Eterm pam_result; +#ifdef ERTS_SMP + ErtsThrPrgrDelayHandle dhndl; +#endif ASSERT(ARE_TRACE_FLAGS_ON(p, F_TRACE_SEND)); @@ -837,6 +840,10 @@ trace_send(Process *p, Eterm to, Eterm msg) } else pam_result = am_true; +#ifdef ERTS_SMP + dhndl = erts_thr_progress_unmanaged_delay(); +#endif + if (is_internal_pid(to)) { if (!erts_proc_lookup(to)) goto send_to_non_existing_process; @@ -852,6 +859,11 @@ trace_send(Process *p, Eterm to, Eterm msg) send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_SEND, operation, msg, to, pam_result); } + +#ifdef ERTS_SMP + erts_thr_progress_unmanaged_continue(dhndl); +#endif + erts_match_set_release_result_trace(p, pam_result); } @@ -1167,6 +1179,8 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, Eterm transformed_args[MAX_ARG]; ErtsTracer pre_ms_tracer = erts_tracer_nil; + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) & ERTS_PROC_LOCK_MAIN); + ASSERT(tracer); if (ERTS_TRACER_COMPARE(*tracer, erts_tracer_true)) { /* Breakpoint trace enabled without specifying tracer => diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 1423973739..d7dd6371b7 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -61,9 +61,12 @@ struct enif_environment_t /* ErlNifEnv */ extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); +#ifdef ERTS_DIRTY_SCHEDULERS extern void erts_pre_dirty_nif(ErtsSchedulerData *, struct enif_environment_t*, Process*, - struct erl_module_nif*, Process* tracee); + struct erl_module_nif*); +extern void erts_post_dirty_nif(struct enif_environment_t* env); +#endif extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(int to, void* to_arg); void erts_unload_nif(struct erl_module_nif* nif); @@ -998,17 +1001,30 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); /* beam_bif_load.c */ #define ERTS_CPC_ALLOW_GC (1 << 0) -#define ERTS_CPC_COPY_LITERALS (1 << 1) -#define ERTS_CPC_ALL (ERTS_CPC_ALLOW_GC | ERTS_CPC_COPY_LITERALS) +#define ERTS_CPC_ALL ERTS_CPC_ALLOW_GC Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls); +#ifdef ERTS_NEW_PURGE_STRATEGY +Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed); +#endif -typedef struct { - Eterm *ptr; - Uint sz; - Eterm pid; -} copy_literals_t; +typedef struct ErtsLiteralArea_ { + struct erl_off_heap_header *off_heap; + Eterm *end; + Eterm start[1]; /* beginning of area */ +} ErtsLiteralArea; -extern copy_literals_t erts_clrange; +#define ERTS_LITERAL_AREA_ALLOC_SIZE(N) \ + (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1)) + +extern ErtsLiteralArea *erts_copy_literal_area; +#ifdef ERTS_NEW_PURGE_STRATEGY +extern Process *erts_literal_area_collector; +#endif +#ifdef ERTS_DIRTY_SCHEDULERS +extern Process *erts_dirty_process_code_checker; +#endif + +extern Process *erts_code_purger; /* beam_load.c */ typedef struct { @@ -1089,15 +1105,22 @@ typedef struct { Uint lit_purge_sz; } erts_shcopy_t; -#define INITIALIZE_SHCOPY(info) \ -do { \ - info.queue_start = info.queue_default; \ - info.bitstore_start = info.bitstore_default; \ - info.shtable_start = info.shtable_default; \ - info.literal_size = 0; \ - info.lit_purge_ptr = erts_clrange.ptr; \ - info.lit_purge_sz = erts_clrange.sz; \ -} while(0) +#define INITIALIZE_SHCOPY(info) \ + do { \ + ErtsLiteralArea *larea__ = erts_copy_literal_area; \ + info.queue_start = info.queue_default; \ + info.bitstore_start = info.bitstore_default; \ + info.shtable_start = info.shtable_default; \ + info.literal_size = 0; \ + if (larea__) { \ + info.lit_purge_ptr = &larea__->start[0]; \ + info.lit_purge_sz = larea__->end - info.lit_purge_ptr; \ + } \ + else { \ + info.lit_purge_ptr = NULL; \ + info.lit_purge_sz = 0; \ + } \ + } while(0) #define DESTROY_SHCOPY(info) \ do { \ @@ -1118,10 +1141,17 @@ typedef struct { Uint lit_purge_sz; } erts_literal_area_t; -#define INITIALIZE_LITERAL_PURGE_AREA(Area) \ - do { \ - (Area).lit_purge_ptr = erts_clrange.ptr; \ - (Area).lit_purge_sz = erts_clrange.sz; \ +#define INITIALIZE_LITERAL_PURGE_AREA(Area) \ + do { \ + ErtsLiteralArea *larea__ = erts_copy_literal_area; \ + if (larea__) { \ + (Area).lit_purge_ptr = &larea__->start[0]; \ + (Area).lit_purge_sz = larea__->end - (Area).lit_purge_ptr; \ + } \ + else { \ + (Area).lit_purge_ptr = NULL; \ + (Area).lit_purge_sz = 0; \ + } \ } while(0) Eterm copy_object_x(Eterm, Process*, Uint); diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 2347a3d4ef..34515efa3d 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -20,7 +20,8 @@ -module(code_SUITE). -export([all/0, suite/0, init_per_suite/1, end_per_suite/1, - versions/1,new_binary_types/1, + versions/1,new_binary_types/1, call_purged_fun_code_gone/1, + call_purged_fun_code_reload/1, call_purged_fun_code_there/1, t_check_process_code/1,t_check_old_code/1, t_check_process_code_ets/1, external_fun/1,get_chunk/1,module_md5/1,make_stub/1, @@ -34,7 +35,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [versions, new_binary_types, t_check_process_code, + [versions, new_binary_types, call_purged_fun_code_gone, + call_purged_fun_code_reload, call_purged_fun_code_there, t_check_process_code, t_check_process_code_ets, t_check_old_code, external_fun, get_chunk, module_md5, make_stub, make_stub_many_funs, constant_pools, constant_refc_binaries, false_dependency, @@ -127,12 +129,169 @@ new_binary_types(Config) when is_list(Config) -> bit_sized_binary(Bin))), ok. +call_purged_fun_code_gone(Config) when is_list(Config) -> + Priv = proplists:get_value(priv_dir, Config), + Data = proplists:get_value(data_dir, Config), + call_purged_fun_test(Priv, Data, code_gone), + ok. + +call_purged_fun_code_reload(Config) when is_list(Config) -> + Priv = proplists:get_value(priv_dir, Config), + Data = proplists:get_value(data_dir, Config), + Path = code:get_path(), + true = code:add_path(Priv), + try + call_purged_fun_test(Priv, Data, code_reload) + after + code:set_path(Path) + end, + ok. + +call_purged_fun_code_there(Config) when is_list(Config) -> + Priv = proplists:get_value(priv_dir, Config), + Data = proplists:get_value(data_dir, Config), + call_purged_fun_test(Priv, Data, code_there), + ok. + +call_purged_fun_test(Priv, Data, Type) -> + File = filename:join(Data, "my_code_test2"), + Code = filename:join(Priv, "my_code_test2"), + + catch erlang:purge_module(my_code_test2), + catch erlang:delete_module(my_code_test2), + catch erlang:purge_module(my_code_test2), + + {ok,my_code_test2} = c:c(File, [{outdir,Priv}]), + + T = ets:new(my_code_test2_fun_table, []), + ets:insert(T, {my_fun,my_code_test2:make_fun(4711)}), + ets:insert(T, {my_fun2,my_code_test2:make_fun2()}), + + spawn(fun () -> + [{my_fun2,F2}] = ets:lookup(T, my_fun2), + F2(fun () -> + receive after infinity -> ok end + end, + fun () -> ok end), + exit(completed) + end), + + PurgeType = case Type of + code_gone -> + ok = file:delete(Code++".beam"), + true; + code_reload -> + true; + code_there -> + false + end, + + true = erlang:delete_module(my_code_test2), + + Purge = start_purge(my_code_test2, PurgeType), + + {P0, M0} = spawn_monitor(fun () -> + [{my_fun,F}] = ets:lookup(T, my_fun), + 4712 = F(1), + exit(completed) + end), + + wait_until(fun () -> + {status, suspended} + == process_info(P0, status) + end), + + ok = continue_purge(Purge), + + {P1, M1} = spawn_monitor(fun () -> + [{my_fun,F}] = ets:lookup(T, my_fun), + 4713 = F(2), + exit(completed) + end), + {P2, M2} = spawn_monitor(fun () -> + [{my_fun,F}] = ets:lookup(T, my_fun), + 4714 = F(3), + exit(completed) + end), + + wait_until(fun () -> + {status, suspended} + == process_info(P1, status) + end), + wait_until(fun () -> + {status, suspended} + == process_info(P2, status) + end), + + {current_function, + {erts_code_purger, + pending_purge_lambda, + 3}} = process_info(P0, current_function), + {current_function, + {erts_code_purger, + pending_purge_lambda, + 3}} = process_info(P1, current_function), + {current_function, + {erts_code_purger, + pending_purge_lambda, + 3}} = process_info(P2, current_function), + + case Type of + code_there -> + false = complete_purge(Purge); + _ -> + {true, true} = complete_purge(Purge) + end, + + case Type of + code_gone -> + receive + {'DOWN', M0, process, P0, Reason0} -> + {undef, _} = Reason0 + end, + receive + {'DOWN', M1, process, P1, Reason1} -> + {undef, _} = Reason1 + end, + receive + {'DOWN', M2, process, P2, Reason2} -> + {undef, _} = Reason2 + end; + _ -> + receive + {'DOWN', M0, process, P0, Reason0} -> + completed = Reason0 + end, + receive + {'DOWN', M1, process, P1, Reason1} -> + completed = Reason1 + end, + receive + {'DOWN', M2, process, P2, Reason2} -> + completed = Reason2 + end, + catch erlang:purge_module(my_code_test2), + catch erlang:delete_module(my_code_test2), + catch erlang:purge_module(my_code_test2) + end, + ok. + t_check_process_code(Config) when is_list(Config) -> + case check_process_code_handle(indirect_references) of + false -> {skipped, "check_process_code() ignores funs"}; + true -> t_check_process_code_test(Config) + end. + +t_check_process_code_test(Config) -> Priv = proplists:get_value(priv_dir, Config), Data = proplists:get_value(data_dir, Config), File = filename:join(Data, "my_code_test"), Code = filename:join(Priv, "my_code_test"), + catch erlang:purge_module(my_code_test), + catch erlang:delete_module(my_code_test), + catch erlang:purge_module(my_code_test), + {ok,my_code_test} = c:c(File, [{outdir,Priv}]), MyFun = fun(X, Y) -> X + Y end, %Confuse things. @@ -231,11 +390,16 @@ gc1() -> ok. %% Test check_process_code/2 in combination with a fun obtained from an ets table. t_check_process_code_ets(Config) when is_list(Config) -> - case test_server:is_native(?MODULE) of - true -> - {skip,"Native code"}; - false -> - do_check_process_code_ets(Config) + case check_process_code_handle(indirect_references) of + false -> + {skipped, "check_process_code() ignores funs"}; + true -> + case test_server:is_native(?MODULE) of + true -> + {skip,"Native code"}; + false -> + do_check_process_code_ets(Config) + end end. do_check_process_code_ets(Config) -> @@ -243,8 +407,9 @@ do_check_process_code_ets(Config) -> Data = proplists:get_value(data_dir, Config), File = filename:join(Data, "my_code_test"), - erlang:purge_module(my_code_test), - erlang:delete_module(my_code_test), + catch erlang:purge_module(my_code_test), + catch erlang:delete_module(my_code_test), + catch erlang:purge_module(my_code_test), {ok,my_code_test} = c:c(File, [{outdir,Priv}]), T = ets:new(my_code_test, []), @@ -295,8 +460,8 @@ t_check_old_code(Config) when is_list(Config) -> Data = proplists:get_value(data_dir, Config), File = filename:join(Data, "my_code_test"), - erlang:purge_module(my_code_test), - erlang:delete_module(my_code_test), + catch erlang:purge_module(my_code_test), + catch erlang:delete_module(my_code_test), catch erlang:purge_module(my_code_test), false = erlang:check_old_code(my_code_test), @@ -971,3 +1136,39 @@ flush() -> receive _ -> flush() after 0 -> ok end. id(I) -> I. + +check_process_code_handle(What) -> + lists:member(What, erlang:system_info(check_process_code)). + +wait_until(Fun) -> + case Fun() of + true -> + ok; + false -> + receive after 100 -> ok end, + wait_until(Fun) + end. + +start_purge(Mod, Type) when is_atom(Mod) + andalso ((Type == true) + orelse (Type == false)) -> + Ref = make_ref(), + erts_code_purger ! {test_purge, Mod, self(), Type, Ref}, + receive + {started, Ref} -> + Ref + end. + +continue_purge(Ref) when is_reference(Ref) -> + erts_code_purger ! {continue, Ref}, + receive + {continued, Ref} -> + ok + end. + +complete_purge(Ref) when is_reference(Ref) -> + erts_code_purger ! {complete, Ref}, + receive + {test_purge, Res, Ref} -> + Res + end. diff --git a/erts/emulator/test/code_SUITE_data/my_code_test.erl b/erts/emulator/test/code_SUITE_data/my_code_test.erl index d2386157d6..9d12aa9897 100644 --- a/erts/emulator/test/code_SUITE_data/my_code_test.erl +++ b/erts/emulator/test/code_SUITE_data/my_code_test.erl @@ -24,5 +24,3 @@ make_fun(A) -> fun(X) -> A + X end. - - diff --git a/erts/emulator/test/code_SUITE_data/my_code_test2.erl b/erts/emulator/test/code_SUITE_data/my_code_test2.erl new file mode 100644 index 0000000000..57973535d4 --- /dev/null +++ b/erts/emulator/test/code_SUITE_data/my_code_test2.erl @@ -0,0 +1,32 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1999-2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(my_code_test2). + +-export([make_fun/1, make_fun2/0]). + +make_fun(A) -> + fun(X) -> A + X end. + +make_fun2() -> + fun (F1,F2) -> + F1(), + F2() + end. diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl index 83b098a704..653b96557e 100644 --- a/erts/emulator/test/dirty_nif_SUITE.erl +++ b/erts/emulator/test/dirty_nif_SUITE.erl @@ -33,7 +33,8 @@ dirty_nif_exception/1, call_dirty_nif_exception/1, dirty_scheduler_exit/1, dirty_call_while_terminated/1, dirty_heap_access/1, dirty_process_info/1, - dirty_process_register/1, dirty_process_trace/1]). + dirty_process_register/1, dirty_process_trace/1, + code_purge/1, dirty_nif_send_traced/1]). -define(nif_stub,nif_stub_error(?LINE)). @@ -48,7 +49,9 @@ all() -> dirty_heap_access, dirty_process_info, dirty_process_register, - dirty_process_trace]. + dirty_process_trace, + code_purge, + dirty_nif_send_traced]. init_per_suite(Config) -> try erlang:system_info(dirty_cpu_schedulers) of @@ -349,6 +352,103 @@ dirty_process_trace(Config) when is_list(Config) -> ok end). +dirty_code_test_code() -> + " +-module(dirty_code_test). + +-export([func/1]). + +func(Fun) -> + Fun(), + blipp:blapp(). + +". + +code_purge(Config) when is_list(Config) -> + Path = ?config(data_dir, Config), + File = filename:join(Path, "dirty_code_test.erl"), + ok = file:write_file(File, dirty_code_test_code()), + {ok, dirty_code_test, Bin} = compile:file(File, [binary]), + {module, dirty_code_test} = erlang:load_module(dirty_code_test, Bin), + Start = erlang:monotonic_time(), + {Pid1, Mon1} = spawn_monitor(fun () -> + dirty_code_test:func(fun () -> + %% Sleep for 6 seconds + %% in dirty nif... + dirty_sleeper() + end) + end), + {module, dirty_code_test} = erlang:load_module(dirty_code_test, Bin), + {Pid2, Mon2} = spawn_monitor(fun () -> + dirty_code_test:func(fun () -> + %% Sleep for 6 seconds + %% in dirty nif... + dirty_sleeper() + end) + end), + receive + {'DOWN', Mon1, process, Pid1, _} -> + ct:fail(premature_death) + after 100 -> + ok + end, + true = erlang:purge_module(dirty_code_test), + receive + {'DOWN', Mon1, process, Pid1, Reason1} -> + killed = Reason1 + end, + receive + {'DOWN', Mon2, process, Pid2, _} -> + ct:fail(premature_death) + after 100 -> + ok + end, + true = erlang:delete_module(dirty_code_test), + receive + {'DOWN', Mon2, process, Pid2, _} -> + ct:fail(premature_death) + after 100 -> + ok + end, + true = erlang:purge_module(dirty_code_test), + receive + {'DOWN', Mon2, process, Pid2, Reason2} -> + killed = Reason2 + end, + End = erlang:monotonic_time(), + Time = erlang:convert_time_unit(End-Start, native, milli_seconds), + io:format("Time=~p~n", [Time]), + true = Time =< 1000, + ok. + +dirty_nif_send_traced(Config) when is_list(Config) -> + Parent = self(), + Rcvr = spawn_link(fun() -> + Self = self(), + receive {ok, Self} -> ok end, + Parent ! {Self, received} + end), + Sndr = spawn_link(fun () -> + receive {Parent, go} -> ok end, + {ok, Rcvr} = send_wait_from_dirty_nif(Rcvr), + Parent ! {self(), sent} + end), + 1 = erlang:trace(Sndr, true, [send]), + Start = erlang:monotonic_time(), + Sndr ! {self(), go}, + receive {trace, Sndr, send, {ok, Rcvr}, Rcvr} -> ok end, + receive {Rcvr, received} -> ok end, + End1 = erlang:monotonic_time(), + Time1 = erlang:convert_time_unit(End1-Start, native, 1000), + io:format("Time1: ~p milliseconds~n", [Time1]), + true = Time1 < 500, + receive {Sndr, sent} -> ok end, + End2 = erlang:monotonic_time(), + Time2 = erlang:convert_time_unit(End2-Start, native, 1000), + io:format("Time2: ~p milliseconds~n", [Time2]), + true = Time2 >= 1900, + ok. + %% %% Internal... %% @@ -431,6 +531,7 @@ mcall(Node, Funs) -> lib_loaded() -> false. call_dirty_nif(_,_,_) -> ?nif_stub. send_from_dirty_nif(_) -> ?nif_stub. +send_wait_from_dirty_nif(_) -> ?nif_stub. call_dirty_nif_exception(_) -> ?nif_stub. call_dirty_nif_zero_args() -> ?nif_stub. dirty_call_while_terminated_nif(_) -> ?nif_stub. diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c index d92933a096..a0019e5d95 100644 --- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c +++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c @@ -100,6 +100,32 @@ static ERL_NIF_TERM send_from_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_ return result; } +static ERL_NIF_TERM send_wait_from_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ERL_NIF_TERM result; + ErlNifPid pid; + ErlNifEnv* menv; + int res; + + if (!enif_get_local_pid(env, argv[0], &pid)) + return enif_make_badarg(env); + result = enif_make_tuple2(env, enif_make_atom(env, "ok"), enif_make_pid(env, &pid)); + menv = enif_alloc_env(); + res = enif_send(env, &pid, menv, result); + enif_free_env(menv); + +#ifdef __WIN32__ + Sleep(2000); +#else + sleep(2); +#endif + + if (!res) + return enif_make_badarg(env); + else + return result; +} + static ERL_NIF_TERM call_dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { switch (argc) { @@ -237,6 +263,7 @@ static ErlNifFunc nif_funcs[] = {"lib_loaded", 0, lib_loaded}, {"call_dirty_nif", 3, call_dirty_nif}, {"send_from_dirty_nif", 1, send_from_dirty_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"send_wait_from_dirty_nif", 1, send_wait_from_dirty_nif, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"call_dirty_nif_exception", 1, call_dirty_nif_exception, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"call_dirty_nif_zero_args", 0, call_dirty_nif_zero_args, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"dirty_sleeper", 0, dirty_sleeper, ERL_NIF_DIRTY_JOB_IO_BOUND}, diff --git a/erts/preloaded/ebin/erts_code_purger.beam b/erts/preloaded/ebin/erts_code_purger.beam Binary files differindex a0da864824..a1eb126098 100644 --- a/erts/preloaded/ebin/erts_code_purger.beam +++ b/erts/preloaded/ebin/erts_code_purger.beam diff --git a/erts/preloaded/ebin/erts_dirty_process_code_checker.beam b/erts/preloaded/ebin/erts_dirty_process_code_checker.beam Binary files differnew file mode 100644 index 0000000000..a7ac116c05 --- /dev/null +++ b/erts/preloaded/ebin/erts_dirty_process_code_checker.beam diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam Binary files differindex d897c8e92f..22817be8f4 100644 --- a/erts/preloaded/ebin/erts_internal.beam +++ b/erts/preloaded/ebin/erts_internal.beam diff --git a/erts/preloaded/ebin/erts_literal_area_collector.beam b/erts/preloaded/ebin/erts_literal_area_collector.beam Binary files differnew file mode 100644 index 0000000000..71f3c2ec8c --- /dev/null +++ b/erts/preloaded/ebin/erts_literal_area_collector.beam diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex f0344fd6ba..849273f746 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/src/Makefile b/erts/preloaded/src/Makefile index 4a447d3a09..2ab9edaf5e 100644 --- a/erts/preloaded/src/Makefile +++ b/erts/preloaded/src/Makefile @@ -44,7 +44,9 @@ PRE_LOADED_ERL_MODULES = \ erts_code_purger \ erlang \ erts_internal \ - erl_tracer + erl_tracer \ + erts_literal_area_collector \ + erts_dirty_process_code_checker PRE_LOADED_BEAM_MODULES = \ prim_eval diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src index e18da28905..7ab06164b4 100644 --- a/erts/preloaded/src/erts.app.src +++ b/erts/preloaded/src/erts.app.src @@ -37,7 +37,7 @@ {registered, []}, {applications, []}, {env, []}, - {runtime_dependencies, ["stdlib-3.0", "kernel-5.0", "sasl-3.0"]} + {runtime_dependencies, ["stdlib-3.0", "kernel-5.0", "sasl-3.0.1"]} ]}. %% vim: ft=erlang diff --git a/erts/preloaded/src/erts_code_purger.erl b/erts/preloaded/src/erts_code_purger.erl index d1e64342e0..ee4fcedd2d 100644 --- a/erts/preloaded/src/erts_code_purger.erl +++ b/erts/preloaded/src/erts_code_purger.erl @@ -22,7 +22,7 @@ %% Purpose : Implement system process erts_code_purger %% to handle code module purging. --export([start/0, purge/1, soft_purge/1]). +-export([start/0, purge/1, soft_purge/1, pending_purge_lambda/3]). -spec start() -> term(). start() -> @@ -40,10 +40,43 @@ loop() -> Res = do_soft_purge(Mod), From ! {reply, soft_purge, Res, Ref}; + {test_purge, Mod, From, Type, Ref} when is_atom(Mod), is_pid(From) -> + do_test_purge(Mod, From, Type, Ref); + _Other -> ignore end, loop(). +%% +%% Processes that tries to call a fun that belongs to +%% a module that currently is being purged will end +%% up here (pending_purge_lambda) in a suspended state. +%% When the purge operation completes or aborts (soft +%% purge that failed) these processes will be resumed. +%% +pending_purge_lambda(_Module, Fun, Args) -> + %% + %% When the process is resumed, the following + %% scenarios exist: + %% * The code that the fun refers to is still + %% there due to a failed soft purge. The + %% call to the fun will succeed via apply/2. + %% * The code was purged, and a current version + %% of the module is loaded which does not + %% contain this fun. The call will result + %% in an exception being raised. + %% * The code was purged, and no current + %% version of the module is loaded. An attempt + %% to load the module (via the error_handler) + %% will be made. This may or may not succeed. + %% If the module is loaded, it may or may + %% not contain the fun. The call will + %% succeed if the error_handler was able + %% to load the module and loaded module + %% contains this fun; otherwise, an exception + %% will be raised. + %% + apply(Fun, Args). %% purge(Module) %% Kill all processes running code from *old* Module, and then purge the @@ -60,16 +93,14 @@ purge(Mod) when is_atom(Mod) -> Result end. - do_purge(Mod) -> - case erts_internal:copy_literals(Mod, true) of - false -> - {false, false}; - true -> - DidKill = check_proc_code(erlang:processes(), Mod, true), - true = erts_internal:copy_literals(Mod, false), - WasPurged = erts_internal:purge_module(Mod), - {WasPurged, DidKill} + case erts_internal:purge_module(Mod, prepare) of + false -> + {false, false}; + true -> + DidKill = check_proc_code(erlang:processes(), Mod, true), + true = erts_internal:purge_module(Mod, complete), + {true, DidKill} end. %% soft_purge(Module) @@ -85,21 +116,17 @@ soft_purge(Mod) -> Result end. - do_soft_purge(Mod) -> - case erts_internal:copy_literals(Mod, true) of + case erts_internal:purge_module(Mod, prepare) of false -> true; true -> - DoPurge = check_proc_code(erlang:processes(), Mod, false), - true = erts_internal:copy_literals(Mod, false), - case DoPurge of - false -> - false; - true -> - erts_internal:purge_module(Mod), - true - end + Res = check_proc_code(erlang:processes(), Mod, false), + erts_internal:purge_module(Mod, + case Res of + false -> abort; + true -> complete + end) end. %% @@ -283,8 +310,7 @@ cpc_sched_kill(Pid, cpc_request(#cpc_static{tag = Tag, module = Mod}, Pid, AllowGc) -> erts_internal:check_process_code(Pid, Mod, [{async, {Tag, Pid, AllowGc}}, - {allow_gc, AllowGc}, - {copy_literals, true}]). + {allow_gc, AllowGc}]). cpc_request_gc(CpcS, [Pid|Pids]) -> cpc_request(CpcS, Pid, true), @@ -297,3 +323,72 @@ cpc_init(CpcS, [Pid|Pids], NoReqs) -> cpc_init(CpcS, Pids, NoReqs+1). % end of check_proc_code() implementation. + +%% +%% FOR TESTING ONLY +%% +%% do_test_purge() is for testing only. The purge is done +%% as usual, but the tester can control when to enter the +%% specific phases. +%% +do_test_purge(Mod, From, Type, Ref) when Type == true; Type == false -> + Mon = erlang:monitor(process, From), + Res = case Type of + true -> do_test_hard_purge(Mod, From, Ref, Mon); + false -> do_test_soft_purge(Mod, From, Ref, Mon) + end, + From ! {test_purge, Res, Ref}, + erlang:demonitor(Mon, [flush]), + ok; +do_test_purge(_, _, _, _) -> + ok. + +do_test_soft_purge(Mod, From, Ref, Mon) -> + PrepRes = erts_internal:purge_module(Mod, prepare), + TestRes = test_progress(started, From, Mon, Ref, ok), + case PrepRes of + false -> + _ = test_progress(continued, From, Mon, Ref, TestRes), + true; + true -> + Res = check_proc_code(erlang:processes(), Mod, false), + _ = test_progress(continued, From, Mon, Ref, TestRes), + erts_internal:purge_module(Mod, + case Res of + false -> abort; + true -> complete + end) + end. + +do_test_hard_purge(Mod, From, Ref, Mon) -> + PrepRes = erts_internal:purge_module(Mod, prepare), + TestRes = test_progress(started, From, Mon, Ref, ok), + case PrepRes of + false -> + _ = test_progress(continued, From, Mon, Ref, TestRes), + {false, false}; + true -> + DidKill = check_proc_code(erlang:processes(), Mod, true), + _ = test_progress(continued, From, Mon, Ref, TestRes), + true = erts_internal:purge_module(Mod, complete), + {true, DidKill} + end. + +test_progress(_State, _From, _Mon, _Ref, died) -> + %% Test process died; continue so we wont + %% leave the system in an inconsistent + %% state... + died; +test_progress(started, From, Mon, Ref, ok) -> + From ! {started, Ref}, + receive + {'DOWN', Mon, process, From, _} -> died; + {continue, Ref} -> ok + end; +test_progress(continued, From, Mon, Ref, ok) -> + From ! {continued, Ref}, + receive + {'DOWN', Mon, process, From, _} -> died; + {complete, Ref} -> ok + end. + diff --git a/erts/preloaded/src/erts_dirty_process_code_checker.erl b/erts/preloaded/src/erts_dirty_process_code_checker.erl new file mode 100644 index 0000000000..911642082c --- /dev/null +++ b/erts/preloaded/src/erts_dirty_process_code_checker.erl @@ -0,0 +1,82 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(erts_dirty_process_code_checker). + +-export([start/0]). + +%% +%% The erts_dirty_process_code_checker is started at +%% VM boot by the VM. It is a spawned as a system +%% process, i.e, the whole VM will terminate if +%% this process terminates. +%% +start() -> + process_flag(trap_exit, true), + msg_loop(). + +msg_loop() -> + _ = receive + Request -> + handle_request(Request) + end, + msg_loop(). + +check_process(Requester, Target, ReqId, Module) -> + Result = erts_internal:check_dirty_process_code(Target, Module), + Requester ! {check_process_code, ReqId, Result}. + +handle_request({Requester, + Target, + Prio, + {check_process_code, + ReqId, + Module, + _Flags} = Op}) -> + %% + %% Target may have stopped executing dirty since the + %% initial request was made. Check its current state + %% and try to send the request if possible; otherwise, + %% check the dirty executing process and send the result... + %% + try + case erts_internal:is_process_executing_dirty(Target) of + true -> + check_process(Requester, Target, ReqId, Module); + false -> + case erts_internal:request_system_task(Requester, + Target, + Prio, + Op) of + ok -> + ok; + dirty_execution -> + check_process(Requester, Target, ReqId, Module) + end + end + catch + _ : _ -> + ok %% Ignore all failures; someone passed us garbage... + end; +handle_request(_Garbage) -> + ignore. + + + diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl index 2459ea2a2c..6229754c8c 100644 --- a/erts/preloaded/src/erts_internal.erl +++ b/erts/preloaded/src/erts_internal.erl @@ -38,11 +38,13 @@ -export([system_check/1, gather_system_check_result/1]). --export([request_system_task/3]). +-export([request_system_task/3, request_system_task/4]). -export([check_process_code/3]). --export([copy_literals/2]). --export([purge_module/1]). +-export([check_dirty_process_code/2]). +-export([is_process_executing_dirty/1]). +-export([release_literal_area_switch/0]). +-export([purge_module/2]). -export([flush_monitor_messages/3]). @@ -204,14 +206,25 @@ port_info(_Result, _Item) -> -spec request_system_task(Pid, Prio, Request) -> 'ok' when Prio :: 'max' | 'high' | 'normal' | 'low', Request :: {'garbage_collect', term()} - | {'check_process_code', term(), module(), non_neg_integer()}, + | {'check_process_code', term(), module(), non_neg_integer()} + | {'copy_literals', term(), boolean()}, Pid :: pid(). request_system_task(_Pid, _Prio, _Request) -> erlang:nif_error(undefined). +-spec request_system_task(RequesterPid, TargetPid, Prio, Request) -> 'ok' | 'dirty_execution' when + Prio :: 'max' | 'high' | 'normal' | 'low', + Request :: {'garbage_collect', term()} + | {'check_process_code', term(), module(), non_neg_integer()} + | {'copy_literals', term(), boolean()}, + RequesterPid :: pid(), + TargetPid :: pid(). + +request_system_task(_RequesterPid, _TargetPid, _Prio, _Request) -> + erlang:nif_error(undefined). + -define(ERTS_CPC_ALLOW_GC, (1 bsl 0)). --define(ERTS_CPC_COPY_LITERALS, (1 bsl 1)). -spec check_process_code(Module, Flags) -> boolean() when Module :: module(), @@ -223,7 +236,7 @@ check_process_code(_Module, _Flags) -> Pid :: pid(), Module :: module(), RequestId :: term(), - Option :: {async, RequestId} | {allow_gc, boolean()} | {copy_literals, boolean()}, + Option :: {async, RequestId} | {allow_gc, boolean()}, OptionList :: [Option], CheckResult :: boolean() | aborted. check_process_code(Pid, Module, OptionList) -> @@ -265,8 +278,6 @@ get_cpc_opts([{async, _ReqId} = AsyncTuple | Options], _OldAsync, Flags) -> get_cpc_opts(Options, AsyncTuple, Flags); get_cpc_opts([{allow_gc, AllowGC} | Options], Async, Flags) -> get_cpc_opts(Options, Async, cpc_flags(Flags, ?ERTS_CPC_ALLOW_GC, AllowGC)); -get_cpc_opts([{copy_literals, CopyLit} | Options], Async, Flags) -> - get_cpc_opts(Options, Async, cpc_flags(Flags, ?ERTS_CPC_COPY_LITERALS, CopyLit)); get_cpc_opts([], Async, Flags) -> {Async, Flags}. @@ -275,15 +286,26 @@ cpc_flags(OldFlags, Bit, true) -> cpc_flags(OldFlags, Bit, false) -> OldFlags band (bnot Bit). --spec copy_literals(Module,Bool) -> 'true' | 'false' | 'aborted' when - Module :: module(), - Bool :: boolean(). -copy_literals(_Mod, _Bool) -> +-spec check_dirty_process_code(Pid,Module) -> 'true' | 'false' when + Pid :: pid(), + Module :: module(). +check_dirty_process_code(_Pid,_Module) -> erlang:nif_error(undefined). --spec purge_module(Module) -> boolean() when - Module :: module(). -purge_module(_Module) -> +-spec is_process_executing_dirty(Pid) -> 'true' | 'false' when + Pid :: pid(). +is_process_executing_dirty(_Pid) -> + erlang:nif_error(undefined). + +-spec release_literal_area_switch() -> 'true' | 'false'. + +release_literal_area_switch() -> + erlang:nif_error(undefined). + +-spec purge_module(Module, Op) -> boolean() when + Module :: module(), + Op :: 'prepare' | 'abort' | 'complete'. +purge_module(_Module, _Op) -> erlang:nif_error(undefined). -spec system_check(Type) -> 'ok' when diff --git a/erts/preloaded/src/erts_literal_area_collector.erl b/erts/preloaded/src/erts_literal_area_collector.erl new file mode 100644 index 0000000000..3befad8dfb --- /dev/null +++ b/erts/preloaded/src/erts_literal_area_collector.erl @@ -0,0 +1,113 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(erts_literal_area_collector). + +-export([start/0]). + +%% Currently we only allow two outstanding literal +%% copying jobs that garbage collect in order to +%% copy the literals. Maybe we could allow more +%% than two outstanding processes, but for now we +%% play it safe... +-define(MAX_GC_OUTSTND, 2). + +%% +%% The erts_literal_area_collector is started at +%% VM boot by the VM. It is a spawned as a system +%% process, i.e, the whole VM will terminate if +%% this process terminates. +%% +start() -> + process_flag(trap_exit, true), + msg_loop(undefined, 0, 0, []). + +%% +%% The VM will send us a 'copy_literals' message +%% when it has a new literal area that needs to +%% be handled is added. We will also be informed +%% about more areas when we call +%% erts_internal:release_literal_area_switch(). +%% +msg_loop(Area, Outstnd, GcOutstnd, NeedGC) -> + receive + + %% A new area to handle has arrived... + copy_literals when Outstnd == 0 -> + switch_area(); + + %% Process (_Pid) has completed the request... + {copy_literals, {Area, _GcAllowed, _Pid}, ok} when Outstnd == 1 -> + switch_area(); %% Last process completed... + {copy_literals, {Area, false, _Pid}, ok} -> + msg_loop(Area, Outstnd-1, GcOutstnd, NeedGC); + {copy_literals, {Area, true, _Pid}, ok} when NeedGC == [] -> + msg_loop(Area, Outstnd-1, GcOutstnd-1, []); + {copy_literals, {Area, true, _Pid}, ok} -> + send_copy_req(hd(NeedGC), Area, true), + msg_loop(Area, Outstnd-1, GcOutstnd, tl(NeedGC)); + + %% Process (Pid) failed to complete the request + %% since it needs to garbage collect in order to + %% complete the request... + {copy_literals, {Area, false, Pid}, need_gc} when GcOutstnd < ?MAX_GC_OUTSTND -> + send_copy_req(Pid, Area, true), + msg_loop(Area, Outstnd, GcOutstnd+1, NeedGC); + {copy_literals, {Area, false, Pid}, need_gc} -> + msg_loop(Area, Outstnd, GcOutstnd, [Pid|NeedGC]); + + %% Not handled message regarding the area that we + %% currently are working with. Crash the VM so + %% we notice this bug... + {copy_literals, {Area, _, _}, _} = Msg when erlang:is_reference(Area) -> + exit({not_handled_message, Msg}); + + %% Unexpected garbage message. Get rid of it... + _Ignore -> + msg_loop(Area, Outstnd, GcOutstnd, NeedGC) + + end. + +switch_area() -> + Res = erts_internal:release_literal_area_switch(), + erlang:garbage_collect(), %% Almost no live data now... + case Res of + false -> + %% No more areas to handle... + msg_loop(undefined, 0, 0, []); + true -> + %% Send requests to all processes to copy + %% all live data they have referring to the + %% literal area that is to be released... + Area = make_ref(), + Outstnd = send_copy_reqs(erlang:processes(), Area, false), + msg_loop(Area, Outstnd, 0, []) + end. + +send_copy_reqs(Ps, Area, GC) -> + send_copy_reqs(Ps, Area, GC, 0). + +send_copy_reqs([], _Area, _GC, N) -> + N; +send_copy_reqs([P|Ps], Area, GC, N) -> + send_copy_req(P, Area, GC), + send_copy_reqs(Ps, Area, GC, N+1). + +send_copy_req(P, Area, GC) -> + erts_internal:request_system_task(P, normal, {copy_literals, {Area, GC, P}, GC}). diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index 67929c53c2..962528f7ab 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -669,9 +669,9 @@ unload(_) -> do_unload(sub([heart|erlang:pre_loaded()],erlang:loaded())). do_unload([M|Mods]) -> - catch erts_internal:purge_module(M), + catch erlang:purge_module(M), catch erlang:delete_module(M), - catch erts_internal:purge_module(M), + catch erlang:purge_module(M), do_unload(Mods); do_unload([]) -> purge_all_hipe_refs(), |