diff options
author | Rickard Green <[email protected]> | 2016-07-27 19:45:41 +0200 |
---|---|---|
committer | Rickard Green <[email protected]> | 2016-08-26 15:50:22 +0200 |
commit | 2fe03e832adb11c50bcfc62679cf17779b284124 (patch) | |
tree | b338e0f77a25091ebfd3e2c183c33edb219ff42c /erts | |
parent | aeb645a709b73e1bda0281f87dda2af3ce92dfe7 (diff) | |
download | otp-2fe03e832adb11c50bcfc62679cf17779b284124.tar.gz otp-2fe03e832adb11c50bcfc62679cf17779b284124.tar.bz2 otp-2fe03e832adb11c50bcfc62679cf17779b284124.zip |
Reclaim literal area after purge has completed
Diffstat (limited to 'erts')
-rw-r--r-- | erts/configure.in | 29 | ||||
-rw-r--r-- | erts/emulator/Makefile.in | 30 | ||||
-rw-r--r-- | erts/emulator/beam/atom.names | 2 | ||||
-rw-r--r-- | erts/emulator/beam/beam_bif_load.c | 478 | ||||
-rw-r--r-- | erts/emulator/beam/beam_load.h | 16 | ||||
-rw-r--r-- | erts/emulator/beam/bif.tab | 2 | ||||
-rw-r--r-- | erts/emulator/beam/erl_alloc.types | 1 | ||||
-rw-r--r-- | erts/emulator/beam/erl_init.c | 28 | ||||
-rw-r--r-- | erts/emulator/beam/erl_lock_check.c | 3 | ||||
-rw-r--r-- | erts/emulator/beam/erl_process.c | 44 | ||||
-rw-r--r-- | erts/emulator/beam/global.h | 37 | ||||
-rw-r--r-- | erts/preloaded/ebin/erts_code_purger.beam | bin | 8696 -> 9904 bytes | |||
-rw-r--r-- | erts/preloaded/ebin/erts_internal.beam | bin | 10536 -> 10576 bytes | |||
-rw-r--r-- | erts/preloaded/ebin/erts_literal_area_collector.beam | bin | 0 -> 3304 bytes | |||
-rw-r--r-- | erts/preloaded/src/Makefile | 3 | ||||
-rw-r--r-- | erts/preloaded/src/erts.app.src | 2 | ||||
-rw-r--r-- | erts/preloaded/src/erts_code_purger.erl | 49 | ||||
-rw-r--r-- | erts/preloaded/src/erts_internal.erl | 11 | ||||
-rw-r--r-- | erts/preloaded/src/erts_literal_area_collector.erl | 113 |
19 files changed, 768 insertions, 80 deletions
diff --git a/erts/configure.in b/erts/configure.in index 81ecad4f51..cdd144be9d 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -121,6 +121,7 @@ AS_HELP_STRING([--enable-bootstrap-only], enable_hipe=no enable_sctp=no enable_dirty_schedulers=no + enable_new_purge=no fi ]) @@ -139,6 +140,13 @@ AS_HELP_STRING([--enable-dirty-schedulers], [enable dirty scheduler support]), *) enable_dirty_schedulers=yes ;; esac ], enable_dirty_schedulers=default) +AC_ARG_ENABLE(new-purge-strategy, +AS_HELP_STRING([--enable-new-purge-strategy], [enable new code purge strategy]), +[ case "$enableval" in + no) enable_new_purge=no ;; + *) enable_new_purge=yes ;; + esac ], enable_new_purge=default) + AC_ARG_ENABLE(smp-support, AS_HELP_STRING([--enable-smp-support], [enable smp support]) AS_HELP_STRING([--disable-smp-support], [disable smp support]), @@ -1023,6 +1031,27 @@ esac AC_MSG_RESULT($DIRTY_SCHEDULER_SUPPORT) AC_SUBST(DIRTY_SCHEDULER_SUPPORT) +AC_MSG_CHECKING(whether the new code purge strategy should be enabled) +case $enable_new_purge-$enable_dirty_schedulers in + yes-*) + AC_MSG_RESULT(yes) + enable_new_purge=yes;; + default-yes) + AC_MSG_RESULT(yes; forced by dirty scheduler support) + enable_new_purge=yes;; + no-yes) + AC_MSG_ERROR([Dirty schedulers enabled, but new code purge strategy disabled]);; + *) + AC_MSG_RESULT(no) + enable_new_purge=no;; +esac + +if test $enable_new_purge = yes; then + AC_DEFINE(ERTS_NEW_PURGE_STRATEGY, 1, [Define if you want to use the new code purge strategy]) +fi +NEW_PURGE_STRATEGY=$enable_new_purge +AC_SUBST(NEW_PURGE_STRATEGY) + if test $ERTS_BUILD_SMP_EMU = yes; then if test $found_threads = no; then diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 2212aed5e0..e3e0e70476 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -51,6 +51,7 @@ ARFLAGS=rc OMIT_OMIT_FP=no DIRTY_SCHEDULER_SUPPORT=@DIRTY_SCHEDULER_SUPPORT@ +NEW_PURGE_STRATEGY=@NEW_PURGE_STRATEGY@ ifeq ($(TYPE),debug) PURIFY = @@ -591,11 +592,8 @@ GENERATE += $(TTF_DIR)/driver_tab.c # # This list must be consistent with PRE_LOADED_MODULES in # erts/preloaded/src/Makefile. -ifeq ($(TARGET),win32) -# On windows the preloaded objects are in a resource object. -PRELOAD_OBJ = $(OBJDIR)/beams.$(RES_EXT) -PRELOAD_SRC = $(TARGET)/beams.rc -$(PRELOAD_SRC): $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ + +PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ $(ERL_TOP)/erts/preloaded/ebin/init.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \ @@ -606,23 +604,19 @@ $(PRELOAD_SRC): $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/erl_prim_loader.beam \ $(ERL_TOP)/erts/preloaded/ebin/erlang.beam \ $(ERL_TOP)/erts/preloaded/ebin/erts_internal.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam + $(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam \ + $(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam + +ifeq ($(TARGET),win32) +# On windows the preloaded objects are in a resource object. +PRELOAD_OBJ = $(OBJDIR)/beams.$(RES_EXT) +PRELOAD_SRC = $(TARGET)/beams.rc +$(PRELOAD_SRC): $(PRELOAD_BEAM) $(gen_verbose)LANG=C $(PERL) utils/make_preload $(MAKE_PRELOAD_EXTRA) -rc $^ > $@ else PRELOAD_OBJ = $(OBJDIR)/preload.o PRELOAD_SRC = $(TARGET)/preload.c -$(PRELOAD_SRC): $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erts_code_purger.beam \ - $(ERL_TOP)/erts/preloaded/ebin/init.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ - $(ERL_TOP)/erts/preloaded/ebin/zlib.beam \ - $(ERL_TOP)/erts/preloaded/ebin/prim_zip.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erl_prim_loader.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erlang.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erts_internal.beam \ - $(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam +$(PRELOAD_SRC): $(PRELOAD_BEAM) $(gen_verbose)LANG=C $(PERL) utils/make_preload -old $^ > $@ endif diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index badd69856e..dd37c682b1 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -176,6 +176,7 @@ atom const atom context_switches atom control atom copy +atom copy_literals atom counters atom cpu atom cpu_timestamp @@ -407,6 +408,7 @@ atom named_table atom namelist atom native atom native_addresses +atom need_gc atom Neq='=/=' atom Neqeq='/=' atom net_kernel diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index a94a3cc3d3..484198b339 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -37,12 +37,41 @@ #include "erl_bits.h" #include "erl_thr_progress.h" +Process *erts_code_purger = NULL; + +ErtsLiteralArea *erts_copy_literal_area = NULL; +#ifdef ERTS_NEW_PURGE_STRATEGY +Process *erts_literal_area_collector = NULL; + +typedef struct ErtsLiteralAreaRef_ ErtsLiteralAreaRef; +struct ErtsLiteralAreaRef_ { + ErtsLiteralAreaRef *next; + ErtsLiteralArea *literal_area; +}; + +struct { + erts_smp_mtx_t mtx; + ErtsLiteralAreaRef *first; + ErtsLiteralAreaRef *last; +} release_literal_areas; +#endif + static void set_default_trace_pattern(Eterm module); static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls); static void delete_code(Module* modp); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); +void +erts_beam_bif_load_init(void) +{ +#ifdef ERTS_NEW_PURGE_STRATEGY + erts_smp_mtx_init(&release_literal_areas.mtx, "release_literal_areas"); + release_literal_areas.first = NULL; + release_literal_areas.last = NULL; +#endif +} + BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) { Module* modp; @@ -767,6 +796,311 @@ static void hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, Eterm *start, Eterm *end, char *lit_start, Uint lit_size); +#ifdef ERTS_NEW_PURGE_STRATEGY + +Eterm +erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed) +{ + ErtsLiteralArea *la; + ErtsMessage *msgp; + struct erl_off_heap_header* oh; + char *literals; + Uint lit_bsize; + ErlHeapFragment *hfrag; + + la = erts_copy_literal_area; + if (!la) + return am_ok; + + oh = la->off_heap; + literals = (char *) &la->start[0]; + lit_bsize = (char *) la->end - literals; + + /* + * If a literal is in the message queue we make an explicit copy of + * it and attach it to the heap fragment. Each message needs to be + * self contained, we cannot save the literal in the old_heap or + * any other heap than the message it self. + */ + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ); + + for (msgp = c_p->msg.first; msgp; msgp = msgp->next) { + ErlHeapFragment *hf; + Uint lit_sz = 0; + + *redsp += 1; + + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) + hfrag = &msgp->hfrag; + else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag) + hfrag = msgp->data.heap_frag; + else + continue; /* Content on heap or in external term format... */ + + for (hf = hfrag; hf; hf = hf->next) { + lit_sz += hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size], + literals, lit_bsize); + *redsp += 1; + } + + *redsp += lit_sz / 16; /* Better value needed... */ + if (lit_sz > 0) { + ErlHeapFragment *bp = new_message_buffer(lit_sz); + Eterm *hp = bp->mem; + + for (hf = hfrag; hf; hf = hf->next) { + hfrag_literal_copy(&hp, &bp->off_heap, + &hf->mem[0], &hf->mem[hf->used_size], + literals, lit_bsize); + hfrag = hf; + } + + /* link new hfrag last */ + ASSERT(hfrag->next == NULL); + hfrag->next = bp; + bp->next = NULL; + } + } + + if (gc_allowed) { + /* + * Current implementation first tests without + * allowing GC, and then restarts the operation + * allowing GC if it is needed. It is therfore + * very likely that we will need the GC (although + * this is not completely certain). We go for + * the GC directly instead of scanning everything + * one more time... + */ + goto literal_gc; + } + + *redsp += 2; + if (any_heap_ref_ptrs(&c_p->fvalue, &c_p->fvalue+1, literals, lit_bsize)) { + c_p->freason = EXC_NULL; + c_p->fvalue = NIL; + c_p->ftrace = NIL; + } + + if (any_heap_ref_ptrs(c_p->stop, c_p->hend, literals, lit_bsize)) + goto literal_gc; + *redsp += 1; + if (any_heap_refs(c_p->heap, c_p->htop, literals, lit_bsize)) + goto literal_gc; + *redsp += 1; + if (any_heap_refs(c_p->old_heap, c_p->old_htop, literals, lit_bsize)) + goto literal_gc; + + /* Check dictionary */ + *redsp += 1; + if (c_p->dictionary) { + Eterm* start = ERTS_PD_START(c_p->dictionary); + Eterm* end = start + ERTS_PD_SIZE(c_p->dictionary); + + if (any_heap_ref_ptrs(start, end, literals, lit_bsize)) + goto literal_gc; + } + + /* Check heap fragments */ + for (hfrag = c_p->mbuf; hfrag; hfrag = hfrag->next) { + Eterm *hp, *hp_end; + + *redsp += 1; + + hp = &hfrag->mem[0]; + hp_end = &hfrag->mem[hfrag->used_size]; + if (any_heap_refs(hp, hp_end, literals, lit_bsize)) + goto literal_gc; + } + + /* + * Message buffer fragments (matched messages) + * - off heap lists should already have been moved into + * process off heap structure. + * - Check for literals + */ + for (msgp = c_p->msg_frag; msgp; msgp = msgp->next) { + hfrag = erts_message_to_heap_frag(msgp); + for (; hfrag; hfrag = hfrag->next) { + Eterm *hp, *hp_end; + ASSERT(!check_mod_funs(c_p, &hfrag->off_heap, mod_start, mod_size)); + + *redsp += 1; + + hp = &hfrag->mem[0]; + hp_end = &hfrag->mem[hfrag->used_size]; + + if (any_heap_refs(hp, hp_end, literals, lit_bsize)) + goto literal_gc; + } + } + + return am_ok; + +literal_gc: + + if (!gc_allowed) + return am_need_gc; + + if (c_p->flags & F_DISABLE_GC) + return THE_NON_VALUE; + + FLAGS(c_p) |= F_NEED_FULLSWEEP; + + *redsp += erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, c_p->arity, fcalls); + + erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize, oh); + + *redsp += lit_bsize / 64; /* Need, better value... */ + + return am_ok; +} + +static Eterm +check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) +{ + BeamInstr* start; + char* mod_start; + Uint mod_size; + Eterm* sp; + int done_gc = 0; + ErtsMessage *msgp; + ErlHeapFragment *hfrag; + + /* + * Pick up limits for the module. + */ + start = (BeamInstr*) modp->old.code_hdr; + mod_start = (char *) start; + mod_size = modp->old.code_length; + + /* + * Check if current instruction or continuation pointer points into module. + */ + if (ErtsInArea(rp->i, mod_start, mod_size) + || ErtsInArea(rp->cp, mod_start, mod_size)) { + return am_true; + } + + /* + * Check all continuation pointers stored on the stack. + */ + for (sp = rp->stop; sp < STACK_START(rp); sp++) { + if (is_CP(*sp) && ErtsInArea(cp_val(*sp), mod_start, mod_size)) { + return am_true; + } + } + + /* + * Check all continuation pointers stored in stackdump + * and clear exception stackdump if there is a pointer + * to the module. + */ + if (rp->ftrace != NIL) { + struct StackTrace *s; + ASSERT(is_list(rp->ftrace)); + s = (struct StackTrace *) big_val(CDR(list_val(rp->ftrace))); + if ((s->pc && ErtsInArea(s->pc, mod_start, mod_size)) || + (s->current && ErtsInArea(s->current, mod_start, mod_size))) { + rp->freason = EXC_NULL; + rp->fvalue = NIL; + rp->ftrace = NIL; + } else { + int i; + for (i = 0; i < s->depth; i++) { + if (ErtsInArea(s->trace[i], mod_start, mod_size)) { + rp->freason = EXC_NULL; + rp->fvalue = NIL; + rp->ftrace = NIL; + break; + } + } + } + } + + /* + * Message queue can contains funs. + */ + + erts_smp_proc_lock(rp, ERTS_PROC_LOCK_MSGQ); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); + + for (msgp = rp->msg.first; msgp; msgp = msgp->next) { + ErlHeapFragment *hf; + if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) + hfrag = &msgp->hfrag; + else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag) + hfrag = msgp->data.heap_frag; + else + continue; + + for (hf = hfrag; hf; hf = hf->next) { + if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)) + return am_true; + } + } + + while (1) { + + /* Check heap, stack etc... */ + if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size)) + goto try_gc; + +#ifdef DEBUG + /* Check heap fragments */ + for (hfrag = rp->mbuf; hfrag; hfrag = hfrag->next) { + /* Off heap lists should already have been moved into process */ + ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); + } + + /* + * Message buffer fragments (matched messages) + * - off heap lists should already have been moved into + * process off heap structure. + * - Check for literals + */ + for (msgp = rp->msg_frag; msgp; msgp = msgp->next) { + hfrag = erts_message_to_heap_frag(msgp); + for (; hfrag; hfrag = hfrag->next) { + Eterm *hp, *hp_end; + ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); + } + } +#endif + + return am_false; + + try_gc: + + if (rp->flags & F_DISABLE_GC) { + /* + * Cannot proceed. Process has disabled gc in order to + * safely leave inconsistent data on the heap and/or + * off heap lists. Need to wait for gc to be enabled + * again. + */ + return THE_NON_VALUE; + } + + if (done_gc) + return am_true; + + if (!(flags & ERTS_CPC_ALLOW_GC)) + return am_aborted; + + FLAGS(rp) |= F_NEED_FULLSWEEP; + *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity, fcalls); + done_gc = 1; + } + +} + +#else /* !ERTS_NEW_PURGE_STRATEGY, i.e, old style purge... */ + static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) { @@ -860,8 +1194,14 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); - literals = (char*) modp->old.code_hdr->literal_area->start; - lit_bsize = (char*) modp->old.code_hdr->literal_area->end - literals; + if (modp->old.code_hdr->literal_area) { + literals = (char*) modp->old.code_hdr->literal_area->start; + lit_bsize = (char*) modp->old.code_hdr->literal_area->end - literals; + } + else { + literals = NULL; + lit_bsize = 0; + } for (msgp = rp->msg.first; msgp; msgp = msgp->next) { if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) @@ -902,12 +1242,6 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls /* Check heap, stack etc... */ if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size)) goto try_gc; - if (!(flags & ERTS_CPC_COPY_LITERALS)) { - /* Process ok. May contain old literals but we will be called - * again before module is purged. - */ - return am_false; - } if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, literals, lit_bsize)) { rp->freason = EXC_NULL; rp->fvalue = NIL; @@ -1006,6 +1340,8 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } +#endif /* !ERTS_NEW_PURGE_STRATEGY */ + static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) { @@ -1136,12 +1472,73 @@ hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, #undef in_area +#ifdef ERTS_NEW_PURGE_STRATEGY + +ErtsThrPrgrLaterOp later_literal_area_switch; + +static void +complete_literal_area_switch(void *unused) +{ + Process *p = erts_literal_area_collector; + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + erts_resume(p, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); +} + +#endif /* ERTS_NEW_PURGE_STRATEGY */ + +BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) +{ +#ifndef ERTS_NEW_PURGE_STRATEGY + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else + ErtsLiteralAreaRef *la_ref; + + if (BIF_P != erts_literal_area_collector) + BIF_ERROR(BIF_P, EXC_NOTSUP); + + erts_smp_mtx_lock(&release_literal_areas.mtx); + + la_ref = release_literal_areas.first; + if (la_ref) { + release_literal_areas.first = la_ref->next; + if (!release_literal_areas.first) + release_literal_areas.last = NULL; + } + + erts_smp_mtx_unlock(&release_literal_areas.mtx); + + if (erts_copy_literal_area) + erts_release_literal_area(erts_copy_literal_area); + + if (!la_ref) { + erts_copy_literal_area = NULL; + BIF_RET(am_false); + } + + erts_copy_literal_area = la_ref->literal_area; + + erts_free(ERTS_ALC_T_LITERAL_REF, la_ref); + +#ifndef ERTS_SMP + BIF_RET(am_true); +#else + erts_schedule_thr_prgr_later_op(complete_literal_area_switch, + NULL, + &later_literal_area_switch); + erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); + ERTS_BIF_YIELD_RETURN(BIF_P, am_true); +#endif + +#endif /* ERTS_NEW_PURGE_STRATEGY */ +} + +#ifndef ERTS_NEW_PURGE_STRATEGY + #ifdef ERTS_SMP static void copy_literals_commit(void*); #endif -copy_literals_t erts_clrange = {NULL, 0, THE_NON_VALUE}; - /* copy literals * * copy_literals.ptr = LitPtr @@ -1157,12 +1554,19 @@ copy_literals_t erts_clrange = {NULL, 0, THE_NON_VALUE}; * ... */ +#endif BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2) { +#ifdef ERTS_NEW_PURGE_STRATEGY + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else ErtsCodeIndex code_ix; Eterm res = am_true; + if (BIF_P != erts_code_purger) + BIF_ERROR(BIF_P, EXC_NOTSUP); + if (is_not_atom(BIF_ARG_1) || (am_true != BIF_ARG_2 && am_false != BIF_ARG_2)) { BIF_ERROR(BIF_P, BADARG); } @@ -1180,22 +1584,13 @@ BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2) res = am_false; goto done; } - if (erts_clrange.ptr != NULL - && !(BIF_P->static_flags & ERTS_STC_FLG_SYSTEM_PROC)) { + if (erts_copy_literal_area) { res = am_aborted; goto done; } - erts_clrange.ptr = modp->old.code_hdr->literal_area->start; - erts_clrange.sz = modp->old.code_hdr->literal_area->end - erts_clrange.ptr; - erts_clrange.pid = BIF_P->common.id; + erts_copy_literal_area = modp->old.code_hdr->literal_area; } else if (BIF_ARG_2 == am_false) { - if (erts_clrange.pid != BIF_P->common.id) { - res = am_false; - goto done; - } - erts_clrange.ptr = NULL; - erts_clrange.sz = 0; - erts_clrange.pid = THE_NON_VALUE; + erts_copy_literal_area = NULL; } #ifdef ERTS_SMP @@ -1209,8 +1604,11 @@ BIF_RETTYPE erts_internal_copy_literals_2(BIF_ALIST_2) done: erts_release_code_write_permission(); BIF_RET(res); +#endif } +#ifndef ERTS_NEW_PURGE_STRATEGY + #ifdef ERTS_SMP static void copy_literals_commit(void* null) { Process* p = committer_state.stager; @@ -1227,6 +1625,7 @@ static void copy_literals_commit(void* null) { } #endif /* ERTS_SMP */ +#endif /* Do the actualy module purging and return: * true for success @@ -1241,6 +1640,7 @@ BIF_RETTYPE erts_internal_purge_module_1(BIF_ALIST_1) Module* modp; int is_blocking = 0; Eterm ret; + ErtsLiteralArea *literals = NULL; if (is_not_atom(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); @@ -1294,7 +1694,7 @@ BIF_RETTYPE erts_internal_purge_module_1(BIF_ALIST_1) erts_cleanup_funs_on_purge(code, end); beam_catches_delmod(modp->old.catches, code, modp->old.code_length, code_ix); - erts_release_literal_area(modp->old.code_hdr->literal_area); + literals = modp->old.code_hdr->literal_area; modp->old.code_hdr->literal_area = NULL; erts_free(ERTS_ALC_T_CODE, (void *) code); modp->old.code_hdr = NULL; @@ -1310,6 +1710,38 @@ BIF_RETTYPE erts_internal_purge_module_1(BIF_ALIST_1) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); } erts_release_code_write_permission(); + +#ifdef ERTS_NEW_PURGE_STRATEGY + + if (literals) { + ErtsLiteralAreaRef *ref; + ref = erts_alloc(ERTS_ALC_T_LITERAL_REF, + sizeof(ErtsLiteralAreaRef)); + ref->literal_area = literals; + ref->next = NULL; + erts_smp_mtx_lock(&release_literal_areas.mtx); + if (release_literal_areas.last) { + release_literal_areas.last->next = ref; + release_literal_areas.last = ref; + } + else { + release_literal_areas.first = ref; + release_literal_areas.last = ref; + } + erts_smp_mtx_unlock(&release_literal_areas.mtx); + erts_queue_message(erts_literal_area_collector, + 0, + erts_alloc_message(0, NULL), + am_copy_literals, + BIF_P->common.id); + } + +#else /* !ERTS_NEW_PURGE_STRATEGY */ + + erts_release_literal_area(literals); + +#endif + return ret; } diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 336f097f0f..d52f12e6d0 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -50,6 +50,7 @@ extern BeamInstr* em_call_error_handler; extern BeamInstr* em_apply_bif; extern BeamInstr* em_call_nif; +struct ErtsLiteralArea_; /* * The following variables keep a sorted list of address ranges for @@ -60,15 +61,6 @@ extern BeamInstr* em_call_nif; /* Total code size in bytes */ extern Uint erts_total_code_size; -typedef struct { - struct erl_off_heap_header *off_heap; - Eterm *end; - Eterm start[1]; /* beginning of area */ -} ErtsLiteralArea; - -#define ERTS_LITERAL_AREA_ALLOC_SIZE(N) \ - (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1)) - typedef struct BeamCodeLineTab_ BeamCodeLineTab; /* @@ -98,7 +90,7 @@ typedef struct beam_code_header { /* * Literal area (constant pool). */ - ErtsLiteralArea *literal_area; + struct ErtsLiteralArea_ *literal_area; /* * Pointer to the on_load function (or NULL if none). @@ -127,9 +119,11 @@ typedef struct beam_code_header { }BeamCodeHeader; -void erts_release_literal_area(ErtsLiteralArea* literal_area); +void erts_release_literal_area(struct ErtsLiteralArea_* literal_area); int erts_is_module_native(BeamCodeHeader* code); +void erts_beam_bif_load_init(void); + /* * Layout of the line table. */ diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 065018514a..7ffcb26567 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -174,6 +174,8 @@ bif erts_internal:is_system_process/1 bif erts_internal:system_check/1 +bif erts_internal:release_literal_area_switch/0 + # inet_db support bif erlang:port_set_data/2 bif erlang:port_get_data/1 diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 227fedfb69..0a4a34a174 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -367,6 +367,7 @@ type MONITOR_LH STANDARD PROCESSES monitor_lh type NLINK_LH STANDARD PROCESSES nlink_lh type CODE LONG_LIVED CODE code type LITERAL LITERAL CODE literal +type LITERAL_REF SHORT_LIVED CODE literal_area_ref type DB_HEIR_DATA STANDARD ETS db_heir_data type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index fbdafec4ef..788dbc3328 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -384,6 +384,7 @@ erl_init(int ncpu, erts_init_unicode(); /* after RE to get access to PCRE unicode */ erts_init_external(); erts_init_map(); + erts_beam_bif_load_init(); erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2); erts_late_init_process(); #if HAVE_ERTS_MSEG @@ -2250,7 +2251,27 @@ erl_start(int argc, char **argv) otp_ring0_pid = erl_first_process_otp("otp_ring0", NULL, 0, boot_argc, boot_argv); - (void) erl_system_process_otp(otp_ring0_pid, "erts_code_purger"); + { + /* + * The erts_code_purger and the erts_literal_area_collector + * system processes are *always* alive. If they terminate + * they bring the whole VM down. + */ + Eterm pid; + + pid = erl_system_process_otp(otp_ring0_pid, "erts_code_purger"); + erts_code_purger = erts_proc_lookup(pid); + ASSERT(erts_code_purger); + erts_proc_inc_refc(erts_code_purger); + +#ifdef ERTS_NEW_PURGE_STRATEGY + pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector"); + erts_literal_area_collector = erts_proc_lookup(pid); + ASSERT(erts_literal_area_collector); + erts_proc_inc_refc(erts_literal_area_collector); +#endif + + } #ifdef ERTS_SMP erts_start_schedulers(); @@ -2350,14 +2371,15 @@ erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) if (erts_mtrace_enabled) erts_mtrace_exit((Uint32) n); + if (fmt != NULL && *fmt != '\0') + erl_error(fmt, args2); /* Print error message. */ + /* Produce an Erlang core dump if error */ if (((n == ERTS_ERROR_EXIT && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) && erts_initialized) { erl_crash_dump_v((char*) NULL, 0, fmt, args1); } - if (fmt != NULL && *fmt != '\0') - erl_error(fmt, args2); /* Print error message. */ sys_tty_reset(n); if (n == ERTS_INTR_EXIT) diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 39c0617143..4192cf389f 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -112,6 +112,9 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "export_tab", NULL }, { "fun_tab", NULL }, { "environ", NULL }, +#ifdef ERTS_NEW_PURGE_STRATEGY + { "release_literal_areas", NULL }, +#endif #endif { "efile_drv", "address" }, { "drv_ev_state_grow", NULL, }, diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 66f22979ad..42c30b935e 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -448,6 +448,9 @@ int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP; typedef enum { ERTS_PSTT_GC, /* Garbage Collect */ ERTS_PSTT_CPC, /* Check Process Code */ +#ifdef ERTS_NEW_PURGE_STRATEGY + ERTS_PSTT_CLA, /* Copy Literal Area */ +#endif ERTS_PSTT_COHMQ, /* Change off heap message queue */ ERTS_PSTT_FTMQ /* Flush trace msg queue */ } ErtsProcSysTaskType; @@ -10393,6 +10396,27 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } break; } +#ifdef ERTS_NEW_PURGE_STRATEGY + case ERTS_PSTT_CLA: { + int fcalls; + int cla_reds = 0; + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + fcalls = reds; + else + fcalls = reds - CONTEXT_REDS; + st_res = erts_proc_copy_literal_area(c_p, + &cla_reds, + fcalls, + st->arg[0] == am_true); + reds -= cla_reds; + if (is_non_value(st_res)) { + /* Needed gc, but gc was disabled */ + save_gc_task(c_p, st, st_prio); + st = NULL; + } + break; + } +#endif case ERTS_PSTT_COHMQ: reds -= erts_complete_off_heap_message_queue_change(c_p); st_res = am_true; @@ -10443,14 +10467,15 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) switch (st->type) { case ERTS_PSTT_GC: - st_res = am_false; - break; case ERTS_PSTT_CPC: - st_res = am_false; - break; case ERTS_PSTT_COHMQ: st_res = am_false; break; +#ifdef ERTS_NEW_PURGE_STRATEGY + case ERTS_PSTT_CLA: + st_res = am_ok; + break; +#endif #ifdef ERTS_SMP case ERTS_PSTT_FTMQ: reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN); @@ -10572,6 +10597,17 @@ erts_internal_request_system_task_3(BIF_ALIST_3) goto noproc; break; +#ifdef ERTS_NEW_PURGE_STRATEGY + case am_copy_literals: + if (st->arg[0] != am_true && st->arg[0] != am_false) + goto badarg; + st->type = ERTS_PSTT_CLA; + noproc_res = am_ok; + if (!rp) + goto noproc; + break; +#endif + default: goto badarg; } diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index f3d4ac56cd..00dfc421d5 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -999,17 +999,27 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); /* beam_bif_load.c */ #define ERTS_CPC_ALLOW_GC (1 << 0) -#define ERTS_CPC_COPY_LITERALS (1 << 1) -#define ERTS_CPC_ALL (ERTS_CPC_ALLOW_GC | ERTS_CPC_COPY_LITERALS) +#define ERTS_CPC_ALL ERTS_CPC_ALLOW_GC Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls); +#ifdef ERTS_NEW_PURGE_STRATEGY +Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed); +#endif -typedef struct { - Eterm *ptr; - Uint sz; - Eterm pid; -} copy_literals_t; +typedef struct ErtsLiteralArea_ { + struct erl_off_heap_header *off_heap; + Eterm *end; + Eterm start[1]; /* beginning of area */ +} ErtsLiteralArea; -extern copy_literals_t erts_clrange; +#define ERTS_LITERAL_AREA_ALLOC_SIZE(N) \ + (sizeof(ErtsLiteralArea) + sizeof(Eterm)*((N) - 1)) + +extern ErtsLiteralArea *erts_copy_literal_area; +#ifdef ERTS_NEW_PURGE_STRATEGY +extern Process *erts_literal_area_collector; +#endif + +extern Process *erts_code_purger; /* beam_load.c */ typedef struct { @@ -1092,12 +1102,19 @@ typedef struct { #define INITIALIZE_SHCOPY(info) \ do { \ + ErtsLiteralArea *larea__ = erts_copy_literal_area; \ info.queue_start = info.queue_default; \ info.bitstore_start = info.bitstore_default; \ info.shtable_start = info.shtable_default; \ info.literal_size = 0; \ - info.range_ptr = erts_clrange.ptr; \ - info.range_sz = erts_clrange.sz; \ + if (larea__) { \ + info.range_ptr = &larea__->start[0]; \ + info.range_sz = larea__->end - info.range_ptr; \ + } \ + else { \ + info.range_ptr = NULL; \ + info.range_sz = 0; \ + } \ } while(0) #define DESTROY_SHCOPY(info) \ diff --git a/erts/preloaded/ebin/erts_code_purger.beam b/erts/preloaded/ebin/erts_code_purger.beam Binary files differindex a0da864824..9133fd9853 100644 --- a/erts/preloaded/ebin/erts_code_purger.beam +++ b/erts/preloaded/ebin/erts_code_purger.beam diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam Binary files differindex d897c8e92f..3349d05385 100644 --- a/erts/preloaded/ebin/erts_internal.beam +++ b/erts/preloaded/ebin/erts_internal.beam diff --git a/erts/preloaded/ebin/erts_literal_area_collector.beam b/erts/preloaded/ebin/erts_literal_area_collector.beam Binary files differnew file mode 100644 index 0000000000..71f3c2ec8c --- /dev/null +++ b/erts/preloaded/ebin/erts_literal_area_collector.beam diff --git a/erts/preloaded/src/Makefile b/erts/preloaded/src/Makefile index 4a447d3a09..df4645070b 100644 --- a/erts/preloaded/src/Makefile +++ b/erts/preloaded/src/Makefile @@ -44,7 +44,8 @@ PRE_LOADED_ERL_MODULES = \ erts_code_purger \ erlang \ erts_internal \ - erl_tracer + erl_tracer \ + erts_literal_area_collector PRE_LOADED_BEAM_MODULES = \ prim_eval diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src index e18da28905..7ab06164b4 100644 --- a/erts/preloaded/src/erts.app.src +++ b/erts/preloaded/src/erts.app.src @@ -37,7 +37,7 @@ {registered, []}, {applications, []}, {env, []}, - {runtime_dependencies, ["stdlib-3.0", "kernel-5.0", "sasl-3.0"]} + {runtime_dependencies, ["stdlib-3.0", "kernel-5.0", "sasl-3.0.1"]} ]}. %% vim: ft=erlang diff --git a/erts/preloaded/src/erts_code_purger.erl b/erts/preloaded/src/erts_code_purger.erl index d1e64342e0..f9208624b7 100644 --- a/erts/preloaded/src/erts_code_purger.erl +++ b/erts/preloaded/src/erts_code_purger.erl @@ -28,7 +28,36 @@ start() -> register(erts_code_purger, self()), process_flag(trap_exit, true), - loop(). + try + %% Pass bad arguments to copy_literals() in + %% order to determine purge strategy used + %% by the VM... + Res = erts_internal:copy_literals(4711, badarg), + exit({copy_literals_returned, Res}) + catch + error : badarg -> %% VM use old purge strategy + old_loop(); + error : notsup -> %% VM use new purge strategy + loop(); + Type : Reason -> + %% This should not be possible... + exit({"Unexpected copy_literals() behaviour", + {Type, Reason}}) + end. + +old_loop() -> + _ = receive + {purge,Mod,From,Ref} when is_atom(Mod), is_pid(From) -> + Res = do_old_purge(Mod), + From ! {reply, purge, Res, Ref}; + + {soft_purge,Mod,From,Ref} when is_atom(Mod), is_pid(From) -> + Res = do_old_soft_purge(Mod), + From ! {reply, soft_purge, Res, Ref}; + + _Other -> ignore + end, + old_loop(). loop() -> _ = receive @@ -61,7 +90,7 @@ purge(Mod) when is_atom(Mod) -> end. -do_purge(Mod) -> +do_old_purge(Mod) -> case erts_internal:copy_literals(Mod, true) of false -> {false, false}; @@ -72,6 +101,11 @@ do_purge(Mod) -> {WasPurged, DidKill} end. +do_purge(Mod) -> + DidKill = check_proc_code(erlang:processes(), Mod, true), + WasPurged = erts_internal:purge_module(Mod), + {WasPurged, DidKill}. + %% soft_purge(Module) %% Purge old code only if no procs remain that run old code. %% Return true in that case, false if procs remain (in this @@ -86,7 +120,7 @@ soft_purge(Mod) -> end. -do_soft_purge(Mod) -> +do_old_soft_purge(Mod) -> case erts_internal:copy_literals(Mod, true) of false -> true; @@ -102,6 +136,12 @@ do_soft_purge(Mod) -> end end. +do_soft_purge(Mod) -> + case check_proc_code(erlang:processes(), Mod, false) of + false -> false; + true -> erts_internal:purge_module(Mod) + end. + %% %% check_proc_code(Pids, Mod, Hard) - Send asynchronous %% requests to all processes to perform a check_process_code @@ -283,8 +323,7 @@ cpc_sched_kill(Pid, cpc_request(#cpc_static{tag = Tag, module = Mod}, Pid, AllowGc) -> erts_internal:check_process_code(Pid, Mod, [{async, {Tag, Pid, AllowGc}}, - {allow_gc, AllowGc}, - {copy_literals, true}]). + {allow_gc, AllowGc}]). cpc_request_gc(CpcS, [Pid|Pids]) -> cpc_request(CpcS, Pid, true), diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl index 2459ea2a2c..2bf430d857 100644 --- a/erts/preloaded/src/erts_internal.erl +++ b/erts/preloaded/src/erts_internal.erl @@ -42,6 +42,7 @@ -export([check_process_code/3]). -export([copy_literals/2]). +-export([release_literal_area_switch/0]). -export([purge_module/1]). -export([flush_monitor_messages/3]). @@ -211,7 +212,6 @@ request_system_task(_Pid, _Prio, _Request) -> erlang:nif_error(undefined). -define(ERTS_CPC_ALLOW_GC, (1 bsl 0)). --define(ERTS_CPC_COPY_LITERALS, (1 bsl 1)). -spec check_process_code(Module, Flags) -> boolean() when Module :: module(), @@ -223,7 +223,7 @@ check_process_code(_Module, _Flags) -> Pid :: pid(), Module :: module(), RequestId :: term(), - Option :: {async, RequestId} | {allow_gc, boolean()} | {copy_literals, boolean()}, + Option :: {async, RequestId} | {allow_gc, boolean()}, OptionList :: [Option], CheckResult :: boolean() | aborted. check_process_code(Pid, Module, OptionList) -> @@ -265,8 +265,6 @@ get_cpc_opts([{async, _ReqId} = AsyncTuple | Options], _OldAsync, Flags) -> get_cpc_opts(Options, AsyncTuple, Flags); get_cpc_opts([{allow_gc, AllowGC} | Options], Async, Flags) -> get_cpc_opts(Options, Async, cpc_flags(Flags, ?ERTS_CPC_ALLOW_GC, AllowGC)); -get_cpc_opts([{copy_literals, CopyLit} | Options], Async, Flags) -> - get_cpc_opts(Options, Async, cpc_flags(Flags, ?ERTS_CPC_COPY_LITERALS, CopyLit)); get_cpc_opts([], Async, Flags) -> {Async, Flags}. @@ -281,6 +279,11 @@ cpc_flags(OldFlags, Bit, false) -> copy_literals(_Mod, _Bool) -> erlang:nif_error(undefined). +-spec release_literal_area_switch() -> 'true' | 'false'. + +release_literal_area_switch() -> + erlang:nif_error(undefined). + -spec purge_module(Module) -> boolean() when Module :: module(). purge_module(_Module) -> diff --git a/erts/preloaded/src/erts_literal_area_collector.erl b/erts/preloaded/src/erts_literal_area_collector.erl new file mode 100644 index 0000000000..3befad8dfb --- /dev/null +++ b/erts/preloaded/src/erts_literal_area_collector.erl @@ -0,0 +1,113 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2016. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(erts_literal_area_collector). + +-export([start/0]). + +%% Currently we only allow two outstanding literal +%% copying jobs that garbage collect in order to +%% copy the literals. Maybe we could allow more +%% than two outstanding processes, but for now we +%% play it safe... +-define(MAX_GC_OUTSTND, 2). + +%% +%% The erts_literal_area_collector is started at +%% VM boot by the VM. It is a spawned as a system +%% process, i.e, the whole VM will terminate if +%% this process terminates. +%% +start() -> + process_flag(trap_exit, true), + msg_loop(undefined, 0, 0, []). + +%% +%% The VM will send us a 'copy_literals' message +%% when it has a new literal area that needs to +%% be handled is added. We will also be informed +%% about more areas when we call +%% erts_internal:release_literal_area_switch(). +%% +msg_loop(Area, Outstnd, GcOutstnd, NeedGC) -> + receive + + %% A new area to handle has arrived... + copy_literals when Outstnd == 0 -> + switch_area(); + + %% Process (_Pid) has completed the request... + {copy_literals, {Area, _GcAllowed, _Pid}, ok} when Outstnd == 1 -> + switch_area(); %% Last process completed... + {copy_literals, {Area, false, _Pid}, ok} -> + msg_loop(Area, Outstnd-1, GcOutstnd, NeedGC); + {copy_literals, {Area, true, _Pid}, ok} when NeedGC == [] -> + msg_loop(Area, Outstnd-1, GcOutstnd-1, []); + {copy_literals, {Area, true, _Pid}, ok} -> + send_copy_req(hd(NeedGC), Area, true), + msg_loop(Area, Outstnd-1, GcOutstnd, tl(NeedGC)); + + %% Process (Pid) failed to complete the request + %% since it needs to garbage collect in order to + %% complete the request... + {copy_literals, {Area, false, Pid}, need_gc} when GcOutstnd < ?MAX_GC_OUTSTND -> + send_copy_req(Pid, Area, true), + msg_loop(Area, Outstnd, GcOutstnd+1, NeedGC); + {copy_literals, {Area, false, Pid}, need_gc} -> + msg_loop(Area, Outstnd, GcOutstnd, [Pid|NeedGC]); + + %% Not handled message regarding the area that we + %% currently are working with. Crash the VM so + %% we notice this bug... + {copy_literals, {Area, _, _}, _} = Msg when erlang:is_reference(Area) -> + exit({not_handled_message, Msg}); + + %% Unexpected garbage message. Get rid of it... + _Ignore -> + msg_loop(Area, Outstnd, GcOutstnd, NeedGC) + + end. + +switch_area() -> + Res = erts_internal:release_literal_area_switch(), + erlang:garbage_collect(), %% Almost no live data now... + case Res of + false -> + %% No more areas to handle... + msg_loop(undefined, 0, 0, []); + true -> + %% Send requests to all processes to copy + %% all live data they have referring to the + %% literal area that is to be released... + Area = make_ref(), + Outstnd = send_copy_reqs(erlang:processes(), Area, false), + msg_loop(Area, Outstnd, 0, []) + end. + +send_copy_reqs(Ps, Area, GC) -> + send_copy_reqs(Ps, Area, GC, 0). + +send_copy_reqs([], _Area, _GC, N) -> + N; +send_copy_reqs([P|Ps], Area, GC, N) -> + send_copy_req(P, Area, GC), + send_copy_reqs(Ps, Area, GC, N+1). + +send_copy_req(P, Area, GC) -> + erts_internal:request_system_task(P, normal, {copy_literals, {Area, GC, P}, GC}). |