diff options
Diffstat (limited to 'erts/emulator')
96 files changed, 5611 insertions, 1900 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index ce50022683..18fd7f320b 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -52,6 +52,7 @@ OMIT_OMIT_FP=no TYPE_LIBS= DIRTY_SCHEDULER_SUPPORT=@DIRTY_SCHEDULER_SUPPORT@ +DIRTY_SCHEDULER_TEST=@DIRTY_SCHEDULER_TEST@ ifeq ($(TYPE),debug) PURIFY = @@ -181,9 +182,24 @@ ENABLE_ALLOC_TYPE_VARS += smp nofrag M4FLAGS += -DERTS_SMP=1 ifeq ($(DIRTY_SCHEDULER_SUPPORT),yes) THR_DEFS += -DERTS_DIRTY_SCHEDULERS -endif +DS_SUPPORT=yes -else +ifeq ($(DIRTY_SCHEDULER_TEST),yes) +DS_TEST=yes +THR_DEFS += -DERTS_DIRTY_SCHEDULERS_TEST +else # DIRTY_SCHEDULER_TEST +DS_TEST=no +endif # DIRTY_SCHEDULER_TEST + +else # DIRTY_SCHEDULER_SUPPORT +DS_SUPPORT=no +DS_TEST=no +endif # DIRTY_SCHEDULER_SUPPORT + +else # FLAVOR + +DS_SUPPORT=no +DS_TEST=no # If flavor isn't one of the above, it *is* plain flavor... override FLAVOR=plain @@ -548,8 +564,10 @@ GENERATE += $(TTF_DIR)/OPCODES-GENERATED # bif and atom table ATOMS= beam/atom.names +DIRTY_BIFS = beam/erl_dirty_bif.tab BIFS = beam/bif.tab ifdef HIPE_ENABLED +HIPE=yes HIPE_ARCH64_TAB=hipe/hipe_bif64.tab HIPE_x86_TAB=hipe/hipe_x86.tab HIPE_amd64_TAB=hipe/hipe_amd64.tab $(HIPE_ARCH64_TAB) @@ -559,20 +577,26 @@ HIPE_ppc64_TAB=hipe/hipe_ppc64.tab $(HIPE_ARCH64_TAB) HIPE_arm_TAB=hipe/hipe_arm.tab HIPE_ARCH_TAB=$(HIPE_$(ARCH)_TAB) BIFS += hipe/hipe_bif0.tab hipe/hipe_bif1.tab hipe/hipe_bif2.tab $(HIPE_ARCH_TAB) -endif - -$(TARGET)/erl_bif_table.c \ -$(TARGET)/erl_bif_table.h \ -$(TARGET)/erl_bif_wrap.c \ -$(TARGET)/erl_bif_list.h \ -$(TARGET)/erl_gc_bifs.c \ -$(TARGET)/erl_atom_table.c \ -$(TARGET)/erl_atom_table.h \ - : $(TARGET)/TABLES-GENERATED -$(TARGET)/TABLES-GENERATED: $(ATOMS) $(BIFS) utils/make_tables - $(gen_verbose)LANG=C $(PERL) utils/make_tables -src $(TARGET) -include $(TARGET)\ - $(ATOMS) $(BIFS) && echo $? >$(TARGET)/TABLES-GENERATED -GENERATE += $(TARGET)/TABLES-GENERATED +HIPE_NBIF_FILES=$(TTF_DIR)/hipe_nbif_impl.h $(TTF_DIR)/hipe_nbif_impl.c +else +HIPE=no +HIPE_NBIF_FILES= +endif + +$(TTF_DIR)/erl_bif_table.c \ +$(TTF_DIR)/erl_bif_table.h \ +$(TTF_DIR)/erl_bif_wrap.c \ +$(TTF_DIR)/erl_bif_list.h \ +$(TTF_DIR)/erl_atom_table.c \ +$(TTF_DIR)/erl_atom_table.h \ +$(TTF_DIR)/erl_guard_bifs.c \ +$(TTF_DIR)/erl_dirty_bif_wrap.c \ +$(HIPE_NBIF_FILES) \ + : $(TTF_DIR)/TABLES-GENERATED +$(TTF_DIR)/TABLES-GENERATED: $(ATOMS) $(DIRTY_BIFS) $(BIFS) utils/make_tables + $(gen_verbose)LANG=C $(PERL) utils/make_tables -src $(TTF_DIR) -include $(TTF_DIR)\ + -ds $(DS_SUPPORT) -dst $(DS_TEST) -hipe $(HIPE) $(ATOMS) $(DIRTY_BIFS) $(BIFS) && echo $? >$(TTF_DIR)/TABLES-GENERATED +GENERATE += $(TTF_DIR)/TABLES-GENERATED $(TTF_DIR)/erl_alloc_types.h: beam/erl_alloc.types utils/make_alloc_types $(gen_verbose)LANG=C $(PERL) utils/make_alloc_types -src $< -dst $@ $(ENABLE_ALLOC_TYPE_VARS) @@ -750,8 +774,8 @@ RUN_OBJS = \ $(OBJDIR)/erl_bif_info.o $(OBJDIR)/erl_bif_op.o \ $(OBJDIR)/erl_bif_os.o $(OBJDIR)/erl_bif_lists.o \ $(OBJDIR)/erl_bif_trace.o $(OBJDIR)/erl_bif_unique.o \ - $(OBJDIR)/erl_bif_wrap.o \ - $(OBJDIR)/erl_gc_bifs.o \ + $(OBJDIR)/erl_bif_wrap.o $(OBJDIR)/erl_nfunc_sched.o \ + $(OBJDIR)/erl_guard_bifs.o $(OBJDIR)/erl_dirty_bif_wrap.o \ $(OBJDIR)/erl_trace.o $(OBJDIR)/copy.o \ $(OBJDIR)/utils.o $(OBJDIR)/bif.o \ $(OBJDIR)/io.o $(OBJDIR)/erl_printf_term.o\ @@ -882,6 +906,7 @@ HIPE_noarch_OBJS= HIPE_ARCH_OBJS=$(HIPE_$(ARCH)_OBJS) HIPE_OBJS= \ + $(OBJDIR)/hipe_nbif_impl.o \ $(OBJDIR)/hipe_bif0.o \ $(OBJDIR)/hipe_bif1.o \ $(OBJDIR)/hipe_bif2.o \ @@ -917,7 +942,7 @@ $(OBJS): $(TTF_DIR)/GENERATED ######################################## # HiPE section -M4FLAGS += -DTARGET=$(TARGET) -DOPSYS=$(OPSYS) -DARCH=$(ARCH) +M4FLAGS += -DTARGET=$(TARGET) -DTTF_DIR=$(TTF_DIR) -DOPSYS=$(OPSYS) -DARCH=$(ARCH) $(TTF_DIR)/%.S: hipe/%.m4 $(m4_verbose)m4 $(M4FLAGS) $< > $@ @@ -938,7 +963,7 @@ $(BINDIR)/hipe_mkliterals$(TF_MARKER): $(OBJDIR)/hipe_mkliterals.o $(ld_verbose)$(CC) $(LDFLAGS) -o $@ $< $(TYPE_LIBS) $(OBJDIR)/hipe_mkliterals.o: $(HIPE_ASM) $(TTF_DIR)/erl_alloc_types.h $(DTRACE_HEADERS) \ - $(TTF_DIR)/OPCODES-GENERATED $(TARGET)/TABLES-GENERATED + $(TTF_DIR)/OPCODES-GENERATED $(TTF_DIR)/TABLES-GENERATED $(TTF_DIR)/hipe_literals.h: $(BINDIR)/hipe_mkliterals$(TF_MARKER) $(gen_verbose)$(BINDIR)/hipe_mkliterals$(TF_MARKER) -c > $@ @@ -947,7 +972,7 @@ $(OBJDIR)/hipe_x86_glue.o: hipe/hipe_x86_glue.S \ $(TTF_DIR)/hipe_x86_asm.h $(TTF_DIR)/hipe_literals.h \ hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_x86_bifs.S: hipe/hipe_x86_bifs.m4 hipe/hipe_x86_asm.m4 \ - hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h + hipe/hipe_bif_list.m4 $(TTF_DIR)/erl_bif_list.h hipe/hipe_gbif_list.h $(OBJDIR)/hipe_x86_bifs.o: $(TTF_DIR)/hipe_x86_bifs.S \ $(TTF_DIR)/hipe_literals.h @@ -955,7 +980,7 @@ $(OBJDIR)/hipe_amd64_glue.o: hipe/hipe_amd64_glue.S \ $(TTF_DIR)/hipe_amd64_asm.h $(TTF_DIR)/hipe_literals.h \ hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_amd64_bifs.S: hipe/hipe_amd64_bifs.m4 hipe/hipe_amd64_asm.m4 \ - hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h + hipe/hipe_bif_list.m4 $(TTF_DIR)/erl_bif_list.h hipe/hipe_gbif_list.h $(OBJDIR)/hipe_amd64_bifs.o: $(TTF_DIR)/hipe_amd64_bifs.S \ $(TTF_DIR)/hipe_literals.h @@ -963,21 +988,21 @@ $(OBJDIR)/hipe_sparc_glue.o: hipe/hipe_sparc_glue.S \ $(TTF_DIR)/hipe_sparc_asm.h hipe/hipe_mode_switch.h \ $(TTF_DIR)/hipe_literals.h $(TTF_DIR)/hipe_sparc_bifs.S: hipe/hipe_sparc_bifs.m4 hipe/hipe_sparc_asm.m4 \ - hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h + hipe/hipe_bif_list.m4 $(TTF_DIR)/erl_bif_list.h hipe/hipe_gbif_list.h $(OBJDIR)/hipe_sparc_bifs.o: $(TTF_DIR)/hipe_sparc_bifs.S \ $(TTF_DIR)/hipe_literals.h $(OBJDIR)/hipe_ppc_glue.o: hipe/hipe_ppc_glue.S $(TTF_DIR)/hipe_ppc_asm.h \ hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h $(TTF_DIR)/hipe_ppc_bifs.S: hipe/hipe_ppc_bifs.m4 hipe/hipe_ppc_asm.m4 \ - hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h + hipe/hipe_bif_list.m4 $(TTF_DIR)/erl_bif_list.h hipe/hipe_gbif_list.h $(OBJDIR)/hipe_ppc_bifs.o: $(TTF_DIR)/hipe_ppc_bifs.S \ $(TTF_DIR)/hipe_literals.h $(OBJDIR)/hipe_arm_glue.o: hipe/hipe_arm_glue.S $(TTF_DIR)/hipe_arm_asm.h \ hipe/hipe_mode_switch.h $(TTF_DIR)/hipe_literals.h $(TTF_DIR)/hipe_arm_bifs.S: hipe/hipe_arm_bifs.m4 hipe/hipe_arm_asm.m4 \ - hipe/hipe_bif_list.m4 $(TARGET)/erl_bif_list.h hipe/hipe_gbif_list.h + hipe/hipe_bif_list.m4 $(TTF_DIR)/erl_bif_list.h hipe/hipe_gbif_list.h $(OBJDIR)/hipe_arm_bifs.o: $(TTF_DIR)/hipe_arm_bifs.S \ $(TTF_DIR)/hipe_literals.h diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index b47739059b..2b5ad097a0 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -199,7 +199,7 @@ atom_alloc(Atom* tmpl) static void atom_free(Atom* obj) { - erts_free(ERTS_ALC_T_ATOM, (void*) obj); + ASSERT(obj->slot.index == atom_val(am_ErtsSecretAtom)); } static void latin1_to_utf8(byte* conv_buf, const byte** srcp, int* lenp) @@ -467,6 +467,9 @@ init_atom_table(void) atom_space -= a.len; atom_tab(ix)->name = (byte*)erl_atom_names[i]; } + + /* Hide am_ErtsSecretAtom */ + hash_erase(&erts_atom_table.htable, atom_tab(atom_val(am_ErtsSecretAtom))); } void diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index f3b21e1687..abd3b44993 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -133,8 +133,6 @@ int atom_table_sz(void); /* table size in bytes, excluding stored objects */ Eterm am_atom_put(const char*, int); /* ONLY 7-bit ascii! */ Eterm erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc); -int atom_erase(byte*, int); -int atom_static_put(byte*, int); void init_atom_table(void); void atom_info(fmtfn_t, void *); void dump_atoms(fmtfn_t, void *); diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 66af05c1f2..dd0a42b5ba 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -59,6 +59,9 @@ atom nocatch atom undefined_function atom undefined_lambda +# Secret internal atom that can never be found by string lookup +# and should never leak out to be seen by the user. +atom ErtsSecretAtom='3RT$' # All other atoms. Try to keep the order alphabetic. # @@ -73,6 +76,7 @@ atom ac atom accessor atom active atom active_tasks +atom alive atom all atom all_but_first atom all_names @@ -196,10 +200,15 @@ atom dexit atom depth atom dgroup_leader atom dictionary +atom dirty_bif_exception +atom dirty_bif_result +atom dirty_bif_trap atom dirty_cpu atom dirty_cpu_schedulers_online atom dirty_execution atom dirty_io +atom dirty_nif_exception +atom dirty_nif_finalizer atom disable_trace atom disabled atom discard @@ -240,6 +249,7 @@ atom ERROR='ERROR' atom error_handler atom error_logger atom erts_code_purger +atom erts_debug atom erts_internal atom ets atom ETS_TRANSFER='ETS-TRANSFER' @@ -564,12 +574,15 @@ atom safe atom save_calls atom scheduler atom scheduler_id +atom scheduler_wall_time +atom scheduler_wall_time_all atom schedulers_online atom scheme atom scientific atom scope atom second atom seconds +atom send atom send_to_non_existing_process atom sensitive atom sequential_tracer @@ -662,11 +675,11 @@ atom value atom values atom version atom visible +atom wait atom waiting atom wall_clock atom warning atom warning_msg -atom scheduler_wall_time atom wordsize atom write_concurrency atom xor diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 5c3565f498..c8dde8caf8 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -36,6 +36,7 @@ #include "erl_nif.h" #include "erl_bits.h" #include "erl_thr_progress.h" +#include "erl_nfunc_sched.h" #ifdef HIPE # include "hipe_bif0.h" # define IF_HIPE(X) (X) @@ -670,7 +671,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } else if (modp->old.code_hdr) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Module %T must be purged before loading\n", + erts_dsprintf(dsbufp, "Module %T must be purged before deleting\n", BIF_ARG_1); erts_send_error_to_logger(BIF_P->group_leader, dsbufp); ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); @@ -806,7 +807,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) } if (BIF_ARG_2 == am_true) { - int i; + int i, num_exps; /* * Make the code with the on_load function current. @@ -822,7 +823,8 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) /* * The on_load function succeded. Fix up export entries. */ - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i,code_ix); if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; @@ -845,14 +847,15 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) hipe_redirect_to_module(modp); #endif } else if (BIF_ARG_2 == am_false) { - int i; + int i, num_exps; /* * The on_load function failed. Remove references to the * code that is about to be purged from the export entries. */ - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i,code_ix); if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; @@ -912,7 +915,7 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed la = ERTS_COPY_LITERAL_AREA(); if (!la) - return am_ok; + goto return_ok; oh = la->off_heap; literals = (char *) &la->start[0]; @@ -976,6 +979,11 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed * this is not completely certain). We go for * the GC directly instead of scanning everything * one more time... + * + * Also note that calling functions expect a + * major GC to be performed if gc_allowed is set + * to true. If you change this, you need to fix + * callers... */ goto literal_gc; } @@ -998,6 +1006,12 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed if (any_heap_refs(c_p->heap, c_p->htop, literals, lit_bsize)) goto literal_gc; *redsp += 1; + if (c_p->abandoned_heap) { + if (any_heap_refs(c_p->abandoned_heap, c_p->abandoned_heap + c_p->heap_sz, + literals, lit_bsize)) + goto literal_gc; + *redsp += 1; + } if (any_heap_refs(c_p->old_heap, c_p->old_htop, literals, lit_bsize)) goto literal_gc; @@ -1044,6 +1058,13 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed } } +return_ok: + +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))) + c_p->flags &= ~F_DIRTY_CLA; +#endif + return am_ok; literal_gc: @@ -1054,13 +1075,13 @@ literal_gc: if (c_p->flags & F_DISABLE_GC) return THE_NON_VALUE; - FLAGS(c_p) |= F_NEED_FULLSWEEP; - - *redsp += erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, c_p->arity, fcalls); - - erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize, oh); + *redsp += erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize, + oh, fcalls); - *redsp += lit_bsize / 64; /* Need, better value... */ +#ifdef ERTS_DIRTY_SCHEDULERS + if (c_p->flags & F_DIRTY_CLA) + return THE_NON_VALUE; +#endif return am_ok; } @@ -1099,6 +1120,11 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) return am_true; + *redsp += 1; + + if (erts_check_nif_export_in_area(rp, mod_start, mod_size)) + return am_true; + *redsp += (STACK_START(rp) - rp->stop) / 32; /* @@ -1777,9 +1803,9 @@ delete_code(Module* modp) { ErtsCodeIndex code_ix = erts_staging_code_ix(); Eterm module = make_atom(modp->module); - int i; + int i, num_exps = export_list_size(code_ix); - for (i = 0; i < export_list_size(code_ix); i++) { + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i, code_ix); if (ep != NULL && (ep->info.mfa.module == module)) { if (ep->addressv[code_ix] == ep->beam) { diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 73158205b3..27329a339e 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -32,6 +32,7 @@ #include "erl_binary.h" #include "beam_bp.h" #include "erl_term.h" +#include "erl_nfunc_sched.h" /* ************************************************************************* ** Macros @@ -74,6 +75,9 @@ extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ erts_smp_atomic32_t erts_active_bp_index; erts_smp_atomic32_t erts_staging_bp_index; +#ifdef ERTS_DIRTY_SCHEDULERS +erts_smp_mtx_t erts_dirty_bp_ix_mtx; +#endif /* * Inlined helpers @@ -85,6 +89,31 @@ get_mtime(Process *c_p) return erts_get_monotonic_time(erts_proc_sched_data(c_p)); } +static ERTS_INLINE Uint32 +acquire_bp_sched_ix(Process *c_p) +{ + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); + ASSERT(esdp); +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { + erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx); + return (Uint32) erts_no_schedulers; + } +#endif + return (Uint32) esdp->no - 1; +} + +static ERTS_INLINE void +release_bp_sched_ix(Uint32 ix) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + if (ix == (Uint32) erts_no_schedulers) + erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx); +#endif +} + + + /* ************************************************************************* ** Local prototypes */ @@ -135,6 +164,9 @@ void erts_bp_init(void) { erts_smp_atomic32_init_nob(&erts_active_bp_index, 0); erts_smp_atomic32_init_nob(&erts_staging_bp_index, 1); +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_mtx_init(&erts_dirty_bp_ix_mtx, "dirty_break_point_index"); +#endif } @@ -774,6 +806,30 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) result = func(p, args, I); + if (erts_nif_export_check_save_trace(p, result, + applying, ep, + cp, flags, + flags_meta, I, + meta_tracer)) { + /* + * erts_bif_trace_epilogue() will be called + * later when appropriate via the NIF export + * scheduling functionality... + */ + return result; + } + + return erts_bif_trace_epilogue(p, result, applying, ep, cp, + flags, flags_meta, I, + meta_tracer); +} + +Eterm +erts_bif_trace_epilogue(Process *p, Eterm result, int applying, + Export* ep, BeamInstr *cp, Uint32 flags, + Uint32 flags_meta, BeamInstr* I, + ErtsTracer meta_tracer) +{ if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) { BeamInstr i_return_trace = beam_return_trace[0]; BeamInstr i_return_to_trace = beam_return_to_trace[0]; @@ -983,6 +1039,7 @@ erts_trace_time_call(Process* c_p, ErtsCodeInfo *info, BpDataTime* bdt) bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; BpDataTime *pbdt = NULL; + Uint32 six = acquire_bp_sched_ix(c_p); ASSERT(c_p); ASSERT(erts_smp_atomic32_read_acqb(&c_p->state) & (ERTS_PSFLG_RUNNING @@ -990,7 +1047,7 @@ erts_trace_time_call(Process* c_p, ErtsCodeInfo *info, BpDataTime* bdt) /* get previous timestamp and breakpoint * from the process psd */ - + pbt = ERTS_PROC_GET_CALL_TIME(c_p); time = get_mtime(c_p); @@ -1016,7 +1073,7 @@ erts_trace_time_call(Process* c_p, ErtsCodeInfo *info, BpDataTime* bdt) /* if null then the breakpoint was removed */ if (pbdt) { - h = &(pbdt->hash[bp_sched2ix_proc(c_p)]); + h = &(pbdt->hash[six]); ASSERT(h); ASSERT(h->item); @@ -1037,7 +1094,7 @@ erts_trace_time_call(Process* c_p, ErtsCodeInfo *info, BpDataTime* bdt) /* this breakpoint */ ASSERT(bdt); - h = &(bdt->hash[bp_sched2ix_proc(c_p)]); + h = &(bdt->hash[six]); ASSERT(h); ASSERT(h->item); @@ -1051,6 +1108,8 @@ erts_trace_time_call(Process* c_p, ErtsCodeInfo *info, BpDataTime* bdt) pbt->ci = info; pbt->time = time; + + release_bp_sched_ix(six); } void @@ -1061,6 +1120,7 @@ erts_trace_time_return(Process *p, ErtsCodeInfo *ci) bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; BpDataTime *pbdt = NULL; + Uint32 six = acquire_bp_sched_ix(p); ASSERT(p); ASSERT(erts_smp_atomic32_read_acqb(&p->state) & (ERTS_PSFLG_RUNNING @@ -1081,6 +1141,7 @@ erts_trace_time_return(Process *p, ErtsCodeInfo *ci) */ if (pbt) { + /* might have been removed due to * trace_pattern(false) */ @@ -1095,7 +1156,8 @@ erts_trace_time_return(Process *p, ErtsCodeInfo *ci) /* beware, the trace_pattern might have been removed */ if (pbdt) { - h = &(pbdt->hash[bp_sched2ix_proc(p)]); + + h = &(pbdt->hash[six]); ASSERT(h); ASSERT(h->item); @@ -1106,11 +1168,15 @@ erts_trace_time_return(Process *p, ErtsCodeInfo *ci) } else { BP_TIME_ADD(item, &sitem); } + } pbt->ci = ci; pbt->time = time; + } + + release_bp_sched_ix(six); } int @@ -1362,6 +1428,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { bp_data_time_item_t sitem, *item = NULL; bp_time_hash_t *h = NULL; BpDataTime *pbdt = NULL; + Uint32 six = acquire_bp_sched_ix(p); ASSERT(p); @@ -1384,7 +1451,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { sitem.pid = p->common.id; sitem.count = 0; - h = &(pbdt->hash[bp_sched2ix_proc(p)]); + h = &(pbdt->hash[six]); ASSERT(h); ASSERT(h->item); @@ -1410,6 +1477,8 @@ void erts_schedule_time_break(Process *p, Uint schedule) { break; } } /* pbt */ + + release_bp_sched_ix(six); } /* ************************************************************************* @@ -1526,7 +1595,11 @@ set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0); bdt = Alloc(sizeof(BpDataTime)); erts_refc_init(&bdt->refc, 1); - bdt->n = erts_no_total_schedulers; +#ifdef ERTS_DIRTY_SCHEDULERS + bdt->n = erts_no_schedulers + 1; +#else + bdt->n = erts_no_schedulers; +#endif bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n)); for (i = 0; i < bdt->n; i++) { bp_hash_init(&(bdt->hash[i]), 32); diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 224b46407d..cccd395e0a 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -79,10 +79,8 @@ typedef struct generic_bp { #define ERTS_BP_CALL_TIME_SCHEDULE_OUT (1) #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2) -#ifdef ERTS_SMP -#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->thr_id - 1) -#else -#define bp_sched2ix_proc(p) (0) +#ifdef ERTS_DIRTY_SCHEDULERS +extern erts_smp_mtx_t erts_dirty_bp_ix_mtx; #endif enum erts_break_op{ diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index e72d7f8de4..8326d348af 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -39,6 +39,7 @@ #include "beam_bp.h" #include "erl_binary.h" #include "erl_thr_progress.h" +#include "erl_nfunc_sched.h" #ifdef ARCH_64 # define HEXF "%016bpX" @@ -764,3 +765,356 @@ static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif) erts_print(to, to_arg, "%T/%u", name, arity); } } + +/* + * Dirty BIF testing. + * + * The erts_debug:dirty_cpu/2, erts_debug:dirty_io/1, and + * erts_debug:dirty/3 BIFs are used by the dirty_bif_SUITE + * test suite. + */ + +#ifdef ERTS_DIRTY_SCHEDULERS +static int ms_wait(Process *c_p, Eterm etimeout, int busy); +static int dirty_send_message(Process *c_p, Eterm to, Eterm tag); +#endif +static BIF_RETTYPE dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I); + +/* + * erts_debug:dirty_cpu/2 is statically determined to execute on + * a dirty CPU scheduler (see erts_dirty_bif.tab). + */ +BIF_RETTYPE +erts_debug_dirty_cpu_2(BIF_ALIST_2) +{ + return dirty_test(BIF_P, am_dirty_cpu, BIF_ARG_1, BIF_ARG_2, BIF_I); +} + +/* + * erts_debug:dirty_io/2 is statically determined to execute on + * a dirty I/O scheduler (see erts_dirty_bif.tab). + */ +BIF_RETTYPE +erts_debug_dirty_io_2(BIF_ALIST_2) +{ + return dirty_test(BIF_P, am_dirty_io, BIF_ARG_1, BIF_ARG_2, BIF_I); +} + +/* + * erts_debug:dirty/3 executes on a normal scheduler. + */ +BIF_RETTYPE +erts_debug_dirty_3(BIF_ALIST_3) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + Eterm argv[2]; + switch (BIF_ARG_1) { + case am_normal: + return dirty_test(BIF_P, am_normal, BIF_ARG_2, BIF_ARG_3, BIF_I); + case am_dirty_cpu: + argv[0] = BIF_ARG_2; + argv[1] = BIF_ARG_3; + return erts_schedule_bif(BIF_P, + argv, + BIF_I, + erts_debug_dirty_cpu_2, + ERTS_SCHED_DIRTY_CPU, + am_erts_debug, + am_dirty_cpu, + 2); + case am_dirty_io: + argv[0] = BIF_ARG_2; + argv[1] = BIF_ARG_3; + return erts_schedule_bif(BIF_P, + argv, + BIF_I, + erts_debug_dirty_io_2, + ERTS_SCHED_DIRTY_IO, + am_erts_debug, + am_dirty_io, + 2); + default: + BIF_ERROR(BIF_P, EXC_BADARG); + } +#else + BIF_ERROR(BIF_P, EXC_UNDEF); +#endif +} + + +static BIF_RETTYPE +dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I) +{ + BIF_RETTYPE ret; +#ifdef ERTS_DIRTY_SCHEDULERS + if (am_scheduler == arg1) { + ErtsSchedulerData *esdp; + if (arg2 != am_type) + goto badarg; + esdp = erts_proc_sched_data(c_p); + if (!esdp) + ERTS_BIF_PREP_RET(ret, am_error); + else if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + ERTS_BIF_PREP_RET(ret, am_normal); + else if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp)) + ERTS_BIF_PREP_RET(ret, am_dirty_cpu); + else if (ERTS_SCHEDULER_IS_DIRTY_IO(esdp)) + ERTS_BIF_PREP_RET(ret, am_dirty_io); + else + ERTS_BIF_PREP_RET(ret, am_error); + } + else if (am_error == arg1) { + switch (arg2) { + case am_notsup: + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_NOTSUP); + break; + case am_undef: + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_UNDEF); + break; + case am_badarith: + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_BADARITH); + break; + case am_noproc: + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_NOPROC); + break; + case am_system_limit: + ERTS_BIF_PREP_ERROR(ret, c_p, SYSTEM_LIMIT); + break; + case am_badarg: + default: + goto badarg; + } + } + else if (am_copy == arg1) { + int i; + Eterm res; + + for (res = NIL, i = 0; i < 1000; i++) { + Eterm *hp, sz; + Eterm cpy; + /* We do not want this to be optimized, + but rather the oposite... */ + sz = size_object(arg2); + hp = HAlloc(c_p, sz); + cpy = copy_struct(arg2, sz, &hp, &c_p->off_heap); + hp = HAlloc(c_p, 2); + res = CONS(hp, cpy, res); + } + + ERTS_BIF_PREP_RET(ret, res); + } + else if (am_send == arg1) { + dirty_send_message(c_p, arg2, am_ok); + ERTS_BIF_PREP_RET(ret, am_ok); + } + else if (ERTS_IS_ATOM_STR("wait", arg1)) { + if (!ms_wait(c_p, arg2, type == am_dirty_cpu)) + goto badarg; + ERTS_BIF_PREP_RET(ret, am_ok); + } + else if (ERTS_IS_ATOM_STR("reschedule", arg1)) { + /* + * Reschedule operation after decrement of two until we reach + * zero. Switch between dirty scheduler types when 'n' is + * evenly divided by 4. If the initial value wasn't evenly + * dividable by 2, throw badarg exception. + */ + Eterm next_type; + Sint n; + if (!term_to_Sint(arg2, &n) || n < 0) + goto badarg; + if (n == 0) + ERTS_BIF_PREP_RET(ret, am_ok); + else { + Eterm argv[3]; + Eterm eint = erts_make_integer((Uint) (n - 2), c_p); + if (n % 4 != 0) + next_type = type; + else { + switch (type) { + case am_dirty_cpu: next_type = am_dirty_io; break; + case am_dirty_io: next_type = am_normal; break; + case am_normal: next_type = am_dirty_cpu; break; + default: goto badarg; + } + } + switch (next_type) { + case am_dirty_io: + argv[0] = arg1; + argv[1] = eint; + ret = erts_schedule_bif(c_p, + argv, + I, + erts_debug_dirty_io_2, + ERTS_SCHED_DIRTY_IO, + am_erts_debug, + am_dirty_io, + 2); + break; + case am_dirty_cpu: + argv[0] = arg1; + argv[1] = eint; + ret = erts_schedule_bif(c_p, + argv, + I, + erts_debug_dirty_cpu_2, + ERTS_SCHED_DIRTY_CPU, + am_erts_debug, + am_dirty_cpu, + 2); + break; + case am_normal: + argv[0] = am_normal; + argv[1] = arg1; + argv[2] = eint; + ret = erts_schedule_bif(c_p, + argv, + I, + erts_debug_dirty_3, + ERTS_SCHED_NORMAL, + am_erts_debug, + am_dirty, + 3); + break; + default: + goto badarg; + } + } + } + else if (ERTS_IS_ATOM_STR("ready_wait6_done", arg1)) { + ERTS_DECL_AM(ready); + ERTS_DECL_AM(done); + dirty_send_message(c_p, arg2, AM_ready); + ms_wait(c_p, make_small(6000), 0); + dirty_send_message(c_p, arg2, AM_done); + ERTS_BIF_PREP_RET(ret, am_ok); + } + else if (ERTS_IS_ATOM_STR("alive_waitexiting", arg1)) { + Process *real_c_p = erts_proc_shadow2real(c_p); + Eterm *hp, *hp2; + Uint sz; + int i; + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); + int dirty_io = esdp->type == ERTS_SCHED_DIRTY_IO; + + if (ERTS_PROC_IS_EXITING(real_c_p)) + goto badarg; + dirty_send_message(c_p, arg2, am_alive); + + /* Wait until dead */ + while (!ERTS_PROC_IS_EXITING(real_c_p)) { + if (dirty_io) + ms_wait(c_p, make_small(100), 0); + else + erts_thr_yield(); + } + + ms_wait(c_p, make_small(1000), 0); + + /* Should still be able to allocate memory */ + hp = HAlloc(c_p, 3); /* Likely on heap */ + sz = 10000; + hp2 = HAlloc(c_p, sz); /* Likely in heap fragment */ + *hp2 = make_pos_bignum_header(sz); + for (i = 1; i < sz; i++) + hp2[i] = (Eterm) 4711; + ERTS_BIF_PREP_RET(ret, TUPLE2(hp, am_ok, make_big(hp2))); + } + else { + badarg: + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + } +#else + ERTS_BIF_PREP_ERROR(ret, c_p, EXC_UNDEF); +#endif + return ret; +} + +#ifdef ERTS_DIRTY_SCHEDULERS + +static int +dirty_send_message(Process *c_p, Eterm to, Eterm tag) +{ + ErtsProcLocks c_p_locks, rp_locks; + Process *rp, *real_c_p; + Eterm msg, *hp; + ErlOffHeap *ohp; + ErtsMessage *mp; + + ASSERT(is_immed(tag)); + + real_c_p = erts_proc_shadow2real(c_p); + if (real_c_p != c_p) + c_p_locks = 0; + else + c_p_locks = ERTS_PROC_LOCK_MAIN; + + ASSERT(real_c_p->common.id == c_p->common.id); + + rp = erts_pid2proc_opt(real_c_p, c_p_locks, + to, 0, + ERTS_P2P_FLG_INC_REFC); + + if (!rp) + return 0; + + rp_locks = 0; + mp = erts_alloc_message_heap(rp, &rp_locks, 3, &hp, &ohp); + + msg = TUPLE2(hp, tag, c_p->common.id); + erts_queue_message(rp, rp_locks, mp, msg, c_p->common.id); + + if (rp == real_c_p) + rp_locks &= ~c_p_locks; + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + erts_proc_dec_refc(rp); + + return 1; +} + +static int +ms_wait(Process *c_p, Eterm etimeout, int busy) +{ + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); + ErtsMonotonicTime time, timeout_time; + Sint64 ms; + + if (!term_to_Sint64(etimeout, &ms)) + return 0; + + time = erts_get_monotonic_time(esdp); + + if (ms < 0) + timeout_time = time; + else + timeout_time = time + ERTS_MSEC_TO_MONOTONIC(ms); + + while (time < timeout_time) { + if (busy) + erts_thr_yield(); + else { + ErtsMonotonicTime timeout = timeout_time - time; + +#ifdef __WIN32__ + Sleep((DWORD) ERTS_MONOTONIC_TO_MSEC(timeout)); +#else + { + ErtsMonotonicTime to = ERTS_MONOTONIC_TO_USEC(timeout); + struct timeval tv; + + tv.tv_sec = (long) to / (1000*1000); + tv.tv_usec = (long) to % (1000*1000); + + select(0, NULL, NULL, NULL, &tv); + } +#endif + } + + time = erts_get_monotonic_time(esdp); + } + return 1; +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 613bd5c14e..85d92321b8 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -38,6 +38,7 @@ #include "beam_bp.h" #include "beam_catches.h" #include "erl_thr_progress.h" +#include "erl_nfunc_sched.h" #ifdef HIPE #include "hipe_mode_switch.h" #include "hipe_bif1.h" @@ -214,6 +215,7 @@ BeamInstr beam_continue_exit[1]; BeamInstr* em_call_error_handler; BeamInstr* em_apply_bif; BeamInstr* em_call_nif; +BeamInstr* em_call_bif_e; /* NOTE These should be the only variables containing trace instructions. @@ -633,21 +635,34 @@ void** beam_ops; y[4] = xt4; \ } while (0) +#define DispatchReturn \ +do { \ + if (FCALLS > 0 || FCALLS > neg_o_reds) { \ + FCALLS--; \ + Goto(*I); \ + } \ + else { \ + c_p->current = NULL; \ + c_p->arity = 1; \ + goto context_switch3; \ + } \ +} while (0) + #define MoveReturn(Src) \ x(0) = (Src); \ I = c_p->cp; \ ASSERT(VALID_INSTR(*c_p->cp)); \ c_p->cp = 0; \ CHECK_TERM(r(0)); \ - Goto(*I) + DispatchReturn #define DeallocateReturn(Deallocate) \ do { \ int words_to_pop = (Deallocate); \ - SET_I((BeamInstr *) cp_val(*E)); \ + SET_I((BeamInstr *) cp_val(*E)); \ E = ADD_BYTE_OFFSET(E, words_to_pop); \ CHECK_TERM(r(0)); \ - Goto(*I); \ + DispatchReturn; \ } while (0) #define MoveDeallocateReturn(Src, Deallocate) \ @@ -1042,14 +1057,17 @@ void** beam_ops; * The following functions are called directly by process_main(). * Don't inline them. */ -static BifFunction translate_gc_bif(void* gcf) NOINLINE; +static ErtsCodeMFA *ubif2mfa(void* uf) NOINLINE; +static ErtsCodeMFA *gcbif2mfa(void* gcf) NOINLINE; static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, - Eterm* reg, BifFunction bf) NOINLINE; + Eterm* reg, ErtsCodeMFA* bif_mfa) NOINLINE; static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) NOINLINE; -static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE; +static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity, + BeamInstr *I, Uint offs) NOINLINE; static BeamInstr* apply(Process* p, Eterm module, Eterm function, - Eterm args, Eterm* reg) NOINLINE; + Eterm args, Eterm* reg, + BeamInstr *I, Uint offs) NOINLINE; static BeamInstr* call_fun(Process* p, int arity, Eterm* reg, Eterm args) NOINLINE; static BeamInstr* apply_fun(Process* p, Eterm fun, @@ -1071,7 +1089,7 @@ static BeamInstr* next_catch(Process* c_p, Eterm *reg); static void terminate_proc(Process* c_p, Eterm Value); static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc); static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, - BifFunction bf, Eterm args); + ErtsCodeMFA *bif_mfa, Eterm args); static struct StackTrace * get_trace_from_exc(Eterm exc); static Eterm make_arglist(Process* c_p, Eterm* reg, int a); @@ -1679,7 +1697,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) c_p->cp = 0; CHECK_TERM(r(0)); HEAP_SPACE_VERIFIED(0); - Goto(*I); + DispatchReturn; } /* @@ -2566,7 +2584,7 @@ do { \ OpCase(bif1_fbsd): { - Eterm (*bf)(Process*, Eterm*); + ErtsBifFunc bf; Eterm tmp_reg[1]; Eterm result; @@ -2577,7 +2595,7 @@ do { \ PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg); + result = (*bf)(c_p, tmp_reg, I); ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -2598,19 +2616,19 @@ do { \ OpCase(bif1_body_bsd): { - Eterm (*bf)(Process*, Eterm*); + ErtsBifFunc bf; Eterm tmp_reg[1]; Eterm result; GetArg1(1, tmp_reg[0]); - bf = (BifFunction) Arg(0); + bf = (ErtsBifFunc) Arg(0); ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg); + result = (*bf)(c_p, tmp_reg, I); ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -2623,7 +2641,7 @@ do { \ } reg[0] = tmp_reg[0]; SWAPOUT; - I = handle_error(c_p, I, reg, bf); + I = handle_error(c_p, I, reg, ubif2mfa((void *) bf)); goto post_error_handling; } @@ -2659,7 +2677,7 @@ do { \ Goto(*I); } x(0) = x(live); - I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); goto post_error_handling; } @@ -2704,7 +2722,7 @@ do { \ live--; x(0) = x(live); x(1) = x(live+1); - I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); goto post_error_handling; } @@ -2750,7 +2768,7 @@ do { \ x(0) = x(live); x(1) = x(live+1); x(2) = x(live+2); - I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf)); goto post_error_handling; } @@ -2760,17 +2778,17 @@ do { \ OpCase(i_bif2_fbssd): { Eterm tmp_reg[2]; - Eterm (*bf)(Process*, Eterm*); + ErtsBifFunc bf; Eterm result; GetArg2(2, tmp_reg[0], tmp_reg[1]); - bf = (BifFunction) Arg(1); + bf = (ErtsBifFunc) Arg(1); ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS; PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg); + result = (*bf)(c_p, tmp_reg, I); ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -2791,15 +2809,15 @@ do { \ OpCase(i_bif2_body_bssd): { Eterm tmp_reg[2]; - Eterm (*bf)(Process*, Eterm*); + ErtsBifFunc bf; Eterm result; GetArg2(1, tmp_reg[0], tmp_reg[1]); - bf = (BifFunction) Arg(0); + bf = (ErtsBifFunc) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); ERTS_CHK_MBUF_SZ(c_p); - result = (*bf)(c_p, tmp_reg); + result = (*bf)(c_p, tmp_reg, I); ERTS_CHK_MBUF_SZ(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -2812,7 +2830,7 @@ do { \ reg[0] = tmp_reg[0]; reg[1] = tmp_reg[1]; SWAPOUT; - I = handle_error(c_p, I, reg, bf); + I = handle_error(c_p, I, reg, ubif2mfa((void *) bf)); goto post_error_handling; } @@ -2822,7 +2840,7 @@ do { \ */ OpCase(call_bif_e): { - Eterm (*bf)(Process*, Eterm*, BeamInstr*); + ErtsBifFunc bf; Eterm result; BeamInstr *next; ErlHeapFragment *live_hf_end; @@ -2889,7 +2907,7 @@ do { \ * Error handling. SWAPOUT is not needed because it was done above. */ ASSERT(c_p->stop == E); - I = handle_error(c_p, I, reg, bf); + I = handle_error(c_p, I, reg, &export->info.mfa); goto post_error_handling; } @@ -3186,21 +3204,21 @@ do { \ OpCase(i_apply): { BeamInstr *next; HEAVY_SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg); + next = apply(c_p, r(0), x(1), x(2), reg, NULL, 0); HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, I+1); SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } OpCase(i_apply_last_P): { BeamInstr *next; HEAVY_SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg); + next = apply(c_p, r(0), x(1), x(2), reg, I, Arg(0)); HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, (BeamInstr *) E[0]); @@ -3208,20 +3226,20 @@ do { \ SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } OpCase(i_apply_only): { BeamInstr *next; HEAVY_SWAPOUT; - next = apply(c_p, r(0), x(1), x(2), reg); + next = apply(c_p, r(0), x(1), x(2), reg, I, 0); HEAVY_SWAPIN; if (next != NULL) { SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } @@ -3229,14 +3247,14 @@ do { \ BeamInstr *next; HEAVY_SWAPOUT; - next = fixed_apply(c_p, reg, Arg(0)); + next = fixed_apply(c_p, reg, Arg(0), NULL, 0); HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, I+2); SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } @@ -3244,7 +3262,7 @@ do { \ BeamInstr *next; HEAVY_SWAPOUT; - next = fixed_apply(c_p, reg, Arg(0)); + next = fixed_apply(c_p, reg, Arg(0), I, Arg(1)); HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, (BeamInstr *) E[0]); @@ -3252,7 +3270,7 @@ do { \ SET_I(next); Dispatch(); } - I = handle_error(c_p, I, reg, apply_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa); goto post_error_handling; } @@ -3547,12 +3565,6 @@ do { \ ErlHeapFragment *live_hf_end; ErtsCodeMFA *codemfa; - if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { - /* If we have run out of reductions, we do a context - switch before calling the nif */ - goto context_switch; - } - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); codemfa = erts_code_to_codemfa(I); @@ -3561,8 +3573,8 @@ do { \ DTRACE_NIF_ENTRY(c_p, codemfa); - SWAPOUT; - c_p->fcalls = FCALLS - 1; + HEAVY_SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); bif_nif_arity = codemfa->arity; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); @@ -3635,7 +3647,7 @@ do { \ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); { - Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf; + ErtsBifFunc bf = vbf; ASSERT(!ERTS_PROC_IS_EXITING(c_p)); live_hf_end = c_p->mbuf; ERTS_CHK_MBUF_SZ(c_p); @@ -3679,7 +3691,7 @@ do { \ } Dispatch(); } - I = handle_error(c_p, c_p->cp, reg, vbf); + I = handle_error(c_p, c_p->cp, reg, c_p->current); goto post_error_handling; } } @@ -5034,7 +5046,7 @@ do { \ goto do_schedule; } else { HEAVY_SWAPIN; - I = handle_error(c_p, I, reg, hibernate_3); + I = handle_error(c_p, I, reg, &bif_export[BIF_hibernate_3]->info.mfa); goto post_error_handling; } } @@ -5128,6 +5140,7 @@ do { \ em_call_error_handler = OpCode(call_error_handler); em_apply_bif = OpCode(apply_bif); em_call_nif = OpCode(call_nif); + em_call_bif_e = OpCode(call_bif_e); beam_apply[0] = (BeamInstr) OpCode(i_apply); beam_apply[1] = (BeamInstr) OpCode(normal_exit); @@ -5314,10 +5327,25 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) ASSERT(!(c_p->flags & F_HIPE_MODE)); ERTS_MSACC_UPDATE_CACHE_X(); - reg = esdp->x_reg_array; - { + /* + * Set fcalls even though we ignore it, so we don't + * confuse code accessing it... + */ + if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) + c_p->fcalls = 0; + else + c_p->fcalls = CONTEXT_REDS; + + if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_DIRTY_RUNNING_SYS) { + erts_execute_dirty_system_task(c_p); + goto do_dirty_schedule; + } + else { + ErtsCodeMFA *codemfa; Eterm* argp; - int i; + int i, exiting; + + reg = esdp->x_reg_array; argp = c_p->arg_reg; for (i = c_p->arity - 1; i >= 0; i--) { @@ -5333,17 +5361,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) I = c_p->i; - ASSERT(em_call_nif == (BeamInstr *) *I); - - /* - * Set fcalls even though we ignore it, so we don't - * confuse code accessing it... - */ - if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) - c_p->fcalls = 0; - else - c_p->fcalls = CONTEXT_REDS; - SWAPIN; #ifdef USE_VM_PROBES @@ -5367,96 +5384,81 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) DTRACE2(process_scheduled, process_buf, fun_buf); } #endif - } - - { -#ifdef DEBUG - Eterm result; -#endif - Eterm arity; - { - /* - * call_nif is always first instruction in function: - * - * I[-3]: Module - * I[-2]: Function - * I[-1]: Arity - * I[0]: &&call_nif - * I[1]: Function pointer to NIF function - * I[2]: Pointer to erl_module_nif - * I[3]: Function pointer to dirty NIF - * - * This layout is determined by the NifExport struct - */ - BifFunction vbf; - ErtsCodeMFA *codemfa; - - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); + /* + * call_nif is always first instruction in function: + * + * I[-3]: Module + * I[-2]: Function + * I[-1]: Arity + * I[0]: &&call_nif + * I[1]: Function pointer to NIF function + * I[2]: Pointer to erl_module_nif + * I[3]: Function pointer to dirty NIF + * + * This layout is determined by the NifExport struct + */ - codemfa = erts_code_to_codemfa(I); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); - DTRACE_NIF_ENTRY(c_p, codemfa); - /* current and vbf set to please handle_error */ - c_p->current = codemfa; - SWAPOUT; - PROCESS_MAIN_CHK_LOCKS(c_p); - arity = codemfa->arity; - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + codemfa = erts_code_to_codemfa(I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - { - typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); - NifF* fp = vbf = (NifF*) I[1]; - struct enif_environment_t env; - ASSERT(!c_p->scheduler_data); - - erts_pre_dirty_nif(esdp, &env, c_p, - (struct erl_module_nif*)I[2]); - -#ifdef DEBUG - result = -#else - (void) -#endif - (*fp)(&env, arity, reg); + DTRACE_NIF_ENTRY(c_p, codemfa); + c_p->current = codemfa; + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - erts_post_dirty_nif(&env); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + if (em_apply_bif == (BeamInstr *) *I) { + exiting = erts_call_dirty_bif(esdp, c_p, I, reg); + } + else { + ASSERT(em_call_nif == (BeamInstr *) *I); + exiting = erts_call_dirty_nif(esdp, c_p, I, reg); + } - ASSERT(!is_value(result)); - ASSERT(c_p->freason == TRAP); - ASSERT(!(c_p->flags & F_HIBERNATE_SCHED)); + ASSERT(!(c_p->flags & F_HIBERNATE_SCHED)); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - if (env.exiting) - goto do_dirty_schedule; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - } + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + if (exiting) + goto do_dirty_schedule; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - DTRACE_NIF_RETURN(c_p, codemfa); - ERTS_HOLE_CHECK(c_p); - SWAPIN; - I = c_p->i; - goto context_switch; - } + DTRACE_NIF_RETURN(c_p, codemfa); + ERTS_HOLE_CHECK(c_p); + SWAPIN; + I = c_p->i; + goto context_switch; } #endif /* ERTS_DIRTY_SCHEDULERS */ } -static BifFunction -translate_gc_bif(void* gcf) +static ErtsCodeMFA * +gcbif2mfa(void* gcf) { - const ErtsGcBif* p; - - for (p = erts_gc_bifs; p->bif != 0; p++) { - if (p->gc_bif == gcf) { - return p->bif; - } + int i; + for (i = 0; erts_gc_bifs[i].bif; i++) { + if (erts_gc_bifs[i].gc_bif == gcf) + return &bif_export[erts_gc_bifs[i].exp_ix]->info.mfa; } erts_exit(ERTS_ERROR_EXIT, "bad gc bif"); + return NULL; +} + +static ErtsCodeMFA * +ubif2mfa(void* uf) +{ + int i; + for (i = 0; erts_u_bifs[i].bif; i++) { + if (erts_u_bifs[i].bif == uf) + return &bif_export[erts_u_bifs[i].exp_ix]->info.mfa; + } + erts_exit(ERTS_ERROR_EXIT, "bad u bif"); + return NULL; } /* @@ -5515,15 +5517,27 @@ Eterm error_atom[NUMBER_EXIT_CODES] = { */ static BeamInstr* -handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) +handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa) { Eterm* hp; Eterm Value = c_p->fvalue; Eterm Args = am_true; - c_p->i = pc; /* In case we call erts_exit(). */ ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */ + if (c_p->freason & EXF_RESTORE_NIF) + erts_nif_export_restore_error(c_p, &pc, reg, &bif_mfa); + +#ifdef DEBUG + if (bif_mfa) { + /* Verify that bif_mfa does not point into our nif export */ + NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + ASSERT(!nep || !ErtsInArea(bif_mfa, (char *)nep, sizeof(NifExport))); + } +#endif + + c_p->i = pc; /* In case we call erts_exit(). */ + /* * Check if we have an arglist for the top level call. If so, this * is encoded in Value, so we have to dig out the real Value as well @@ -5546,7 +5560,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) * more modular. */ if (c_p->freason & EXF_SAVETRACE) { - save_stacktrace(c_p, pc, reg, bf, Args); + save_stacktrace(c_p, pc, reg, bif_mfa, Args); } /* @@ -5821,11 +5835,12 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) { */ static void -save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, - Eterm args) { +save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, + ErtsCodeMFA *bif_mfa, Eterm args) { struct StackTrace* s; int sz; int depth = erts_backtrace_depth; /* max depth (never negative) */ + if (depth > 0) { /* There will always be a current function */ depth --; @@ -5841,33 +5856,30 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, s->depth = 0; /* - * If the failure was in a BIF other than 'error', 'exit' or - * 'throw', find the bif-table index and save the argument + * If the failure was in a BIF other than 'error/1', 'error/2', + * 'exit/1' or 'throw/1', save BIF-MFA and save the argument * registers by consing up an arglist. */ - if (bf != NULL && bf != error_1 && bf != error_2 && - bf != exit_1 && bf != throw_1) { - int i; - int a = 0; - for (i = 0; i < BIF_SIZE; i++) { - if (bf == bif_table[i].f || bf == bif_table[i].traced) { - Export *ep = bif_export[i]; - s->current = &ep->info.mfa; - a = bif_table[i].arity; + if (bif_mfa) { + if (bif_mfa->module == am_erlang) { + switch (bif_mfa->function) { + case am_error: + if (bif_mfa->arity == 1 || bif_mfa->arity == 2) + goto non_bif_stacktrace; + break; + case am_exit: + if (bif_mfa->arity == 1) + goto non_bif_stacktrace; + break; + case am_throw: + if (bif_mfa->arity == 1) + goto non_bif_stacktrace; + break; + default: break; } } - if (i >= BIF_SIZE) { - /* - * The Bif does not really exist (no BIF entry). It is a - * TRAP and traps are called through apply_bif, which also - * sets c_p->current (luckily). - * OR it is a NIF called by call_nif where current is also set. - */ - ASSERT(c_p->current); - s->current = c_p->current; - a = s->current->arity; - } + s->current = bif_mfa; /* Save first stack entry */ ASSERT(pc); if (depth > 0) { @@ -5880,8 +5892,11 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, depth--; } s->pc = NULL; - args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */ + args = make_arglist(c_p, reg, bif_mfa->arity); /* Overwrite CAR(c_p->ftrace) */ } else { + + non_bif_stacktrace: + s->current = c_p->current; /* * For a function_clause error, the arguments are in the beam @@ -5942,12 +5957,30 @@ erts_save_stacktrace(Process* p, struct StackTrace* s, int depth) p->cp) { /* Cannot follow cp here - code may be unloaded */ BeamInstr *cpp = p->cp; + int trace_cp; if (cpp == beam_exception_trace || cpp == beam_return_trace) { /* Skip return_trace parameters */ ptr += 2; + trace_cp = 1; } else if (cpp == beam_return_to_trace) { /* Skip return_to_trace parameters */ ptr += 1; + trace_cp = 1; + } + else { + trace_cp = 0; + } + if (trace_cp && s->pc == cpp) { + /* + * If process 'cp' points to a return/exception trace + * instruction and 'cp' has been saved as 'pc' in + * stacktrace, we need to update 'pc' in stacktrace + * with the actual 'cp' located on the top of the + * stack; otherwise, we will lose the top stackframe + * when building the stack trace. + */ + ASSERT(is_CP(p->stop[0])); + s->pc = cp_val(p->stop[0]); } } while (ptr < STACK_START(p) && depth > 0) { @@ -6067,12 +6100,15 @@ build_stacktrace(Process* c_p, Eterm exc) { erts_set_current_function(&fi, s->current); } + depth = s->depth; /* - * If fi.current is still NULL, default to the initial function + * If fi.current is still NULL, and we have no + * stack at all, default to the initial function * (e.g. spawn_link(erlang, abs, [1])). */ if (fi.mfa == NULL) { - erts_set_current_function(&fi, &c_p->u.initial); + if (depth <= 0) + erts_set_current_function(&fi, &c_p->u.initial); args = am_true; /* Just in case */ } else { args = get_args_from_exc(exc); @@ -6082,10 +6118,9 @@ build_stacktrace(Process* c_p, Eterm exc) { * Look up all saved continuation pointers and calculate * needed heap space. */ - depth = s->depth; stk = stkp = (FunctionInfo *) erts_alloc(ERTS_ALC_T_TMP, depth*sizeof(FunctionInfo)); - heap_size = fi.needed + 2; + heap_size = fi.mfa ? fi.needed + 2 : 0; for (i = 0; i < depth; i++) { erts_lookup_function_info(stkp, s->trace[i], 1); if (stkp->mfa) { @@ -6104,8 +6139,10 @@ build_stacktrace(Process* c_p, Eterm exc) { res = CONS(hp, mfa, res); hp += 2; } - hp = erts_build_mfa_item(&fi, hp, args, &mfa); - res = CONS(hp, mfa, res); + if (fi.mfa) { + hp = erts_build_mfa_item(&fi, hp, args, &mfa); + res = CONS(hp, mfa, res); + } erts_free(ERTS_ALC_T_TMP, (void *) stk); return res; @@ -6203,8 +6240,107 @@ apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, return ep; } +static ERTS_INLINE void +apply_bif_error_adjustment(Process *p, Export *ep, + Eterm *reg, Uint arity, + BeamInstr *I, Uint stack_offset) +{ + /* + * I is only set when the apply is a tail call, i.e., + * from the instructions i_apply_only, i_apply_last_P, + * and apply_last_IP. + */ + if (I + && ep->beam[0] == (BeamInstr) em_apply_bif + && (ep == bif_export[BIF_error_1] + || ep == bif_export[BIF_error_2] + || ep == bif_export[BIF_exit_1] + || ep == bif_export[BIF_throw_1])) { + /* + * We are about to tail apply one of the BIFs + * erlang:error/1, erlang:error/2, erlang:exit/1, + * or erlang:throw/1. Error handling of these BIFs is + * special! + * + * We need 'p->cp' to point into the calling + * function when handling the error after the BIF has + * been applied. This in order to get the topmost + * stackframe correct. Without the following adjustment, + * 'p->cp' will point into the function that called + * current function when handling the error. We add a + * dummy stackframe in order to achive this. + * + * Note that these BIFs unconditionally will cause + * an exception to be raised. That is, our modifications + * of 'p->cp' as well as the stack will be corrected by + * the error handling code. + * + * If we find an exception/return-to trace continuation + * pointer as the topmost continuation pointer, we do not + * need to do anything since the information already will + * be available for generation of the stacktrace. + */ + int apply_only = stack_offset == 0; + BeamInstr *cpp; + + if (apply_only) { + ASSERT(p->cp != NULL); + cpp = p->cp; + } + else { + ASSERT(is_CP(p->stop[0])); + cpp = cp_val(p->stop[0]); + } + + if (cpp != beam_exception_trace + && cpp != beam_return_trace + && cpp != beam_return_to_trace) { + Uint need = stack_offset /* bytes */ / sizeof(Eterm); + if (need == 0) + need = 1; /* i_apply_only */ + if (p->stop - p->htop < need) + erts_garbage_collect(p, (int) need, reg, arity+1); + p->stop -= need; + + if (apply_only) { + /* + * Called from the i_apply_only instruction. + * + * 'p->cp' contains continuation pointer pointing + * into the function that called current function. + * We push that continuation pointer onto the stack, + * and set 'p->cp' to point into current function. + */ + + p->stop[0] = make_cp(p->cp); + p->cp = I; + } + else { + /* + * Called from an i_apply_last_p, or apply_last_IP, + * instruction. + * + * Calling instruction will after we return read + * a continuation pointer from the stack and write + * it to 'p->cp', and then remove the topmost + * stackframe of size 'stack_offset'. + * + * We have sized the dummy-stackframe so that it + * will be removed by the instruction we currently + * are executing, and leave the stackframe that + * normally would have been removed intact. + * + */ + p->stop[0] = make_cp(I); + } + } + } +} + static BeamInstr* -apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) +apply( +Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg, +BeamInstr *I, Uint stack_offset) { int arity; Export* ep; @@ -6229,21 +6365,54 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) return 0; } - /* The module argument may be either an atom or an abstract module - * (currently implemented using tuples, but this might change). - */ - this = THE_NON_VALUE; - if (is_not_atom(module)) { - Eterm* tp; + while (1) { + Eterm m, f, a; + /* The module argument may be either an atom or an abstract module + * (currently implemented using tuples, but this might change). + */ + this = THE_NON_VALUE; + if (is_not_atom(module)) { + Eterm* tp; + + if (is_not_tuple(module)) goto error; + tp = tuple_val(module); + if (arityval(tp[0]) < 1) goto error; + this = module; + module = tp[1]; + if (is_not_atom(module)) goto error; + } - if (is_not_tuple(module)) goto error; - tp = tuple_val(module); - if (arityval(tp[0]) < 1) goto error; - this = module; - module = tp[1]; - if (is_not_atom(module)) goto error; + if (module != am_erlang || function != am_apply) + break; + + /* Adjust for multiple apply of apply/3... */ + + a = args; + if (is_list(a)) { + Eterm *consp = list_val(a); + m = CAR(consp); + a = CDR(consp); + if (is_list(a)) { + consp = list_val(a); + f = CAR(consp); + a = CDR(consp); + if (is_list(a)) { + consp = list_val(a); + a = CAR(consp); + if (is_nil(CDR(consp))) { + /* erlang:apply/3 */ + module = m; + function = f; + args = a; + if (is_not_atom(f)) + goto error; + continue; + } + } + } + } + break; /* != erlang:apply/3 */ } - /* * Walk down the 3rd parameter of apply (the argument list) and copy * the parameters to the x registers (reg[]). If the module argument @@ -6281,12 +6450,14 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg) } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { save_calls(p, ep); } + apply_bif_error_adjustment(p, ep, reg, arity, I, stack_offset); DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep); return ep->addressv[erts_active_code_ix()]; } static BeamInstr* -fixed_apply(Process* p, Eterm* reg, Uint arity) +fixed_apply(Process* p, Eterm* reg, Uint arity, + BeamInstr *I, Uint stack_offset) { Export* ep; Eterm module; @@ -6316,6 +6487,10 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) if (is_not_atom(module)) goto error; ++arity; } + + /* Handle apply of apply/3... */ + if (module == am_erlang && function == am_apply && arity == 3) + return apply(p, reg[0], reg[1], reg[2], reg, I, stack_offset); /* * Get the index into the export table, or failing that the export @@ -6330,6 +6505,7 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) { save_calls(p, ep); } + apply_bif_error_adjustment(p, ep, reg, arity, I, stack_offset); DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep); return ep->addressv[erts_active_code_ix()]; } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 3cd395c2c1..309465bcd3 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -800,14 +800,14 @@ erts_finish_loading(Binary* magic, Process* c_p, } else { ErtsCodeIndex code_ix = erts_staging_code_ix(); Eterm module = stp->module; - int i; + int i, num_exps; /* * There is an -on_load() function. We will keep the current * code, but we must turn off any tracing. */ - - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export *ep = export_list(i, code_ix); if (ep == NULL || ep->info.mfa.module != module) { continue; @@ -2546,15 +2546,10 @@ load_code(LoaderState* stp) if (stp->may_load_nif) { const int finfo_ix = ci - FUNC_INFO_SZ; -#ifdef ERTS_DIRTY_SCHEDULERS - enum { MIN_FUNC_SZ = 4 }; -#else - enum { MIN_FUNC_SZ = 3 }; -#endif - if (finfo_ix - last_func_start < MIN_FUNC_SZ && last_func_start) { + if (finfo_ix - last_func_start < BEAM_NIF_MIN_FUNC_SZ && last_func_start) { /* Must make room for call_nif op */ - int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start); - ASSERT(pad > 0 && pad < MIN_FUNC_SZ); + int pad = BEAM_NIF_MIN_FUNC_SZ - (finfo_ix - last_func_start); + ASSERT(pad > 0 && pad < BEAM_NIF_MIN_FUNC_SZ); CodeNeed(pad); sys_memmove(&code[finfo_ix+pad], &code[finfo_ix], FUNC_INFO_SZ*sizeof(BeamInstr)); @@ -5729,12 +5724,13 @@ exported_from_module(Process* p, /* Process whose heap to use. */ ErtsCodeIndex code_ix, Eterm mod) /* Tagged atom for module. */ { - int i; + int i, num_exps; Eterm* hp = NULL; Eterm* hend = NULL; Eterm result = NIL; - for (i = 0; i < export_list_size(code_ix); i++) { + num_exps = export_list_size(code_ix); + for (i = 0; i < num_exps; i++) { Export* ep = export_list(i,code_ix); if (ep->info.mfa.module == mod) { diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index d420a4346c..659b9c303f 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -111,6 +111,12 @@ typedef struct beam_code_header { }BeamCodeHeader; +#ifdef ERTS_DIRTY_SCHEDULERS +# define BEAM_NIF_MIN_FUNC_SZ 4 +#else +# define BEAM_NIF_MIN_FUNC_SZ 3 +#endif + void erts_release_literal_area(struct ErtsLiteralArea_* literal_area); int erts_is_module_native(BeamCodeHeader* code); void erts_beam_bif_load_init(void); diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index d886c2985e..65c370c55b 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -3865,13 +3865,6 @@ BIF_RETTYPE now_0(BIF_ALIST_0) /**********************************************************************/ -BIF_RETTYPE garbage_collect_0(BIF_ALIST_0) -{ - FLAGS(BIF_P) |= F_NEED_FULLSWEEP; - erts_garbage_collect(BIF_P, 0, NULL, 0); - return am_true; -} - /* * Pass atom 'minor' for relaxed generational GC run. This is only * recommendation, major run may still be chosen by VM. @@ -4324,7 +4317,7 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2) erts_smp_proc_unlock(new_member, ERTS_PROC_LOCK_STATUS); if (new_member == BIF_P || !(erts_smp_atomic32_read_nob(&new_member->state) - & (ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS))) { + & ERTS_PSFLG_DIRTY_RUNNING)) { new_member->group_leader = STORE_NC_IN_PROC(new_member, BIF_ARG_1); } @@ -4608,7 +4601,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) erts_aint32_t new = BIF_ARG_2 == am_true ? 1 : 0; erts_aint32_t old = erts_smp_atomic32_xchg_nob(&sched_wall_time, new); - Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new); + Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new, 0, 0); ASSERT(is_value(ref)); BIF_TRAP2(await_sched_wall_time_mod_trap, BIF_P, @@ -4958,7 +4951,7 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, Export bif_return_trap_export; void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, - Eterm (*bif)(BIF_ALIST_0)) + Eterm (*bif)(BIF_ALIST)) { int i; sys_memset((void *) ep, 0, sizeof(Export)); @@ -5019,6 +5012,314 @@ void erts_init_bif(void) erts_smp_atomic32_init_nob(&msacc, ERTS_MSACC_IS_ENABLED()); } +/* + * Scheduling of BIFs via NifExport... + */ +#define ERTS_WANT_NFUNC_SCHED_INTERNALS__ +#include "erl_nfunc_sched.h" + +#define ERTS_SCHED_BIF_TRAP_MARKER ((void *) (UWord) 1) + +static ERTS_INLINE void +schedule(Process *c_p, Process *dirty_shadow_proc, + ErtsCodeMFA *mfa, BeamInstr *pc, + ErtsBifFunc dfunc, void *ifunc, + Eterm module, Eterm function, + int argc, Eterm *argv) +{ + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); + (void) erts_nif_export_schedule(c_p, dirty_shadow_proc, + mfa, pc, (BeamInstr) em_apply_bif, + dfunc, ifunc, + module, function, + argc, argv); +} + +#ifdef ERTS_DIRTY_SCHEDULERS + +static BIF_RETTYPE dirty_bif_result(BIF_ALIST_1) +{ + NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P); + erts_nif_export_restore(BIF_P, nep, BIF_ARG_1); + BIF_RET(BIF_ARG_1); +} + +static BIF_RETTYPE dirty_bif_trap(BIF_ALIST) +{ + NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P); + + /* + * Arity and argument registers already set + * correct by call to dirty_bif_trap()... + */ + + ASSERT(BIF_P->arity == nep->exp.info.mfa.arity); + + erts_nif_export_restore(BIF_P, nep, THE_NON_VALUE); + + BIF_P->i = (BeamInstr *) nep->func; + BIF_P->freason = TRAP; + return THE_NON_VALUE; +} + +static BIF_RETTYPE dirty_bif_exception(BIF_ALIST_2) +{ + Eterm freason; + + ASSERT(is_small(BIF_ARG_1)); + + freason = signed_val(BIF_ARG_1); + + /* Restore orig info for error and clear nif export in handle_error() */ + freason |= EXF_RESTORE_NIF; + + BIF_P->fvalue = BIF_ARG_2; + + BIF_ERROR(BIF_P, freason); +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ + +extern BeamInstr* em_call_bif_e; +static BIF_RETTYPE call_bif(Process *c_p, Eterm *reg, BeamInstr *I); + +BIF_RETTYPE +erts_schedule_bif(Process *proc, + Eterm *argv, + BeamInstr *i, + ErtsBifFunc bif, + ErtsSchedType sched_type, + Eterm mod, + Eterm func, + int argc) +{ + Process *c_p, *dirty_shadow_proc; + ErtsCodeMFA *mfa; + +#ifdef ERTS_DIRTY_SCHEDULERS + if (proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + dirty_shadow_proc = proc; + c_p = proc->next; + ASSERT(c_p->common.id == dirty_shadow_proc->common.id); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + } + else +#endif + { + dirty_shadow_proc = NULL; + c_p = proc; + } + + if (!ERTS_PROC_IS_EXITING(c_p)) { + Export *exp; + BifFunction dbif, ibif; + BeamInstr *pc; + + /* + * dbif - direct bif + * ibif - indirect bif + */ + +#ifdef ERTS_DIRTY_SCHEDULERS + erts_aint32_t set, mask; + mask = (ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC); + switch (sched_type) { + case ERTS_SCHED_DIRTY_CPU: + set = ERTS_PSFLG_DIRTY_CPU_PROC; + dbif = bif; + ibif = NULL; + break; + case ERTS_SCHED_DIRTY_IO: + set = ERTS_PSFLG_DIRTY_IO_PROC; + dbif = bif; + ibif = NULL; + break; + case ERTS_SCHED_NORMAL: + default: + set = 0; + dbif = call_bif; + ibif = bif; + break; + } + + (void) erts_smp_atomic32_read_bset_nob(&c_p->state, mask, set); +#else + dbif = call_bif; + ibif = bif; +#endif + + if (i == NULL) { + ERTS_INTERNAL_ERROR("Missing instruction pointer"); + } +#ifdef HIPE + else if (proc->flags & F_HIPE_MODE) { + /* Pointer to bif export in i */ + exp = (Export *) i; + pc = c_p->cp; + mfa = &exp->info.mfa; + } +#endif + else if (em_call_bif_e == (BeamInstr *) *i) { + /* Pointer to bif export in i+1 */ + exp = (Export *) i[1]; + pc = i; + mfa = &exp->info.mfa; + } + else if (em_apply_bif == (BeamInstr *) *i) { + /* Pointer to bif in i+1, and mfa in i-3 */ + pc = c_p->cp; + mfa = erts_code_to_codemfa(i); + } + else { + ERTS_INTERNAL_ERROR("erts_schedule_bif() called " + "from unexpected instruction"); + } + ASSERT(bif); + + if (argc < 0) { /* reschedule original call */ + mod = mfa->module; + func = mfa->function; + argc = (int) mfa->arity; + } + + schedule(c_p, dirty_shadow_proc, mfa, pc, dbif, ibif, + mod, func, argc, argv); + } + + if (dirty_shadow_proc) + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + + return THE_NON_VALUE; +} + +static BIF_RETTYPE +call_bif(Process *c_p, Eterm *reg, BeamInstr *I) +{ + NifExport *nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); + ErtsBifFunc bif = (ErtsBifFunc) nep->func; + BIF_RETTYPE ret; + + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); + + nep->func = ERTS_SCHED_BIF_TRAP_MARKER; + + ASSERT(bif); + + ret = (*bif)(c_p, reg, I); + + if (is_value(ret)) + erts_nif_export_restore(c_p, nep, ret); + else if (c_p->freason != TRAP) + c_p->freason |= EXF_RESTORE_NIF; /* restore in handle_error() */ + else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) { + /* BIF did an ordinary trap... */ + erts_nif_export_restore(c_p, nep, ret); + } + /* else: + * BIF rescheduled itself using erts_schedule_bif(). + */ + + return ret; +} + +#ifdef ERTS_DIRTY_SCHEDULERS + +int +erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg) +{ + BIF_RETTYPE result; + int exiting; + Process *dirty_shadow_proc; + ErtsBifFunc bf; + NifExport *nep; +#ifdef DEBUG + Eterm *c_p_htop; + erts_aint32_t state; + + ASSERT(!c_p->scheduler_data); + state = erts_smp_atomic32_read_nob(&c_p->state); + ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) + && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); + ASSERT(esdp); + +#endif + + nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); + ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p)); + + nep->func = ERTS_SCHED_BIF_TRAP_MARKER; + + bf = (ErtsBifFunc) I[1]; + + erts_smp_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC)); + + dirty_shadow_proc = erts_make_dirty_shadow_proc(esdp, c_p); + + dirty_shadow_proc->freason = c_p->freason; + dirty_shadow_proc->fvalue = c_p->fvalue; + dirty_shadow_proc->ftrace = c_p->ftrace; + dirty_shadow_proc->cp = c_p->cp; + dirty_shadow_proc->i = c_p->i; + +#ifdef DEBUG + c_p_htop = c_p->htop; +#endif + + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + + result = (*bf)(dirty_shadow_proc, reg, I); + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + + ASSERT(c_p_htop == c_p->htop); + ASSERT(dirty_shadow_proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(dirty_shadow_proc->next == c_p); + + exiting = ERTS_PROC_IS_EXITING(c_p); + + if (!exiting) { + if (is_value(result)) + schedule(c_p, dirty_shadow_proc, NULL, NULL, dirty_bif_result, + NULL, am_erts_internal, am_dirty_bif_result, 1, &result); + else if (dirty_shadow_proc->freason != TRAP) { + Eterm argv[2]; + ASSERT(dirty_shadow_proc->freason <= MAX_SMALL); + argv[0] = make_small(dirty_shadow_proc->freason); + argv[1] = dirty_shadow_proc->fvalue; + schedule(c_p, dirty_shadow_proc, NULL, NULL, + dirty_bif_exception, NULL, am_erts_internal, + am_dirty_bif_exception, 2, argv); + } + else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) { + /* Dirty BIF did an ordinary trap... */ + ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state) + & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC))); + schedule(c_p, dirty_shadow_proc, NULL, NULL, + dirty_bif_trap, (void *) dirty_shadow_proc->i, + am_erts_internal, am_dirty_bif_trap, + dirty_shadow_proc->arity, reg); + } + /* else: + * BIF rescheduled itself using erts_schedule_bif(). + */ + c_p->freason = dirty_shadow_proc->freason; + c_p->fvalue = dirty_shadow_proc->fvalue; + c_p->ftrace = dirty_shadow_proc->ftrace; + c_p->cp = dirty_shadow_proc->cp; + c_p->i = dirty_shadow_proc->i; + c_p->arity = dirty_shadow_proc->arity; + } + + erts_flush_dirty_shadow_proc(dirty_shadow_proc); + + return exiting; +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ + + #ifdef HARDDEBUG /* You'll need this line in bif.tab to be able to use this debug bif diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 0c85e19ef0..01cca90a7a 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -29,17 +29,35 @@ extern Export *erts_convert_time_unit_trap; #define BIF_P A__p -#define BIF_ALIST_0 Process* A__p, Eterm* BIF__ARGS -#define BIF_ALIST_1 Process* A__p, Eterm* BIF__ARGS -#define BIF_ALIST_2 Process* A__p, Eterm* BIF__ARGS -#define BIF_ALIST_3 Process* A__p, Eterm* BIF__ARGS -#define BIF_ALIST_4 Process* A__p, Eterm* BIF__ARGS +#define BIF_ALIST Process* A__p, Eterm* BIF__ARGS, BeamInstr *A__I +#define BIF_CALL_ARGS A__p, BIF__ARGS, A__I + +#define BIF_ALIST_0 BIF_ALIST +#define BIF_ALIST_1 BIF_ALIST +#define BIF_ALIST_2 BIF_ALIST +#define BIF_ALIST_3 BIF_ALIST +#define BIF_ALIST_4 BIF_ALIST #define BIF_ARG_1 (BIF__ARGS[0]) #define BIF_ARG_2 (BIF__ARGS[1]) #define BIF_ARG_3 (BIF__ARGS[2]) #define BIF_ARG_4 (BIF__ARGS[3]) +#define BIF_I A__I + +/* NBIF_* is for bif calls from native code... */ + +#define NBIF_ALIST Process* A__p, Eterm* BIF__ARGS +#define NBIF_CALL_ARGS A__p, BIF__ARGS + +#define NBIF_ALIST_0 NBIF_ALIST +#define NBIF_ALIST_1 NBIF_ALIST +#define NBIF_ALIST_2 NBIF_ALIST +#define NBIF_ALIST_3 NBIF_ALIST +#define NBIF_ALIST_4 NBIF_ALIST + +typedef BIF_RETTYPE (*ErtsBifFunc)(BIF_ALIST); + #define ERTS_IS_PROC_OUT_OF_REDS(p) \ ((p)->fcalls > 0 \ ? 0 \ @@ -480,6 +498,43 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, Eterm args[], int nargs); +#ifdef ERTS_DIRTY_SCHEDULERS +int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, + BeamInstr *I, Eterm *reg); +#endif + +BIF_RETTYPE +erts_schedule_bif(Process *proc, + Eterm *argv, + BeamInstr *i, + ErtsBifFunc dbf, + ErtsSchedType sched_type, + Eterm mod, + Eterm func, + int argc); + +ERTS_GLB_INLINE BIF_RETTYPE +erts_reschedule_bif(Process *proc, + Eterm *argv, + BeamInstr *i, + ErtsBifFunc dbf, + ErtsSchedType sched_type); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE BIF_RETTYPE +erts_reschedule_bif(Process *proc, + Eterm *argv, + BeamInstr *i, + ErtsBifFunc dbf, + ErtsSchedType sched_type) +{ + return erts_schedule_bif(proc, argv, i, dbf, sched_type, + THE_NON_VALUE, THE_NON_VALUE, -1); +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + #ifdef ERL_WANT_HIPE_BIF_WRAPPER__ #ifndef HIPE @@ -510,16 +565,16 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p, #define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY) \ -BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p, \ - Eterm* args); \ -BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p, \ - Eterm* args) \ +BIF_RETTYPE \ +nbif_impl_hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (NBIF_ALIST); \ +BIF_RETTYPE \ +nbif_impl_hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (NBIF_ALIST) \ { \ BIF_RETTYPE res; \ - hipe_reserve_beam_trap_frame(c_p, args, ARITY); \ - res = BIF_NAME ## _ ## ARITY (c_p, args); \ - if (is_value(res) || c_p->freason != TRAP) { \ - hipe_unreserve_beam_trap_frame(c_p); \ + hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, ARITY); \ + res = nbif_impl_ ## BIF_NAME ## _ ## ARITY (NBIF_CALL_ARGS); \ + if (is_value(res) || BIF_P->freason != TRAP) { \ + hipe_unreserve_beam_trap_frame(BIF_P); \ } \ return res; \ } diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 32600f4338..47fdcfa7a4 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -68,7 +68,6 @@ gcbif erlang:float/1 bif erlang:float_to_list/1 bif erlang:float_to_list/2 bif erlang:fun_info/2 -bif erlang:garbage_collect/0 bif erts_internal:garbage_collect/1 bif erlang:get/0 bif erlang:get/1 @@ -420,6 +419,9 @@ bif erts_debug:set_internal_state/2 bif erts_debug:display/1 bif erts_debug:dist_ext_to_term/2 bif erts_debug:instructions/0 +bif erts_debug:dirty_cpu/2 +bif erts_debug:dirty_io/2 +bif erts_debug:dirty/3 # # Monitor testing bif's... diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 3ddf7a53e2..43c22e2e7c 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -44,6 +44,7 @@ #include "erl_hl_timer.h" #include "erl_cpu_topology.h" #include "erl_thr_queue.h" +#include "erl_nfunc_sched.h" #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE) #include "erl_check_io.h" #endif @@ -681,6 +682,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_ABIF_TIMER)] = erts_timer_type_size(ERTS_ALC_T_ABIF_TIMER); #endif + fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_EXP_TRACE)] + = sizeof(NifExportTrace); #ifdef HARD_DEBUG hdbg_init(); @@ -2439,6 +2442,10 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) fi, ERTS_ALC_T_ABIF_TIMER); #endif + add_fix_values(&size.processes, + &size.processes_used, + fi, + ERTS_ALC_T_NIF_EXP_TRACE); } if (want.atom || want.atom_used) { diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 7ea8c98008..1f7e4400e1 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -376,7 +376,8 @@ type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data type LL_TEMP_TERM LONG_LIVED SYSTEM ll_temp_term -type NIF_TRAP_EXPORT STANDARD CODE nif_trap_export_entry +type NIF_TRAP_EXPORT STANDARD PROCESSES nif_trap_export_entry +type NIF_EXP_TRACE FIXED_SIZE PROCESSES nif_export_trace type EXPORT LONG_LIVED CODE export_entry type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh type NLINK_SH FIXED_SIZE PROCESSES nlink_sh diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 61cce224dd..d346f8c8b6 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -6134,18 +6134,18 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef USE_THREADS if (init->ts) { allctr->thread_safe = 1; - + #ifdef ERTS_ENABLE_LOCK_COUNT erts_mtx_init_x_opt(&allctr->mutex, "alcu_allocator", make_small(allctr->alloc_no), - ERTS_LCNT_LT_ALLOC,1); + ERTS_LCNT_LT_ALLOC); #else erts_mtx_init_x(&allctr->mutex, "alcu_allocator", - make_small(allctr->alloc_no),1); + make_small(allctr->alloc_no)); #endif /*ERTS_ENABLE_LOCK_COUNT*/ - + #ifdef DEBUG allctr->debug.saved_tid = 0; #endif diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 0e0842e139..5ee19aead8 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -94,6 +94,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE #if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP) " [ds:%beu:%beu:%beu]" #endif +#if defined(ERTS_DIRTY_SCHEDULERS_TEST) + " [dirty-schedulers-TEST]" +#endif " [async-threads:%d]" #endif #ifdef HIPE @@ -1673,11 +1676,11 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp) Eterm mfa; Eterm res = NIL; - depth = 8; + depth = erts_backtrace_depth; sz = offsetof(struct StackTrace, trace) + sizeof(BeamInstr *)*depth; s = (struct StackTrace *) erts_alloc(ERTS_ALC_T_TMP, sz); s->depth = 0; - if (rp->i) { + if (depth > 0 && rp->i) { s->trace[s->depth++] = rp->i; depth--; } @@ -2677,20 +2680,30 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) erts_schedulers_state(NULL, NULL, &active, NULL, NULL, NULL, NULL, NULL); BIF_RET(make_small(active)); #endif -#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS) } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers", BIF_ARG_1)) { Uint dirty_cpu; +#ifdef ERTS_DIRTY_SCHEDULERS erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, NULL, NULL); +#else + dirty_cpu = 0; +#endif BIF_RET(make_small(dirty_cpu)); } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers_online", BIF_ARG_1)) { Uint dirty_cpu_onln; +#ifdef ERTS_DIRTY_SCHEDULERS erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, NULL, NULL); +#else + dirty_cpu_onln = 0; +#endif BIF_RET(make_small(dirty_cpu_onln)); } else if (ERTS_IS_ATOM_STR("dirty_io_schedulers", BIF_ARG_1)) { Uint dirty_io; +#ifdef ERTS_DIRTY_SCHEDULERS erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, NULL, &dirty_io, NULL); - BIF_RET(make_small(dirty_io)); +#else + dirty_io = 0; #endif + BIF_RET(make_small(dirty_io)); } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) { res = make_small(erts_no_run_queues); BIF_RET(res); @@ -2863,6 +2876,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("atom_limit",BIF_ARG_1)) { BIF_RET(make_small(erts_get_atom_limit())); } + else if (ERTS_IS_ATOM_STR("atom_count",BIF_ARG_1)) { + BIF_RET(make_small(atom_table_size())); + } else if (ERTS_IS_ATOM_STR("tolerant_timeofday",BIF_ARG_1)) { if (erts_has_time_correction() && erts_time_offset_state() == ERTS_TIME_OFFSET_FINAL) { @@ -3369,7 +3385,12 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) Eterm* hp; if (BIF_ARG_1 == am_scheduler_wall_time) { - res = erts_sched_wall_time_request(BIF_P, 0, 0); + res = erts_sched_wall_time_request(BIF_P, 0, 0, 1, 0); + if (is_non_value(res)) + BIF_RET(am_undefined); + BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res); + } else if (BIF_ARG_1 == am_scheduler_wall_time_all) { + res = erts_sched_wall_time_request(BIF_P, 0, 0, 1, 1); if (is_non_value(res)) BIF_RET(am_undefined); BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res); diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index b1c2c427b0..a480754bdb 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1061,9 +1061,16 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) erts_smp_thr_progress_block(); } #endif +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx); +#endif + r = function_is_traced(p, mfa, &ms, &ms_meta, &meta, &count, &call_time); +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx); +#endif #ifdef ERTS_SMP if ( (key == am_call_time) || (key == am_all)) { erts_smp_thr_progress_unblock(); diff --git a/erts/emulator/beam/erl_dirty_bif.tab b/erts/emulator/beam/erl_dirty_bif.tab new file mode 100644 index 0000000000..69421dcfcc --- /dev/null +++ b/erts/emulator/beam/erl_dirty_bif.tab @@ -0,0 +1,82 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2016. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# + +# +# Static declaration of BIFs that should execute on dirty schedulers. +# +# <dirty-bif-decl> ::= <type> <bif> +# <bif> ::= <module> ":" <name> "/" <arity> +# <type> ::= dirty-cpu | dirty-io | dirty-cpu-test | dirty-io-test +# +# When dirty scheduler support is available, a BIF declared with the +# 'dirty-cpu' type will unconditionally execute on a dirty CPU scheduler, +# and a BIF declared with the type 'dirty-io' will unconditionally execute +# on a dirty IO scheduler. When dirty scheduler support is not available +# all BIFs will of course execute on normal schedulers. +# +# When the emulator has been configured with the debug option +# '--enable-dirty-schedulers-test', BIFs with the types 'dirty-cpu-test', +# and 'dirty-io-test' will unconditionally execute on dirty schedulers. +# When this debug option has not been enabled, these BIFs will be executed +# on normal schedulers. +# +# BIFs marked as 'ubif' in ./bif.tab will be ignored, i.e., will always +# execute on normal schedulers. +# + +# --- Dirty BIFs --- + +dirty-cpu erts_debug:dirty_cpu/2 +dirty-io erts_debug:dirty_io/2 + +# --- TEST of Dirty BIF functionality --- +# Functions below will execute on dirty schedulers when emulator has +# been configured for testing dirty schedulers. This is used for test +# and debug purposes only. We really do *not* want to execute these +# on dirty schedulers on a real system. + +dirty-cpu-test erlang:'++'/2 +dirty-cpu-test erlang:append/2 +dirty-cpu-test erlang:'--'/2 +dirty-cpu-test erlang:subtract/2 +dirty-cpu-test erlang:iolist_size/1 +dirty-cpu-test erlang:make_tuple/2 +dirty-cpu-test erlang:make_tuple/3 +dirty-cpu-test erlang:append_element/2 +dirty-cpu-test erlang:insert_element/3 +dirty-cpu-test erlang:delete_element/2 +dirty-cpu-test erlang:atom_to_list/1 +dirty-cpu-test erlang:list_to_atom/1 +dirty-cpu-test erlang:list_to_existing_atom/1 +dirty-cpu-test erlang:integer_to_list/1 +dirty-cpu-test erlang:string_to_integer/1 +dirty-cpu-test erlang:list_to_integer/1 +dirty-cpu-test erlang:list_to_integer/2 +dirty-cpu-test erlang:float_to_list/1 +dirty-cpu-test erlang:float_to_list/2 +dirty-cpu-test erlang:float_to_binary/1 +dirty-cpu-test erlang:float_to_binary/2 +dirty-cpu-test erlang:string_to_float/1 +dirty-cpu-test erlang:list_to_float/1 +dirty-cpu-test erlang:binary_to_float/1 +dirty-cpu-test erlang:tuple_to_list/1 +dirty-cpu-test erlang:list_to_tuple/1 +dirty-cpu-test erlang:display/1 +dirty-cpu-test erlang:display_string/1 diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 92edce5176..0e6aadf568 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -54,6 +54,9 @@ fatal_error(int err, char *func) struct ErlDrvMutex_ { ethr_mutex mtx; +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_t lcnt; +#endif char *name; }; @@ -64,6 +67,9 @@ struct ErlDrvCond_ { struct ErlDrvRWLock_ { ethr_rwmutex rwmtx; +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_t lcnt; +#endif char *name; }; @@ -161,14 +167,17 @@ erl_drv_mutex_create(char *name) opt.posix_compliant = 1; if (ethr_mutex_init_opt(&dmtx->mtx, &opt) != 0) { erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx); - dmtx = NULL; + return NULL; } - else if (!name) - dmtx->name = no_name; - else { + if (name) { dmtx->name = ((char *) dmtx) + sizeof(ErlDrvMutex); sys_strcpy(dmtx->name, name); + } else { + dmtx->name = no_name; } +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_init_lock(&dmtx->lcnt, dmtx->name, ERTS_LCNT_LT_MUTEX); +#endif } return dmtx; #else @@ -180,7 +189,11 @@ void erl_drv_mutex_destroy(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_destroy(&dmtx->mtx) : EINVAL; + int res; +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_destroy_lock(&dmtx->lcnt); +#endif + res = dmtx ? ethr_mutex_destroy(&dmtx->mtx) : EINVAL; if (res != 0) fatal_error(res, "erl_drv_mutex_destroy()"); erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx); @@ -202,9 +215,14 @@ int erl_drv_mutex_trylock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS + int res; if (!dmtx) fatal_error(EINVAL, "erl_drv_mutex_trylock()"); - return ethr_mutex_trylock(&dmtx->mtx); + res = ethr_mutex_trylock(&dmtx->mtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_trylock(&dmtx->lcnt, res); +#endif + return res; #else return 0; #endif @@ -216,8 +234,14 @@ erl_drv_mutex_lock(ErlDrvMutex *dmtx) #ifdef USE_THREADS if (!dmtx) fatal_error(EINVAL, "erl_drv_mutex_lock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock(&dmtx->lcnt); +#endif ethr_mutex_lock(&dmtx->mtx); #endif +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_post(&dmtx->lcnt); +#endif } void @@ -226,6 +250,9 @@ erl_drv_mutex_unlock(ErlDrvMutex *dmtx) #ifdef USE_THREADS if (!dmtx) fatal_error(EINVAL, "erl_drv_mutex_unlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_unlock(&dmtx->lcnt); +#endif ethr_mutex_unlock(&dmtx->mtx); #endif } @@ -306,10 +333,18 @@ erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx) if (!dcnd || !dmtx) { fatal_error(EINVAL, "erl_drv_cond_wait()"); } +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_unlock(&dmtx->lcnt); +#endif while (1) { int res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx); - if (res == 0) + if (res == 0) { +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock(&dmtx->lcnt); + erts_lcnt_lock_post(&dmtx->lcnt); +#endif break; + } } #endif } @@ -324,14 +359,17 @@ erl_drv_rwlock_create(char *name) if (drwlck) { if (ethr_rwmutex_init(&drwlck->rwmtx) != 0) { erts_free(ERTS_ALC_T_DRV_RWLCK, (void *) drwlck); - drwlck = NULL; + return NULL; } - else if (!name) - drwlck->name = no_name; - else { + if (name) { drwlck->name = ((char *) drwlck) + sizeof(ErlDrvRWLock); sys_strcpy(drwlck->name, name); + } else { + drwlck->name = no_name; } +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_init_lock(&drwlck->lcnt, drwlck->name, ERTS_LCNT_LT_RWMUTEX); +#endif } return drwlck; #else @@ -343,7 +381,11 @@ void erl_drv_rwlock_destroy(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_destroy(&drwlck->rwmtx) : EINVAL; + int res; +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_destroy_lock(&drwlck->lcnt); +#endif + res = drwlck ? ethr_rwmutex_destroy(&drwlck->rwmtx) : EINVAL; if (res != 0) fatal_error(res, "erl_drv_rwlock_destroy()"); erts_free(ERTS_ALC_T_DRV_RWLCK, (void *) drwlck); @@ -364,9 +406,14 @@ int erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS + int res; if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()"); - return ethr_rwmutex_tryrlock(&drwlck->rwmtx); + res = ethr_rwmutex_tryrlock(&drwlck->rwmtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LCNT_LO_READ); +#endif + return res; #else return 0; #endif @@ -378,7 +425,13 @@ erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck) #ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_rlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ); +#endif ethr_rwmutex_rlock(&drwlck->rwmtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_post(&drwlck->lcnt); +#endif #endif } @@ -388,6 +441,9 @@ erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck) #ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_runlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ); +#endif ethr_rwmutex_runlock(&drwlck->rwmtx); #endif } @@ -396,9 +452,14 @@ int erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS + int res; if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()"); - return ethr_rwmutex_tryrwlock(&drwlck->rwmtx); + res = ethr_rwmutex_tryrwlock(&drwlck->rwmtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LCNT_LO_READ_WRITE); +#endif + return res; #else return 0; #endif @@ -410,7 +471,13 @@ erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck) #ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_rwlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ_WRITE); +#endif ethr_rwmutex_rwlock(&drwlck->rwmtx); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_lock_post(&drwlck->lcnt); +#endif #endif } @@ -420,6 +487,9 @@ erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck) #ifdef USE_THREADS if (!drwlck) fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()"); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ_WRITE); +#endif ethr_rwmutex_rwunlock(&drwlck->rwmtx); #endif } diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 28b2b02914..6815d76776 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -42,6 +42,7 @@ #include "dtrace-wrapper.h" #include "erl_bif_unique.h" #include "dist.h" +#include "erl_nfunc_sched.h" #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20 @@ -59,6 +60,10 @@ # define ERTS_GC_ASSERT(B) ((void) 1) #endif +#if defined(DEBUG) && 0 +# define HARDDEBUG 1 +#endif + /* * Returns number of elements in an array. */ @@ -110,18 +115,20 @@ typedef struct { static Uint setup_rootset(Process*, Eterm*, int, Rootset*); static void cleanup_rootset(Rootset *rootset); -static void remove_message_buffers(Process* p); static Eterm *full_sweep_heaps(Process *p, int hibernate, Eterm *n_heap, Eterm* n_htop, char *oh, Uint oh_size, Eterm *objv, int nobj); static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj, int fcalls); + int need, Eterm* objv, int nobj, int fcalls, + Uint max_young_gen_usage); static int major_collection(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj, Uint *recl); + int need, Eterm* objv, int nobj, + Uint ygen_usage, Uint *recl); static int minor_collection(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj, Uint *recl); + int need, Eterm* objv, int nobj, + Uint ygen_usage, Uint *recl); static void do_minor(Process *p, ErlHeapFragment *live_hf_end, char *mature, Uint mature_size, Uint new_sz, Eterm* objv, int nobj); @@ -153,7 +160,7 @@ static void init_gc_info(ErtsGCInfo *gcip); static Uint64 next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz); #ifdef HARDDEBUG -static void disallow_heap_frag_ref_in_heap(Process* p); +static void disallow_heap_frag_ref_in_heap(Process *p, Eterm *heap, Eterm *htop); static void disallow_heap_frag_ref_in_old_heap(Process* p); #endif @@ -176,6 +183,13 @@ typedef struct { erts_smp_atomic32_t refc; } ErtsGCInfoReq; +#ifdef ERTS_DIRTY_SCHEDULERS +static struct { + erts_mtx_t mtx; + ErtsGCInfo info; +} dirty_gc; +#endif + static ERTS_INLINE int gc_cost(Uint gc_moved_live_words, Uint resize_moved_words) { @@ -259,6 +273,11 @@ erts_init_gc(void) init_gc_info(&esdp->gc_info); } +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_mtx_init(&dirty_gc.mtx, "dirty_gc_info"); + init_gc_info(&dirty_gc.info); +#endif + init_gcireq_alloc(); } @@ -400,15 +419,15 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end, regs = erts_proc_sched_data(p)->x_reg_array; } #endif - cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls); + cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls, 0); } else { - cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls); + cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls, 0); } } else { Eterm val[1]; val[0] = result; - cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls); + cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls, 0); result = val[0]; } BUMP_REDS(p, cost); @@ -477,24 +496,26 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int hsz = ssz + need + ERTS_DELAY_GC_EXTRA_FREE; hfrag = new_message_buffer(hsz); - hfrag->next = p->mbuf; - p->mbuf = hfrag; - p->mbuf_sz += hsz; p->heap = p->htop = &hfrag->mem[0]; p->hend = hend = &hfrag->mem[hsz]; p->stop = stop = hend - ssz; sys_memcpy((void *) stop, (void *) orig_stop, ssz * sizeof(Eterm)); if (p->abandoned_heap) { - /* Active heap already in a fragment; adjust it... */ - ErlHeapFragment *hfrag = ((ErlHeapFragment *) - (((char *) orig_heap) - - offsetof(ErlHeapFragment, mem))); - Uint unused = orig_hend - orig_htop; - ASSERT(hfrag->used_size == hfrag->alloc_size); - ASSERT(hfrag->used_size >= unused); - hfrag->used_size -= unused; - p->mbuf_sz -= unused; + /* + * Active heap already in a fragment; adjust it and + * save it into mbuf list... + */ + ErlHeapFragment *hfrag = ((ErlHeapFragment *) + (((char *) orig_heap) + - offsetof(ErlHeapFragment, mem))); + Uint used = orig_htop - orig_heap; + hfrag->used_size = used; + p->mbuf_sz += used; + ASSERT(hfrag->used_size <= hfrag->alloc_size); + ASSERT(!hfrag->off_heap.first && !hfrag->off_heap.overhead); + hfrag->next = p->mbuf; + p->mbuf = hfrag; } else { /* Do not leave a hole in the abandoned heap... */ @@ -556,17 +577,14 @@ young_gen_usage(Process *p) } } + hsz += p->htop - p->heap; aheap = p->abandoned_heap; - if (!aheap) - hsz += p->htop - p->heap; - else { + if (aheap) { /* used in orig heap */ if (p->flags & F_ABANDONED_HEAP_USE) hsz += aheap[p->heap_sz-1]; else hsz += p->heap_sz; - /* Remove unused part in latest fragment */ - hsz -= p->hend - p->htop; } return hsz; } @@ -587,6 +605,32 @@ young_gen_usage(Process *p) } \ } while (0) +#ifdef ERTS_DIRTY_SCHEDULERS + +static ERTS_INLINE void +check_for_possibly_long_gc(Process *p, Uint ygen_usage) +{ + int major; + Uint sz; + + major = (p->flags & F_NEED_FULLSWEEP) || GEN_GCS(p) >= MAX_GEN_GCS(p); + + sz = ygen_usage; + sz += p->hend - p->stop; + if (p->flags & F_ON_HEAP_MSGQ) + sz += p->msg.len; + if (major) + sz += p->old_htop - p->old_heap; + + if (sz >= ERTS_POTENTIALLY_LONG_GC_HSIZE) { + ASSERT(!(p->flags & (F_DISABLE_GC|F_DELAY_GC))); + p->flags |= major ? F_DIRTY_MAJOR_GC : F_DIRTY_MINOR_GC; + erts_schedule_dirty_sys_execution(p); + } +} + +#endif + /* * Garbage collect a process. * @@ -597,28 +641,44 @@ young_gen_usage(Process *p) */ static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj, int fcalls) + int need, Eterm* objv, int nobj, int fcalls, + Uint max_young_gen_usage) { Uint reclaimed_now = 0; + Uint ygen_usage; Eterm gc_trace_end_tag; int reds; - ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */ - ErtsSchedulerData *esdp; + ErtsMonotonicTime start_time; + ErtsSchedulerData *esdp = erts_proc_sched_data(p); erts_aint32_t state; ERTS_MSACC_PUSH_STATE_M(); #ifdef USE_VM_PROBES DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); #endif + ERTS_UNDEF(start_time, 0); ERTS_CHK_MBUF_SZ(p); - ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) - >= erts_proc_sched_data(p)->virtual_reds); + ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) >= esdp->virtual_reds); state = erts_smp_atomic32_read_nob(&p->state); - if (p->flags & (F_DISABLE_GC|F_DELAY_GC) || state & ERTS_PSFLG_EXITING) + if ((p->flags & (F_DISABLE_GC|F_DELAY_GC)) || state & ERTS_PSFLG_EXITING) { +#ifdef ERTS_DIRTY_SCHEDULERS + delay_gc_before_start: +#endif return delay_garbage_collection(p, live_hf_end, need, fcalls); + } + + ygen_usage = max_young_gen_usage ? max_young_gen_usage : young_gen_usage(p); + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + check_for_possibly_long_gc(p, ygen_usage); + if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) + goto delay_gc_before_start; + } +#endif if (p->abandoned_heap) live_hf_end = ERTS_INVALID_HFRAG_PTR; @@ -627,8 +687,6 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_GC); - esdp = erts_get_scheduler_data(); - erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); if (erts_system_monitor_long_gc != 0) start_time = erts_get_monotonic_time(esdp); @@ -655,14 +713,25 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, trace_gc(p, am_gc_minor_start, need, THE_NON_VALUE); } DTRACE2(gc_minor_start, pidbuf, need); - reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); + reds = minor_collection(p, live_hf_end, need, objv, nobj, + ygen_usage, &reclaimed_now); DTRACE2(gc_minor_end, pidbuf, reclaimed_now); if (reds == -1) { if (IS_TRACED_FL(p, F_TRACE_GC)) { trace_gc(p, am_gc_minor_end, reclaimed_now, THE_NON_VALUE); } +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + p->flags |= F_NEED_FULLSWEEP; + check_for_possibly_long_gc(p, ygen_usage); + if (p->flags & F_DIRTY_MAJOR_GC) + goto delay_gc_after_start; + } +#endif goto do_major_collection; } + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + p->flags &= ~F_DIRTY_MINOR_GC; gc_trace_end_tag = am_gc_minor_end; } else { do_major_collection: @@ -671,7 +740,10 @@ do_major_collection: trace_gc(p, am_gc_major_start, need, THE_NON_VALUE); } DTRACE2(gc_major_start, pidbuf, need); - reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); + reds = major_collection(p, live_hf_end, need, objv, nobj, + ygen_usage, &reclaimed_now); + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + p->flags &= ~(F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC); DTRACE2(gc_major_end, pidbuf, reclaimed_now); gc_trace_end_tag = am_gc_major_end; ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC); @@ -701,6 +773,9 @@ do_major_collection: am_kill, NIL, NULL, 0); erts_smp_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR); +#ifdef ERTS_DIRTY_SCHEDULERS + delay_gc_after_start: +#endif /* erts_send_exit_signal looks for ERTS_PSFLG_GC, so we have to remove it after the signal is sent */ erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); @@ -735,8 +810,19 @@ do_major_collection: monitor_large_heap(p); } - esdp->gc_info.garbage_cols++; - esdp->gc_info.reclaimed += reclaimed_now; +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { + erts_mtx_lock(&dirty_gc.mtx); + dirty_gc.info.garbage_cols++; + dirty_gc.info.reclaimed += reclaimed_now; + erts_mtx_unlock(&dirty_gc.mtx); + } + else +#endif + { + esdp->gc_info.garbage_cols++; + esdp->gc_info.reclaimed += reclaimed_now; + } FLAGS(p) &= ~F_FORCE_GC; p->live_hf_end = ERTS_INVALID_HFRAG_PTR; @@ -765,6 +851,7 @@ do_major_collection: ASSERT(!p->mbuf); ASSERT(!ERTS_IS_GC_DESIRED(p)); + ASSERT(need <= HEAP_LIMIT(p) - HEAP_TOP(p)); return reds; } @@ -772,7 +859,7 @@ do_major_collection: int erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fcalls) { - int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls); + int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls, 0); int reds_left = ERTS_REDS_LEFT(p, fcalls); if (reds > reds_left) reds = reds_left; @@ -783,12 +870,13 @@ erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fca void erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) { - int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls); + int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls, 0); BUMP_REDS(p, reds); ASSERT(CONTEXT_REDS - ERTS_BIF_REDS_LEFT(p) >= erts_proc_sched_data(p)->virtual_reds); } + /* * Place all living data on a the new heap; deallocate any old heap. * Meant to be used by hibernate/3. @@ -808,6 +896,20 @@ erts_garbage_collect_hibernate(Process* p) if (p->flags & F_DISABLE_GC) ERTS_INTERNAL_ERROR("GC disabled"); +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p))) + p->flags &= ~(F_DIRTY_GC_HIBERNATE|F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC); + else { + Uint flags = p->flags; + p->flags |= F_NEED_FULLSWEEP; + check_for_possibly_long_gc(p, (p->htop - p->heap) + p->mbuf_sz); + if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) { + p->flags = flags|F_DIRTY_GC_HIBERNATE; + return; + } + p->flags = flags; + } +#endif /* * Preliminaries. */ @@ -819,7 +921,6 @@ erts_garbage_collect_hibernate(Process* p) * Do it. */ - heap_size = p->heap_sz + (p->old_htop - p->old_heap) + p->mbuf_sz; heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_TMP_HEAP, @@ -835,11 +936,11 @@ erts_garbage_collect_hibernate(Process* p) p->arg_reg, p->arity); - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (p->abandoned_heap - ? p->abandoned_heap - : p->heap), - p->heap_sz * sizeof(Eterm)); +#ifdef HARDDEBUG + disallow_heap_frag_ref_in_heap(p, heap, htop); +#endif + + erts_deallocate_young_generation(p); p->heap = heap; p->high_water = htop; @@ -874,8 +975,6 @@ erts_garbage_collect_hibernate(Process* p) sys_memcpy((void *) heap, (void *) p->heap, actual_size*sizeof(Eterm)); ERTS_HEAP_FREE(ERTS_ALC_T_TMP_HEAP, p->heap, p->heap_sz*sizeof(Eterm)); - remove_message_buffers(p); - p->stop = p->hend = heap + heap_size; offs = heap - p->heap; @@ -962,10 +1061,11 @@ static ERTS_INLINE void offset_nstack(Process* p, Sint offs, #endif /* HIPE */ -void +int erts_garbage_collect_literals(Process* p, Eterm* literals, Uint byte_lit_size, - struct erl_off_heap_header* oh) + struct erl_off_heap_header* oh, + int fcalls) { Uint lit_size = byte_lit_size / sizeof(Eterm); Uint old_heap_size; @@ -977,20 +1077,49 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint area_size; Eterm* old_htop; Uint n; + Uint ygen_usage = 0; struct erl_off_heap_header** prev = NULL; + Sint64 reds; + + if (p->flags & (F_DISABLE_GC|F_DELAY_GC)) + ERTS_INTERNAL_ERROR("GC disabled"); + + /* + * First an ordinary major collection... + */ + + p->flags |= F_NEED_FULLSWEEP; + +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p))) + p->flags &= ~F_DIRTY_CLA; + else { + ygen_usage = young_gen_usage(p); + check_for_possibly_long_gc(p, + (byte_lit_size/sizeof(Uint) + + 2*ygen_usage)); + if (p->flags & F_DIRTY_MAJOR_GC) { + p->flags |= F_DIRTY_CLA; + return 10; + } + } +#endif + + reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, 0, + p->arg_reg, p->arity, fcalls, + ygen_usage); + + ASSERT(!(p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC))); - if (p->flags & F_DISABLE_GC) - return; /* * Set GC state. */ erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC); /* - * We assume that the caller has already done a major collection - * (which has discarded the old heap), so that we don't have to cope - * with pointer to literals on the old heap. We will now allocate - * an old heap to contain the literals. + * Just did a major collection (which has discarded the old heap), + * so that we don't have to cope with pointer to literals on the + * old heap. We will now allocate an old heap to contain the literals. */ ASSERT(p->old_heap == 0); /* Must NOT have an old heap yet. */ @@ -1133,15 +1262,21 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, * Restore status. */ erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + + reds += (Sint64) gc_cost((p->htop - p->heap) + byte_lit_size/sizeof(Uint), 0); + if (reds > INT_MAX) + return INT_MAX; + return (int) reds; } static int minor_collection(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj, Uint *recl) + int need, Eterm* objv, int nobj, + Uint ygen_usage, Uint *recl) { Eterm *mature = p->abandoned_heap ? p->abandoned_heap : p->heap; Uint mature_size = p->high_water - mature; - Uint size_before = young_gen_usage(p); + Uint size_before = ygen_usage; /* * Check if we have gone past the max heap size limit @@ -1494,22 +1629,16 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, } #endif - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (p->abandoned_heap - ? p->abandoned_heap - : HEAP_START(p)), - HEAP_SIZE(p) * sizeof(Eterm)); - p->abandoned_heap = NULL; - p->flags &= ~F_ABANDONED_HEAP_USE; +#ifdef HARDDEBUG + disallow_heap_frag_ref_in_heap(p, n_heap, n_htop); +#endif + + erts_deallocate_young_generation(p); + HEAP_START(p) = n_heap; HEAP_TOP(p) = n_htop; HEAP_SIZE(p) = new_sz; HEAP_END(p) = n_heap + new_sz; - -#ifdef HARDDEBUG - disallow_heap_frag_ref_in_heap(p); -#endif - remove_message_buffers(p); } /* @@ -1518,7 +1647,8 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end, static int major_collection(Process* p, ErlHeapFragment *live_hf_end, - int need, Eterm* objv, int nobj, Uint *recl) + int need, Eterm* objv, int nobj, + Uint ygen_usage, Uint *recl) { Uint size_before, size_after, stack_size; Eterm* n_heap; @@ -1536,7 +1666,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, * to receive all live data. */ - size_before = young_gen_usage(p); + size_before = ygen_usage; size_before += p->old_htop - p->old_heap; stack_size = p->hend - p->stop; @@ -1598,13 +1728,12 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, } #endif - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, - (p->abandoned_heap - ? p->abandoned_heap - : HEAP_START(p)), - p->heap_sz * sizeof(Eterm)); - p->abandoned_heap = NULL; - p->flags &= ~F_ABANDONED_HEAP_USE; +#ifdef HARDDEBUG + disallow_heap_frag_ref_in_heap(p, n_heap, n_htop); +#endif + + erts_deallocate_young_generation(p); + HEAP_START(p) = n_heap; HEAP_TOP(p) = n_htop; HEAP_SIZE(p) = new_sz; @@ -1613,11 +1742,6 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, HIGH_WATER(p) = HEAP_TOP(p); -#ifdef HARDDEBUG - disallow_heap_frag_ref_in_heap(p); -#endif - remove_message_buffers(p); - if (p->flags & F_ON_HEAP_MSGQ) move_msgq_to_heap(p); @@ -1770,22 +1894,56 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) return adjusted; } -/* - * Remove all message buffers. - */ -static void -remove_message_buffers(Process* p) +void +erts_deallocate_young_generation(Process *c_p) { - if (MBUF(p) != NULL) { - free_message_buffer(MBUF(p)); - MBUF(p) = NULL; + Eterm *orig_heap; + + if (!c_p->abandoned_heap) { + orig_heap = c_p->heap; + ASSERT(!(c_p->flags & F_ABANDONED_HEAP_USE)); + } + else { + ErlHeapFragment *hfrag; + + orig_heap = c_p->abandoned_heap; + c_p->abandoned_heap = NULL; + c_p->flags &= ~F_ABANDONED_HEAP_USE; + + /* + * Temporary heap located in heap fragment + * only referred to by 'c_p->heap'. Add it to + * 'c_p->mbuf' list and deallocate it as any + * other heap fragment... + */ + hfrag = ((ErlHeapFragment *) + (((char *) c_p->heap) + - offsetof(ErlHeapFragment, mem))); + + ASSERT(!hfrag->off_heap.first); + ASSERT(!hfrag->off_heap.overhead); + ASSERT(!hfrag->next); + ASSERT(c_p->htop - c_p->heap <= hfrag->alloc_size); + + hfrag->next = c_p->mbuf; + c_p->mbuf = hfrag; + } + + if (c_p->mbuf) { + free_message_buffer(c_p->mbuf); + c_p->mbuf = NULL; } - if (p->msg_frag) { - erts_cleanup_messages(p->msg_frag); - p->msg_frag = NULL; + if (c_p->msg_frag) { + erts_cleanup_messages(c_p->msg_frag); + c_p->msg_frag = NULL; } - MBUF_SIZE(p) = 0; + c_p->mbuf_sz = 0; + + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, + orig_heap, + c_p->heap_sz * sizeof(Eterm)); } + #ifdef HARDDEBUG /* @@ -1797,19 +1955,15 @@ remove_message_buffers(Process* p) */ static void -disallow_heap_frag_ref_in_heap(Process* p) +disallow_heap_frag_ref_in_heap(Process *p, Eterm *heap, Eterm *htop) { Eterm* hp; - Eterm* htop; - Eterm* heap; Uint heap_size; if (p->mbuf == 0) { return; } - htop = p->htop; - heap = p->heap; heap_size = (htop - heap)*sizeof(Eterm); hp = heap; @@ -2384,17 +2538,10 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) } /* - * If a NIF has saved arguments, they need to be added + * If a NIF or BIF has saved arguments, they need to be added */ - if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) { - Eterm* argv; - int argc; - if (erts_setup_nif_gc(p, &argv, &argc)) { - roots[n].v = argv; - roots[n].sz = argc; - n++; - } - } + if (erts_setup_nif_export_rootset(p, &roots[n].v, &roots[n].sz)) + n++; ASSERT(n <= rootset->size); @@ -2946,6 +3093,8 @@ static void ERTS_INLINE offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, Eterm* objv, int nobj) { + Eterm *v; + Uint sz; if (p->dictionary) { offset_heap(ERTS_PD_START(p->dictionary), ERTS_PD_SIZE(p->dictionary), @@ -2966,12 +3115,8 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, offset_heap_ptr(objv, nobj, offs, area, area_size); } offset_off_heap(p, offs, area, area_size); - if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) { - Eterm* argv; - int argc; - if (erts_setup_nif_gc(p, &argv, &argc)) - offset_heap_ptr(argv, argc, offs, area, area_size); - } + if (erts_setup_nif_export_rootset(p, &v, &sz)) + offset_heap_ptr(v, sz, offs, area, area_size); } static void @@ -3009,6 +3154,18 @@ reply_gc_info(void *vgcirp) reclaimed = esdp->gc_info.reclaimed; garbage_cols = esdp->gc_info.garbage_cols; +#ifdef ERTS_DIRTY_SCHEDULERS + /* + * Add dirty schedulers info on requesting + * schedulers info + */ + if (gcirp->req_sched == esdp->no) { + erts_mtx_lock(&dirty_gc.mtx); + reclaimed += dirty_gc.info.reclaimed; + garbage_cols += dirty_gc.info.garbage_cols; + erts_mtx_unlock(&dirty_gc.mtx); + } +#endif sz = 0; hpp = NULL; @@ -3289,13 +3446,15 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop) ErtsMessage* mp; Eterm *htop, *heap; - if (p->abandoned_heap) + if (p->abandoned_heap) { ERTS_GET_ORIG_HEAP(p, heap, htop); - else { - heap = p->heap; - htop = real_htop ? real_htop : HEAP_TOP(p); + if (heap <= ptr && ptr < htop) + return 1; } + heap = p->heap; + htop = real_htop ? real_htop : HEAP_TOP(p); + if (OLD_HEAP(p) && (OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p))) { return 1; } diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 54ea9ca3c0..f4cbe732ce 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -25,11 +25,9 @@ /* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */ -#include "erl_map.h" +#define ERTS_POTENTIALLY_LONG_GC_HSIZE (128*1024) /* Words */ -#if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF -# define HARDDEBUG 1 -#endif +#include "erl_map.h" #define IS_MOVED_BOXED(x) (!is_header((x))) #define IS_MOVED_CONS(x) (is_non_value((x))) @@ -145,9 +143,10 @@ void erts_garbage_collect_hibernate(struct process* p); Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end, Eterm result, Eterm* regs, Uint arity); Eterm erts_gc_after_bif_call(struct process* p, Eterm result, Eterm* regs, Uint arity); -void erts_garbage_collect_literals(struct process* p, Eterm* literals, - Uint lit_size, - struct erl_off_heap_header* oh); +int erts_garbage_collect_literals(struct process* p, Eterm* literals, + Uint lit_size, + struct erl_off_heap_header* oh, + int fcalls); Uint erts_next_heap_size(Uint, Uint); Eterm erts_heap_sizes(struct process* p); @@ -157,5 +156,6 @@ void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*); void erts_free_heap_frags(struct process* p); Eterm erts_max_heap_size_map(Sint, Uint, Eterm **, Uint *); int erts_max_heap_size(Eterm, Uint *, Uint *); +void erts_deallocate_young_generation(Process *c_p); #endif /* __ERL_GC_H__ */ diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index d29d079fc5..647fa26811 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -2666,7 +2666,7 @@ erts_start_timer_callback(ErtsMonotonicTime tmo, tmo); twt = tmo < ERTS_TIMER_WHEEL_MSEC; - if (esdp) + if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) start_callback_timer(esdp, twt, timeout_pos, @@ -2865,7 +2865,8 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp) if (tmr->timeout <= btmp->now) left = 0; - left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now); + else + left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now); receiver = ((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) ? tmr->receiver.name diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h index 705be94532..9cdcd581a0 100644 --- a/erts/emulator/beam/erl_hl_timer.h +++ b/erts/emulator/beam/erl_hl_timer.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2015. All Rights Reserved. + * Copyright Ericsson AB 2015-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 356f5ca71e..08dcbed91c 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -127,6 +127,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "run_queue", "address" }, #ifdef ERTS_DIRTY_SCHEDULERS { "dirty_run_queue_sleep_list", "address" }, + { "dirty_gc_info", NULL }, + { "dirty_break_point_index", NULL }, #endif { "process_table", NULL }, { "cpu_info", NULL }, diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index 481e92b2cd..6354fc8663 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -55,11 +55,11 @@ static erts_lcnt_thread_data_t *lcnt_thread_data[2048]; /* local functions */ static ERTS_INLINE void lcnt_lock(void) { - ethr_mutex_lock(&lcnt_data_lock); + ethr_mutex_lock(&lcnt_data_lock); } static ERTS_INLINE void lcnt_unlock(void) { - ethr_mutex_unlock(&lcnt_data_lock); + ethr_mutex_unlock(&lcnt_data_lock); } const int log2_tab64[64] = { @@ -159,7 +159,7 @@ static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) { lcnt_thread_data[eltd->id] = eltd; return eltd; -} +} static erts_lcnt_thread_data_t *lcnt_get_thread_data(void) { return (erts_lcnt_thread_data_t *)ethr_tsd_get(lcnt_thr_data_key); @@ -254,9 +254,9 @@ void erts_lcnt_init() { /* init lock */ if (ethr_mutex_init(&lcnt_data_lock) != 0) abort(); - /* init tsd */ + /* init tsd */ lcnt_n_thr = 0; - ethr_tsd_key_create(&lcnt_thr_data_key,"lcnt_data"); + ethr_tsd_key_create(&lcnt_thr_data_key, "lcnt_data"); lcnt_lock(); @@ -352,11 +352,11 @@ void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) /* interface to erl_threads.h */ /* only lock on init and destroy, all others should use atomics */ -void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { +void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { erts_lcnt_init_lock_x(lock, name, flag, NIL); } -void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { +void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { int i; if (name == NULL) { ERTS_LCNT_CLEAR_FLAG(lock); return; } lcnt_lock(); @@ -382,7 +382,10 @@ void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eter erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock); lcnt_unlock(); } -/* init empty, instead of zero struct */ + +/* init empty, instead of zero struct + * used by process locks probes + */ void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock) { lock->next = NULL; lock->prev = NULL; @@ -444,7 +447,7 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { } /* we cannot acquire w_lock if either w or r are taken */ - /* we cannot acquire r_lock if w_lock is taken */ + /* we cannot acquire r_lock if w_lock is taken */ if ((w_state > 0) || (r_state > 0)) { eltd->lock_in_conflict = 1; @@ -561,7 +564,7 @@ void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; #ifdef DEBUG { - erts_aint_t w_state; + erts_aint_t w_state; erts_aint_t flowstate; /* flowstate */ @@ -647,7 +650,7 @@ Uint16 erts_lcnt_set_rt_opt(Uint16 opt) { return prev; } -Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) { +Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) { Uint16 prev; prev = (erts_lcnt_rt_options & opt); erts_lcnt_rt_options &= ~opt; diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index 3e8dcefe69..4f838f7faa 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -148,13 +148,13 @@ typedef struct erts_lcnt_lock_stats_s { typedef struct erts_lcnt_lock_s { char *name; /* lock name */ Uint16 flag; /* lock type */ - Eterm id; /* id if possible */ + Eterm id; /* id if possible */ #ifdef DEBUG ethr_atomic_t flowstate; #endif - /* lock states */ + /* lock states */ ethr_atomic_t w_state; /* 0 not taken, otherwise n threads waiting */ ethr_atomic_t r_state; /* 0 not taken, > 0 -> writes will wait */ diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 118adc0c1b..792b69bb37 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -193,7 +193,7 @@ free_message_buffer(ErlHeapFragment* bp) erts_cleanup_offheap(&bp->off_heap); ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp, - ERTS_HEAP_FRAG_SIZE(bp->size)); + ERTS_HEAP_FRAG_SIZE(bp->alloc_size)); bp = next_bp; }while (bp != NULL); } diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c index 421445fbad..66bb55e6c8 100644 --- a/erts/emulator/beam/erl_msacc.c +++ b/erts/emulator/beam/erl_msacc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2014-2015. All Rights Reserved. + * Copyright Ericsson AB 2014-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -137,8 +137,8 @@ void erts_msacc_init_thread(char *type, int id, int managed) { void erts_msacc_set_bif_state(ErtsMsAcc *__erts_msacc_cache, Eterm mod, void *fn) { #ifdef ERTS_MSACC_EXTENDED_BIFS -#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num) \ - if (fn == &FuncAddr) { \ +#define BIF_LIST(Mod,Func,Arity,BifFuncAddr,FuncAddr,Num) \ + if (fn == &BifFuncAddr) { \ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATIC_STATE_COUNT + Num); \ } else #include "erl_bif_list.h" diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index 7e02d8e101..d64ef8c8b9 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2014-2015. All Rights Reserved. + * Copyright Ericsson AB 2014-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -122,7 +122,7 @@ static char *erts_msacc_states[] = { "sleep", "timers" #ifdef ERTS_MSACC_EXTENDED_BIFS -#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num) \ +#define BIF_LIST(Mod,Func,Arity,BifFuncAddr,FuncAddr,Num) \ ,"bif_" #Mod "_" #Func "_" #Arity #include "erl_bif_list.h" #undef BIF_LIST diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c new file mode 100644 index 0000000000..1bebc1eda4 --- /dev/null +++ b/erts/emulator/beam/erl_nfunc_sched.c @@ -0,0 +1,180 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#define ERTS_WANT_NFUNC_SCHED_INTERNALS__ + +#include "global.h" +#include "erl_process.h" +#include "bif.h" +#include "erl_nfunc_sched.h" +#include "erl_trace.h" + +NifExport * +erts_new_proc_nif_export(Process *c_p, int argc) +{ + size_t size; + int i; + NifExport *nep, *old_nep; + + size = sizeof(NifExport) + (argc-1)*sizeof(Eterm); + nep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, size); + + for (i = 0; i < ERTS_NUM_CODE_IX; i++) + nep->exp.addressv[i] = &nep->exp.beam[0]; + + nep->argc = -1; /* unused marker */ + nep->argv_size = argc; + nep->trace = NULL; + old_nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(c_p, nep); + if (old_nep) { + ASSERT(!nep->trace); + erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, old_nep); + } + return nep; +} + +void +erts_destroy_nif_export(Process *p) +{ + NifExport *nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL); + if (nep) { + if (nep->m) + erts_nif_export_cleanup_nif_mod(nep); + erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, nep); + } +} + +void +erts_nif_export_save_trace(Process *c_p, NifExport *nep, int applying, + Export* ep, BeamInstr *cp, Uint32 flags, + Uint32 flags_meta, BeamInstr* I, + ErtsTracer meta_tracer) +{ + NifExportTrace *netp; + ASSERT(nep && nep->argc >= 0); + ASSERT(!nep->trace); + netp = erts_alloc(ERTS_ALC_T_NIF_EXP_TRACE, + sizeof(NifExportTrace)); + netp->applying = applying; + netp->ep = ep; + netp->cp = cp; + netp->flags = flags; + netp->flags_meta = flags_meta; + netp->I = I; + netp->meta_tracer = NIL; + erts_tracer_update(&netp->meta_tracer, meta_tracer); + nep->trace = netp; +} + +void +erts_nif_export_restore_trace(Process *c_p, Eterm result, NifExport *nep) +{ + NifExportTrace *netp = nep->trace; + nep->trace = NULL; + erts_bif_trace_epilogue(c_p, result, netp->applying, netp->ep, + netp->cp, netp->flags, netp->flags_meta, + netp->I, netp->meta_tracer); + erts_tracer_update(&netp->meta_tracer, NIL); + erts_free(ERTS_ALC_T_NIF_EXP_TRACE, netp); +} + +NifExport * +erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, + ErtsCodeMFA *mfa, BeamInstr *pc, + BeamInstr instr, + void *dfunc, void *ifunc, + Eterm mod, Eterm func, + int argc, const Eterm *argv) +{ + Process *used_proc; + ErtsSchedulerData *esdp; + Eterm* reg; + NifExport* nep; + int i; + + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + if (dirty_shadow_proc) { + esdp = erts_get_scheduler_data(); + ASSERT(esdp && ERTS_SCHEDULER_IS_DIRTY(esdp)); + + used_proc = dirty_shadow_proc; + } + else { + esdp = erts_proc_sched_data(c_p); + ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)); + + used_proc = c_p; + ERTS_VBUMP_ALL_REDS(c_p); + } + + reg = esdp->x_reg_array; + + if (mfa) + nep = erts_get_proc_nif_export(c_p, (int) mfa->arity); + else { + /* If no mfa, this is not the first schedule... */ + nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + ASSERT(nep && nep->argc >= 0); + } + + if (nep->argc < 0) { + /* + * First schedule; save things that might + * need to be restored... + */ + for (i = 0; i < (int) mfa->arity; i++) + nep->argv[i] = reg[i]; + nep->pc = pc; + nep->cp = c_p->cp; + nep->mfa = mfa; + nep->current = c_p->current; + ASSERT(argc >= 0); + nep->argc = (int) mfa->arity; + nep->m = NULL; + + ASSERT(!erts_check_nif_export_in_area(c_p, + (char *) nep, + (sizeof(NifExport) + + (sizeof(Eterm) + *(nep->argc-1))))); + } + /* Copy new arguments into register array if necessary... */ + if (reg != argv) { + for (i = 0; i < argc; i++) + reg[i] = argv[i]; + } + ASSERT(is_atom(mod) && is_atom(func)); + nep->exp.info.mfa.module = mod; + nep->exp.info.mfa.function = func; + nep->exp.info.mfa.arity = (Uint) argc; + nep->exp.beam[0] = (BeamInstr) instr; /* call_nif || apply_bif */ + nep->exp.beam[1] = (BeamInstr) dfunc; + nep->func = ifunc; + used_proc->arity = argc; + used_proc->freason = TRAP; + used_proc->i = (BeamInstr*) nep->exp.addressv[0]; + return nep; +} diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h new file mode 100644 index 0000000000..55a3a6dbf6 --- /dev/null +++ b/erts/emulator/beam/erl_nfunc_sched.h @@ -0,0 +1,332 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef ERL_NFUNC_SCHED_H__ +#define ERL_NFUNC_SCHED_H__ + +#include "erl_process.h" +#include "bif.h" +#include "error.h" + +typedef struct { + int applying; + Export* ep; + BeamInstr *cp; + Uint32 flags; + Uint32 flags_meta; + BeamInstr* I; + ErtsTracer meta_tracer; +} NifExportTrace; + +/* + * NIF exports need a few more items than the Export struct provides, + * including the erl_module_nif* and a NIF function pointer, so the + * NifExport below adds those. The Export member must be first in the + * struct. A number of values are stored for error handling purposes + * only. + * + * 'argc' is >= 0 when NifExport is in use, and < 0 when not. + */ + +typedef struct { + Export exp; + struct erl_module_nif* m; /* NIF module, or NULL if BIF */ + void *func; /* Indirect NIF or BIF to execute (may be unused) */ + ErtsCodeMFA *current;/* Current as set when originally called */ + NifExportTrace *trace; + /* --- The following is only used on error --- */ + BeamInstr *pc; /* Program counter */ + BeamInstr *cp; /* Continuation pointer */ + ErtsCodeMFA *mfa; /* MFA of original call */ + int argc; /* Number of arguments in original call */ + int argv_size; /* Allocated size of argv */ + Eterm argv[1]; /* Saved arguments from the original call */ +} NifExport; + +NifExport *erts_new_proc_nif_export(Process *c_p, int argc); +void erts_nif_export_save_trace(Process *c_p, NifExport *nep, int applying, + Export* ep, BeamInstr *cp, Uint32 flags, + Uint32 flags_meta, BeamInstr* I, + ErtsTracer meta_tracer); +void erts_nif_export_restore_trace(Process *c_p, Eterm result, NifExport *nep); +void erts_destroy_nif_export(Process *p); +NifExport *erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc, + ErtsCodeMFA *mfa, BeamInstr *pc, + BeamInstr instr, + void *dfunc, void *ifunc, + Eterm mod, Eterm func, + int argc, const Eterm *argv); +void erts_nif_export_cleanup_nif_mod(NifExport *ep); /* erl_nif.c */ +ERTS_GLB_INLINE NifExport *erts_get_proc_nif_export(Process *c_p, int extra); +ERTS_GLB_INLINE int erts_setup_nif_export_rootset(Process* proc, Eterm** objv, + Uint* nobj); +ERTS_GLB_INLINE int erts_check_nif_export_in_area(Process *p, + char *start, Uint size); +ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep, + Eterm result); +ERTS_GLB_INLINE void erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, + Eterm *reg, ErtsCodeMFA **nif_mfa); +ERTS_GLB_INLINE int erts_nif_export_check_save_trace(Process *c_p, Eterm result, + int applying, Export* ep, + BeamInstr *cp, Uint32 flags, + Uint32 flags_meta, BeamInstr* I, + ErtsTracer meta_tracer); +ERTS_GLB_INLINE Process *erts_proc_shadow2real(Process *c_p); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE NifExport * +erts_get_proc_nif_export(Process *c_p, int argc) +{ + NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + if (!nep || (nep->argc < 0 && nep->argv_size < argc)) + return erts_new_proc_nif_export(c_p, argc); + return nep; +} + +/* + * If a process has saved arguments, they need to be part of the GC + * rootset. The function below is called from setup_rootset() in + * erl_gc.c. Any exception term saved in the NifExport is also made + * part of the GC rootset here; it always resides in rootset[0]. + */ +ERTS_GLB_INLINE int +erts_setup_nif_export_rootset(Process* proc, Eterm** objv, Uint* nobj) +{ + NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + + if (!ep || ep->argc <= 0) + return 0; + + *objv = ep->argv; + *nobj = ep->argc; + return 1; +} + +/* + * Check if nif export points into code area... + */ +ERTS_GLB_INLINE int +erts_check_nif_export_in_area(Process *p, char *start, Uint size) +{ + NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p); + if (!nep || nep->argc < 0) + return 0; + if (ErtsInArea(nep->pc, start, size)) + return 1; + if (ErtsInArea(nep->cp, start, size)) + return 1; + if (ErtsInArea(nep->mfa, start, size)) + return 1; + if (ErtsInArea(nep->current, start, size)) + return 1; + return 0; +} + +ERTS_GLB_INLINE void +erts_nif_export_restore(Process *c_p, NifExport *ep, Eterm result) +{ + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); + ERTS_SMP_LC_ASSERT(!(c_p->static_flags + & ERTS_STC_FLG_SHADOW_PROC)); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + c_p->current = ep->current; + ep->argc = -1; /* Unused nif-export marker... */ + if (ep->trace) + erts_nif_export_restore_trace(c_p, result, ep); +} + +ERTS_GLB_INLINE void +erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, + Eterm *reg, ErtsCodeMFA **nif_mfa) +{ + NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + int ix; + + ASSERT(nep); + *pc = nep->pc; + c_p->cp = nep->cp; + *nif_mfa = nep->mfa; + for (ix = 0; ix < nep->argc; ix++) + reg[ix] = nep->argv[ix]; + erts_nif_export_restore(c_p, nep, THE_NON_VALUE); +} + +ERTS_GLB_INLINE int +erts_nif_export_check_save_trace(Process *c_p, Eterm result, + int applying, Export* ep, + BeamInstr *cp, Uint32 flags, + Uint32 flags_meta, BeamInstr* I, + ErtsTracer meta_tracer) +{ + if (is_non_value(result) && c_p->freason == TRAP) { + NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p); + if (nep && nep->argc >= 0) { + erts_nif_export_save_trace(c_p, nep, applying, ep, + cp, flags, flags_meta, + I, meta_tracer); + return 1; + } + } + return 0; +} + +ERTS_GLB_INLINE Process * +erts_proc_shadow2real(Process *c_p) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + if (c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC) { + Process *real_c_p = c_p->next; + ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); + ASSERT(real_c_p->common.id == c_p->common.id); + return real_c_p; + } + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())); +#endif + return c_p; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERL_NFUNC_SCHED_H__ */ + +#if defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__) +#define ERTS_NFUNC_SCHED_INTERNALS__ + +#define ERTS_I_BEAM_OP_TO_NIF_EXPORT(I) \ + (ASSERT(BeamOp(op_apply_bif) == (BeamInstr *) (*(I)) \ + || BeamOp(op_call_nif) == (BeamInstr *) (*(I))), \ + ((NifExport *) (((char *) (I)) - offsetof(NifExport, exp.beam[0])))) + +#ifdef ERTS_DIRTY_SCHEDULERS + +#include "erl_message.h" +#include <stddef.h> + +ERTS_GLB_INLINE void erts_flush_dirty_shadow_proc(Process *sproc); +ERTS_GLB_INLINE void erts_cache_dirty_shadow_proc(Process *sproc); +ERTS_GLB_INLINE Process *erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp, + Process *c_p); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE void +erts_flush_dirty_shadow_proc(Process *sproc) +{ + Process *c_p = sproc->next; + + ASSERT(sproc->common.id == c_p->common.id); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + ASSERT(c_p->stop == sproc->stop); + ASSERT(c_p->hend == sproc->hend); + ASSERT(c_p->heap == sproc->heap); + ASSERT(c_p->abandoned_heap == sproc->abandoned_heap); + ASSERT(c_p->heap_sz == sproc->heap_sz); + ASSERT(c_p->high_water == sproc->high_water); + ASSERT(c_p->old_heap == sproc->old_heap); + ASSERT(c_p->old_htop == sproc->old_htop); + ASSERT(c_p->old_hend == sproc->old_hend); + + ASSERT(c_p->htop <= sproc->htop && sproc->htop <= c_p->stop); + + c_p->htop = sproc->htop; + + if (!c_p->mbuf) + c_p->mbuf = sproc->mbuf; + else if (sproc->mbuf) { + ErlHeapFragment *bp; + for (bp = sproc->mbuf; bp->next; bp = bp->next) + ASSERT(!bp->off_heap.first); + bp->next = c_p->mbuf; + c_p->mbuf = sproc->mbuf; + } + + c_p->mbuf_sz += sproc->mbuf_sz; + + if (!c_p->off_heap.first) + c_p->off_heap.first = sproc->off_heap.first; + else if (sproc->off_heap.first) { + struct erl_off_heap_header *ohhp; + for (ohhp = sproc->off_heap.first; ohhp->next; ohhp = ohhp->next) + ; + ohhp->next = c_p->off_heap.first; + c_p->off_heap.first = sproc->off_heap.first; + } + + c_p->off_heap.overhead += sproc->off_heap.overhead; +} + +ERTS_GLB_INLINE void +erts_cache_dirty_shadow_proc(Process *sproc) +{ + Process *c_p = sproc->next; + ASSERT(c_p); + ASSERT(sproc->common.id == c_p->common.id); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) + & ERTS_PROC_LOCK_MAIN); + + sproc->htop = c_p->htop; + sproc->stop = c_p->stop; + sproc->hend = c_p->hend; + sproc->heap = c_p->heap; + sproc->abandoned_heap = c_p->abandoned_heap; + sproc->heap_sz = c_p->heap_sz; + sproc->high_water = c_p->high_water; + sproc->old_hend = c_p->old_hend; + sproc->old_htop = c_p->old_htop; + sproc->old_heap = c_p->old_heap; + sproc->mbuf = NULL; + sproc->mbuf_sz = 0; + ERTS_INIT_OFF_HEAP(&sproc->off_heap); +} + +ERTS_GLB_INLINE Process * +erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp, Process *c_p) +{ + Process *sproc; + + ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); + + sproc = esdp->dirty_shadow_process; + ASSERT(sproc); + ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(erts_smp_atomic32_read_nob(&sproc->state) + == (ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_PROXY)); + + sproc->next = c_p; + sproc->common.id = c_p->common.id; + + erts_cache_dirty_shadow_proc(sproc); + + return sproc; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ERTS_DIRTY_SCHEDULERS */ + +#endif /* defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__) */ + diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 4e41944ccb..85fa53a886 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -55,6 +55,9 @@ #include "dtrace-wrapper.h" #include "erl_process.h" #include "erl_bif_unique.h" +#undef ERTS_WANT_NFUNC_SCHED_INTERNALS__ +#define ERTS_WANT_NFUNC_SCHED_INTERNALS__ +#include "erl_nfunc_sched.h" #if defined(USE_DYNAMIC_TRACE) && (defined(USE_DTRACE) || defined(USE_SYSTEMTAP)) #define HAVE_USE_DTRACE 1 #endif @@ -79,8 +82,11 @@ struct erl_module_nif { ErlNifFunc _funcs_copy_[1]; /* only used for old libs */ }; +typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]); + #ifdef DEBUG # define READONLY_CHECK +# define ERTS_DBG_NIF_NOT_SCHED_MARKER ((void *) (UWord) 1) #endif #ifdef READONLY_CHECK # define ADD_READONLY_CHECK(ENV,PTR,SIZE) add_readonly_check(ENV,PTR,SIZE) @@ -219,38 +225,6 @@ static void cache_env(ErlNifEnv* env); static void full_flush_env(ErlNifEnv *env); static void flush_env(ErlNifEnv* env); -#ifdef ERTS_DIRTY_SCHEDULERS -void erts_pre_dirty_nif(ErtsSchedulerData *esdp, - ErlNifEnv* env, Process* p, - struct erl_module_nif* mod_nif) -{ - Process *sproc; -#ifdef DEBUG - erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); - - ASSERT(!p->scheduler_data); - ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) - && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); - ASSERT(esdp); -#endif - - erts_pre_nif(env, p, mod_nif, NULL); - - sproc = esdp->dirty_shadow_process; - ASSERT(sproc); - ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC); - ASSERT(erts_smp_atomic32_read_nob(&sproc->state) - == (ERTS_PSFLG_ACTIVE - | ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_PROXY)); - - sproc->next = p; - sproc->common.id = p->common.id; - env->proc = sproc; - full_cache_env(env); -} -#endif - /* Temporary object header, auto-deallocated when NIF returns * or when independent environment is cleared. */ @@ -278,115 +252,154 @@ void erts_post_nif(ErlNifEnv* env) env->exiting = ERTS_PROC_IS_EXITING(env->proc); } -#ifdef ERTS_DIRTY_SCHEDULERS -void erts_post_dirty_nif(ErlNifEnv* env) + +/* + * Initialize a NifExport struct. Create it if needed and store it in the + * proc. The direct_fp function is what will be invoked by op_call_nif, and + * the indirect_fp function, if not NULL, is what the direct_fp function + * will call. If the allocated NifExport isn't enough to hold all of argv, + * allocate a larger one. Save 'current' and registers if first time this + * call is scheduled. + */ + +static ERTS_INLINE ERL_NIF_TERM +schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, + Eterm mod, Eterm func_name, int argc, const ERL_NIF_TERM argv[]) { - Process *c_p; - ASSERT(env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); - ASSERT(env->proc->next); - erts_unblock_fpe(env->fpe_was_unmasked); - full_flush_env(env); - free_tmp_objs(env); - c_p = env->proc->next; - env->exiting = ERTS_PROC_IS_EXITING(c_p); - ERTS_VBUMP_ALL_REDS(c_p); + NifExport *ep; + Process *c_p, *dirty_shadow_proc; + + execution_state(env, &c_p, NULL); + if (c_p == env->proc) + dirty_shadow_proc = NULL; + else + dirty_shadow_proc = env->proc; + + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p)); + + ep = erts_nif_export_schedule(c_p, dirty_shadow_proc, + c_p->current, + c_p->cp, + (BeamInstr) em_call_nif, + direct_fp, indirect_fp, + mod, func_name, + argc, (const Eterm *) argv); + if (!ep->m) { + /* First time this call is scheduled... */ + erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1); + ep->m = env->mod_nif; + } + return (ERL_NIF_TERM) THE_NON_VALUE; } -#endif -static void full_flush_env(ErlNifEnv* env) -{ #ifdef ERTS_DIRTY_SCHEDULERS - if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { - /* Dirty nif call using shadow process struct */ - Process *c_p = env->proc->next; - - ASSERT(is_scheduler() < 0); - ASSERT(env->proc->common.id == c_p->common.id); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) - & ERTS_PROC_LOCK_MAIN); - - if (!env->heap_frag) { - ASSERT(env->hp_end == HEAP_LIMIT(c_p)); - ASSERT(env->hp >= HEAP_TOP(c_p)); - ASSERT(env->hp <= HEAP_LIMIT(c_p)); - HEAP_TOP(c_p) = env->hp; + +static ERL_NIF_TERM dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); +static ERL_NIF_TERM dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]); + +int +erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg) +{ + int exiting; + ERL_NIF_TERM *argv = (ERL_NIF_TERM *) reg; + NifExport *nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I); + ErtsCodeMFA *codemfa = erts_code_to_codemfa(I); + NativeFunPtr dirty_nif = (NativeFunPtr) I[1]; + ErlNifEnv env; + ERL_NIF_TERM result; +#ifdef DEBUG + erts_aint32_t state = erts_smp_atomic32_read_nob(&c_p->state); + + ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p)); + + ASSERT(!c_p->scheduler_data); + ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) + && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); + ASSERT(esdp); + + nep->func = ERTS_DBG_NIF_NOT_SCHED_MARKER; +#endif + + erts_pre_nif(&env, c_p, nep->m, NULL); + + env.proc = erts_make_dirty_shadow_proc(esdp, c_p); + + env.proc->freason = EXC_NULL; + env.proc->fvalue = NIL; + env.proc->ftrace = NIL; + env.proc->i = c_p->i; + + ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))); + + erts_smp_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC)); + + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN); + + result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */ + + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); + + ASSERT(env.proc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(env.proc->next == c_p); + + exiting = ERTS_PROC_IS_EXITING(c_p); + + if (!exiting) { + if (env.exception_thrown) { + schedule_exception: + schedule(&env, dirty_nif_exception, NULL, + am_erts_internal, am_dirty_nif_exception, + 1, &env.proc->fvalue); + } + else if (is_value(result)) { + schedule(&env, dirty_nif_finalizer, NULL, + am_erts_internal, am_dirty_nif_finalizer, + 1, &result); + } + else if (env.proc->freason != TRAP) { /* user returned garbage... */ + ERTS_DECL_AM(badreturn); + (void) enif_raise_exception(&env, AM_badreturn); + goto schedule_exception; } else { - Uint usz; - ASSERT(env->hp_end != HEAP_LIMIT(c_p)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); - - HEAP_TOP(c_p) = HEAP_TOP(env->proc); - usz = env->hp - env->heap_frag->mem; - env->proc->mbuf_sz += usz - env->heap_frag->used_size; - env->heap_frag->used_size = usz; - - ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); - - if (c_p->mbuf) { - ErlHeapFragment *bp; - for (bp = env->proc->mbuf; bp->next; bp = bp->next) - ; - bp->next = c_p->mbuf; - } + /* Rescheduled by dirty NIF call... */ + ASSERT(nep->func != ERTS_DBG_NIF_NOT_SCHED_MARKER); + } + c_p->i = env.proc->i; + c_p->arity = env.proc->arity; + } - c_p->mbuf = env->proc->mbuf; - c_p->mbuf_sz += env->proc->mbuf_sz; +#ifdef DEBUG + if (nep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER) + nep->func = NULL; +#endif - } + erts_unblock_fpe(env.fpe_was_unmasked); + full_flush_env(&env); + free_tmp_objs(&env); - if (!c_p->off_heap.first) - c_p->off_heap.first = env->proc->off_heap.first; - else if (env->proc->off_heap.first) { - struct erl_off_heap_header *ohhp; - for (ohhp = env->proc->off_heap.first; ohhp->next; ohhp = ohhp->next) - ; - ohhp->next = c_p->off_heap.first; - c_p->off_heap.first = env->proc->off_heap.first; - } - c_p->off_heap.overhead += env->proc->off_heap.overhead; + return exiting; +} - return; - } #endif +static void full_flush_env(ErlNifEnv* env) +{ flush_env(env); +#ifdef ERTS_DIRTY_SCHEDULERS + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) + /* Dirty nif call using shadow process struct */ + erts_flush_dirty_shadow_proc(env->proc); +#endif } static void full_cache_env(ErlNifEnv* env) { #ifdef ERTS_DIRTY_SCHEDULERS - if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { - /* Dirty nif call using shadow process struct */ - Process *sproc = env->proc; - Process *c_p = sproc->next; - ASSERT(c_p); - ASSERT(is_scheduler() < 0); - ASSERT(env->proc->common.id == c_p->common.id); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) - & ERTS_PROC_LOCK_MAIN); - - sproc->htop = c_p->htop; - sproc->stop = c_p->stop; - sproc->hend = c_p->hend; - sproc->heap = c_p->heap; - sproc->abandoned_heap = c_p->abandoned_heap; - sproc->heap_sz = c_p->heap_sz; - sproc->high_water = c_p->high_water; - sproc->old_hend = c_p->old_hend; - sproc->old_htop = c_p->old_htop; - sproc->old_heap = c_p->old_heap; - sproc->mbuf = NULL; - sproc->mbuf_sz = 0; - ERTS_INIT_OFF_HEAP(&sproc->off_heap); - - env->hp_end = HEAP_LIMIT(c_p); - env->hp = HEAP_TOP(c_p); - env->heap_frag = NULL; - return; - } + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) + erts_cache_dirty_shadow_proc(env->proc); #endif - cache_env(env); } @@ -1303,22 +1316,15 @@ Eterm enif_make_badarg(ErlNifEnv* env) Eterm enif_raise_exception(ErlNifEnv* env, ERL_NIF_TERM reason) { - Process *c_p; - - execution_state(env, &c_p, NULL); - env->exception_thrown = 1; - c_p->fvalue = reason; - BIF_ERROR(c_p, EXC_ERROR); + env->proc->fvalue = reason; + BIF_ERROR(env->proc, EXC_ERROR); } int enif_has_pending_exception(ErlNifEnv* env, ERL_NIF_TERM* reason) { - if (env->exception_thrown && reason != NULL) { - Process *c_p; - execution_state(env, &c_p, NULL); - *reason = c_p->fvalue; - } + if (env->exception_thrown && reason != NULL) + *reason = env->proc->fvalue; return env->exception_thrown; } @@ -2279,188 +2285,28 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent) return ERTS_BIF_REDS_LEFT(proc) == 0; } -/* - * NIF exports need a few more items than the Export struct provides, - * including the erl_module_nif* and a NIF function pointer, so the - * NifExport below adds those. The Export member must be first in the - * struct. The saved_current, exception_thrown, saved_argc, rootset_extra, and - * rootset members are used to track the MFA, any pending exception, and - * arguments of the top NIF in case a chain of one or more - * enif_schedule_nif() calls results in an exception, since in that case - * the original MFA and registers have to be restored before returning to - * Erlang to ensure stacktrace information associated with the exception is - * correct. - */ -typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]); - -typedef struct { - Export exp; - struct erl_module_nif* m; - NativeFunPtr fp; - ErtsCodeMFA *saved_current; - int exception_thrown; - int saved_argc; - int rootset_extra; - Eterm rootset[1]; -} NifExport; - -/* - * If a process has saved arguments, they need to be part of the GC - * rootset. The function below is called from setup_rootset() in - * erl_gc.c. This function is declared in erl_process.h. Any exception term - * saved in the NifExport is also made part of the GC rootset here; it - * always resides in rootset[0]. - */ -int -erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj) -{ - NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - int gc = ep && (ep->saved_argc > 0 || ep->rootset[0] != NIL); - - if (gc) { - *objv = ep->rootset; - *nobj = 1 + ep->saved_argc; - } - return gc; -} - -int -erts_check_nif_export_in_area(Process *p, char *start, Uint size) -{ - NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p); - if (!nep || !nep->saved_current) - return 0; - if (ErtsInArea(nep->saved_current, start, size)) - return 1; - return 0; -} - -/* - * Allocate a NifExport and set it in proc specific data - */ -static NifExport* -allocate_nif_sched_data(Process* proc, int argc) -{ - NifExport* ep; - size_t total; - int i; - - total = sizeof(NifExport) + argc*sizeof(Eterm); - ep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, total); - sys_memset((void*) ep, 0, total); - ep->rootset_extra = argc; - ep->rootset[0] = NIL; - for (i=0; i<ERTS_NUM_CODE_IX; i++) { - ep->exp.addressv[i] = &ep->exp.beam[0]; - } - ep->exp.beam[0] = (BeamInstr) em_call_nif; - (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep); - return ep; -} - static ERTS_INLINE void -destroy_nif_export(NifExport *nif_export) +nif_export_cleanup_nif_mod(NifExport *ep) { - erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, (void *) nif_export); + if (erts_refc_dectest(&ep->m->rt_dtor_cnt, 0) == 0 && ep->m->mod == NULL) + close_lib(ep->m); + ep->m = NULL; } void -erts_destroy_nif_export(void *nif_export) +erts_nif_export_cleanup_nif_mod(NifExport *ep) { - destroy_nif_export((NifExport *) nif_export); + nif_export_cleanup_nif_mod(ep); } -/* - * Initialize a NifExport struct. Create it if needed and store it in the - * proc. The direct_fp function is what will be invoked by op_call_nif, and - * the indirect_fp function, if not NULL, is what the direct_fp function - * will call. If the allocated NifExport isn't enough to hold all of argv, - * allocate a larger one. Save MFA and registers only if the need_save - * parameter is true. - */ -static ERL_NIF_TERM -init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, - int need_save, int argc, const ERL_NIF_TERM argv[]) +static ERTS_INLINE void +nif_export_restore(Process *c_p, NifExport *ep, Eterm res) { - Process* proc; - Eterm* reg; - NifExport* ep; - int i, scheduler; - int orig_argc; - - execution_state(env, &proc, &scheduler); - - ASSERT(scheduler); - - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc) - & ERTS_PROC_LOCK_MAIN); - - reg = erts_proc_sched_data(proc)->x_reg_array; - - ASSERT(!need_save || proc->current); - orig_argc = need_save ? (int) proc->current->arity : 0; - - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - if (!ep) - ep = allocate_nif_sched_data(proc, orig_argc); - else if (need_save && ep->rootset_extra < orig_argc) { - NifExport* new_ep = allocate_nif_sched_data(proc, orig_argc); - destroy_nif_export(ep); - ep = new_ep; - } - if (env->exception_thrown) { - ep->exception_thrown = 1; - ep->rootset[0] = proc->fvalue; - } else { - ep->exception_thrown = 0; - ep->rootset[0] = NIL; - } - if (scheduler > 0) - ERTS_VBUMP_ALL_REDS(proc); - if (need_save) { - ep->saved_current = proc->current; - ep->saved_argc = orig_argc; - for (i = 0; i < orig_argc; i++) - ep->rootset[i+1] = reg[i]; - } - for (i = 0; i < argc; i++) - reg[i] = (Eterm) argv[i]; - proc->i = (BeamInstr*) ep->exp.addressv[0]; - ep->exp.info.mfa.module = proc->current->module; - ep->exp.info.mfa.function = proc->current->function; - ep->exp.info.mfa.arity = argc; - ep->exp.beam[1] = (BeamInstr) direct_fp; - ep->m = env->mod_nif; - ep->fp = indirect_fp; - proc->freason = TRAP; - proc->arity = argc; - return THE_NON_VALUE; + erts_nif_export_restore(c_p, ep, res); + ASSERT(ep->m); + nif_export_cleanup_nif_mod(ep); } -/* - * Restore saved MFA and registers. Registers are restored only when the - * exception flag is true. - */ -static void -restore_nif_mfa(Process* proc, NifExport* ep, int exception) -{ - int i; - - ERTS_SMP_LC_ASSERT(!(proc->static_flags - & ERTS_STC_FLG_SHADOW_PROC)); - ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc) - & ERTS_PROC_LOCK_MAIN); - - ASSERT(ep->saved_current != &ep->exp.info.mfa); - proc->current = ep->saved_current; - ep->saved_current = NULL; - if (exception) { - Eterm* reg = erts_proc_sched_data(proc)->x_reg_array; - for (i = 0; i < ep->saved_argc; i++) - reg[i] = ep->rootset[i+1]; - } - ep->saved_argc = 0; -} #ifdef ERTS_DIRTY_SCHEDULERS @@ -2469,7 +2315,7 @@ restore_nif_mfa(Process* proc, NifExport* ep, int exception) * switch the process off a dirty scheduler thread and back onto a regular * scheduler thread, and then return the result from the dirty NIF. It also * restores the original NIF MFA when necessary based on the value of - * ep->fp set by execute_dirty_nif via init_nif_sched_data -- non-NULL + * ep->func set by execute_dirty_nif via init_nif_sched_data -- non-NULL * means restore, NULL means do not restore. */ static ERL_NIF_TERM @@ -2484,9 +2330,7 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); - ASSERT(!ep->exception_thrown); - if (ep->fp) - restore_nif_mfa(proc, ep, 0); + nif_export_restore(proc, ep, argv[0]); return argv[0]; } @@ -2496,148 +2340,100 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) static ERL_NIF_TERM dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + ERL_NIF_TERM ret; Process* proc; NifExport* ep; + Eterm exception; execution_state(env, &proc, NULL); + ASSERT(argc == 1); ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); - ASSERT(ep->exception_thrown); - if (ep->fp) - restore_nif_mfa(proc, ep, 1); - return enif_raise_exception(env, ep->rootset[0]); + exception = argv[0]; /* argv overwritten by restore below... */ + nif_export_cleanup_nif_mod(ep); + ret = enif_raise_exception(env, exception); + + /* Restore orig info for error and clear nif export in handle_error() */ + proc->freason |= EXF_RESTORE_NIF; + return ret; } /* - * Dirty NIF execution wrapper function. Invoke an application's dirty NIF, - * then check the result and schedule the appropriate finalizer function - * where needed. Also restore the original NIF MFA when appropriate. + * Dirty NIF scheduling wrapper function. Schedule a dirty NIF to execute. + * The dirty scheduler thread type (CPU or I/O) is indicated in flags + * parameter. */ -static ERL_NIF_TERM -execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +static ERTS_INLINE ERL_NIF_TERM +schedule_dirty_nif(ErlNifEnv* env, int flags, NativeFunPtr fp, + Eterm func_name, int argc, const ERL_NIF_TERM argv[]) { Process* proc; - NativeFunPtr fp; - NifExport* ep; - ERL_NIF_TERM result; - - execution_state(env, &proc, NULL); - - ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); - fp = ep->fp; - ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); - - /* - * Set ep->fp to NULL before the native call so we know later whether it scheduled another NIF for execution - */ - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - ASSERT(ep && fp); - - ep->fp = NULL; - erts_smp_atomic32_read_band_mb(&proc->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC - | ERTS_PSFLG_DIRTY_IO_PROC)); + ASSERT(is_atom(func_name)); + ASSERT(fp); - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); + ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND); - result = (*fp)(env, argc, argv); + execution_state(env, &proc, NULL); - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); + (void) erts_smp_atomic32_read_bset_nob(&proc->state, + (ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC), + (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND + ? ERTS_PSFLG_DIRTY_CPU_PROC + : ERTS_PSFLG_DIRTY_IO_PROC)); - if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0 && env->mod_nif->mod == NULL) - close_lib(env->mod_nif); - /* - * If no more NIFs were scheduled by the native call via - * enif_schedule_nif(), then ep->fp will still be NULL as set above, in - * which case we need to restore the original NIF calling - * context. Reuse fp essentially as a boolean for this, passing it to - * init_nif_sched_data below. Both dirty_nif_exception and - * dirty_nif_finalizer then check ep->fp to decide whether or not to - * restore the original calling context. - */ - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - ASSERT(ep); - if (ep->fp) - fp = NULL; - if (is_non_value(result) || env->exception_thrown) { - if (proc->freason != TRAP) { - return init_nif_sched_data(env, dirty_nif_exception, fp, 0, argc, argv); - } else { - if (ep->fp == NULL) - restore_nif_mfa(proc, ep, 1); - return THE_NON_VALUE; - } - } - else - return init_nif_sched_data(env, dirty_nif_finalizer, fp, 0, 1, &result); + return schedule(env, fp, NULL, proc->current->module, func_name, argc, argv); } -/* - * Dirty NIF scheduling wrapper function. Schedule a dirty NIF to execute - * via the execute_dirty_nif() wrapper function. The dirty scheduler thread - * type (CPU or I/O) is indicated in flags parameter. - */ static ERTS_INLINE ERL_NIF_TERM -schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[]) +static_schedule_dirty_nif(ErlNifEnv* env, erts_aint32_t dirty_psflg, + int argc, const ERL_NIF_TERM argv[]) { - ERL_NIF_TERM result; - erts_aint32_t act, dirty_flag; - Process* proc; + Process *proc; + NifExport *ep; + Eterm mod, func; NativeFunPtr fp; - NifExport* ep; - int need_save, scheduler; - execution_state(env, &proc, &scheduler); - if (scheduler <= 0) { - ASSERT(scheduler < 0); - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); - } + execution_state(env, &proc, NULL); + + /* + * Called in order to schedule statically determined + * dirty NIF calls... + * + * Note that 'current' does not point into a NifExport + * structure; only a structure with similar + * parts (located in code). + */ ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); - fp = ep->fp; + mod = proc->current->module; + func = proc->current->function; + fp = (NativeFunPtr) ep->func; + ASSERT(is_atom(mod) && is_atom(func)); ASSERT(fp); - ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND); - - if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND) - dirty_flag = ERTS_PSFLG_DIRTY_CPU_PROC; - else - dirty_flag = ERTS_PSFLG_DIRTY_IO_PROC; - - act = erts_smp_atomic32_read_bor_nob(&proc->state, dirty_flag); - if (!(act & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC))) - erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1); - else if ((act & (ERTS_PSFLG_DIRTY_CPU_PROC - | ERTS_PSFLG_DIRTY_IO_PROC)) & ~dirty_flag) { - /* clear other flag... */ - if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND) - dirty_flag = ERTS_PSFLG_DIRTY_IO_PROC; - else - dirty_flag = ERTS_PSFLG_DIRTY_CPU_PROC; - erts_smp_atomic32_read_band_nob(&proc->state, ~dirty_flag); - } + (void) erts_smp_atomic32_read_bset_nob(&proc->state, + (ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC), + dirty_psflg); - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - need_save = (ep == NULL || !ep->saved_current); - result = init_nif_sched_data(env, execute_dirty_nif, fp, need_save, argc, argv); - if (scheduler <= 0) - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); - return result; + return schedule(env, fp, NULL, mod, func, argc, argv); } static ERL_NIF_TERM -schedule_dirty_io_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +static_schedule_dirty_io_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_IO_BOUND, argc, argv); + return static_schedule_dirty_nif(env, ERTS_PSFLG_DIRTY_IO_PROC, argc, argv); } static ERL_NIF_TERM -schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +static_schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_CPU_BOUND, argc, argv); + return static_schedule_dirty_nif(env, ERTS_PSFLG_DIRTY_CPU_PROC, argc, argv); } #endif /* ERTS_DIRTY_SCHEDULERS */ @@ -2656,23 +2452,42 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ERL_NIF_TERM result; execution_state(env, &proc, NULL); - ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); - fp = ep->fp; - ASSERT(!env->exception_thrown); - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->func; ASSERT(ep); - ep->fp = NULL; + ASSERT(!env->exception_thrown); + + fp = (NativeFunPtr) ep->func; + +#ifdef DEBUG + ep->func = ERTS_DBG_NIF_NOT_SCHED_MARKER; +#endif + result = (*fp)(env, argc, argv); - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - ASSERT(ep); - /* - * If no NIFs were scheduled by the native call via - * enif_schedule_nif(), then ep->fp will still be NULL as set above, in - * which case we need to restore the original NIF MFA. - */ - if (ep->fp == NULL) - restore_nif_mfa(proc, ep, env->exception_thrown); + + ASSERT(ep == ERTS_PROC_GET_NIF_TRAP_EXPORT(proc)); + + if (is_value(result) || proc->freason != TRAP) { + /* Done (not rescheduled)... */ + ASSERT(ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER); + if (!env->exception_thrown) + nif_export_restore(proc, ep, result); + else { + nif_export_cleanup_nif_mod(ep); + /* + * Restore orig info for error and clear nif + * export in handle_error() + */ + proc->freason |= EXF_RESTORE_NIF; + } + } + +#ifdef DEBUG + if (ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER) + ep->func = NULL; +#endif + return result; } @@ -2682,9 +2497,8 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, int argc, const ERL_NIF_TERM argv[]) { Process* proc; - NifExport* ep; ERL_NIF_TERM fun_name_atom, result; - int need_save, scheduler; + int scheduler; if (argc > MAX_ARG) return enif_make_badarg(env); @@ -2699,35 +2513,16 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); } - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - need_save = (ep == NULL || !ep->saved_current); - - if (flags) { + if (flags == 0) + result = schedule(env, execute_nif, fp, proc->current->module, + fun_name_atom, argc, argv); #ifdef ERTS_DIRTY_SCHEDULERS - NativeFunPtr sched_fun; - int chkflgs = (flags & (ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND)); - if (chkflgs == ERL_NIF_DIRTY_JOB_IO_BOUND) - sched_fun = schedule_dirty_io_nif; - else if (chkflgs == ERL_NIF_DIRTY_JOB_CPU_BOUND) - sched_fun = schedule_dirty_cpu_nif; - else { - result = enif_make_badarg(env); - goto done; - } - result = init_nif_sched_data(env, sched_fun, fp, need_save, argc, argv); -#else - result = enif_make_badarg(env); + else if (!(flags & ~(ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND))) + result = schedule_dirty_nif(env, flags, fp, fun_name_atom, argc, argv); #endif - goto done; - } else - result = init_nif_sched_data(env, execute_nif, fp, need_save, argc, argv); - - ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - ASSERT(ep); - ep->exp.info.mfa.function = (BeamInstr) fun_name_atom; + result = enif_make_badarg(env); -done: if (scheduler < 0) erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); @@ -3356,11 +3151,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) mod_atom, f->name, f->arity); #endif } -#ifdef ERTS_DIRTY_SCHEDULERS - else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (4)) -#else - else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (3)) -#endif + else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) + < BEAM_NIF_MIN_FUNC_SZ) { ret = load_nif_error(BIF_P,bad_lib,"No explicit call to load_nif" " in module (%T:%s/%u too small)", @@ -3434,8 +3226,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) if (f->flags) { code_ptr[3] = (BeamInstr) f->fptr; code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? - (BeamInstr) schedule_dirty_io_nif : - (BeamInstr) schedule_dirty_cpu_nif; + (BeamInstr) static_schedule_dirty_io_nif : + (BeamInstr) static_schedule_dirty_cpu_nif; } else #endif @@ -3546,8 +3338,8 @@ Eterm erts_nif_call_function(Process *p, Process *tracee, #endif if (p) { /* This is almost a normal nif call like in beam_emu, - except that any heap fragment created in the nif will be - discarded without checking if anything in it is live. + except that any heap consumed by the nif will be + released without checking if anything in it is live. This is because we cannot do a GC here as we don't know the number of live registers that have to be preserved. This means that any heap part of the returned term may diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 3102e44c11..4836b9e2d3 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -936,7 +936,7 @@ enqueue_port(ErtsRunQueue *runq, Port *pp) erts_smp_inc_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL); #ifdef ERTS_SMP - if (runq->halt_in_progress) + if (ERTS_RUNQ_FLGS_GET_NOB(runq) & ERTS_RUNQ_FLG_HALTING) erts_non_empty_runq(runq); #endif } diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index 2a6bd165a3..e3550e878e 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -188,13 +188,11 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id) ptsp->taskq.in.last = NULL; erts_smp_atomic32_init_nob(&ptsp->flags, 0); #ifdef ERTS_SMP - erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id, #ifdef ERTS_ENABLE_LOCK_COUNT - (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) -#else - 1 + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)) + lock_str = NULL; #endif - ); + erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id); #endif } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 4fb0f9e975..f80ebdf31b 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -48,6 +48,7 @@ #include "erl_bif_unique.h" #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" +#include "erl_nfunc_sched.h" #define ERTS_CHECK_TIME_REDS CONTEXT_REDS #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0) @@ -106,9 +107,31 @@ #define LOW_BIT (1 << PRIORITY_LOW) #define PORT_BIT (1 << ERTS_PORT_PRIO_LEVEL) -#define ERTS_EMPTY_RUNQ(RQ) \ - ((ERTS_RUNQ_FLGS_GET_NOB((RQ)) & ERTS_RUNQ_FLGS_QMASK) == 0 \ - && (RQ)->misc.start == NULL) +#define ERTS_IS_RUNQ_EMPTY_FLGS(FLGS) \ + (!((FLGS) & (ERTS_RUNQ_FLGS_QMASK|ERTS_RUNQ_FLG_MISC_OP))) + +#define ERTS_IS_RUNQ_EMPTY_PORTS_FLGS(FLGS) \ + (!((FLGS) & (PORT_BIT|ERTS_RUNQ_FLG_MISC_OP))) + +#define ERTS_EMPTY_RUNQ(RQ) \ + ERTS_IS_RUNQ_EMPTY_FLGS(ERTS_RUNQ_FLGS_GET_NOB((RQ))) + +#define ERTS_EMPTY_RUNQ_PORTS(RQ) \ + ERTS_IS_RUNQ_EMPTY_FLGS(ERTS_RUNQ_FLGS_GET_NOB((RQ))) + +static ERTS_INLINE int +runq_got_work_to_execute_flags(Uint32 flags) +{ + if (flags & ERTS_RUNQ_FLG_HALTING) + return !ERTS_IS_RUNQ_EMPTY_PORTS_FLGS(flags); + return !ERTS_IS_RUNQ_EMPTY_FLGS(flags); +} + +static ERTS_INLINE int +runq_got_work_to_execute(ErtsRunQueue *rq) +{ + return runq_got_work_to_execute_flags(ERTS_RUNQ_FLGS_GET_NOB(rq)); +} #undef RUNQ_READ_RQ #undef RUNQ_SET_RQ @@ -140,9 +163,6 @@ do { \ # define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) #endif -#define ERTS_EMPTY_RUNQ_PORTS(RQ) \ - (RUNQ_READ_LEN(&(RQ)->ports.info.len) == 0 && (RQ)->misc.start == NULL) - const Process erts_invalid_process = {{ERTS_INVALID_PID}}; extern BeamInstr beam_apply[]; @@ -193,170 +213,145 @@ static ErtsAuxWorkData *aux_thread_aux_work_data; #define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2) #define ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN (((erts_aint32_t) 1) << 3) -typedef enum { - ERTS_SCHED_NORMAL, - ERTS_SCHED_DIRTY_CPU, - ERTS_SCHED_DIRTY_IO -} ErtsSchedType; - typedef struct { int ongoing; ErtsProcList *blckrs; ErtsProcList *chngq; } ErtsMultiSchedulingBlock; +typedef struct { + Uint32 normal; +#ifdef ERTS_DIRTY_SCHEDULERS + Uint32 dirty_cpu; + Uint32 dirty_io; +#endif +} ErtsSchedTypeCounters; + static struct { erts_smp_mtx_t mtx; - Uint32 online; - Uint32 curr_online; - Uint32 active; + ErtsSchedTypeCounters online; + ErtsSchedTypeCounters curr_online; + ErtsSchedTypeCounters active; erts_smp_atomic32_t changing; ErtsProcList *chngq; Eterm changer; ErtsMultiSchedulingBlock nmsb; /* Normal multi Scheduling Block */ ErtsMultiSchedulingBlock msb; /* Multi Scheduling Block */ +#ifdef ERTS_DIRTY_SCHEDULERS + ErtsSchedType last_msb_dirty_type; +#endif } schdlr_sspnd; -#define ERTS_SCHDLR_SSPND_S_BITS 10 -#define ERTS_SCHDLR_SSPND_DCS_BITS 11 -#define ERTS_SCHDLR_SSPND_DIS_BITS 11 - -#define ERTS_SCHDLR_SSPND_S_MASK ((1 << ERTS_SCHDLR_SSPND_S_BITS)-1) -#define ERTS_SCHDLR_SSPND_DCS_MASK ((1 << ERTS_SCHDLR_SSPND_DCS_BITS)-1) -#define ERTS_SCHDLR_SSPND_DIS_MASK ((1 << ERTS_SCHDLR_SSPND_DIS_BITS)-1) - -#define ERTS_SCHDLR_SSPND_S_SHIFT 0 -#define ERTS_SCHDLR_SSPND_DCS_SHIFT (ERTS_SCHDLR_SSPND_S_SHIFT \ - + ERTS_SCHDLR_SSPND_S_BITS) -#define ERTS_SCHDLR_SSPND_DIS_SHIFT (ERTS_SCHDLR_SSPND_DCS_SHIFT \ - + ERTS_SCHDLR_SSPND_DCS_BITS) - -#if (ERTS_SCHDLR_SSPND_S_BITS \ - + ERTS_SCHDLR_SSPND_DCS_BITS \ - + ERTS_SCHDLR_SSPND_DIS_BITS) > 32 -# error Wont fit in Uint32 -#endif +static void init_scheduler_suspend(void); -#if (ERTS_MAX_NO_OF_SCHEDULERS-1) > ERTS_SCHDLR_SSPND_S_MASK -# error Max no schedulers wont fit in its bit-field -#endif -#if ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS > ERTS_SCHDLR_SSPND_DCS_MASK -# error Max no dirty cpu schedulers wont fit in its bit-field -#endif -#if ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS > ERTS_SCHDLR_SSPND_DIS_MASK -# error Max no dirty io schedulers wont fit in its bit-field +static ERTS_INLINE Uint32 +schdlr_sspnd_eq_nscheds(ErtsSchedTypeCounters *val1p, ErtsSchedTypeCounters *val2p) +{ + int res = val1p->normal == val2p->normal; +#ifdef ERTS_DIRTY_SCHEDULERS + res &= val1p->dirty_cpu == val2p->dirty_cpu; + res &= val1p->dirty_io == val2p->dirty_io; #endif - -#define ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(S, DCS, DIS) \ - ((((Uint32) (((S) & ERTS_SCHDLR_SSPND_S_MASK))-1) \ - << ERTS_SCHDLR_SSPND_S_SHIFT) \ - | ((((Uint32) ((DCS) & ERTS_SCHDLR_SSPND_DCS_MASK)) \ - << ERTS_SCHDLR_SSPND_DCS_SHIFT)) \ - | ((((Uint32) ((DIS) & ERTS_SCHDLR_SSPND_DIS_MASK)) \ - << ERTS_SCHDLR_SSPND_DIS_SHIFT))) - -static void init_scheduler_suspend(void); + return res; +} static ERTS_INLINE Uint32 -schdlr_sspnd_get_nscheds(Uint32 *valp, ErtsSchedType type) +schdlr_sspnd_get_nscheds(ErtsSchedTypeCounters *valp, + ErtsSchedType type) { - Uint32 res = (Uint32) (*valp); switch (type) { case ERTS_SCHED_NORMAL: - res >>= ERTS_SCHDLR_SSPND_S_SHIFT; - res &= (Uint32) ERTS_SCHDLR_SSPND_S_MASK; - res++; - break; + return valp->normal; +#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: - res >>= ERTS_SCHDLR_SSPND_DCS_SHIFT; - res &= (Uint32) ERTS_SCHDLR_SSPND_DCS_MASK; - break; + return valp->dirty_cpu; case ERTS_SCHED_DIRTY_IO: - res >>= ERTS_SCHDLR_SSPND_DIS_SHIFT; - res &= (Uint32) ERTS_SCHDLR_SSPND_DIS_MASK; - break; + return valp->dirty_io; +#else + case ERTS_SCHED_DIRTY_CPU: + case ERTS_SCHED_DIRTY_IO: + return 0; +#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); return 0; } +} +static ERTS_INLINE Uint32 +schdlr_sspnd_get_nscheds_tot(ErtsSchedTypeCounters *valp) +{ + Uint32 res = valp->normal; +#ifdef ERTS_DIRTY_SCHEDULERS + res += valp->dirty_cpu; + res += valp->dirty_io; +#endif return res; } static ERTS_INLINE void -schdlr_sspnd_dec_nscheds(Uint32 *valp, ErtsSchedType type) +schdlr_sspnd_dec_nscheds(ErtsSchedTypeCounters *valp, + ErtsSchedType type) { ASSERT(schdlr_sspnd_get_nscheds(valp, type) > 0); switch (type) { case ERTS_SCHED_NORMAL: - *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT; + valp->normal--; break; +#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: - *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT; + valp->dirty_cpu--; break; case ERTS_SCHED_DIRTY_IO: - *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT; + valp->dirty_io--; break; +#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } } static ERTS_INLINE void -schdlr_sspnd_inc_nscheds(Uint32 *valp, ErtsSchedType type) +schdlr_sspnd_inc_nscheds(ErtsSchedTypeCounters *valp, + ErtsSchedType type) { switch (type) { case ERTS_SCHED_NORMAL: - ASSERT(schdlr_sspnd_get_nscheds(valp, type) - < ERTS_MAX_NO_OF_SCHEDULERS-1); - *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT; + valp->normal++; break; +#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: - ASSERT(schdlr_sspnd_get_nscheds(valp, type) - < ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS); - *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT; + valp->dirty_cpu++; break; case ERTS_SCHED_DIRTY_IO: - ASSERT(schdlr_sspnd_get_nscheds(valp, type) - < ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS); - *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT; + valp->dirty_io++; break; +#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } } static ERTS_INLINE void -schdlr_sspnd_set_nscheds(Uint32 *valp, ErtsSchedType type, Uint32 no) +schdlr_sspnd_set_nscheds(ErtsSchedTypeCounters *valp, + ErtsSchedType type, Uint32 no) { - Uint32 val = *valp; - switch (type) { case ERTS_SCHED_NORMAL: - ASSERT(no > 0); - val &= ~(((Uint32) ERTS_SCHDLR_SSPND_S_MASK) - << ERTS_SCHDLR_SSPND_S_SHIFT); - val |= (((no-1) & ((Uint32) ERTS_SCHDLR_SSPND_S_MASK)) - << ERTS_SCHDLR_SSPND_S_SHIFT); + valp->normal = no; break; +#ifdef ERTS_DIRTY_SCHEDULERS case ERTS_SCHED_DIRTY_CPU: - val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK) - << ERTS_SCHDLR_SSPND_DCS_SHIFT); - val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK)) - << ERTS_SCHDLR_SSPND_DCS_SHIFT); + valp->dirty_cpu = no; break; case ERTS_SCHED_DIRTY_IO: - val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK) - << ERTS_SCHDLR_SSPND_DIS_SHIFT); - val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK)) - << ERTS_SCHDLR_SSPND_DIS_SHIFT); + valp->dirty_io = no; break; +#endif default: ERTS_INTERNAL_ERROR("Invalid scheduler type"); } - - *valp = val; } static struct { @@ -809,15 +804,28 @@ erts_late_init_process(void) } +#define ERTS_SCHED_WTIME_IDLE ~((Uint64) 0) + static void -init_sched_wall_time(ErtsSchedWallTime *swtp) +init_sched_wall_time(ErtsSchedulerData *esdp, Uint64 time_stamp) { - swtp->need = erts_sched_balance_util; - swtp->enabled = 0; - swtp->start = 0; - swtp->working.total = 0; - swtp->working.start = 0; - swtp->working.currently = 0; +#ifdef ERTS_DIRTY_SCHEDULERS + if (esdp->type != ERTS_SCHED_NORMAL) { + erts_atomic32_init_nob(&esdp->sched_wall_time.u.mod, 0); + esdp->sched_wall_time.enabled = 1; + esdp->sched_wall_time.start = time_stamp; + esdp->sched_wall_time.working.total = 0; + esdp->sched_wall_time.working.start = ERTS_SCHED_WTIME_IDLE; + } + else +#endif + { + esdp->sched_wall_time.u.need = erts_sched_balance_util; + esdp->sched_wall_time.enabled = 0; + esdp->sched_wall_time.start = 0; + esdp->sched_wall_time.working.total = 0; + esdp->sched_wall_time.working.start = 0; + } } static ERTS_INLINE Uint64 @@ -1008,31 +1016,145 @@ init_runq_sched_util(ErtsRunQueueSchedUtil *rqsu, int enabled) #endif /* ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT */ -static ERTS_INLINE void +#ifdef ERTS_DIRTY_SCHEDULERS + +typedef struct { + Uint64 working; + Uint64 total; +} ErtsDirtySchedWallTime; + +static void +read_dirty_sched_wall_time(ErtsSchedulerData *esdp, ErtsDirtySchedWallTime *info) +{ + erts_aint32_t mod1; + Uint64 working, start, ts; + + mod1 = erts_atomic32_read_nob(&esdp->sched_wall_time.u.mod); + + while (1) { + erts_aint32_t mod2; + + /* Spin until values are not written... */ + while (1) { + if ((mod1 & 1) == 0) + break; + ERTS_SPIN_BODY; + mod1 = erts_atomic32_read_nob(&esdp->sched_wall_time.u.mod); + } + + ERTS_THR_READ_MEMORY_BARRIER; + + working = esdp->sched_wall_time.working.total; + start = esdp->sched_wall_time.working.start; + + ERTS_THR_READ_MEMORY_BARRIER; + + mod2 = erts_atomic32_read_nob(&esdp->sched_wall_time.u.mod); + if (mod1 == mod2) + break; + mod1 = mod2; + } + + ts = sched_wall_time_ts(); + ts -= esdp->sched_wall_time.start; + + info->total = ts; + + if (start == ERTS_SCHED_WTIME_IDLE || ts < start) + info->working = working; + else + info->working = working + (ts - start); + + if (info->working > info->total) + info->working = info->total; +} + +#endif + +#ifdef ERTS_SMP + +static void +dirty_sched_wall_time_change(ErtsSchedulerData *esdp, int working) +{ + erts_aint32_t mod; + Uint64 ts = sched_wall_time_ts(); + + ts -= esdp->sched_wall_time.start; + + /* + * This thread is the only thread writing in + * this sched_wall_time struct. We set 'mod' to + * an odd value while writing... + */ + mod = erts_atomic32_read_dirty(&esdp->sched_wall_time.u.mod); + ASSERT((mod & 1) == 0); + mod++; + + erts_atomic32_set_nob(&esdp->sched_wall_time.u.mod, mod); + ERTS_THR_WRITE_MEMORY_BARRIER; + + if (working) { + ASSERT(esdp->sched_wall_time.working.start + == ERTS_SCHED_WTIME_IDLE); + + esdp->sched_wall_time.working.start = ts; + + } + else { + Uint64 total; + + ASSERT(esdp->sched_wall_time.working.start + != ERTS_SCHED_WTIME_IDLE); + + total = esdp->sched_wall_time.working.total; + total += ts - esdp->sched_wall_time.working.start; + + esdp->sched_wall_time.working.total = total; + esdp->sched_wall_time.working.start = ERTS_SCHED_WTIME_IDLE; + + + } + + ERTS_THR_WRITE_MEMORY_BARRIER; + mod++; + erts_atomic32_set_nob(&esdp->sched_wall_time.u.mod, mod); + +#if 0 + if (!working) { + ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_BUSY_WAIT); + } else { + ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_OTHER); + } +#endif +} + +#endif /* ERTS_SMP */ + +static void sched_wall_time_change(ErtsSchedulerData *esdp, int working) { - if (esdp->sched_wall_time.need) { + if (esdp->sched_wall_time.u.need) { Uint64 ts = sched_wall_time_ts(); #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT - update_avg_sched_util(esdp, ts, working); + update_avg_sched_util(esdp, ts, working); #endif if (esdp->sched_wall_time.enabled) { if (working) { -#ifdef DEBUG - ASSERT(!esdp->sched_wall_time.working.currently); - esdp->sched_wall_time.working.currently = 1; -#endif + ASSERT(esdp->sched_wall_time.working.start + == ERTS_SCHED_WTIME_IDLE); ts -= esdp->sched_wall_time.start; esdp->sched_wall_time.working.start = ts; } else { -#ifdef DEBUG - ASSERT(esdp->sched_wall_time.working.currently); - esdp->sched_wall_time.working.currently = 0; -#endif + ASSERT(esdp->sched_wall_time.working.start + != ERTS_SCHED_WTIME_IDLE); ts -= esdp->sched_wall_time.start; ts -= esdp->sched_wall_time.working.start; esdp->sched_wall_time.working.total += ts; +#ifdef DEBUG + esdp->sched_wall_time.working.start + = ERTS_SCHED_WTIME_IDLE; +#endif } } } @@ -1052,6 +1174,10 @@ typedef struct { Eterm ref_heap[REF_THING_SIZE]; Uint req_sched; erts_smp_atomic32_t refc; +#ifdef ERTS_DIRTY_SCHEDULERS + int want_dirty_cpu; + int want_dirty_io; +#endif } ErtsSchedWallTimeReq; typedef struct { @@ -1073,6 +1199,7 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(screq, 5, ERTS_ALC_T_SYS_CHECK_REQ) + static void reply_sched_wall_time(void *vswtrp) { @@ -1096,23 +1223,23 @@ reply_sched_wall_time(void *vswtrp) #endif if (swtrp->set) { if (!swtrp->enable && esdp->sched_wall_time.enabled) { - esdp->sched_wall_time.need = erts_sched_balance_util; + esdp->sched_wall_time.u.need = erts_sched_balance_util; esdp->sched_wall_time.enabled = 0; } else if (swtrp->enable && !esdp->sched_wall_time.enabled) { Uint64 ts = sched_wall_time_ts(); - esdp->sched_wall_time.need = 1; + esdp->sched_wall_time.u.need = 1; esdp->sched_wall_time.enabled = 1; esdp->sched_wall_time.start = ts; esdp->sched_wall_time.working.total = 0; esdp->sched_wall_time.working.start = 0; - esdp->sched_wall_time.working.currently = 1; } } if (esdp->sched_wall_time.enabled) { Uint64 ts = sched_wall_time_ts(); - ASSERT(esdp->sched_wall_time.working.currently); + ASSERT(esdp->sched_wall_time.working.start + != ERTS_SCHED_WTIME_IDLE); ts -= esdp->sched_wall_time.start; total = ts; ts -= esdp->sched_wall_time.working.start; @@ -1123,30 +1250,117 @@ reply_sched_wall_time(void *vswtrp) hpp = NULL; szp = &sz; - while (1) { - if (hpp) - ref_copy = STORE_NC(hpp, ohp, swtrp->ref); - else - *szp += REF_THING_SIZE; +#ifdef ERTS_DIRTY_SCHEDULERS + if (esdp->sched_wall_time.enabled + && swtrp->req_sched == esdp->no + && (swtrp->want_dirty_cpu || swtrp->want_dirty_io)) { + /* Reply with info about this scheduler and all dirty schedulers... */ + ErtsDirtySchedWallTime *dswt; + int ix, no_dirty_scheds, want_dcpu, want_dio, soffset; + + want_dcpu = swtrp->want_dirty_cpu; + want_dio = swtrp->want_dirty_io; + + no_dirty_scheds = 0; + if (want_dcpu) + no_dirty_scheds += erts_no_dirty_cpu_schedulers; + if (want_dio) + no_dirty_scheds += erts_no_dirty_io_schedulers; + + ASSERT(no_dirty_scheds); + + dswt = erts_alloc(ERTS_ALC_T_TMP, + sizeof(ErtsDirtySchedWallTime) + * no_dirty_scheds); + + for (ix = 0; ix < no_dirty_scheds; ix++) { + ErtsSchedulerData *esdp; + if (want_dcpu && ix < erts_no_dirty_cpu_schedulers) + esdp = &erts_aligned_dirty_cpu_scheduler_data[ix].esd; + else { + int dio_ix = ix - erts_no_dirty_cpu_schedulers; + esdp = &erts_aligned_dirty_io_scheduler_data[dio_ix].esd; + } + read_dirty_sched_wall_time(esdp, &dswt[ix]); + } - if (swtrp->set) - msg = ref_copy; - else { - msg = (!esdp->sched_wall_time.enabled - ? am_notsup - : erts_bld_tuple(hpp, szp, 3, - make_small(esdp->no), - erts_bld_uint64(hpp, szp, working), - erts_bld_uint64(hpp, szp, total))); + soffset = erts_no_schedulers + 1; - msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); - } - if (hpp) - break; + if (!want_dcpu) { + ASSERT(want_dio); + soffset += erts_no_dirty_cpu_schedulers; + } - mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); - szp = NULL; - hpp = &hp; + while (1) { + if (hpp) + ref_copy = STORE_NC(hpp, ohp, swtrp->ref); + else + *szp += REF_THING_SIZE; + + ASSERT(!swtrp->set); + + /* info about dirty schedulers... */ + msg = NIL; + for (ix = no_dirty_scheds-1; ix >= 0; ix--) { + msg = erts_bld_cons(hpp, szp, + erts_bld_tuple(hpp, szp, 3, + make_small(ix+soffset), + erts_bld_uint64(hpp, szp, + dswt[ix].working), + erts_bld_uint64(hpp, szp, + dswt[ix].total)), + msg); + } + /* info about this scheduler... */ + msg = erts_bld_cons(hpp, szp, + erts_bld_tuple(hpp, szp, 3, + make_small(esdp->no), + erts_bld_uint64(hpp, szp, working), + erts_bld_uint64(hpp, szp, total)), + msg); + + msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); + + if (hpp) + break; + + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); + szp = NULL; + hpp = &hp; + } + + erts_free(ERTS_ALC_T_TMP, dswt); + } + else +#endif + { + /* Reply with info about this scheduler only... */ + + while (1) { + if (hpp) + ref_copy = STORE_NC(hpp, ohp, swtrp->ref); + else + *szp += REF_THING_SIZE; + + if (swtrp->set) + msg = ref_copy; + else { + msg = (!esdp->sched_wall_time.enabled + ? am_undefined + : erts_bld_tuple(hpp, szp, 3, + make_small(esdp->no), + erts_bld_uint64(hpp, szp, working), + erts_bld_uint64(hpp, szp, total))); + + msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); + } + if (hpp) + break; + + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); + szp = NULL; + hpp = &hp; + } } erts_queue_message(rp, rp_locks, mp, msg, am_system); @@ -1164,7 +1378,8 @@ reply_sched_wall_time(void *vswtrp) } Eterm -erts_sched_wall_time_request(Process *c_p, int set, int enable) +erts_sched_wall_time_request(Process *c_p, int set, int enable, + int want_dirty_cpu, int want_dirty_io) { ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); Eterm ref; @@ -1186,6 +1401,10 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable) swtrp->proc = c_p; swtrp->ref = STORE_NC(&hp, NULL, ref); swtrp->req_sched = esdp->no; +#ifdef ERTS_DIRTY_SCHEDULERS + swtrp->want_dirty_cpu = want_dirty_cpu; + swtrp->want_dirty_io = want_dirty_io; +#endif erts_smp_atomic32_init_nob(&swtrp->refc, (erts_aint32_t) erts_no_schedulers); @@ -2059,6 +2278,7 @@ handle_thr_prgr_later_op(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int wait #endif for (lops = 0; lops < ERTS_MAX_THR_PRGR_LATER_OPS; lops++) { ErtsThrPrgrLaterOp *lop = awdp->later_op.first; + if (!erts_thr_progress_has_reached_this(current, lop->later)) return aux_work & ~ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP; awdp->later_op.first = lop->next; @@ -2297,7 +2517,8 @@ static ERTS_INLINE erts_aint32_t handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS); - awdp->esdp->run_queue->halt_in_progress = 1; + ERTS_RUNQ_FLGS_SET(awdp->esdp->run_queue, ERTS_RUNQ_FLG_HALTING); + if (erts_smp_atomic32_dec_read_acqb(&erts_halt_progress) == 0) { int i, max = erts_ptab_max(&erts_port); erts_smp_atomic32_set_nob(&erts_halt_progress, 1); @@ -2837,11 +3058,12 @@ static erts_aint32_t sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi) { erts_aint32_t oflgs; - erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING); + erts_aint32_t nflgs; erts_aint32_t xflgs = 0; do { + nflgs = (xflgs & ERTS_SSI_FLG_MSB_EXEC); + nflgs |= ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; @@ -2863,7 +3085,7 @@ sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi) if (oflgs == xflgs) return nflgs; xflgs = oflgs; - nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + nflgs |= oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC); } while (oflgs & ERTS_SSI_FLG_WAITING); return oflgs; } @@ -2913,7 +3135,7 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) return oflgs; } xflgs = oflgs; - nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + nflgs |= oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC); } } @@ -3048,6 +3270,8 @@ aux_thread(void *unused) return NULL; } +static void suspend_scheduler(ErtsSchedulerData *esdp); + #endif /* ERTS_SMP */ static void @@ -3106,8 +3330,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) tse_wait: - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && thr_prgr_active != working) - sched_wall_time_change(esdp, thr_prgr_active); + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + dirty_sched_wall_time_change(esdp, working = 0); + else if (thr_prgr_active != working) + sched_wall_time_change(esdp, working = thr_prgr_active); while (1) { ErtsMonotonicTime current_time = 0; @@ -3213,13 +3439,17 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) } - if (flgs & ~ERTS_SSI_FLG_SUSPENDED) - erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) + erts_smp_atomic32_read_band_nob(&ssi->flags, + (ERTS_SSI_FLG_SUSPENDED + | ERTS_SSI_FLG_MSB_EXEC)); - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + dirty_sched_wall_time_change(esdp, working = 1); + else if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); - } + } erts_smp_runq_lock(rq); sched_active(esdp->no, rq); @@ -3413,8 +3643,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_smp_runq_lock(rq); } clear_sys_scheduling(); - if (flgs & ~ERTS_SSI_FLG_SUSPENDED) - erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) + erts_smp_atomic32_read_band_nob(&ssi->flags, + (ERTS_SSI_FLG_SUSPENDED + | ERTS_SSI_FLG_MSB_EXEC)); #endif if (!working) sched_wall_time_change(esdp, working = 1); @@ -3437,17 +3669,54 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return oflgs; - nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED; + nflgs = oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC); xflgs = oflgs; } } +static ERTS_INLINE void +ssi_wake(ErtsSchedulerSleepInfo *ssi) +{ + erts_sched_finish_poke(ssi, ssi_flags_set_wake(ssi)); +} + +#ifdef ERTS_DIRTY_SCHEDULERS + static void -wake_scheduler(ErtsRunQueue *rq) +dcpu_sched_ix_suspend_wake(Uint ix) { - ErtsSchedulerSleepInfo *ssi; - erts_aint32_t flgs; + ErtsSchedulerSleepInfo* ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + ssi_wake(ssi); +} + +static void +dio_sched_ix_suspend_wake(Uint ix) +{ + ErtsSchedulerSleepInfo* ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + ssi_wake(ssi); +} + +static void +dcpu_sched_ix_wake(Uint ix) +{ + ssi_wake(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix)); +} +#if 0 +static void +dio_sched_ix_wake(Uint ix) +{ + ssi_wake(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix)); +} +#endif + +#endif + +static void +wake_scheduler(ErtsRunQueue *rq) +{ /* * The unlocked run queue is not strictly necessary * from a thread safety or deadlock prevention @@ -3459,10 +3728,7 @@ wake_scheduler(ErtsRunQueue *rq) ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq) || ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); - ssi = rq->scheduler->ssi; - - flgs = ssi_flags_set_wake(ssi); - erts_sched_finish_poke(ssi, flgs); + ssi_wake(rq->scheduler->ssi); } #ifdef ERTS_DIRTY_SCHEDULERS @@ -3509,6 +3775,13 @@ wake_dirty_schedulers(ErtsRunQueue *rq, int one) } while (ssi); } } + +static void +wake_dirty_scheduler(ErtsRunQueue *rq) +{ + wake_dirty_schedulers(rq, 1); +} + #endif #define ERTS_NO_USED_RUNQS_SHIFT 16 @@ -3649,7 +3922,7 @@ smp_notify_inc_runq(ErtsRunQueue *runq) if (runq) { #ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) - wake_dirty_schedulers(runq, 1); + wake_dirty_scheduler(runq); else #endif wake_scheduler(runq); @@ -3953,8 +4226,7 @@ suspend_run_queue(ErtsRunQueue *rq) wake_scheduler(rq); } -static void scheduler_ix_resume_wake(Uint ix); -static void scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi); +static void nrml_sched_ix_resume_wake(Uint ix); static ERTS_INLINE void resume_run_queue(ErtsRunQueue *rq) @@ -3962,16 +4234,19 @@ resume_run_queue(ErtsRunQueue *rq) int pix; Uint32 oflgs; + ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); + erts_smp_runq_lock(rq); oflgs = ERTS_RUNQ_FLGS_READ_BSET(rq, (ERTS_RUNQ_FLG_OUT_OF_WORK | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK - | ERTS_RUNQ_FLG_SUSPENDED), + | ERTS_RUNQ_FLG_SUSPENDED + | ERTS_RUNQ_FLG_MSB_EXEC), (ERTS_RUNQ_FLG_OUT_OF_WORK | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); - if (oflgs & ERTS_RUNQ_FLG_SUSPENDED) { + if (oflgs & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_MSB_EXEC)) { erts_aint32_t len; rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; @@ -3990,10 +4265,7 @@ resume_run_queue(ErtsRunQueue *rq) erts_smp_runq_unlock(rq); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) -#endif - scheduler_ix_resume_wake(rq->ix); + nrml_sched_ix_resume_wake(rq->ix); } typedef struct { @@ -4051,28 +4323,22 @@ evacuate_run_queue(ErtsRunQueue *rq, int prio_q; ErtsRunQueue *to_rq; ErtsMigrationPaths *mps; - ErtsMigrationPath *mp = NULL; + ErtsMigrationPath *mp; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) -#endif - { - mps = erts_get_migration_paths_managed(); - mp = &mps->mpath[rq->ix]; - } + ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); + + mps = erts_get_migration_paths_managed(); + mp = &mps->mpath[rq->ix]; /* Evacuate scheduled misc ops */ if (rq->misc.start) { ErtsMiscOpList *start, *end; -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); -#endif to_rq = mp->misc_evac_runq; if (!to_rq) return; @@ -4081,6 +4347,7 @@ evacuate_run_queue(ErtsRunQueue *rq, end = rq->misc.end; rq->misc.start = NULL; rq->misc.end = NULL; + ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP); erts_smp_runq_unlock(rq); erts_smp_runq_lock(to_rq); @@ -4101,9 +4368,6 @@ evacuate_run_queue(ErtsRunQueue *rq, if (rq->ports.start) { Port *prt; -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); -#endif to_rq = mp->prio[ERTS_PORT_PRIO_LEVEL].runq; if (!to_rq) return; @@ -4141,15 +4405,10 @@ evacuate_run_queue(ErtsRunQueue *rq, int notify = 0; to_rq = NULL; -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) -#endif - { - if (!mp->prio[prio_q].runq) - return; - if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq) - return; - } + if (!mp->prio[prio_q].runq) + return; + if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq) + return; proc = dequeue_process(rq, prio_q, &state); while (proc) { @@ -4205,11 +4464,6 @@ evacuate_run_queue(ErtsRunQueue *rq, goto handle_next_proc; } -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) - clear_proc_dirty_queue_bit(real_proc, rq, qbit); -#endif - if (ERTS_PSFLG_BOUND & real_state) { /* Bound processes get stuck here... */ proc->next = NULL; @@ -4223,16 +4477,7 @@ evacuate_run_queue(ErtsRunQueue *rq, int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); erts_smp_runq_unlock(rq); -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) - /* - * dirty run queues evacuate only to run - * queue 0 during multi-scheduling blocking - */ - to_rq = ERTS_RUNQ_IX(0); - else -#endif - to_rq = mp->prio[prio].runq; + to_rq = mp->prio[prio].runq; RUNQ_SET_RQ(&proc->run_queue, to_rq); erts_smp_runq_lock(to_rq); @@ -4267,7 +4512,7 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq, erts_smp_runq_lock(vrq); - if (rq->halt_in_progress) + if (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_HALTING) goto no_procs; /* @@ -4366,8 +4611,7 @@ check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix) { ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix); Uint32 flags = ERTS_RUNQ_FLGS_GET(vrq); - if ((flags & (ERTS_RUNQ_FLG_NONEMPTY - | ERTS_RUNQ_FLG_PROTECTED)) == ERTS_RUNQ_FLG_NONEMPTY) + if (runq_got_work_to_execute_flags(flags) & (!(flags & ERTS_RUNQ_FLG_PROTECTED))) return try_steal_task_from_victim(rq, rq_lockedp, vrq, flags); else return 0; @@ -4435,11 +4679,9 @@ try_steal_task(ErtsRunQueue *rq) if (!rq_locked) erts_smp_runq_lock(rq); - if (!res) - res = rq->halt_in_progress ? - !ERTS_EMPTY_RUNQ_PORTS(rq) : !ERTS_EMPTY_RUNQ(rq); - - return res; + if (res) + return res; + return runq_got_work_to_execute(rq); } /* Run queue balancing */ @@ -5351,7 +5593,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) #ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { if (rq->waiting) { - wake_dirty_schedulers(rq, 1); + wake_dirty_scheduler(rq); } } else #endif @@ -5698,7 +5940,8 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, ErtsSchedulerSleepInfo* ssi, ErtsRunQueue* runq, char** daww_ptr, size_t daww_sz, - Process *shadow_proc) + Process *shadow_proc, + Uint64 time_stamp) { esdp->timer_wheel = NULL; #ifdef ERTS_SMP @@ -5714,13 +5957,29 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, MAX_REG * sizeof(FloatDef)); #ifdef ERTS_DIRTY_SCHEDULERS + esdp->run_queue = runq; if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) { esdp->no = 0; + if (runq == ERTS_DIRTY_CPU_RUNQ) + esdp->type = ERTS_SCHED_DIRTY_CPU; + else { + ASSERT(runq == ERTS_DIRTY_IO_RUNQ); + esdp->type = ERTS_SCHED_DIRTY_IO; + } ERTS_DIRTY_SCHEDULER_NO(esdp) = (Uint) num; + if (num == 1) { + /* + * Multi-scheduling block functionality depends + * on finding dirty scheduler number 1 here... + */ + runq->scheduler = esdp; + } } else { + esdp->type = ERTS_SCHED_NORMAL; esdp->no = (Uint) num; ERTS_DIRTY_SCHEDULER_NO(esdp) = 0; + runq->scheduler = esdp; } esdp->dirty_shadow_process = shadow_proc; if (shadow_proc) { @@ -5732,6 +5991,8 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, shadow_proc->static_flags = ERTS_STC_FLG_SHADOW_PROC; } #else + runq->scheduler = esdp; + esdp->run_queue = runq; esdp->no = (Uint) num; #endif @@ -5744,9 +6005,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, erts_init_atom_cache_map(&esdp->atom_cache_map); - esdp->run_queue = runq; - esdp->run_queue->scheduler = esdp; - esdp->last_monotonic_time = 0; esdp->check_time_reds = 0; @@ -5765,7 +6023,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->reductions = 0; - init_sched_wall_time(&esdp->sched_wall_time); + init_sched_wall_time(esdp, time_stamp); erts_port_task_handle_init(&esdp->nosuspend_port_task_handle); } @@ -5862,7 +6120,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online erts_smp_atomic32_set_nob(&rq->len, 0); rq->wakeup_other = 0; rq->wakeup_other_reds = 0; - rq->halt_in_progress = 0; rq->procs.pending_exiters = NULL; rq->procs.context_switches = 0; @@ -5980,17 +6237,18 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online erts_aligned_scheduler_data = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, - n*sizeof(ErtsAlignedSchedulerData)); + n*sizeof(ErtsAlignedSchedulerData)); for (ix = 0; ix < n; ix++) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); init_scheduler_data(esdp, ix+1, ERTS_SCHED_SLEEP_INFO_IX(ix), ERTS_RUNQ_IX(ix), &daww_ptr, daww_sz, - NULL); + NULL, 0); } #ifdef ERTS_DIRTY_SCHEDULERS { + Uint64 ts = sched_wall_time_ts(); int dirty_scheds = no_dirty_cpu_schedulers + no_dirty_io_schedulers; int adspix = 0; ErtsAlignedDirtyShadowProcess *adsp = @@ -6010,13 +6268,13 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); init_scheduler_data(esdp, ix+1, ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix), ERTS_DIRTY_CPU_RUNQ, NULL, 0, - &adsp[adspix++].dsp); + &adsp[adspix++].dsp, ts); } for (ix = 0; ix < no_dirty_io_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); init_scheduler_data(esdp, ix+1, ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix), ERTS_DIRTY_IO_RUNQ, NULL, 0, - &adsp[adspix++].dsp); + &adsp[adspix++].dsp, ts); } } #endif @@ -6233,6 +6491,12 @@ check_dirty_enqueue_in_prio_queue(Process *c_p, int queue; erts_aint32_t dact, max_qbit; + /* Do not enqueue free process... */ + if (actual & ERTS_PSFLG_FREE) { + *newp &= ~ERTS_PSFLGS_DIRTY_WORK; + return ERTS_ENQUEUE_NOT; + } + /* Termination should be done on an ordinary scheduler */ if ((*newp) & ERTS_PSFLG_EXITING) { *newp &= ~ERTS_PSFLGS_DIRTY_WORK; @@ -6425,6 +6689,9 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st /* * schedule_out_process() return with c_rq locked. + * + * Return non-zero value if caller should decrease + * reference count on the process when done with it... */ static ERTS_INLINE int schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, @@ -6434,10 +6701,30 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, int enqueue; /* < 0 -> use proxy */ ErtsRunQueue* runq; - if (is_normal_sched) - running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS; - else + if (!is_normal_sched) running_flgs = ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS; + else { + running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS; +#ifdef ERTS_DIRTY_SCHEDULERS + if (state & ERTS_PSFLG_DIRTY_ACTIVE_SYS + && (p->flags & (F_DELAY_GC|F_DISABLE_GC))) { + /* + * Delay dirty GC; will be enabled automatically + * again by next GC... + */ + + /* + * No normal execution until dirty CLA or hibernat has + * been handled... + */ + ASSERT(!(p->flags & (F_DIRTY_CLA | F_DIRTY_GC_HIBERNATE))); + + state = erts_smp_atomic32_read_band_nob(&p->state, + ~ERTS_PSFLG_DIRTY_ACTIVE_SYS); + state &= ~ERTS_PSFLG_DIRTY_ACTIVE_SYS; + } +#endif + } a = state; @@ -6479,12 +6766,18 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, erts_smp_runq_lock(c_rq); - return 0; - +#if !defined(ERTS_SMP) + /* Decrement refc if process struct is free... */ + return !!(n & ERTS_PSFLG_FREE); +#else + /* Decrement refc if scheduled out from dirty scheduler... */ + return !is_normal_sched; +#endif } else { Process* sched_p; + ASSERT(!(n & ERTS_PSFLG_FREE)); ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & (ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_DIRTY_ACTIVE_SYS))); @@ -6500,11 +6793,14 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, erts_smp_runq_lock(runq); + if (is_normal_sched && sched_p == p && ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) + erts_proc_inc_refc(p); /* Needs to be done before enqueue_process() */ + /* Enqueue the process */ enqueue_process(runq, (int) enq_prio, sched_p); if (runq == c_rq) - return 1; + return 0; erts_smp_runq_unlock(runq); @@ -6512,9 +6808,15 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, erts_smp_runq_lock(c_rq); - return 1; + /* + * Decrement refc if process is scheduled out by a + * dirty scheduler, and we have not just scheduled + * the process using the ordinary process struct + * on a dirty run-queue again... + */ + return !is_normal_sched && (sched_p != p + || !ERTS_RUNQ_IX_IS_DIRTY(runq->ix)); } - } static ERTS_INLINE void @@ -6529,8 +6831,17 @@ add2runq(int enqueue, erts_aint32_t prio, if (runq) { Process *sched_p; - if (enqueue > 0) + if (enqueue > 0) { sched_p = proc; + /* + * Refc on process struct (i.e. true struct, + * not proxy-struct) increased while in a + * dirty run-queue or executing on a dirty + * scheduler. + */ + if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) + erts_proc_inc_refc(proc); + } else { Process *pxy; @@ -6925,15 +7236,8 @@ resume_process(Process *p, ErtsProcLocks locks) #ifdef ERTS_SMP -static void -scheduler_ix_resume_wake(Uint ix) -{ - ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); -} - -static void -scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi) +static ERTS_INLINE void +sched_resume_wake__(ErtsSchedulerSleepInfo *ssi) { erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING | ERTS_SSI_FLG_TSE_SLEEPING @@ -6947,9 +7251,31 @@ scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi) break; } xflgs = oflgs; - } while (oflgs & ERTS_SSI_FLG_SUSPENDED); + } while (oflgs & (ERTS_SSI_FLG_MSB_EXEC|ERTS_SSI_FLG_SUSPENDED)); } +static void +nrml_sched_ix_resume_wake(Uint ix) +{ + sched_resume_wake__(ERTS_SCHED_SLEEP_INFO_IX(ix)); +} + +#ifdef ERTS_DIRTY_SCHEDULERS + +static void +dcpu_sched_ix_resume_wake(Uint ix) +{ + sched_resume_wake__(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix)); +} + +static void +dio_sched_ix_resume_wake(Uint ix) +{ + sched_resume_wake__(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix)); +} + +#endif + static erts_aint32_t sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct) { @@ -7029,9 +7355,18 @@ static void init_scheduler_suspend(void) { erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); - schdlr_sspnd.online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); - schdlr_sspnd.curr_online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); - schdlr_sspnd.active = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); + schdlr_sspnd.online.normal = 1; + schdlr_sspnd.curr_online.normal = 1; + schdlr_sspnd.active.normal = 1; +#ifdef ERTS_DIRTY_SCHEDULERS + schdlr_sspnd.online.dirty_cpu = 0; + schdlr_sspnd.curr_online.dirty_cpu = 0; + schdlr_sspnd.active.dirty_cpu = 0; + schdlr_sspnd.online.dirty_io = 0; + schdlr_sspnd.curr_online.dirty_io = 0; + schdlr_sspnd.active.dirty_io = 0; + schdlr_sspnd.last_msb_dirty_type = ERTS_SCHED_DIRTY_IO; +#endif erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0); schdlr_sspnd.chngq = NULL; schdlr_sspnd.changer = am_false; @@ -7054,12 +7389,18 @@ typedef struct { } ErtsSchdlrSspndResume; static void -schdlr_sspnd_resume_proc(Eterm pid) +schdlr_sspnd_resume_proc(ErtsSchedType sched_type, Eterm pid) { - Process *p = erts_pid2proc(NULL, 0, pid, ERTS_PROC_LOCK_STATUS); + Process *p; + p = erts_pid2proc_opt(NULL, 0, pid, ERTS_PROC_LOCK_STATUS, + (sched_type != ERTS_SCHED_NORMAL + ? ERTS_P2P_FLG_INC_REFC + : 0)); if (p) { resume_process(p, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + if (sched_type != ERTS_SCHED_NORMAL) + erts_proc_dec_refc(p); } } @@ -7068,21 +7409,270 @@ schdlr_sspnd_resume_procs(ErtsSchedType sched_type, ErtsSchdlrSspndResume *resume) { if (is_internal_pid(resume->onln.chngr)) { - schdlr_sspnd_resume_proc(resume->onln.chngr); + schdlr_sspnd_resume_proc(sched_type, + resume->onln.chngr); resume->onln.chngr = NIL; } if (is_internal_pid(resume->onln.nxt)) { - schdlr_sspnd_resume_proc(resume->onln.nxt); + schdlr_sspnd_resume_proc(sched_type, + resume->onln.nxt); resume->onln.nxt = NIL; } while (resume->msb.chngrs) { ErtsProcList *plp = resume->msb.chngrs; resume->msb.chngrs = plp->next; - schdlr_sspnd_resume_proc(plp->pid); + schdlr_sspnd_resume_proc(sched_type, + plp->pid); proclist_destroy(plp); } } +#ifdef ERTS_DIRTY_SCHEDULERS + +static ERTS_INLINE int +have_dirty_work(void) +{ + return !(ERTS_EMPTY_RUNQ(ERTS_DIRTY_CPU_RUNQ) + | ERTS_EMPTY_RUNQ(ERTS_DIRTY_IO_RUNQ)); +} + +#define ERTS_MSB_NONE_PRIO_BIT PORT_BIT + +static ERTS_INLINE Uint32 +msb_runq_prio_bit(Uint32 flgs) +{ + int pbit; + + pbit = (int) (flgs & ERTS_RUNQ_FLGS_PROCS_QMASK); + if (flgs & PORT_BIT) { + /* rate ports as proc prio high */ + pbit |= HIGH_BIT; + } + if (flgs & ERTS_RUNQ_FLG_MISC_OP) { + /* rate misc ops as proc prio normal */ + pbit |= NORMAL_BIT; + } + if (flgs & LOW_BIT) { + /* rate low prio as normal (avoid starvation) */ + pbit |= NORMAL_BIT; + } + if (!pbit) + pbit = (int) ERTS_MSB_NONE_PRIO_BIT; + else + pbit &= -pbit; /* least significant bit set... */ + ASSERT(pbit); + + /* High prio low value; low prio high value... */ + return (Uint32) pbit; +} + +static ERTS_INLINE void +msb_runq_prio_bits(Uint32 *nrmlp, Uint32 *dcpup, Uint32 *diop) +{ + Uint32 flgs = ERTS_RUNQ_FLGS_GET(ERTS_RUNQ_IX(0)); + if (flgs & ERTS_RUNQ_FLG_HALTING) { + /* + * Emulator is halting; only execute port jobs + * on normal scheduler. Ensure that we switch + * to the normal scheduler. + */ + *nrmlp = HIGH_BIT; + *dcpup = ERTS_MSB_NONE_PRIO_BIT; + *diop = ERTS_MSB_NONE_PRIO_BIT; + } + else { + *nrmlp = msb_runq_prio_bit(flgs); + + flgs = ERTS_RUNQ_FLGS_GET(ERTS_DIRTY_CPU_RUNQ); + *dcpup = msb_runq_prio_bit(flgs); + + flgs = ERTS_RUNQ_FLGS_GET(ERTS_DIRTY_IO_RUNQ); + *diop = msb_runq_prio_bit(flgs); + } +} + +static int +msb_scheduler_type_switch(ErtsSchedType sched_type, + ErtsSchedulerData *esdp, + long no) +{ + Uint32 nrml_prio, dcpu_prio, dio_prio; + ErtsSchedType exec_type; + ErtsRunQueue *exec_rq; +#ifdef DEBUG + erts_aint32_t dbg_val; +#endif + + ASSERT(schdlr_sspnd.msb.ongoing); + + /* + * This function determines how to switch + * between scheduler types when multi-scheduling + * is blocked. + * + * If no dirty work exist, we always select + * execution of normal scheduler. If nothing + * executes, normal scheduler 1 should be waiting + * in sys_schedule(), otherwise we cannot react + * on I/O events. + * + * We unconditionally switch back to normal + * scheduler after executing dirty in order to + * make sure we check for I/O... + */ + + msb_runq_prio_bits(&nrml_prio, &dcpu_prio, &dio_prio); + + exec_type = ERTS_SCHED_NORMAL; + if (sched_type == ERTS_SCHED_NORMAL) { + + /* + * Check priorities of work in the + * different run-queues and determine + * run-queue with highest prio job... + */ + + if ((dcpu_prio == ERTS_MSB_NONE_PRIO_BIT) + & (dio_prio == ERTS_MSB_NONE_PRIO_BIT)) { + /* + * No dirty work exist; continue on normal + * scheduler... + */ + return 0; + } + + if (dcpu_prio < nrml_prio) { + exec_type = ERTS_SCHED_DIRTY_CPU; + if (dio_prio < dcpu_prio) + exec_type = ERTS_SCHED_DIRTY_IO; + } + else { + if (dio_prio < nrml_prio) + exec_type = ERTS_SCHED_DIRTY_IO; + } + + /* + * Make sure to alternate between dirty types + * inbetween normal execution if highest + * priorities are equal. + */ + + if (exec_type == ERTS_SCHED_NORMAL) { + if (dcpu_prio == nrml_prio) + exec_type = ERTS_SCHED_DIRTY_CPU; + else if (dio_prio == nrml_prio) + exec_type = ERTS_SCHED_DIRTY_IO; + else { + /* + * Normal work has higher prio than + * dirty work; continue on normal + * scheduler... + */ + return 0; + } + } + + ASSERT(exec_type != ERTS_SCHED_NORMAL); + if (dio_prio == dcpu_prio) { + /* Alter between dirty types... */ + if (schdlr_sspnd.last_msb_dirty_type == ERTS_SCHED_DIRTY_IO) + exec_type = ERTS_SCHED_DIRTY_CPU; + else + exec_type = ERTS_SCHED_DIRTY_IO; + } + } + + ASSERT(sched_type != exec_type); + + if (exec_type != ERTS_SCHED_NORMAL) + schdlr_sspnd.last_msb_dirty_type = exec_type; + else { + erts_aint32_t calls; + /* + * Going back to normal scheduler after + * dirty execution; make sure it will check + * for I/O... + */ + if (ERTS_USE_MODIFIED_TIMING()) + calls = ERTS_MODIFIED_TIMING_INPUT_REDS + 1; + else + calls = INPUT_REDUCTIONS + 1; + erts_smp_atomic32_set_nob(&function_calls, calls); + + if ((nrml_prio == ERTS_MSB_NONE_PRIO_BIT) + & ((dcpu_prio != ERTS_MSB_NONE_PRIO_BIT) + | (dio_prio != ERTS_MSB_NONE_PRIO_BIT))) { + /* + * We have dirty work, but an empty + * normal run-queue. + * + * Since the normal run-queue is + * empty, the normal scheduler will + * go to sleep when selected for + * execution. We have dirty work to + * do, so we only want it to check + * I/O, and then come back here and + * switch to dirty execution. + * + * To prevent the scheduler from going + * to sleep we trick it into believing + * it has work to do... + */ + ERTS_RUNQ_FLGS_SET_NOB(ERTS_RUNQ_IX(0), + ERTS_RUNQ_FLG_MISC_OP); + } + } + + /* + * Suspend this scheduler and wake up scheduler + * number one of another type... + */ +#ifdef DEBUG + dbg_val = +#else + (void) +#endif + erts_smp_atomic32_read_bset_mb(&esdp->ssi->flags, + (ERTS_SSI_FLG_SUSPENDED + | ERTS_SSI_FLG_MSB_EXEC), + ERTS_SSI_FLG_SUSPENDED); + ASSERT(dbg_val & ERTS_SSI_FLG_MSB_EXEC); + + switch (exec_type) { + case ERTS_SCHED_NORMAL: + exec_rq = ERTS_RUNQ_IX(0); + break; + case ERTS_SCHED_DIRTY_CPU: + exec_rq = ERTS_DIRTY_CPU_RUNQ; + break; + case ERTS_SCHED_DIRTY_IO: + exec_rq = ERTS_DIRTY_IO_RUNQ; + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + exec_rq = NULL; + break; + } + +#ifdef DEBUG + dbg_val = +#else + (void) +#endif + erts_smp_atomic32_read_bset_mb(&exec_rq->scheduler->ssi->flags, + (ERTS_SSI_FLG_SUSPENDED + | ERTS_SSI_FLG_MSB_EXEC), + ERTS_SSI_FLG_MSB_EXEC); + ASSERT(dbg_val & ERTS_SSI_FLG_SUSPENDED); + + wake_scheduler(exec_rq); + + return 1; /* suspend this scheduler... */ + +} + +#endif + static void suspend_scheduler(ErtsSchedulerData *esdp) { @@ -7108,40 +7698,51 @@ suspend_scheduler(ErtsSchedulerData *esdp) * Regardless of why a scheduler is suspended, it ends up here. */ -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { + +#if !defined(ERTS_DIRTY_SCHEDULERS) + + sched_type = ERTS_SCHED_NORMAL; + online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN; + no = esdp->no; + ASSERT(no != 1); + +#else + + sched_type = esdp->type; + switch (sched_type) { + case ERTS_SCHED_NORMAL: + online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN; + no = esdp->no; + break; + case ERTS_SCHED_DIRTY_CPU: + online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; + no = ERTS_DIRTY_SCHEDULER_NO(esdp); + break; + case ERTS_SCHED_DIRTY_IO: + online_flag = 0; no = ERTS_DIRTY_SCHEDULER_NO(esdp); - if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) { - online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; - sched_type = ERTS_SCHED_DIRTY_CPU; - } - else { - online_flag = 0; - sched_type = ERTS_SCHED_DIRTY_IO; - } + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + return; } - else -#endif - { - online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN; - no = esdp->no; - sched_type = ERTS_SCHED_NORMAL; + + if (erts_smp_atomic32_read_nob(&ssi->flags) & ERTS_SSI_FLG_MSB_EXEC) { + ASSERT(no == 1); + if (!msb_scheduler_type_switch(sched_type, esdp, no)) + return; + /* Suspend and let scheduler 1 of another type execute... */ } - ASSERT(sched_type != ERTS_SCHED_NORMAL || no != 1); +#endif if (sched_type != ERTS_SCHED_NORMAL) { - if (erts_smp_mtx_trylock(&schdlr_sspnd.mtx) == EBUSY) { - erts_smp_runq_unlock(esdp->run_queue); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - erts_smp_runq_lock(esdp->run_queue); - } - if (schdlr_sspnd.msb.ongoing) - evacuate_run_queue(esdp->run_queue, &sbp); erts_smp_runq_unlock(esdp->run_queue); + dirty_sched_wall_time_change(esdp, 0); } else { - evacuate_run_queue(esdp->run_queue, &sbp); + if (no != 1) + evacuate_run_queue(esdp->run_queue, &sbp); erts_smp_runq_unlock(esdp->run_queue); @@ -7149,20 +7750,15 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_inactive); - - sched_wall_time_change(esdp, 0); - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); } + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); if (flgs & ERTS_SSI_FLG_SUSPENDED) { schdlr_sspnd_dec_nscheds(&schdlr_sspnd.active, sched_type); - ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, - ERTS_SCHED_NORMAL) >= 1); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); while (1) { @@ -7184,9 +7780,13 @@ suspend_scheduler(ErtsSchedulerData *esdp) ERTS_SCHED_NORMAL) == 1) { clr_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB; } - else if (schdlr_sspnd.active - == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)) { - clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB; + else if (schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) == 1 + && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_CPU) == 0 + && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_IO) == 0) { + clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB; } if (clr_flg) { @@ -7197,15 +7797,18 @@ suspend_scheduler(ErtsSchedulerData *esdp) (void) erts_proclist_fetch(&msb[i]->chngq, &end_plp); /* resume processes that initiated the multi scheduling block... */ plp = msb[i]->chngq; - while (plp) { - erts_proclist_store_last(&msb[i]->blckrs, - proclist_copy(plp)); - plp = plp->next; - } - if (end_plp) + if (plp) { + ASSERT(end_plp); + ASSERT(msb[i]->ongoing); + do { + erts_proclist_store_last(&msb[i]->blckrs, + proclist_copy(plp)); + plp = plp->next; + } while (plp); end_plp->next = resume.msb.chngrs; - resume.msb.chngrs = msb[i]->chngq; - msb[i]->chngq = NULL; + resume.msb.chngrs = msb[i]->chngq; + msb[i]->chngq = NULL; + } } } } @@ -7255,10 +7858,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) } } - if (curr_online - && (sched_type == ERTS_SCHED_NORMAL - ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) - : !schdlr_sspnd.msb.ongoing)) { + if (curr_online) { flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; @@ -7269,28 +7869,16 @@ suspend_scheduler(ErtsSchedulerData *esdp) while (1) { ErtsMonotonicTime current_time; - erts_aint32_t qmask; erts_aint32_t flgs; - qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue) - & ERTS_RUNQ_FLGS_QMASK); - - if (sched_type != ERTS_SCHED_NORMAL) { - if (qmask) { - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - erts_smp_runq_lock(esdp->run_queue); - if (schdlr_sspnd.msb.ongoing) - evacuate_run_queue(esdp->run_queue, &sbp); - erts_smp_runq_unlock(esdp->run_queue); - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - } + if (sched_type != ERTS_SCHED_NORMAL) aux_work = 0; - } else { + int evacuate = no == 1 ? 0 : !ERTS_EMPTY_RUNQ(esdp->run_queue); aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work|qmask) { + if (aux_work|evacuate) { if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); @@ -7302,7 +7890,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); - if (qmask) { + if (evacuate) { erts_smp_runq_lock(esdp->run_queue); evacuate_run_queue(esdp->run_queue, &sbp); erts_smp_runq_unlock(esdp->run_queue); @@ -7414,37 +8002,46 @@ suspend_scheduler(ErtsSchedulerData *esdp) schdlr_sspnd_inc_nscheds(&schdlr_sspnd.active, sched_type); changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) - && schdlr_sspnd.online == schdlr_sspnd.active) { - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); - } - + if (changing) { + if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) + && !schdlr_sspnd.msb.ongoing + && schdlr_sspnd_eq_nscheds(&schdlr_sspnd.online, + &schdlr_sspnd.active)) { + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); + } + if ((changing & ERTS_SCHDLR_SSPND_CHNG_NMSB) + && !schdlr_sspnd.nmsb.ongoing + && (schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL) + == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL))) { + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_NMSB); + } + } ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type)); - ASSERT((sched_type == ERTS_SCHED_NORMAL - ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) - : !schdlr_sspnd.msb.ongoing)); } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - ASSERT(!resume.msb.chngrs); schdlr_sspnd_resume_procs(sched_type, &resume); ASSERT(curr_online); - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(esdp->no), am_active); + if (sched_type != ERTS_SCHED_NORMAL) + dirty_sched_wall_time_change(esdp, 1); + else { + (void) erts_get_monotonic_time(esdp); + if (erts_system_profile_flags.scheduler) + profile_scheduler(make_small(esdp->no), am_active); - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } } - if (sched_type == ERTS_SCHED_NORMAL) - (void) erts_get_monotonic_time(esdp); erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); @@ -7552,7 +8149,7 @@ abort_sched_onln_chng_waitq(Process *p) erts_smp_mtx_unlock(&schdlr_sspnd.mtx); if (is_internal_pid(resume)) - schdlr_sspnd_resume_proc(resume); + schdlr_sspnd_resume_proc(ERTS_SCHED_NORMAL, resume); } ErtsSchedSuspendResult @@ -7714,10 +8311,8 @@ erts_set_schedulers_online(Process *p, erts_sched_poke(ssi); } } else { - for (ix = dirty_online; ix < dirty_no; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - } + for (ix = dirty_online; ix < dirty_no; ix++) + dcpu_sched_ix_resume_wake(ix); } } if (!dirty_only) @@ -7745,19 +8340,19 @@ erts_set_schedulers_online(Process *p, else /* if decrease */ { #ifdef ERTS_DIRTY_SCHEDULERS if (change_dirty) { - ErtsSchedulerSleepInfo* ssi; if (schdlr_sspnd.msb.ongoing) { - for (ix = dirty_no; ix < dirty_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_sched_poke(ssi); - } - } else { - for (ix = dirty_no; ix < dirty_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); + for (ix = dirty_no; ix < dirty_online; ix++) + erts_sched_poke(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix)); + } + else { + for (ix = dirty_no; ix < dirty_online; ix++) + dcpu_sched_ix_suspend_wake(ix); + /* + * Newly suspended scheduler may have just been + * about to handle a task. Make sure someone takes + * care of such a task... + */ + dcpu_sched_ix_wake(0); } } if (!dirty_only) @@ -7820,9 +8415,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal { int resume_proc, ix, res, have_unlocked_plocks = 0; ErtsProcList *plp; -#ifdef ERTS_DIRTY_SCHEDULERS - ErtsSchedulerSleepInfo* ssi; -#endif ErtsMultiSchedulingBlock *msbp; erts_aint32_t chng_flg; int have_blckd_flg; @@ -7869,8 +8461,9 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal erts_proclist_store_last(&msbp->blckrs, plp); p->flags |= have_blckd_flg; ASSERT(normal - ? 1 == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, ERTS_SCHED_NORMAL) - : schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)); + ? 1 == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) + : schdlr_sspnd_get_nscheds_tot(&schdlr_sspnd.active) == 1); ASSERT(erts_proc_sched_data(p)->no == 1); if (schdlr_sspnd.msb.ongoing) res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; @@ -7888,59 +8481,41 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal } ASSERT(!msbp->ongoing); msbp->ongoing = 1; - if (schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0) - || (normal && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, - ERTS_SCHED_NORMAL) == 1)) { - ASSERT(erts_proc_sched_data(p)->no == 1); - plp = proclist_create(p); - erts_proclist_store_last(&msbp->blckrs, plp); - if (schdlr_sspnd.msb.ongoing) - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - else - res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED; - } - else { - erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, - chng_flg); - change_no_used_runqs(1); - for (ix = 1; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); - for (ix = 1; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq); - } + erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, + chng_flg); + change_no_used_runqs(1); + for (ix = 1; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!normal) { - for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); + for (ix = 1; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + wake_scheduler(rq); + } - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); - } +#ifdef ERTS_DIRTY_SCHEDULERS + if (!normal) { + ERTS_RUNQ_FLGS_SET_NOB(ERTS_RUNQ_IX(0), ERTS_RUNQ_FLG_MSB_EXEC); + erts_smp_atomic32_read_bor_nob(&ERTS_RUNQ_IX(0)->scheduler->ssi->flags, + ERTS_SSI_FLG_MSB_EXEC); + for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) + dcpu_sched_ix_suspend_wake(ix); + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) + dio_sched_ix_suspend_wake(ix); + } #endif - wait_until_msb: + wait_until_msb: - ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)); + ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)); - plp = proclist_create(p); - erts_proclist_store_last(&msbp->chngq, plp); - resume_proc = 0; - if (schdlr_sspnd.msb.ongoing) - res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; - else - res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED; - } + plp = proclist_create(p); + erts_proclist_store_last(&msbp->chngq, plp); + resume_proc = 0; + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED; ASSERT(erts_proc_sched_data(p)); } } @@ -7950,21 +8525,20 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal } else { /* ------ UNBLOCK ------ */ if (p->flags & have_blckd_flg) { - ErtsProcList *plps[2]; + ErtsProcList **plpps[3] = {0}; ErtsProcList *plp; - int limit = 0; - plps[limit++] = erts_proclist_peek_first(msbp->blckrs); - if (all) - plps[limit++] = erts_proclist_peek_first(msbp->chngq); + plpps[0] = &msbp->blckrs; + if (all) + plpps[1] = &msbp->chngq; - for (ix = 0; ix < limit; ix++) { - plp = plps[ix]; + for (ix = 0; plpps[ix]; ix++) { + plp = erts_proclist_peek_first(*plpps[ix]); while (plp) { ErtsProcList *tmp_plp = plp; - plp = erts_proclist_peek_next(msbp->blckrs, plp); + plp = erts_proclist_peek_next(*plpps[ix], plp); if (erts_proclist_same(tmp_plp, p)) { - erts_proclist_remove(&msbp->blckrs, tmp_plp); + erts_proclist_remove(plpps[ix], tmp_plp); proclist_destroy(tmp_plp); if (!all) break; @@ -7973,27 +8547,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal } } if (!msbp->blckrs && !msbp->chngq) { - int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, - ERTS_SCHED_NORMAL); + int online; erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, chng_flg); p->flags &= ~have_blckd_flg; msbp->ongoing = 0; - if (online == 1) { - /* No normal schedulers to resume */ - ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, - ERTS_SCHED_NORMAL) == 1); -#ifndef ERTS_DIRTY_SCHEDULERS - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~chng_flg); -#endif - } - else if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) { - if (plocks) { + if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) { + if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } + online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL); change_no_used_runqs(online); /* Resume all online run queues */ @@ -8004,19 +8570,15 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal suspend_run_queue(ERTS_RUNQ_IX(ix)); } #ifdef ERTS_DIRTY_SCHEDULERS - if (!normal) { - ASSERT(!schdlr_sspnd.msb.ongoing); + if (!schdlr_sspnd.msb.ongoing) { + /* Get rid of msb-exec flag in run-queue of scheduler 1 */ + resume_run_queue(ERTS_RUNQ_IX(0)); online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, ERTS_SCHED_DIRTY_CPU); - for (ix = 0; ix < online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - } - - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - } + for (ix = 0; ix < online; ix++) + dcpu_sched_ix_resume_wake(ix); + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) + dio_sched_ix_resume_wake(ix); } #endif } @@ -8190,6 +8752,8 @@ sched_dirty_cpu_thread_func(void *vesdp) callbacks.wait = NULL; callbacks.finalize_wait = NULL; + dirty_sched_wall_time_change(esdp, 1); + esdp->thr_id += erts_no_schedulers; erts_msacc_init_thread("dirty_cpu_scheduler", no, 0); @@ -8237,6 +8801,8 @@ sched_dirty_io_thread_func(void *vesdp) callbacks.wait = NULL; callbacks.finalize_wait = NULL; + dirty_sched_wall_time_change(esdp, 1); + esdp->thr_id += erts_no_schedulers + erts_no_dirty_cpu_schedulers; erts_msacc_init_thread("dirty_io_scheduler", no, 0); @@ -8604,8 +9170,14 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, * from being selected for normal execution regardless * of locks held or not held on it... */ - ASSERT(!((ERTS_PSFLG_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS) - & erts_smp_atomic32_read_nob(&rp->state))); +#ifdef DEBUG + { + erts_aint32_t state; + state = erts_smp_atomic32_read_nob(&rp->state); + ASSERT((state & ERTS_PSFLG_PENDING_EXIT) + || !(state & ERTS_PSFLG_RUNNING)); + } +#endif if (!suspend) resume_process(rp, pid_locks|ERTS_PROC_LOCK_STATUS); @@ -9587,40 +10159,45 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) esdp->reductions += reds; - /* schedule_out_process() returns with rq locked! */ - schedule_out_process(rq, state, p, proxy_p, is_normal_sched); - proxy_p = NULL; + { + int dec_refc; - ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq, - (int) ERTS_PSFLGS_GET_USR_PRIO(state), - reds, - actual_reds); + /* schedule_out_process() returns with rq locked! */ + dec_refc = schedule_out_process(rq, state, p, + proxy_p, is_normal_sched); + proxy_p = NULL; - esdp->current_process = NULL; + ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq, + (int) ERTS_PSFLGS_GET_USR_PRIO(state), + reds, + actual_reds); + + esdp->current_process = NULL; #ifdef ERTS_SMP - p->scheduler_data = NULL; + p->scheduler_data = NULL; #endif - erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN - | ERTS_PROC_LOCK_STATUS - | ERTS_PROC_LOCK_TRACE)); + erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN + | ERTS_PROC_LOCK_STATUS + | ERTS_PROC_LOCK_TRACE)); - ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); - if (state & ERTS_PSFLG_FREE) { - if (!is_normal_sched) { - ASSERT(p->flags & F_DELAYED_DEL_PROC); - erts_proc_dec_refc(p); - } - else { #ifdef ERTS_SMP - ASSERT(esdp->free_process == p); - esdp->free_process = NULL; -#else - erts_proc_dec_refc(p); + if (state & ERTS_PSFLG_FREE) { + if (!is_normal_sched) { + ASSERT(p->flags & F_DELAYED_DEL_PROC); + } + else { + ASSERT(esdp->free_process == p); + esdp->free_process = NULL; + } + } #endif - } - } + + if (dec_refc) + erts_proc_dec_refc(p); + } #ifdef ERTS_SMP ASSERT(!esdp->free_process); @@ -9670,7 +10247,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (!is_normal_sched) { if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) - & ERTS_SSI_FLG_SUSPENDED) { + & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) { suspend_scheduler(esdp); } } @@ -9680,8 +10257,10 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ASSERT(is_normal_sched); - if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) { - if (flags & ERTS_RUNQ_FLG_SUSPENDED) { + if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND + | ERTS_RUNQ_FLG_SUSPENDED + | ERTS_RUNQ_FLG_MSB_EXEC)) { + if (flags & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_MSB_EXEC)) { (void) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_EXEC); suspend_scheduler(esdp); flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); @@ -9720,13 +10299,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) flags = ERTS_RUNQ_FLGS_GET_NOB(rq); - if (!is_normal_sched && rq->halt_in_progress) { + if (!is_normal_sched & !!(flags & ERTS_RUNQ_FLG_HALTING)) { /* Wait for emulator to terminate... */ while (1) erts_milli_sleep(1000*1000); } - else if ((!(flags & ERTS_RUNQ_FLGS_QMASK) && !rq->misc.start) - || (rq->halt_in_progress && ERTS_EMPTY_RUNQ_PORTS(rq))) { + else if (!runq_got_work_to_execute_flags(flags)) { /* Prepare for scheduler wait */ #ifdef ERTS_SMP ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -9740,21 +10318,44 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (flags & ERTS_RUNQ_FLG_INACTIVE) empty_runq(rq); else { - if (is_normal_sched && try_steal_task(rq)) - goto continue_check_activities_to_run; - - empty_runq(rq); - - /* - * Check for ERTS_RUNQ_FLG_SUSPENDED has to be done - * after trying to steal a task. - */ - flags = ERTS_RUNQ_FLGS_GET_NOB(rq); - if (flags & ERTS_RUNQ_FLG_SUSPENDED) { - non_empty_runq(rq); - flags |= ERTS_RUNQ_FLG_NONEMPTY; - goto continue_check_activities_to_run_known_flags; + ASSERT(!runq_got_work_to_execute(rq)); + if (!is_normal_sched) { + /* Dirty scheduler */ + if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) + & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) { + /* Go suspend... */ + goto continue_check_activities_to_run_known_flags; + } + } + else { + /* Normal scheduler */ + if (try_steal_task(rq)) + goto continue_check_activities_to_run; + /* + * Check for suspend has to be done after trying + * to steal a task... + */ + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + if ((flags & ERTS_RUNQ_FLG_SUSPENDED) +#ifdef ERTS_DIRTY_SCHEDULERS + /* If multi scheduling block and we have + * dirty work, suspend and let dirty + * scheduler handle work... */ + || ((((flags & (ERTS_RUNQ_FLG_HALTING + | ERTS_RUNQ_FLG_MSB_EXEC)) + == ERTS_RUNQ_FLG_MSB_EXEC)) + && have_dirty_work()) +#endif + ) { + non_empty_runq(rq); + flags |= ERTS_RUNQ_FLG_NONEMPTY; + /* + * Go suspend... + */ + goto continue_check_activities_to_run_known_flags; + } } + empty_runq(rq); } #endif @@ -9806,7 +10407,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) #endif } - if (rq->misc.start) + if (flags & ERTS_RUNQ_FLG_MISC_OP) exec_misc_ops(rq); #ifdef ERTS_SMP @@ -9817,13 +10418,15 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) * Find a new port to run. */ - if (RUNQ_READ_LEN(&rq->ports.info.len)) { + flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + + if (flags & PORT_BIT) { int have_outstanding_io; have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port); if ((!erts_eager_check_io && have_outstanding_io && fcalls > 2*input_reductions) - || rq->halt_in_progress) { + || (flags & ERTS_RUNQ_FLG_HALTING)) { /* * If we have performed more than 2*INPUT_REDUCTIONS since * last call to erl_sys_schedule() and we still haven't @@ -9896,8 +10499,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) if (!(state & ERTS_PSFLG_PROXY)) psflg_band_mask &= ~ERTS_PSFLG_IN_RUNQ; else { + Eterm pid = p->common.id; proxy_p = p; - p = erts_proc_lookup_raw(proxy_p->common.id); + p = (is_normal_sched + ? erts_proc_lookup_raw(pid) + : erts_pid2proc_opt(NULL, 0, pid, 0, + ERTS_P2P_FLG_INC_REFC)); if (!p) { free_proxy_proc(proxy_p); proxy_p = NULL; @@ -9929,8 +10536,10 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) | ERTS_PSFLG_FREE))) #ifdef ERTS_DIRTY_SCHEDULERS | (((state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_FREE | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING_SYS | ERTS_PSFLG_EXITING)) == ERTS_PSFLG_EXITING) & (!!is_normal_sched)) @@ -9942,7 +10551,14 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) | ERTS_PSFLG_PENDING_EXIT | ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) - != ERTS_PSFLG_SUSPENDED)); + != ERTS_PSFLG_SUSPENDED) +#ifdef ERTS_DIRTY_SCHEDULERS + & (!(state & (ERTS_PSFLG_EXITING + | ERTS_PSFLG_PENDING_EXIT)) + | (!!is_normal_sched)) +#endif + ); + if (run_process) { if (state & (ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) @@ -9961,6 +10577,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) /* free and not queued by proxy */ erts_proc_dec_refc(p); } + if (!is_normal_sched) + erts_proc_dec_refc(p); goto pick_next_process; } state = new; @@ -10045,9 +10663,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) goto sunlock_sched_out_proc; } - ASSERT((state & ERTS_PSFLG_DIRTY_ACTIVE_SYS) - || *p->i == (BeamInstr) em_call_nif); - ASSERT(rq == ERTS_DIRTY_CPU_RUNQ ? (state & (ERTS_PSFLG_DIRTY_CPU_PROC | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) @@ -10084,65 +10699,70 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } } - if (state & (ERTS_PSFLG_RUNNING_SYS - | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { - /* - * GC is normally never delayed when a process - * is scheduled out, but might be when executing - * hand written beam assembly in - * prim_eval:'receive'. If GC is delayed we are - * not allowed to execute system tasks. - */ - if (!(p->flags & F_DELAY_GC)) { - int cost = execute_sys_tasks(p, &state, reds); - calls += cost; - reds -= cost; - if (reds <= 0 + if (is_normal_sched) { + + if (state & ERTS_PSFLG_RUNNING_SYS) { + /* + * GC is normally never delayed when a process + * is scheduled out, but might be when executing + * hand written beam assembly in + * prim_eval:'receive'. If GC is delayed we are + * not allowed to execute system tasks. + */ + if (!(p->flags & F_DELAY_GC)) { + int cost = execute_sys_tasks(p, &state, reds); + calls += cost; + reds -= cost; + if (reds <= 0) + goto sched_out_proc; #ifdef ERTS_DIRTY_SCHEDULERS - || !is_normal_sched - || (state & ERTS_PSFLGS_DIRTY_WORK) + if (state & ERTS_PSFLGS_DIRTY_WORK) + goto sched_out_proc; #endif - ) { - goto sched_out_proc; - } - } + } - ASSERT(state & psflg_running_sys); - ASSERT(!(state & psflg_running)); + ASSERT(state & psflg_running_sys); + ASSERT(!(state & psflg_running)); - while (1) { - erts_aint32_t n, e; + while (1) { + erts_aint32_t n, e; - if (((state & (ERTS_PSFLG_SUSPENDED - | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE) - && !(state & ERTS_PSFLG_EXITING)) { - goto sched_out_proc; - } + if (((state & (ERTS_PSFLG_SUSPENDED + | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE) + && !(state & ERTS_PSFLG_EXITING)) { + goto sched_out_proc; + } - n = e = state; - n &= ~psflg_running_sys; - n |= psflg_running; + n = e = state; + n &= ~psflg_running_sys; + n |= psflg_running; - state = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); - if (state == e) { - state = n; - break; - } + state = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); + if (state == e) { + state = n; + break; + } - ASSERT(state & psflg_running_sys); - ASSERT(!(state & psflg_running)); - } - } + ASSERT(state & psflg_running_sys); + ASSERT(!(state & psflg_running)); + } + } - if (ERTS_IS_GC_DESIRED(p) && !ERTS_SCHEDULER_IS_DIRTY_IO(esdp)) { - if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) { - int cost = scheduler_gc_proc(p, reds); - calls += cost; - reds -= cost; - if (reds <= 0) - goto sched_out_proc; - } - } + if (ERTS_IS_GC_DESIRED(p)) { + if (!(state & ERTS_PSFLG_EXITING) + && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) { + int cost = scheduler_gc_proc(p, reds); + calls += cost; + reds -= cost; + if (reds <= 0) + goto sched_out_proc; +#ifdef ERTS_DIRTY_SCHEDULERS + if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) + goto sched_out_proc; +#endif + } + } + } if (proxy_p) { free_proxy_proc(proxy_p); @@ -10154,7 +10774,13 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* Never run a suspended process */ - ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state))); +#ifdef DEBUG + { + erts_aint32_t dstate = erts_smp_atomic32_read_nob(&p->state); + ASSERT(!(ERTS_PSFLG_SUSPENDED & dstate) + || (ERTS_PSFLG_DIRTY_RUNNING_SYS & dstate)); + } +#endif ASSERT(erts_proc_read_refc(p) > 0); @@ -10175,9 +10801,18 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } static int -notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) +notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, + Eterm st_result, int normal_sched) { - Process *rp = erts_proc_lookup(st->requester); + Process *rp; +#ifdef ERTS_DIRTY_SCHEDULERS + if (!normal_sched) + rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + st->requester, 0, + ERTS_P2P_FLG_INC_REFC); + else +#endif + rp = erts_proc_lookup(st->requester); if (rp) { ErtsProcLocks rp_locks; ErlOffHeap *ohp; @@ -10225,6 +10860,11 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!normal_sched) + erts_proc_dec_refc(rp); +#endif } erts_cleanup_offheap(&st->off_heap); @@ -10380,18 +11020,23 @@ done: } static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio); +#ifdef ERTS_DIRTY_SCHEDULERS +static void save_dirty_task(Process *c_p, ErtsProcSysTask *st); +#endif static int execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) { - int garbage_collected = 0; + int minor_gc = 0, major_gc = 0; erts_aint32_t state = *statep; int reds = in_reds; int qmask = 0; + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p))); ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN); do { + ErtsProcSysTaskType type; ErtsProcSysTask *st; int st_prio; Eterm st_res; @@ -10409,7 +11054,9 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) if (!st) break; - switch (st->type) { + type = st->type; + + switch (type) { case ERTS_PSTT_GC_MAJOR: case ERTS_PSTT_GC_MINOR: if (c_p->flags & F_DISABLE_GC) { @@ -10418,12 +11065,23 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) reds--; } else { - if (!garbage_collected) { - if (st->type == ERTS_PSTT_GC_MAJOR) { + if (!minor_gc + || (!major_gc && type == ERTS_PSTT_GC_MAJOR)) { + if (type == ERTS_PSTT_GC_MAJOR) { FLAGS(c_p) |= F_NEED_FULLSWEEP; } reds -= scheduler_gc_proc(c_p, reds); - garbage_collected = 1; +#ifdef ERTS_DIRTY_SCHEDULERS + if (c_p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) { + save_dirty_task(c_p, st); + st = NULL; + break; + } +#endif + if (type == ERTS_PSTT_GC_MAJOR) + minor_gc = major_gc = 1; + else + minor_gc = 1; } st_res = am_true; } @@ -10450,20 +11108,31 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) case ERTS_PSTT_CLA: { int fcalls; int cla_reds = 0; + int do_gc; + if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) fcalls = reds; else fcalls = reds - CONTEXT_REDS; - st_res = erts_proc_copy_literal_area(c_p, - &cla_reds, - fcalls, - st->arg[0] == am_true); + do_gc = st->arg[0] == am_true; + st_res = erts_proc_copy_literal_area(c_p, &cla_reds, + fcalls, do_gc); reds -= cla_reds; if (is_non_value(st_res)) { +#ifdef ERTS_DIRTY_SCHEDULERS + if (c_p->flags & F_DIRTY_CLA) { + save_dirty_task(c_p, st); + st = NULL; + break; + } +#endif /* Needed gc, but gc was disabled */ save_gc_task(c_p, st, st_prio); st = NULL; + break; } + if (do_gc) /* We did a major gc */ + minor_gc = major_gc = 1; break; } case ERTS_PSTT_COHMQ: @@ -10482,7 +11151,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } if (st) - reds += notify_sys_task_executed(c_p, st, st_res); + reds += notify_sys_task_executed(c_p, st, st_res, 1); state = erts_smp_atomic32_read_acqb(&c_p->state); } while (qmask && reds > 0); @@ -10510,9 +11179,18 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) Eterm st_res; int st_prio; - st = fetch_sys_task(c_p, state, &qmask, &st_prio); - if (!st) - break; +#ifdef ERTS_DIRTY_SCHEDULERS + if (c_p->dirty_sys_tasks) { + st = c_p->dirty_sys_tasks; + c_p->dirty_sys_tasks = st->next; + } + else +#endif + { + st = fetch_sys_task(c_p, state, &qmask, &st_prio); + if (!st) + break; + } switch (st->type) { case ERTS_PSTT_GC_MAJOR: @@ -10536,7 +11214,7 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) break; } - reds += notify_sys_task_executed(c_p, st, st_res); + reds += notify_sys_task_executed(c_p, st, st_res, 1); state = erts_smp_atomic32_read_acqb(&c_p->state); } while (qmask && reds < max_reds); @@ -10546,6 +11224,92 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) #ifdef ERTS_DIRTY_SCHEDULERS +void +erts_execute_dirty_system_task(Process *c_p) +{ + Eterm cla_res = THE_NON_VALUE; + ErtsProcSysTask *stasks; + + /* + * If multiple operations, perform them in the following + * order (in order to avoid unnecessary GC): + * 1. Copy Literal Area (implies major GC). + * 2. GC Hibernate (implies major GC if not woken). + * 3. Major GC (implies minor GC). + * 4. Minor GC. + * + * System task requests are handled after the actual + * operations have been performed... + */ + + ASSERT(!(c_p->flags & (F_DELAY_GC|F_DISABLE_GC))); + + if (c_p->flags & F_DIRTY_CLA) { + int cla_reds = 0; + cla_res = erts_proc_copy_literal_area(c_p, &cla_reds, c_p->fcalls, 1); + ASSERT(is_value(cla_res)); + } + + if (c_p->flags & F_DIRTY_GC_HIBERNATE) { + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p); + if (c_p->msg.len) + c_p->flags &= ~F_DIRTY_GC_HIBERNATE; /* operation aborted... */ + else { + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + c_p->fvalue = NIL; + erts_garbage_collect_hibernate(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + } + erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); + } + + if (c_p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) { + if (c_p->flags & F_DIRTY_MAJOR_GC) + c_p->flags |= F_NEED_FULLSWEEP; + (void) erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, + c_p->arity, c_p->fcalls); + } + + ASSERT(!(c_p->flags & (F_DIRTY_CLA + | F_DIRTY_GC_HIBERNATE + | F_DIRTY_MAJOR_GC + | F_DIRTY_MINOR_GC))); + + stasks = c_p->dirty_sys_tasks; + c_p->dirty_sys_tasks = NULL; + + while (stasks) { + Eterm st_res; + ErtsProcSysTask *st = stasks; + stasks = st->next; + + switch (st->type) { + case ERTS_PSTT_CLA: + ASSERT(is_value(st_res)); + st_res = cla_res; + break; + case ERTS_PSTT_GC_MAJOR: + st_res = am_true; + break; + case ERTS_PSTT_GC_MINOR: + st_res = am_true; + break; + + default: + ERTS_INTERNAL_ERROR("Not supported dirty system task"); + break; + } + + (void) notify_sys_task_executed(c_p, st, st_res, 0); + + } + + erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_DIRTY_ACTIVE_SYS); +} + static BIF_RETTYPE dispatch_system_task(Process *c_p, erts_aint_t fail_state, ErtsProcSysTask *st, Eterm target, @@ -10748,7 +11512,7 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, ERTS_INTERNAL_ERROR("Unknown failure schedule_process_sys_task()"); failure = am_internal_error; } - notify_sys_task_executed(c_p, st, failure); + notify_sys_task_executed(c_p, st, failure, 1); } ERTS_BIF_PREP_RET(ret, am_ok); @@ -10963,6 +11727,15 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) } } +#ifdef ERTS_DIRTY_SCHEDULERS +static void +save_dirty_task(Process *c_p, ErtsProcSysTask *st) +{ + st->next = c_p->dirty_sys_tasks; + c_p->dirty_sys_tasks = st; +} +#endif + int erts_set_gc_state(Process *c_p, int enable) { @@ -11185,6 +11958,8 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) non_empty_runq(rq); #endif + ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP); + erts_smp_runq_unlock(rq); smp_notify_inc_runq(rq); @@ -11215,6 +11990,9 @@ exec_misc_ops(ErtsRunQueue *rq) rq->misc.end = NULL; } + if (!rq->misc.start) + ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP); + erts_smp_runq_unlock(rq); while (molp) { @@ -11299,6 +12077,7 @@ static void early_init_process_struct(void *varg, Eterm data) proc->common.id = make_internal_pid(data); #ifdef ERTS_DIRTY_SCHEDULERS erts_smp_atomic32_init_nob(&proc->dirty_state, 0); + proc->dirty_sys_tasks = NULL; #endif erts_smp_atomic32_init_relb(&proc->state, arg->state); @@ -11807,6 +12586,7 @@ void erts_init_empty_process(Process *p) #ifdef ERTS_DIRTY_SCHEDULERS erts_smp_atomic32_init_nob(&p->dirty_state, 0); + p->dirty_sys_tasks = NULL; #endif erts_smp_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL); @@ -11915,11 +12695,9 @@ erts_cleanup_empty_process(Process* p) static void delete_process(Process* p) { - Eterm *heap; ErtsPSD *psd; struct saved_calls *scb; process_breakpoint_time_t *pbt; - void *nif_export; VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id)); VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id, @@ -11936,9 +12714,7 @@ delete_process(Process* p) if (pbt) erts_free(ERTS_ALC_T_BPD, (void *) pbt); - nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL); - if (nif_export) - erts_destroy_nif_export(nif_export); + erts_destroy_nif_export(p); /* Cleanup psd */ @@ -11970,13 +12746,8 @@ delete_process(Process* p) hipe_delete_process(&p->hipe); #endif - heap = p->abandoned_heap ? p->abandoned_heap : p->heap; + erts_deallocate_young_generation(p); -#ifdef DEBUG - sys_memset(heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm)); -#endif - - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) heap, p->heap_sz*sizeof(Eterm)); if (p->old_heap != NULL) { #ifdef DEBUG @@ -11988,16 +12759,6 @@ delete_process(Process* p) (p->old_hend-p->old_heap)*sizeof(Eterm)); } - /* - * Free all pending message buffers. - */ - if (p->mbuf != NULL) { - free_message_buffer(p->mbuf); - } - - if (p->msg_frag) - erts_cleanup_messages(p->msg_frag); - erts_erase_dicts(p); /* free all pending messages */ @@ -12397,7 +13158,9 @@ send_exit_signal(Process *c_p, /* current process if and only } set_proc_exiting(c_p, state, rsn, NULL); } - else if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) { + else if (!(state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING_SYS))) { /* Process not running ... */ ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL; ErlHeapFragment *bp = NULL; @@ -12424,8 +13187,7 @@ send_exit_signal(Process *c_p, /* current process if and only ErlOffHeap *ohp; Uint rsn_sz = size_object(rsn); #ifdef ERTS_DIRTY_SCHEDULERS - if (state & (ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + if (state & ERTS_PSFLG_DIRTY_RUNNING) { bp = new_message_buffer(rsn_sz); ohp = &bp->off_heap; hp = &bp->mem[0]; @@ -12442,7 +13204,7 @@ send_exit_signal(Process *c_p, /* current process if and only set_proc_exiting(rp, state, rsn_cpy, bp); } else { /* Process running... */ - + /* * The pending exit will be discovered when the process * is scheduled out if not discovered earlier. @@ -12464,8 +13226,35 @@ send_exit_signal(Process *c_p, /* current process if and only &bp->off_heap); rp->pending_exit.bp = bp; } - erts_smp_atomic32_read_bor_relb(&rp->state, - ERTS_PSFLG_PENDING_EXIT); + + /* + * If no dirty work has been scheduled, pending exit will + * be discovered when the process is scheduled. If dirty work + * has been scheduled, we may need to add it to a normal run + * queue... + */ +#ifndef ERTS_DIRTY_SCHEDULERS + (void) erts_smp_atomic32_read_bor_relb(&rp->state, + ERTS_PSFLG_PENDING_EXIT); +#else + { + erts_aint32_t a = erts_smp_atomic32_read_nob(&rp->state); + while (1) { + erts_aint32_t n, e; + int dwork; + n = e = a; + n |= ERTS_PSFLG_PENDING_EXIT; + dwork = !!(n & ERTS_PSFLGS_DIRTY_WORK); + n &= ~ERTS_PSFLGS_DIRTY_WORK; + a = erts_smp_atomic32_cmpxchg_mb(&rp->state, n, e); + if (a == e) { + if (dwork) + erts_schedule_process(rp, n, *rp_locks); + break; + } + } + } +#endif } } /* else: @@ -13094,15 +13883,14 @@ erts_continue_exit_process(Process *p) } #ifdef ERTS_DIRTY_SCHEDULERS - if (a & (ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + if (a & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { p->flags |= F_DELAYED_DEL_PROC; delay_del_proc = 1; /* - * The dirty scheduler will also decrease - * refc when done... + * The dirty scheduler decrease refc + * when done with the process... */ - erts_proc_inc_refc(p); } #endif @@ -13316,6 +14104,8 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { erts_print(to, to_arg, "WAITING"); break; case ERTS_SSI_FLG_SUSPENDED: erts_print(to, to_arg, "SUSPENDED"); break; + case ERTS_SSI_FLG_MSB_EXEC: + erts_print(to, to_arg, "MSB_EXEC"); break; default: erts_print(to, to_arg, "UNKNOWN(%d)", flg); break; } @@ -13425,6 +14215,12 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { erts_print(to, to_arg, "NONEMPTY"); break; case ERTS_RUNQ_FLG_PROTECTED: erts_print(to, to_arg, "PROTECTED"); break; + case ERTS_RUNQ_FLG_EXEC: + erts_print(to, to_arg, "EXEC"); break; + case ERTS_RUNQ_FLG_MSB_EXEC: + erts_print(to, to_arg, "MSB_EXEC"); break; + case ERTS_RUNQ_FLG_MISC_OP: + erts_print(to, to_arg, "MISC_OP"); break; default: erts_print(to, to_arg, "UNKNOWN(%d)", flg); break; } @@ -13474,11 +14270,11 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) { * A nice system halt closing all open port goes as follows: * 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS * on all schedulers, then schedules itself out. - * 2) All shedulers detect this and set the flag halt_in_progress + * 2) All shedulers detect this and set the flag ERTS_RUNQ_FLG_HALTING * on their run queue. The last scheduler sets all non-closed ports * ERTS_PORT_SFLG_HALT. Global atomic erts_halt_progress is used * as refcount to determine which is last. - * 3) While the run ques has flag halt_in_progress no processes + * 3) While the run queues has flag ERTS_RUNQ_FLG_HALTING no processes * will be scheduled, only ports. * 4) When the last port closes that scheduler calls erlang:halt/1. * The same global atomic is used as refcount. @@ -13493,8 +14289,8 @@ void erts_halt(int code) erts_no_schedulers, -1)) { #ifdef ERTS_DIRTY_SCHEDULERS - ERTS_DIRTY_CPU_RUNQ->halt_in_progress = 1; - ERTS_DIRTY_IO_RUNQ->halt_in_progress = 1; + ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_CPU_RUNQ, ERTS_RUNQ_FLG_HALTING); + ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_IO_RUNQ, ERTS_RUNQ_FLG_HALTING); #endif erts_halt_code = code; notify_reap_ports_relb(); @@ -13508,6 +14304,9 @@ erts_dbg_check_halloc_lock(Process *p) ErtsSchedulerData *esdp; if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)) return 1; + if ((p->static_flags & ERTS_STC_FLG_SHADOW_PROC) + && ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data())) + return 1; if (p->common.id == ERTS_INVALID_PID) return 1; esdp = erts_proc_sched_data(p); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 1c4b9f149d..ce0989883c 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -170,8 +170,14 @@ extern int erts_sched_thread_suggested_stack_size; (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6)) #define ERTS_RUNQ_FLG_EXEC \ (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7)) +#define ERTS_RUNQ_FLG_MSB_EXEC \ + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 8)) +#define ERTS_RUNQ_FLG_MISC_OP \ + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 9)) +#define ERTS_RUNQ_FLG_HALTING \ + (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 10)) -#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8) +#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 11) #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS \ (ERTS_RUNQ_FLGS_EMIGRATE_QMASK \ @@ -262,8 +268,9 @@ typedef enum { #define ERTS_SSI_FLG_TSE_SLEEPING (((erts_aint32_t) 1) << 2) #define ERTS_SSI_FLG_WAITING (((erts_aint32_t) 1) << 3) #define ERTS_SSI_FLG_SUSPENDED (((erts_aint32_t) 1) << 4) +#define ERTS_SSI_FLG_MSB_EXEC (((erts_aint32_t) 1) << 5) -#define ERTS_SSI_FLGS_MAX 5 +#define ERTS_SSI_FLGS_MAX 6 #define ERTS_SSI_FLGS_SLEEP_TYPE \ (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING) @@ -274,7 +281,8 @@ typedef enum { #define ERTS_SSI_FLGS_ALL \ (ERTS_SSI_FLGS_SLEEP \ | ERTS_SSI_FLG_WAITING \ - | ERTS_SSI_FLG_SUSPENDED) + | ERTS_SSI_FLG_SUSPENDED \ + | ERTS_SSI_FLG_MSB_EXEC) /* * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency @@ -393,6 +401,12 @@ typedef struct { Process* last; } ErtsRunPrioQueue; +typedef enum { + ERTS_SCHED_NORMAL, + ERTS_SCHED_DIRTY_CPU, + ERTS_SCHED_DIRTY_IO +} ErtsSchedType; + typedef struct ErtsSchedulerData_ ErtsSchedulerData; typedef struct ErtsRunQueue_ ErtsRunQueue; @@ -478,7 +492,6 @@ struct ErtsRunQueue_ { erts_smp_atomic32_t len; int wakeup_other; int wakeup_other_reds; - int halt_in_progress; struct { ErtsProcList *pending_exiters; @@ -537,13 +550,15 @@ do { \ } while (0) typedef struct { - int need; /* "+sbu true" or scheduler_wall_time enabled */ + union { + erts_atomic32_t mod; /* on dirty schedulers */ + int need; /* "+sbu true" or scheduler_wall_time enabled */ + } u; int enabled; Uint64 start; struct { Uint64 total; Uint64 start; - int currently; } working; } ErtsSchedWallTime; @@ -642,6 +657,7 @@ struct ErtsSchedulerData_ { #endif ErtsSchedulerSleepInfo *ssi; Process *current_process; + ErtsSchedType type; Uint no; /* Scheduler number for normal schedulers */ #ifdef ERTS_DIRTY_SCHEDULERS ErtsDirtySchedId dirty_no; /* Scheduler number for dirty schedulers */ @@ -838,8 +854,8 @@ typedef struct { #define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN -#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ((ErtsProcLocks) 0) -#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ((ErtsProcLocks) 0) +#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN typedef struct { ErtsProcLocks get_locks; @@ -1060,6 +1076,9 @@ struct process { Uint64 bin_old_vheap; /* Virtual old heap size for binaries */ ErtsProcSysTaskQs *sys_task_qs; +#ifdef ERTS_DIRTY_SCHEDULERS + ErtsProcSysTask *dirty_sys_tasks; +#endif erts_smp_atomic32_t state; /* Process state flags (see ERTS_PSFLG_*) */ #ifdef ERTS_DIRTY_SCHEDULERS @@ -1384,14 +1403,18 @@ extern int erts_system_profile_ts_type; #define F_FORCE_GC (1 << 10) /* Force gc at process in-scheduling */ #define F_DISABLE_GC (1 << 11) /* Disable GC (see below) */ #define F_OFF_HEAP_MSGQ (1 << 12) /* Off heap msg queue */ -#define F_ON_HEAP_MSGQ (1 << 13) /* Off heap msg queue */ +#define F_ON_HEAP_MSGQ (1 << 13) /* On heap msg queue */ #define F_OFF_HEAP_MSGQ_CHNG (1 << 14) /* Off heap msg queue changing */ #define F_ABANDONED_HEAP_USE (1 << 15) /* Have usage of abandoned heap */ #define F_DELAY_GC (1 << 16) /* Similar to disable GC (see below) */ #define F_SCHDLR_ONLN_WAITQ (1 << 17) /* Process enqueued waiting to change schedulers online */ #define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */ -#define F_HIPE_MODE (1 << 19) +#define F_HIPE_MODE (1 << 19) /* Process is executing in HiPE mode */ #define F_DELAYED_DEL_PROC (1 << 20) /* Delay delete process (dirty proc exit case) */ +#define F_DIRTY_CLA (1 << 21) /* Dirty copy literal area scheduled */ +#define F_DIRTY_GC_HIBERNATE (1 << 22) /* Dirty GC hibernate scheduled */ +#define F_DIRTY_MAJOR_GC (1 << 23) /* Dirty major GC scheduled */ +#define F_DIRTY_MINOR_GC (1 << 24) /* Dirty minor GC scheduled */ /* * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent @@ -1573,19 +1596,18 @@ void erts_init_scheduling(int, int , int, int, int #endif ); - +#ifdef ERTS_DIRTY_SCHEDULERS +void erts_execute_dirty_system_task(Process *c_p); +#endif int erts_set_gc_state(Process *c_p, int enable); -Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable); +Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable, + int dirty_cpu, int want_dirty_io); Eterm erts_system_check_request(Process *c_p); Eterm erts_gc_info_request(Process *c_p); Uint64 erts_get_proc_interval(void); Uint64 erts_ensure_later_proc_interval(Uint64); Uint64 erts_step_proc_interval(void); -int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */ -void erts_destroy_nif_export(void *); /* see erl_nif.c */ -int erts_check_nif_export_in_area(Process *p, char *start, Uint size); - ErtsProcList *erts_proclist_create(Process *); ErtsProcList *erts_proclist_copy(ErtsProcList *); void erts_proclist_destroy(ErtsProcList *); @@ -1681,7 +1703,7 @@ ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list) return NULL; else { ErtsProcList *res = *list; - if (res == *list) + if (res->next == *list) *list = NULL; else *list = res->next; @@ -1922,6 +1944,8 @@ ErtsSchedulerData *erts_get_scheduler_data(void) void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks); ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks); +ERTS_GLB_INLINE void erts_schedule_dirty_sys_execution(Process *c_p); + #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks) @@ -1931,6 +1955,34 @@ erts_proc_notify_new_message(Process *p, ErtsProcLocks locks) if (!(state & ERTS_PSFLG_ACTIVE)) erts_schedule_process(p, state, locks); } + +ERTS_GLB_INLINE void +erts_schedule_dirty_sys_execution(Process *c_p) +{ + erts_aint32_t a, n, e; + + a = erts_smp_atomic32_read_nob(&c_p->state); + + /* + * Only a currently executing process schedules + * itself for dirty-sys execution... + */ + + ASSERT(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)); + + /* Don't set dirty-active-sys if we are about to exit... */ + + while (!(a & (ERTS_PSFLG_DIRTY_ACTIVE_SYS + | ERTS_PSFLG_EXITING + | ERTS_PSFLG_PENDING_EXIT))) { + e = a; + n = a | ERTS_PSFLG_DIRTY_ACTIVE_SYS; + a = erts_smp_atomic32_cmpxchg_mb(&c_p->state, n, e); + if (a == e) + break; /* dirty-active-sys set */ + } +} + #endif #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index e534c33637..7c23b5c76a 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -90,9 +90,12 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) { erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size); erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size); size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm); + if (p->abandoned_heap) + size += (p->hend - p->heap) * sizeof(Eterm); if (p->old_hend && p->old_heap) size += (p->old_hend - p->old_heap) * sizeof(Eterm); + size += p->msg.len * sizeof(ErtsMessage); for (mp = p->msg.first; mp; mp = mp->next) diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index a69185bc5c..a93f1755c8 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -123,7 +123,7 @@ erts_init_proc_lock(int cpus) for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) { #ifdef ERTS_ENABLE_LOCK_COUNT erts_mtx_init_x(&erts_pix_locks[i].u.mtx, - "pix_lock", make_small(i), 1); + "pix_lock", make_small(i)); #else erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock"); #endif @@ -1006,6 +1006,41 @@ erts_pid2proc_opt(Process *c_p, return proc; } +static ERTS_INLINE +Process *proc_lookup_inc_refc(Eterm pid, int allow_exit) +{ + Process *proc; +#ifdef ERTS_SMP + ErtsThrPrgrDelayHandle dhndl; + + dhndl = erts_thr_progress_unmanaged_delay(); +#endif + + proc = erts_proc_lookup_raw(pid); + if (proc) { + if (!allow_exit && ERTS_PROC_IS_EXITING(proc)) + proc = NULL; + else + erts_proc_inc_refc(proc); + } + +#ifdef ERTS_SMP + erts_thr_progress_unmanaged_continue(dhndl); +#endif + + return proc; +} + +Process *erts_proc_lookup_inc_refc(Eterm pid) +{ + return proc_lookup_inc_refc(pid, 0); +} + +Process *erts_proc_lookup_raw_inc_refc(Eterm pid) +{ + return proc_lookup_inc_refc(pid, 1); +} + void erts_proc_lock_init(Process *p) { @@ -1027,39 +1062,32 @@ erts_proc_lock_init(Process *p) #endif #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL -#ifdef ERTS_ENABLE_LOCK_COUNT - int do_lock_count = 1; -#else - int do_lock_count = 0; -#endif - - erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id); ethr_mutex_lock(&p->lock.main.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.main.lc); #endif - erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id); ethr_mutex_lock(&p->lock.link.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.link.lc); #endif - erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id); ethr_mutex_lock(&p->lock.msgq.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.msgq.lc); #endif - erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id); ethr_mutex_lock(&p->lock.btm.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.btm.lc); #endif - erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id, - do_lock_count); + erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id); ethr_mutex_lock(&p->lock.status.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.status.lc); #endif - erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id, do_lock_count); + erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id); ethr_mutex_lock(&p->lock.trace.mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &p->lock.trace.lc); diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 2cccf0697a..773529384f 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -940,6 +940,8 @@ void erts_proc_safelock(Process *a_proc, #define erts_pid2proc(PROC, HL, PID, NL) \ erts_pid2proc_opt((PROC), (HL), (PID), (NL), 0) +Process *erts_proc_lookup_inc_refc(Eterm pid); +Process *erts_proc_lookup_raw_inc_refc(Eterm pid); ERTS_GLB_INLINE Process *erts_pix2proc(int ix); ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid); diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 713ed50b86..14be511f86 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1065,7 +1065,7 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra) { #ifdef ERTS_SMP - erts_mtx_init_x(mtx, name, extra, 1); + erts_mtx_init_x(mtx, name, extra); #endif } @@ -1073,7 +1073,7 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra) { #ifdef ERTS_SMP - erts_mtx_init_locked_x(mtx, name, extra, 1); + erts_mtx_init_locked_x(mtx, name, extra); #endif } diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index eccd49f2a9..9e75f6fee5 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -479,14 +479,9 @@ ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void)); ERTS_GLB_INLINE erts_tid_t erts_thr_self(void); ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len); ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y); -ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, - int enable_lcnt); -ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, - Uint16 opt, int enable_lcnt); -ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, - char *name, - Eterm extra, - int enable_lcnt); +ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra); +ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt); +ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra); ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx); @@ -2164,7 +2159,7 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y) } ERTS_GLB_INLINE void -erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt) +erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra) { #ifdef USE_THREADS int res = ethr_mutex_init(&mtx->mtx); @@ -2174,17 +2169,13 @@ erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt) erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - if (enable_lcnt) - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); - else - erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra); + erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); #endif #endif } ERTS_GLB_INLINE void -erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt, - int enable_lcnt) +erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt) { #ifdef USE_THREADS int res = ethr_mutex_init(&mtx->mtx); @@ -2194,17 +2185,14 @@ erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt, erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - if (enable_lcnt) - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra); - else - erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX | opt, extra); + erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra); #endif #endif } ERTS_GLB_INLINE void -erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt) +erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra) { #ifdef USE_THREADS int res = ethr_mutex_init(&mtx->mtx); @@ -2214,10 +2202,7 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - if (enable_lcnt) - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); - else - erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra); + erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); #endif ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 9be4741ec8..04f3160d42 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1435,6 +1435,7 @@ void trace_gc(Process *p, Eterm what, Uint size, Eterm msg) { ErtsTracerNif *tnif = NULL; + Eterm* o_hp = NULL; Eterm* hp; Uint sz = 0; Eterm tup; @@ -1445,7 +1446,7 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg) if (is_non_value(msg)) { (void) erts_process_gc_info(p, &sz, NULL, 0, 0); - hp = HAlloc(p, sz + 3 + 2); + o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, (sz + 3 + 2) * sizeof(Eterm)); msg = erts_process_gc_info(p, NULL, &hp, 0, 0); tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3; @@ -1454,6 +1455,8 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg) send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_GC, what, msg, THE_NON_VALUE, am_true); + if (o_hp) + erts_free(ERTS_ALC_T_TMP, o_hp); } } diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index 378b6de49c..01fe1e5e23 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -143,6 +143,11 @@ Uint erts_trace_flag2bit(Eterm flag); int erts_trace_flags(Eterm List, Uint *pMask, ErtsTracer *pTracer, int *pCpuTimestamp); Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I); +Eterm +erts_bif_trace_epilogue(Process *p, Eterm result, int applying, + Export* ep, BeamInstr *cp, Uint32 flags, + Uint32 flags_meta, BeamInstr* I, + ErtsTracer meta_tracer); #ifdef ERTS_SMP void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp); diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 93cfe08105..d88cafc5bf 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -36,7 +36,7 @@ #define EMULATOR "BEAM" #define SEQ_TRACE 1 -#define CONTEXT_REDS 2000 /* Swap process out after this number */ +#define CONTEXT_REDS 4000 /* Swap process out after this number */ #define MAX_ARG 255 /* Max number of arguments allowed */ #define MAX_REG 1024 /* Max number of x(N) registers used */ diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index e431c3051b..64c08b1570 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -39,14 +39,11 @@ */ /* - * Bits 0-1 index the 'exception class tag' table. - */ -#define EXC_CLASSBITS 3 -#define GET_EXC_CLASS(x) ((x) & EXC_CLASSBITS) - -/* * Exception class tags (indices into the 'exception_tag' array) */ +#define EXTAG_OFFSET 0 +#define EXTAG_BITS 2 + #define EXTAG_ERROR 0 #define EXTAG_EXIT 1 #define EXTAG_THROWN 2 @@ -54,20 +51,31 @@ #define NUMBER_EXC_TAGS 3 /* The number of exception class tags */ /* - * Exit code flags (bits 2-7) + * Index to the 'exception class tag' table. + */ +#define EXC_CLASSBITS ((1<<EXTAG_BITS)-1) +#define GET_EXC_CLASS(x) ((x) & EXC_CLASSBITS) + +/* + * Exit code flags * * These flags make is easier and quicker to decide what to do with the * exception in the early stages, before a handler is found, and also * maintains some separation between the class tag and the actions. */ -#define EXF_PANIC (1<<2) /* ignore catches */ -#define EXF_THROWN (1<<3) /* nonlocal return */ -#define EXF_LOG (1<<4) /* write to logger on termination */ -#define EXF_NATIVE (1<<5) /* occurred in native code */ -#define EXF_SAVETRACE (1<<6) /* save stack trace in internal form */ -#define EXF_ARGLIST (1<<7) /* has arglist for top of trace */ +#define EXF_OFFSET EXTAG_BITS +#define EXF_BITS 7 -#define EXC_FLAGBITS 0x00fc +#define EXF_PANIC (1<<(0+EXF_OFFSET)) /* ignore catches */ +#define EXF_THROWN (1<<(1+EXF_OFFSET)) /* nonlocal return */ +#define EXF_LOG (1<<(2+EXF_OFFSET)) /* write to logger on termination */ +#define EXF_NATIVE (1<<(3+EXF_OFFSET)) /* occurred in native code */ +#define EXF_SAVETRACE (1<<(4+EXF_OFFSET)) /* save stack trace in internal form */ +#define EXF_ARGLIST (1<<(5+EXF_OFFSET)) /* has arglist for top of trace */ +#define EXF_RESTORE_NIF (1<<(6+EXF_OFFSET)) /* restore original bif/nif */ + +#define EXC_FLAGBITS (((1<<(EXF_BITS+EXF_OFFSET))-1) \ + & ~((1<<(EXF_OFFSET))-1)) /* * The primary fields of an exception code @@ -77,11 +85,16 @@ #define NATIVE_EXCEPTION(x) ((x) | EXF_NATIVE) /* - * Bits 8-12 of the error code are used for indexing into + * Error code used for indexing into * the short-hand error descriptor table. */ -#define EXC_INDEXBITS 0x1f00 -#define GET_EXC_INDEX(x) (((x) & EXC_INDEXBITS) >> 8) +#define EXC_OFFSET (EXF_OFFSET+EXF_BITS) +#define EXC_BITS 5 + +#define EXC_INDEXBITS (((1<<(EXC_BITS+EXC_OFFSET))-1) \ + & ~((1<<(EXC_OFFSET))-1)) + +#define GET_EXC_INDEX(x) (((x) & EXC_INDEXBITS) >> EXC_OFFSET) /* * Exit codes used for raising a fresh exception. The primary exceptions @@ -107,46 +120,46 @@ /* Error with given arglist term * (exit reason in p->fvalue) */ -#define EXC_NORMAL ((1 << 8) | EXC_EXIT) +#define EXC_NORMAL ((1 << EXC_OFFSET) | EXC_EXIT) /* Normal exit (reason 'normal') */ -#define EXC_INTERNAL_ERROR ((2 << 8) | EXC_ERROR | EXF_PANIC) +#define EXC_INTERNAL_ERROR ((2 << EXC_OFFSET) | EXC_ERROR | EXF_PANIC) /* Things that shouldn't happen */ -#define EXC_BADARG ((3 << 8) | EXC_ERROR) +#define EXC_BADARG ((3 << EXC_OFFSET) | EXC_ERROR) /* Bad argument to a BIF */ -#define EXC_BADARITH ((4 << 8) | EXC_ERROR) +#define EXC_BADARITH ((4 << EXC_OFFSET) | EXC_ERROR) /* Bad arithmetic */ -#define EXC_BADMATCH ((5 << 8) | EXC_ERROR) +#define EXC_BADMATCH ((5 << EXC_OFFSET) | EXC_ERROR) /* Bad match in function body */ -#define EXC_FUNCTION_CLAUSE ((6 << 8) | EXC_ERROR) +#define EXC_FUNCTION_CLAUSE ((6 << EXC_OFFSET) | EXC_ERROR) /* No matching function head */ -#define EXC_CASE_CLAUSE ((7 << 8) | EXC_ERROR) +#define EXC_CASE_CLAUSE ((7 << EXC_OFFSET) | EXC_ERROR) /* No matching case clause */ -#define EXC_IF_CLAUSE ((8 << 8) | EXC_ERROR) +#define EXC_IF_CLAUSE ((8 << EXC_OFFSET) | EXC_ERROR) /* No matching if clause */ -#define EXC_UNDEF ((9 << 8) | EXC_ERROR) +#define EXC_UNDEF ((9 << EXC_OFFSET) | EXC_ERROR) /* No farity that matches */ -#define EXC_BADFUN ((10 << 8) | EXC_ERROR) +#define EXC_BADFUN ((10 << EXC_OFFSET) | EXC_ERROR) /* Not an existing fun */ -#define EXC_BADARITY ((11 << 8) | EXC_ERROR) +#define EXC_BADARITY ((11 << EXC_OFFSET) | EXC_ERROR) /* Attempt to call fun with * wrong number of arguments. */ -#define EXC_TIMEOUT_VALUE ((12 << 8) | EXC_ERROR) +#define EXC_TIMEOUT_VALUE ((12 << EXC_OFFSET) | EXC_ERROR) /* Bad time out value */ -#define EXC_NOPROC ((13 << 8) | EXC_ERROR) +#define EXC_NOPROC ((13 << EXC_OFFSET) | EXC_ERROR) /* No process or port */ -#define EXC_NOTALIVE ((14 << 8) | EXC_ERROR) +#define EXC_NOTALIVE ((14 << EXC_OFFSET) | EXC_ERROR) /* Not distributed */ -#define EXC_SYSTEM_LIMIT ((15 << 8) | EXC_ERROR) +#define EXC_SYSTEM_LIMIT ((15 << EXC_OFFSET) | EXC_ERROR) /* Ran out of something */ -#define EXC_TRY_CLAUSE ((16 << 8) | EXC_ERROR) +#define EXC_TRY_CLAUSE ((16 << EXC_OFFSET) | EXC_ERROR) /* No matching try clause */ -#define EXC_NOTSUP ((17 << 8) | EXC_ERROR) +#define EXC_NOTSUP ((17 << EXC_OFFSET) | EXC_ERROR) /* Not supported */ -#define EXC_BADMAP ((18 << 8) | EXC_ERROR) +#define EXC_BADMAP ((18 << EXC_OFFSET) | EXC_ERROR) /* Bad map */ -#define EXC_BADKEY ((19 << 8) | EXC_ERROR) +#define EXC_BADKEY ((19 << EXC_OFFSET) | EXC_ERROR) /* Bad key in map */ #define NUMBER_EXIT_CODES 20 /* The number of exit code indices */ @@ -154,7 +167,7 @@ /* * Internal pseudo-error codes. */ -#define TRAP (1 << 8) /* BIF Trap to erlang code */ +#define TRAP (1 << EXC_OFFSET) /* BIF Trap to erlang code */ /* * Aliases for some common exit codes. diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index f397ab6b00..33ed6d7ec1 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -356,7 +356,7 @@ Export *export_list(int i, ErtsCodeIndex code_ix) int export_list_size(ErtsCodeIndex code_ix) { - return export_tables[code_ix].entries; + return erts_index_num_entries(&export_tables[code_ix]); } int export_table_sz(void) diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index a49b242d7c..10e452fa25 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2163,12 +2163,8 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) * We use this atom as sysname in local pid/port/refs * for the ETS compressed format (DFLAG_INTERNAL_TAGS). * - * We used atom '' earlier but that turned out to cause problems - * for buggy erl_interface/ic usage of c-nodes with empty node names. - * A long atom reduces risk of nodes actually called this and the length - * does not matter anyway as it's encoded with atom index (ATOM_INTERNAL_REF2). */ -#define INTERNAL_LOCAL_SYSNAME am_await_microstate_accounting_modifications +#define INTERNAL_LOCAL_SYSNAME am_ErtsSecretAtom static byte* enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 2decb56544..2158f54a49 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -89,12 +89,6 @@ typedef struct enif_resource_t extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); extern void erts_post_nif(struct enif_environment_t* env); -#ifdef ERTS_DIRTY_SCHEDULERS -extern void erts_pre_dirty_nif(ErtsSchedulerData *, - struct enif_environment_t*, Process*, - struct erl_module_nif*); -extern void erts_post_dirty_nif(struct enif_environment_t* env); -#endif extern void erts_resource_stop(ErlNifResource*, ErlNifEvent, int is_direct_call); extern Eterm erts_nif_taints(Process* p); extern void erts_print_nif_taints(fmtfn_t to, void* to_arg); @@ -107,6 +101,12 @@ extern Eterm erts_nif_call_function(Process *p, Process *tracee, struct enif_func_t *, int argc, Eterm *argv); +#ifdef ERTS_DIRTY_SCHEDULERS +int erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, + BeamInstr *I, Eterm *reg); +#endif /* ERTS_DIRTY_SCHEDULERS */ + + /* Driver handle (wrapper for old plain handle) */ #define ERL_DE_OK 0 #define ERL_DE_UNLOAD 1 @@ -1022,7 +1022,7 @@ void erts_queue_monitor_message(Process *, Eterm, Eterm); void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, - Eterm (*bif)(Process*,Eterm*)); + Eterm (*bif)(Process*, Eterm*, BeamInstr*)); void erts_init_bif(void); Eterm erl_send(Process *p, Eterm to, Eterm msg); diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index c86a2122f6..cd834e2c12 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -91,9 +91,16 @@ index_put_entry(IndexTable* t, void* tmpl) t->seg_table[ix>>INDEX_PAGE_SHIFT] = erts_alloc(t->type, sz); t->size += INDEX_PAGE_SIZE; } - t->entries++; p->index = ix; t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK] = p; + + /* + * Do a write barrier here to allow readers to do lock free iteration. + * erts_index_num_entries() does matching read barrier. + */ + ERTS_SMP_WRITE_MEMORY_BARRIER; + t->entries++; + return p; } diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h index b2e3c0eab5..10f5d1eb39 100644 --- a/erts/emulator/beam/index.h +++ b/erts/emulator/beam/index.h @@ -65,6 +65,7 @@ void index_erase_latest_from(IndexTable*, Uint ix); ERTS_GLB_INLINE int index_put(IndexTable*, void*); ERTS_GLB_INLINE IndexSlot* erts_index_lookup(IndexTable*, Uint); +ERTS_GLB_INLINE int erts_index_num_entries(IndexTable* t); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -78,6 +79,19 @@ erts_index_lookup(IndexTable* t, Uint ix) { return t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK]; } + +ERTS_GLB_INLINE int erts_index_num_entries(IndexTable* t) +{ + int ret = t->entries; + /* + * Do a read barrier here to allow lock free iteration + * on tables where entries are never erased. + * index_put_entry() does matching write barrier. + */ + ERTS_SMP_READ_MEMORY_BARRIER; + return ret; +} + #endif #endif diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 4f131c74de..33b74f30b7 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -259,13 +259,11 @@ static ERTS_INLINE void port_init_instr(Port *prt ASSERT(prt->drv_ptr && prt->lock); if (!prt->drv_ptr->lock) { char *lock_str = "port_lock"; - erts_mtx_init_locked_x(prt->lock, lock_str, id, #ifdef ERTS_ENABLE_LOCK_COUNT - (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK) -#else - 0 + if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)) + lock_str = NULL; #endif - ); + erts_mtx_init_locked_x(prt->lock, lock_str, id); } #endif erts_port_task_init_sched(&prt->sched, id); @@ -7107,7 +7105,7 @@ driver_pdl_create(ErlDrvPort dp) return NULL; pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK, sizeof(struct erl_drv_port_data_lock)); - erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id, 1); + erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id); pdl_init_refc(pdl); erts_port_inc_refc(pp); pdl->prt = pp; @@ -8290,14 +8288,13 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle) erts_mtx_init_x(drv->lock, "driver_lock", #if defined(ERTS_ENABLE_LOCK_CHECK) || defined(ERTS_ENABLE_LOCK_COUNT) - erts_atom_put((byte *) drv->name, - sys_strlen(drv->name), - ERTS_ATOM_ENC_LATIN1, - 1), + erts_atom_put((byte *) drv->name, + sys_strlen(drv->name), + ERTS_ATOM_ENC_LATIN1, + 1) #else - NIL, + NIL #endif - 1 ); } #endif diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 4ef04d020a..ec36b23059 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -802,15 +802,10 @@ call_ext_last u==1 Bif=u$bif:erts_internal:check_process_code/1 D => i_call_ext_ call_ext_only u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext_only Bif # -# The BIFs erlang:garbage_collect/0 must be called like a function, +# The BIFs erts_internal:garbage_collect/1 must be called like a function, # to allow them to invoke the garbage collector. (The stack pointer must # be saved and p->arity must be zeroed, which is not done on ordinary BIF calls.) # - -call_ext u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext Bif -call_ext_last u==0 Bif=u$bif:erlang:garbage_collect/0 D => i_call_ext_last Bif D -call_ext_only u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext_only Bif - call_ext u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext Bif call_ext_last u==1 Bif=u$bif:erts_internal:garbage_collect/1 D => i_call_ext_last Bif D call_ext_only u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext_only Bif diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index ec502d5a78..3fa48da1ec 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -56,6 +56,8 @@ #ifdef HIPE # include "hipe_mode_switch.h" #endif +#define ERTS_WANT_NFUNC_SCHED_INTERNALS__ +#include "erl_nfunc_sched.h" #undef M_TRIM_THRESHOLD #undef M_TOP_PAD diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index 3adb8db661..d64f015a6a 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -1937,7 +1937,8 @@ static void free_sendfile(void *data) { MUTEX_LOCK(d->c.sendfile.q_mtx); driver_deq(d->c.sendfile.port,1); MUTEX_UNLOCK(d->c.sendfile.q_mtx); - driver_select(d->c.sendfile.port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 0); + driver_select(d->c.sendfile.port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, + ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 0); } EF_FREE(data); } @@ -2555,7 +2556,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) desc->sendfile_state = sending; desc->d = d; driver_select(desc->port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, - ERL_DRV_USE_NO_CALLBACK|ERL_DRV_WRITE, 1); + ERL_DRV_USE|ERL_DRV_WRITE, 1); } break; #endif diff --git a/erts/emulator/hipe/hipe_amd64_bifs.m4 b/erts/emulator/hipe/hipe_amd64_bifs.m4 index 21739726bb..dca3887564 100644 --- a/erts/emulator/hipe/hipe_amd64_bifs.m4 +++ b/erts/emulator/hipe/hipe_amd64_bifs.m4 @@ -41,11 +41,11 @@ define(HANDLE_GOT_MBUF,` `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) # define CALL_BIF(F) \ - movq CSYM(F)@GOTPCREL(%rip), %r11; \ + movq CSYM(nbif_impl_##F)@GOTPCREL(%rip), %r11; \ movq %r11, P_BIF_CALLEE(P); \ call CSYM(hipe_debug_bif_wrapper) #else -# define CALL_BIF(F) call CSYM(F) +# define CALL_BIF(F) call CSYM(nbif_impl_##F) #endif' /* @@ -595,13 +595,9 @@ noproc_primop_interface_0(nbif_handle_fp_exception, erts_restore_fpu) #endif /* NO_FPE_SIGNALS */ /* - * Implement gc_bif_interface_0 as nofail_primop_interface_0. - */ -define(gc_bif_interface_0,`nofail_primop_interface_0($1, $2)') - -/* - * Implement gc_bif_interface_N as standard_bif_interface_N (N=1,2,3). + * Implement gc_bif_interface_N as standard_bif_interface_N. */ +define(gc_bif_interface_0,`standard_bif_interface_0($1, $2)') define(gc_bif_interface_1,`standard_bif_interface_1($1, $2)') define(gc_bif_interface_2,`standard_bif_interface_2($1, $2)') define(gc_bif_interface_3,`standard_bif_interface_3($1, $2)') diff --git a/erts/emulator/hipe/hipe_arm_bifs.m4 b/erts/emulator/hipe/hipe_arm_bifs.m4 index d7a2fec04a..a9097dabde 100644 --- a/erts/emulator/hipe/hipe_arm_bifs.m4 +++ b/erts/emulator/hipe/hipe_arm_bifs.m4 @@ -30,9 +30,9 @@ include(`hipe/hipe_arm_asm.m4') .arm `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) -# define CALL_BIF(F) ldr r14, =F; str r14, [r0, #P_BIF_CALLEE]; bl hipe_debug_bif_wrapper +# define CALL_BIF(F) ldr r14, =nbif_impl_##F; str r14, [r0, #P_BIF_CALLEE]; bl hipe_debug_bif_wrapper #else -# define CALL_BIF(F) bl F +# define CALL_BIF(F) bl nbif_impl_##F #endif' define(TEST_GOT_MBUF,`ldr r1, [P, #P_MBUF] /* `TEST_GOT_MBUF' */ diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 57fbbd9403..9c6ac4bd9c 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -732,7 +732,7 @@ struct nbif { }; static struct nbif nbifs[BIF_SIZE] = { -#define BIF_LIST(MOD,FUN,ARY,CFUN,IX) \ +#define BIF_LIST(MOD,FUN,ARY,BIF,CFUN,IX) \ { {0,0}, MOD, FUN, ARY, &nbif_##CFUN }, #include "erl_bif_list.h" #undef BIF_LIST @@ -905,7 +905,8 @@ BIF_RETTYPE hipe_bifs_term_to_word_1(BIF_ALIST_1) } /* XXX: this is really a primop, not a BIF */ -BIF_RETTYPE hipe_conv_big_to_float(BIF_ALIST_1) +/* Called via standard_bif_interface_1 */ +BIF_RETTYPE nbif_impl_hipe_conv_big_to_float(NBIF_ALIST_1) { Eterm res; Eterm *hp; @@ -1121,7 +1122,7 @@ static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa) struct hipe_ref { struct hipe_ref_head head; /* list of refs to same calleee */ void *address; -#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) +#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) void *trampoline; #endif unsigned int flags; @@ -1432,7 +1433,8 @@ void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a) } /* primop, but called like a BIF for error handling purposes */ -BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3) +/* Called via standard_bif_interface_3 */ +BIF_RETTYPE nbif_impl_hipe_find_na_or_make_stub(NBIF_ALIST_3) { Uint arity; void *address; @@ -1457,7 +1459,8 @@ BIF_RETTYPE hipe_bifs_find_na_or_make_stub_1(BIF_ALIST_1) } /* primop, but called like a BIF for error handling purposes */ -BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) +/* Called via standard_bif_interface_2 */ +BIF_RETTYPE nbif_impl_hipe_nonclosure_address(NBIF_ALIST_2) { Eterm hdr, m, f; void *address; @@ -1549,7 +1552,7 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref)); ref->address = address; -#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) +#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) ref->trampoline = trampoline; #endif ref->flags = flags; @@ -1864,7 +1867,7 @@ void hipe_redirect_to_module(Module* modp) if (ref->flags & REF_FLAG_IS_LOAD_MFA) res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa); else { -#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) +#if defined(__arm__) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) void* trampoline = ref->trampoline; #else void* trampoline = NULL; diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index 4a59bacc6e..811c3801c1 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -30,7 +30,7 @@ extern Uint *hipe_bifs_find_pc_from_mfa(Eterm mfa); extern void hipe_mfa_info_table_init(void); extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a); -extern BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3); +extern BIF_RETTYPE nbif_impl_hipe_find_na_or_make_stub(NBIF_ALIST_3); extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a); /* needed in beam_load.c */ diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c index dfd34e31d4..e04d3d32d1 100644 --- a/erts/emulator/hipe/hipe_bif2.c +++ b/erts/emulator/hipe/hipe_bif2.c @@ -155,7 +155,7 @@ BIF_RETTYPE hipe_bifs_modeswitch_debug_off_0(BIF_ALIST_0) #if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) -BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1); +BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1); # define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\ @@ -163,13 +163,13 @@ BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1); # define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN) -BIF_RETTYPE hipe_debug_bif_wrapper(BIF_ALIST_1) +BIF_RETTYPE hipe_debug_bif_wrapper(NBIF_ALIST_1) { - typedef BIF_RETTYPE Bif(BIF_ALIST_1); - Bif* fp = (Bif*) (BIF_P->hipe.bif_callee); + typedef BIF_RETTYPE nBif(NBIF_ALIST_1); + nBif* fp = (nBif*) (BIF_P->hipe.bif_callee); BIF_RETTYPE res; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(BIF_P); - res = (*fp)(BIF_P, BIF__ARGS); + res = (*fp)(NBIF_CALL_ARGS); ERTS_SMP_REQ_PROC_MAIN_LOCK(BIF_P); return res; } diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4 index bb328b5915..f034c4700c 100644 --- a/erts/emulator/hipe/hipe_bif_list.m4 +++ b/erts/emulator/hipe/hipe_bif_list.m4 @@ -71,6 +71,32 @@ ****************************************************************/ /* + * NOTE: + * Beam BIFs have the prototype: + * Eterm (*BIF)(Process *c_p, Eterm *regs, UWord *I) + * Native BIFs have the prototype: + * Eterm (*BIF)(Process *c_p, Eterm *regs) + * + * Beam BIFs expect 'I' to contain current instruction + * pointer when called from beam, and expect 'I' to + * contain a pointer to the export entry of the BIF + * when called from native code. In order to facilitate + * this, beam BIFs are called via wrapper functions + * when called from native code. These wrapper functions + * are auto-generated (by utils/make_tables) and have + * the function names nbif_impl_<BIF>. + * + * The standard_bif_interface_*() and + * gc_bif_interface_*() will add the prefix and + * thus call nbif_impl_<cbif_name>. That is, all + * functions (true BIFs as well as other c-functions) + * called via these interfaces have to be named + * nbif_impl_<FUNC>. + */ + +/* + * See NOTE above! + * * standard_bif_interface_0(nbif_name, cbif_name) * standard_bif_interface_1(nbif_name, cbif_name) * standard_bif_interface_2(nbif_name, cbif_name) @@ -93,6 +119,8 @@ */ /* + * See NOTE above! + * * gc_bif_interface_0(nbif_name, cbif_name) * gc_bif_interface_1(nbif_name, cbif_name) * gc_bif_interface_2(nbif_name, cbif_name) @@ -155,7 +183,7 @@ standard_bif_interface_0(nbif_ports_0, ports_0) */ gc_bif_interface_1(nbif_erts_internal_check_process_code_1, hipe_erts_internal_check_process_code_1) gc_bif_interface_1(nbif_erase_1, erase_1) -gc_bif_interface_0(nbif_garbage_collect_0, garbage_collect_0) +gc_bif_interface_1(nbif_erts_internal_garbage_collect_1, erts_internal_garbage_collect_1) gc_nofail_primop_interface_1(nbif_gc_1, hipe_gc) gc_bif_interface_2(nbif_put_2, put_2) @@ -247,7 +275,7 @@ nocons_nofail_primop_interface_5(nbif_bs_put_big_integer, hipe_bs_put_big_intege noproc_primop_interface_5(nbif_bs_put_big_integer, hipe_bs_put_big_integer) ')dnl -gc_bif_interface_0(nbif_check_get_msg, hipe_check_get_msg) +nofail_primop_interface_0(nbif_check_get_msg, hipe_check_get_msg) #`ifdef' NO_FPE_SIGNALS nocons_nofail_primop_interface_0(nbif_emulate_fpe, hipe_emulate_fpe) @@ -291,8 +319,8 @@ gc_bif_interface_2(nbif_maps_merge_2, hipe_wrapper_maps_merge_2) * BIF_LIST(ModuleAtom,FunctionAtom,Arity,CFun,Index) */ -define(BIF_LIST,`standard_bif_interface_$3(nbif_$4, $4)') -include(TARGET/`erl_bif_list.h') +define(BIF_LIST,`standard_bif_interface_$3(nbif_$5, $5)') +include(TTF_DIR/`erl_bif_list.h') /* * Guard BIFs. diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c index 9439b823ab..9a9252e8b8 100644 --- a/erts/emulator/hipe/hipe_native_bif.c +++ b/erts/emulator/hipe/hipe_native_bif.c @@ -42,8 +42,8 @@ */ /* for -Wmissing-prototypes :-( */ -extern Eterm hipe_erts_internal_check_process_code_1(BIF_ALIST_1); -extern Eterm hipe_show_nstack_1(BIF_ALIST_1); +extern Eterm nbif_impl_hipe_erts_internal_check_process_code_1(NBIF_ALIST_1); +extern Eterm nbif_impl_hipe_show_nstack_1(NBIF_ALIST_1); /* Used when a BIF can trigger a stack walk. */ static __inline__ void hipe_set_narity(Process *p, unsigned int arity) @@ -51,22 +51,24 @@ static __inline__ void hipe_set_narity(Process *p, unsigned int arity) p->hipe.narity = arity; } -Eterm hipe_erts_internal_check_process_code_1(BIF_ALIST_1) +/* Called via standard_bif_interface_2 */ +Eterm nbif_impl_hipe_erts_internal_check_process_code_1(NBIF_ALIST_1) { Eterm ret; hipe_set_narity(BIF_P, 1); - ret = erts_internal_check_process_code_1(BIF_P, BIF__ARGS); + ret = nbif_impl_erts_internal_check_process_code_1(NBIF_CALL_ARGS); hipe_set_narity(BIF_P, 0); return ret; } -Eterm hipe_show_nstack_1(BIF_ALIST_1) +/* Called via standard_bif_interface_1 */ +Eterm nbif_impl_hipe_show_nstack_1(NBIF_ALIST_1) { Eterm ret; hipe_set_narity(BIF_P, 1); - ret = hipe_bifs_show_nstack_1(BIF_P, BIF__ARGS); + ret = nbif_impl_hipe_bifs_show_nstack_1(NBIF_CALL_ARGS); hipe_set_narity(BIF_P, 0); return ret; } @@ -89,7 +91,7 @@ void hipe_gc(Process *p, Eterm need) * has begun. * XXX: BUG: native code should check return status */ -BIF_RETTYPE hipe_set_timeout(BIF_ALIST_1) +BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1) { Process* p = BIF_P; Eterm timeout_value = BIF_ARG_1; @@ -226,11 +228,6 @@ void hipe_handle_exception(Process *c_p) ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */ - if (c_p->mbuf) { - erts_printf("%s line %u: p==%p, p->mbuf==%p\n", __FUNCTION__, __LINE__, c_p, c_p->mbuf); - /* erts_garbage_collect(c_p, 0, NULL, 0); */ - } - /* * Check if we have an arglist for the top level call. If so, this * is encoded in Value, so we have to dig out the real Value as well @@ -259,11 +256,6 @@ void hipe_handle_exception(Process *c_p) /* Synthesized to avoid having to generate code for it. */ c_p->def_arg_reg[0] = exception_tag[GET_EXC_CLASS(c_p->freason)]; - if (c_p->mbuf) { - /* erts_printf("%s line %u: p==%p, p->mbuf==%p, p->lastbif==%p\n", __FUNCTION__, __LINE__, c_p, c_p->mbuf, c_p->hipe.lastbif); */ - erts_garbage_collect(c_p, 0, NULL, 0); - } - hipe_find_handler(c_p); } @@ -280,10 +272,10 @@ static struct StackTrace *get_trace_from_exc(Eterm exc) * This does what the (misnamed) Beam instruction 'raise_ss' does, * namely, a proper re-throw of an exception that was caught by 'try'. */ - -BIF_RETTYPE hipe_rethrow(BIF_ALIST_2) +/* Called via standard_bif_interface_2 */ +BIF_RETTYPE nbif_impl_hipe_rethrow(NBIF_ALIST_2) { - Process* c_p = BIF_P; + Process *c_p = BIF_P; Eterm exc = BIF_ARG_1; Eterm value = BIF_ARG_2; @@ -407,7 +399,7 @@ Eterm hipe_bs_utf8_size(Eterm arg) return make_small(4); } -BIF_RETTYPE hipe_bs_put_utf8(BIF_ALIST_3) +BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3) { Process* p = BIF_P; Eterm arg = BIF_ARG_1; @@ -468,7 +460,7 @@ Eterm hipe_bs_put_utf16(Process *p, Eterm arg, byte *base, unsigned int offset, return new_offset; } -BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3) +BIF_RETTYPE nbif_impl_hipe_bs_put_utf16be(NBIF_ALIST_3) { Process *p = BIF_P; Eterm arg = BIF_ARG_1; @@ -477,7 +469,7 @@ BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3) return hipe_bs_put_utf16(p, arg, base, offset, 0); } -BIF_RETTYPE hipe_bs_put_utf16le(BIF_ALIST_3) +BIF_RETTYPE nbif_impl_hipe_bs_put_utf16le(NBIF_ALIST_3) { Process *p = BIF_P; Eterm arg = BIF_ARG_1; @@ -495,7 +487,7 @@ static int validate_unicode(Eterm arg) return 1; } -BIF_RETTYPE hipe_bs_validate_unicode(BIF_ALIST_1) +BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1) { Process *p = BIF_P; Eterm arg = BIF_ARG_1; @@ -513,7 +505,8 @@ int hipe_bs_validate_unicode_retract(ErlBinMatchBuffer* mb, Eterm arg) return 1; } -BIF_RETTYPE hipe_is_divisible(BIF_ALIST_2) +/* Called via standard_bif_interface_2 */ +BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2) { /* Arguments are Eterm-sized unsigned integers */ Uint dividend = BIF_ARG_1; diff --git a/erts/emulator/hipe/hipe_native_bif.h b/erts/emulator/hipe/hipe_native_bif.h index a02d26087b..38f874888b 100644 --- a/erts/emulator/hipe/hipe_native_bif.h +++ b/erts/emulator/hipe/hipe_native_bif.h @@ -74,27 +74,27 @@ AEXTERN(void,nbif_select_msg,(Process*)); AEXTERN(Eterm,nbif_cmp_2,(void)); AEXTERN(Eterm,nbif_eq_2,(void)); -BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2); -BIF_RETTYPE hipe_conv_big_to_float(BIF_ALIST_1); +BIF_RETTYPE nbif_impl_hipe_nonclosure_address(NBIF_ALIST_2); +BIF_RETTYPE nbif_impl_hipe_conv_big_to_float(NBIF_ALIST_1); void hipe_fclearerror_error(Process*); void hipe_select_msg(Process*); void hipe_gc(Process*, Eterm); -BIF_RETTYPE hipe_set_timeout(BIF_ALIST_1); +BIF_RETTYPE nbif_impl_hipe_set_timeout(NBIF_ALIST_1); void hipe_handle_exception(Process*); -BIF_RETTYPE hipe_rethrow(BIF_ALIST_2); +BIF_RETTYPE nbif_impl_hipe_rethrow(NBIF_ALIST_2); char *hipe_bs_allocate(int); Binary *hipe_bs_reallocate(Binary*, int); int hipe_bs_put_small_float(Process*, Eterm, Uint, byte*, unsigned, unsigned); void hipe_bs_put_bits(Eterm, Uint, byte*, unsigned, unsigned); Eterm hipe_bs_utf8_size(Eterm); -BIF_RETTYPE hipe_bs_put_utf8(BIF_ALIST_3); +BIF_RETTYPE nbif_impl_hipe_bs_put_utf8(NBIF_ALIST_3); Eterm hipe_bs_utf16_size(Eterm); -BIF_RETTYPE hipe_bs_put_utf16be(BIF_ALIST_3); -BIF_RETTYPE hipe_bs_put_utf16le(BIF_ALIST_3); -BIF_RETTYPE hipe_bs_validate_unicode(BIF_ALIST_1); +BIF_RETTYPE nbif_impl_hipe_bs_put_utf16be(NBIF_ALIST_3); +BIF_RETTYPE nbif_impl_hipe_bs_put_utf16le(NBIF_ALIST_3); +BIF_RETTYPE nbif_impl_hipe_bs_validate_unicode(NBIF_ALIST_1); struct erl_bin_match_buffer; int hipe_bs_validate_unicode_retract(struct erl_bin_match_buffer*, Eterm); -BIF_RETTYPE hipe_is_divisible(BIF_ALIST_2); +BIF_RETTYPE nbif_impl_hipe_is_divisible(NBIF_ALIST_2); #ifdef NO_FPE_SIGNALS AEXTERN(void,nbif_emulate_fpe,(Process*)); @@ -129,7 +129,7 @@ void hipe_atomic_inc(int*); void hipe_clear_timeout(Process*); #endif -#define BIF_LIST(M,F,A,C,I) AEXTERN(Eterm,nbif_##C,(void)); +#define BIF_LIST(M,F,A,B,C,I) AEXTERN(Eterm,nbif_##C,(void)); #include "erl_bif_list.h" #undef BIF_LIST diff --git a/erts/emulator/hipe/hipe_ppc_bifs.m4 b/erts/emulator/hipe/hipe_ppc_bifs.m4 index b540562185..79a8bef77d 100644 --- a/erts/emulator/hipe/hipe_ppc_bifs.m4 +++ b/erts/emulator/hipe/hipe_ppc_bifs.m4 @@ -26,9 +26,9 @@ include(`hipe/hipe_ppc_asm.m4') #`include' "hipe_literals.h" `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) -# define CALL_BIF(F) STORE_IA(CSYM(F), P_BIF_CALLEE(P), r29); bl CSYM(hipe_debug_bif_wrapper) +# define CALL_BIF(F) STORE_IA(CSYM(nbif_impl_##F), P_BIF_CALLEE(P), r29); bl CSYM(hipe_debug_bif_wrapper) #else -# define CALL_BIF(F) bl CSYM(F) +# define CALL_BIF(F) bl CSYM(nbif_impl_##F) #endif' .text diff --git a/erts/emulator/hipe/hipe_sparc_bifs.m4 b/erts/emulator/hipe/hipe_sparc_bifs.m4 index 1389beaa61..14330c2f1c 100644 --- a/erts/emulator/hipe/hipe_sparc_bifs.m4 +++ b/erts/emulator/hipe/hipe_sparc_bifs.m4 @@ -29,9 +29,9 @@ include(`hipe/hipe_sparc_asm.m4') .align 4 `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) -# define CALL_BIF(F) set F, %o7; st %o7, [%o0+P_BIF_CALLEE]; call hipe_debug_bif_wrapper +# define CALL_BIF(F) set nbif_impl_##F, %o7; st %o7, [%o0+P_BIF_CALLEE]; call hipe_debug_bif_wrapper #else -# define CALL_BIF(F) call F +# define CALL_BIF(F) call nbif_impl_##F #endif' /* diff --git a/erts/emulator/hipe/hipe_x86_bifs.m4 b/erts/emulator/hipe/hipe_x86_bifs.m4 index c0c149733c..aecf67dc1b 100644 --- a/erts/emulator/hipe/hipe_x86_bifs.m4 +++ b/erts/emulator/hipe/hipe_x86_bifs.m4 @@ -32,9 +32,9 @@ include(`hipe/hipe_x86_asm.m4') #endif' `#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) -# define CALL_BIF(F) movl $CSYM(F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper) +# define CALL_BIF(F) movl $CSYM(nbif_impl_##F), P_BIF_CALLEE(P); call CSYM(hipe_debug_bif_wrapper) #else -# define CALL_BIF(F) call CSYM(F) +# define CALL_BIF(F) call CSYM(nbif_impl_##F) #endif' define(TEST_GOT_MBUF,`movl P_MBUF(P), %edx /* `TEST_GOT_MBUF' */ @@ -666,13 +666,9 @@ noproc_primop_interface_0(nbif_handle_fp_exception, erts_restore_fpu) #endif /* NO_FPE_SIGNALS */ /* - * Implement gc_bif_interface_0 as nofail_primop_interface_0. - */ -define(gc_bif_interface_0,`nofail_primop_interface_0($1, $2)') - -/* - * Implement gc_bif_interface_N as standard_bif_interface_N (N=1,2,3). + * Implement gc_bif_interface_N as standard_bif_interface_N. */ +define(gc_bif_interface_0,`standard_bif_interface_0($1, $2)') define(gc_bif_interface_1,`standard_bif_interface_1($1, $2)') define(gc_bif_interface_2,`standard_bif_interface_2($1, $2)') define(gc_bif_interface_3,`standard_bif_interface_3($1, $2)') diff --git a/erts/emulator/nifs/common/erl_tracer_nif.c b/erts/emulator/nifs/common/erl_tracer_nif.c index c0cc48ff42..0bde60d057 100644 --- a/erts/emulator/nifs/common/erl_tracer_nif.c +++ b/erts/emulator/nifs/common/erl_tracer_nif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson 2015. All Rights Reserved. + * Copyright Ericsson 2015-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 5ce57b7b1b..a852550915 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -417,6 +417,7 @@ erts_sys_pre_init(void) #ifdef ERTS_THR_HAVE_SIG_FUNCS sigemptyset(&thr_create_sigmask); sigaddset(&thr_create_sigmask, SIGINT); /* block interrupt */ + sigaddset(&thr_create_sigmask, SIGTERM); /* block terminate signal */ sigaddset(&thr_create_sigmask, SIGUSR1); /* block user defined signal */ #endif @@ -665,6 +666,40 @@ static RETSIGTYPE request_break(int signum) #endif } +static void stop_requested(void) { + Process* p = NULL; + Eterm msg, *hp; + ErtsProcLocks locks = 0; + ErlOffHeap *ohp; + Eterm id = erts_whereis_name_to_id(NULL, am_init); + + if ((p = (erts_pid2proc_opt(NULL, 0, id, 0, ERTS_P2P_FLG_INC_REFC))) != NULL) { + ErtsMessage *msgp = erts_alloc_message_heap(p, &locks, 3, &hp, &ohp); + + /* init ! {stop,stop} */ + msg = TUPLE2(hp, am_stop, am_stop); + erts_queue_message(p, locks, msgp, msg, am_system); + + if (locks) + erts_smp_proc_unlock(p, locks); + erts_proc_dec_refc(p); + } +} + +#if (defined(SIG_SIGSET) || defined(SIG_SIGNAL)) +static RETSIGTYPE request_stop(void) +#else +static RETSIGTYPE request_stop(int signum) +#endif +{ +#ifdef ERTS_SMP + smp_sig_notify('S'); +#else + stop_requested(); +#endif +} + + static ERTS_INLINE void sigusr1_exit(void) { @@ -761,6 +796,7 @@ static RETSIGTYPE do_quit(int signum) /* Disable break */ void erts_set_ignore_break(void) { sys_signal(SIGINT, SIG_IGN); + sys_signal(SIGTERM, SIG_IGN); sys_signal(SIGQUIT, SIG_IGN); sys_signal(SIGTSTP, SIG_IGN); } @@ -786,6 +822,7 @@ void erts_replace_intr(void) { void init_break_handler(void) { sys_signal(SIGINT, request_break); + sys_signal(SIGTERM, request_stop); #ifndef ETHR_UNUSABLE_SIGUSRX sys_signal(SIGUSR1, user_signal1); #endif /* #ifndef ETHR_UNUSABLE_SIGUSRX */ @@ -1299,6 +1336,9 @@ signal_dispatcher_thread_func(void *unused) switch (buf[i]) { case 0: /* Emulator initialized */ break; + case 'S': /* SIGTERM */ + stop_requested(); + break; case 'I': /* SIGINT */ break_requested(); break; diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index 2e48c475d5..7c9927c4f3 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -53,6 +53,7 @@ MODULES= \ crypto_SUITE \ ddll_SUITE \ decode_packet_SUITE \ + dirty_bif_SUITE \ dirty_nif_SUITE \ distribution_SUITE \ driver_SUITE \ diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index b8d89126fe..f70fb0e501 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -32,7 +32,8 @@ binary_to_atom/1,binary_to_existing_atom/1, atom_to_binary/1,min_max/1, erlang_halt/1, erl_crash_dump_bytes/1, - is_builtin/1]). + is_builtin/1, error_stacktrace/1, + error_stacktrace_during_call_trace/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -44,8 +45,8 @@ all() -> t_list_to_existing_atom, os_env, otp_7526, display, atom_to_binary, binary_to_atom, binary_to_existing_atom, - erl_crash_dump_bytes, - min_max, erlang_halt, is_builtin]. + erl_crash_dump_bytes, min_max, erlang_halt, is_builtin, + error_stacktrace, error_stacktrace_during_call_trace]. %% Uses erlang:display to test that erts_printf does not do deep recursion display(Config) when is_list(Config) -> @@ -727,6 +728,172 @@ is_builtin(_Config) -> ok. +error_stacktrace(Config) when is_list(Config) -> + error_stacktrace_test(). + +error_stacktrace_during_call_trace(Config) when is_list(Config) -> + Tracer = spawn_link(fun () -> + receive after infinity -> ok end + end), + Mprog = [{'_',[],[{exception_trace}]}], + erlang:trace_pattern({?MODULE,'_','_'}, Mprog, [local]), + 1 = erlang:trace_pattern({erlang,error,2}, Mprog, [local]), + 1 = erlang:trace_pattern({erlang,error,1}, Mprog, [local]), + erlang:trace(all, true, [call,return_to,timestamp,{tracer, Tracer}]), + try + error_stacktrace_test() + after + erlang:trace(all, false, [call,return_to,timestamp,{tracer, Tracer}]), + erlang:trace_pattern({erlang,error,2}, false, [local]), + erlang:trace_pattern({erlang,error,1}, false, [local]), + erlang:trace_pattern({?MODULE,'_','_'}, false, [local]), + unlink(Tracer), + exit(Tracer, kill), + Mon = erlang:monitor(process, Tracer), + receive + {'DOWN', Mon, process, Tracer, _} -> ok + end + end, + ok. + + +error_stacktrace_test() -> + Types = [apply_const_last, apply_const, apply_last, + apply, double_apply_const_last, double_apply_const, + double_apply_last, double_apply, multi_apply_const_last, + multi_apply_const, multi_apply_last, multi_apply, + call_const_last, call_last, call_const, call], + lists:foreach(fun (Type) -> + {Pid, Mon} = spawn_monitor( + fun () -> + stk([a,b,c,d], Type, error_2) + end), + receive + {'DOWN', Mon, process, Pid, Reason} -> + {oops, Stack} = Reason, +%% io:format("Type: ~p Stack: ~p~n", +%% [Type, Stack]), + [{?MODULE, do_error_2, [Type], _}, + {?MODULE, stk, 3, _}, + {?MODULE, stk, 3, _}] = Stack + end + end, + Types), + lists:foreach(fun (Type) -> + {Pid, Mon} = spawn_monitor( + fun () -> + stk([a,b,c,d], Type, error_1) + end), + receive + {'DOWN', Mon, process, Pid, Reason} -> + {oops, Stack} = Reason, +%% io:format("Type: ~p Stack: ~p~n", +%% [Type, Stack]), + [{?MODULE, do_error_1, 1, _}, + {?MODULE, stk, 3, _}, + {?MODULE, stk, 3, _}] = Stack + end + end, + Types), + ok. + +stk([], Type, Func) -> + tail(Type, Func, jump), + ok; +stk([_|L], Type, Func) -> + stk(L, Type, Func), + ok. + +tail(Type, Func, jump) -> + tail(Type, Func, do); +tail(Type, error_1, do) -> + do_error_1(Type); +tail(Type, error_2, do) -> + do_error_2(Type). + +do_error_2(apply_const_last) -> + erlang:apply(erlang, error, [oops, [apply_const_last]]); +do_error_2(apply_const) -> + erlang:apply(erlang, error, [oops, [apply_const]]), + ok; +do_error_2(apply_last) -> + erlang:apply(id(erlang), id(error), id([oops, [apply_last]])); +do_error_2(apply) -> + erlang:apply(id(erlang), id(error), id([oops, [apply]])), + ok; +do_error_2(double_apply_const_last) -> + erlang:apply(erlang, apply, [erlang, error, [oops, [double_apply_const_last]]]); +do_error_2(double_apply_const) -> + erlang:apply(erlang, apply, [erlang, error, [oops, [double_apply_const]]]), + ok; +do_error_2(double_apply_last) -> + erlang:apply(id(erlang), id(apply), [id(erlang), id(error), id([oops, [double_apply_last]])]); +do_error_2(double_apply) -> + erlang:apply(id(erlang), id(apply), [id(erlang), id(error), id([oops, [double_apply]])]), + ok; +do_error_2(multi_apply_const_last) -> + erlang:apply(erlang, apply, [erlang, apply, [erlang, apply, [erlang, error, [oops, [multi_apply_const_last]]]]]); +do_error_2(multi_apply_const) -> + erlang:apply(erlang, apply, [erlang, apply, [erlang, apply, [erlang, error, [oops, [multi_apply_const]]]]]), + ok; +do_error_2(multi_apply_last) -> + erlang:apply(id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(error), id([oops, [multi_apply_last]])]]]); +do_error_2(multi_apply) -> + erlang:apply(id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(error), id([oops, [multi_apply]])]]]), + ok; +do_error_2(call_const_last) -> + erlang:error(oops, [call_const_last]); +do_error_2(call_last) -> + erlang:error(id(oops), id([call_last])); +do_error_2(call_const) -> + erlang:error(oops, [call_const]), + ok; +do_error_2(call) -> + erlang:error(id(oops), id([call])). + + +do_error_1(apply_const_last) -> + erlang:apply(erlang, error, [oops]); +do_error_1(apply_const) -> + erlang:apply(erlang, error, [oops]), + ok; +do_error_1(apply_last) -> + erlang:apply(id(erlang), id(error), id([oops])); +do_error_1(apply) -> + erlang:apply(id(erlang), id(error), id([oops])), + ok; +do_error_1(double_apply_const_last) -> + erlang:apply(erlang, apply, [erlang, error, [oops]]); +do_error_1(double_apply_const) -> + erlang:apply(erlang, apply, [erlang, error, [oops]]), + ok; +do_error_1(double_apply_last) -> + erlang:apply(id(erlang), id(apply), [id(erlang), id(error), id([oops])]); +do_error_1(double_apply) -> + erlang:apply(id(erlang), id(apply), [id(erlang), id(error), id([oops])]), + ok; +do_error_1(multi_apply_const_last) -> + erlang:apply(erlang, apply, [erlang, apply, [erlang, apply, [erlang, error, [oops]]]]); +do_error_1(multi_apply_const) -> + erlang:apply(erlang, apply, [erlang, apply, [erlang, apply, [erlang, error, [oops]]]]), + ok; +do_error_1(multi_apply_last) -> + erlang:apply(id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(error), id([oops])]]]); +do_error_1(multi_apply) -> + erlang:apply(id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(apply), [id(erlang), id(error), id([oops])]]]), + ok; +do_error_1(call_const_last) -> + erlang:error(oops); +do_error_1(call_last) -> + erlang:error(id(oops)); +do_error_1(call_const) -> + erlang:error(oops), + ok; +do_error_1(call) -> + erlang:error(id(oops)). + + + %% Helpers diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl index 6ba6301c7c..2e303ba9a8 100644 --- a/erts/emulator/test/call_trace_SUITE.erl +++ b/erts/emulator/test/call_trace_SUITE.erl @@ -45,7 +45,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap, {seconds, 30}}]. -all() -> +all() -> Common = [errors, on_load], NotHipe = [process_specs, basic, flags, pam, change_pam, upgrade, @@ -1090,8 +1090,7 @@ exception_nocatch() -> {trace,t2,exception_from,{erlang,throw,1}, {error,{nocatch,Q2}}}], exception_from, {error,{nocatch,Q2}}), - expect({trace,T2,exit,{{nocatch,Q2},[{erlang,throw,[Q2],[]}, - {?MODULE,deep_4,1, + expect({trace,T2,exit,{{nocatch,Q2},[{?MODULE,deep_4,1, Deep4LocThrow}]}}), Q3 = {dump,[dump,{dump}]}, T3 = @@ -1100,8 +1099,7 @@ exception_nocatch() -> {trace,t3,exception_from,{erlang,error,1}, {error,Q3}}], exception_from, {error,Q3}), - expect({trace,T3,exit,{Q3,[{erlang,error,[Q3],[]}, - {?MODULE,deep_4,1,Deep4LocError}]}}), + expect({trace,T3,exit,{Q3,[{?MODULE,deep_4,1,Deep4LocError}]}}), T4 = exception_nocatch(?LINE, '=', [17,4711], 5, [], exception_from, {error,{badmatch,4711}}), diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 774461c525..b29520ab9f 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -65,9 +65,9 @@ versions(Config) when is_list(Config) -> 2 = versions:version(), %% Kill processes, unload code. - P1 ! P2 ! done, _ = monitor(process, P1), _ = monitor(process, P2), + P1 ! P2 ! done, receive {'DOWN',_,process,P1,normal} -> ok end, @@ -155,7 +155,7 @@ call_purged_fun_code_there(Config) when is_list(Config) -> call_purged_fun_test(Priv, Data, Type) -> OptsList = case erlang:system_info(hipe_architecture) of undefined -> [[]]; - _ -> [[], [native]] + _ -> [[], [native,{d,hipe}]] end, [call_purged_fun_test_do(Priv, Data, Type, CO, FO) || CO <- OptsList, FO <- OptsList]. diff --git a/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl b/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl index 5e031abca8..699f0c1161 100644 --- a/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl +++ b/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl @@ -2,7 +2,27 @@ -export([do/4]). +%% Resurrect line macro when hipe compiled +-ifdef(hipe). +-define(line, put(the_line,?LINE),). do(Priv, Data, Type, Opts) -> + try do_it(Priv, Data, Type, Opts) + catch + C:E -> + ST = erlang:get_stacktrace(), + io:format("Caught exception from line ~p:\n~p\n", + [get(the_line), ST]), + io:format("Message queue: ~p\n", [process_info(self(), messages)]), + erlang:raise(C, E, ST) + end. +-else. +-define(line,). +do(P,D,T,O) -> + do_it(P,D,T,O). +-endif. + + +do_it(Priv, Data, Type, Opts) -> File = filename:join(Data, "my_code_test2"), Code = filename:join(Priv, "my_code_test2"), @@ -10,25 +30,27 @@ do(Priv, Data, Type, Opts) -> catch erlang:delete_module(my_code_test2), catch erlang:purge_module(my_code_test2), - {ok,my_code_test2} = c:c(File, [{outdir,Priv} | Opts]), + ?line {ok,my_code_test2} = c:c(File, [{outdir,Priv} | Opts]), - IsNative = lists:member(native,Opts), - IsNative = code:is_module_native(my_code_test2), + ?line IsNative = lists:member(native,Opts), + ?line IsNative = code:is_module_native(my_code_test2), - T = ets:new(my_code_test2_fun_table, []), + ?line T = ets:new(my_code_test2_fun_table, []), ets:insert(T, {my_fun,my_code_test2:make_fun(4711)}), ets:insert(T, {my_fun2,my_code_test2:make_fun2()}), - spawn(fun () -> - [{my_fun2,F2}] = ets:lookup(T, my_fun2), - F2(fun () -> - receive after infinity -> ok end - end, - fun () -> ok end), - exit(completed) - end), - - PurgeType = case Type of + Papa = self(), + {P0,M0} = spawn_monitor(fun () -> + [{my_fun2,F2}] = ets:lookup(T, my_fun2), + F2(fun () -> + Papa ! {self(),"going to sleep"}, + receive {Papa,"wake up"} -> ok end + end, + fun () -> ok end), + exit(completed) + end), + + ?line PurgeType = case Type of code_gone -> ok = file:delete(Code++".beam"), true; @@ -38,98 +60,95 @@ do(Priv, Data, Type, Opts) -> false end, - true = erlang:delete_module(my_code_test2), + ?line true = erlang:delete_module(my_code_test2), + + ?line ok = receive {P0, "going to sleep"} -> ok + after 1000 -> timeout + end, - Purge = start_purge(my_code_test2, PurgeType), + ?line Purge = start_purge(my_code_test2, PurgeType), - {P0, M0} = spawn_monitor(fun () -> - [{my_fun,F}] = ets:lookup(T, my_fun), - 4712 = F(1), - exit(completed) + ?line {P1, M1} = spawn_monitor(fun () -> + ?line [{my_fun,F}] = ets:lookup(T, my_fun), + ?line 4712 = F(1), + exit(completed) end), - wait_until(fun () -> - {status, suspended} - == process_info(P0, status) - end), + ?line ok = wait_until(fun () -> + {status, suspended} + == process_info(P1, status) + end), - ok = continue_purge(Purge), + ?line ok = continue_purge(Purge), - {P1, M1} = spawn_monitor(fun () -> - [{my_fun,F}] = ets:lookup(T, my_fun), - 4713 = F(2), - exit(completed) + ?line {P2, M2} = spawn_monitor(fun () -> + ?line [{my_fun,F}] = ets:lookup(T, my_fun), + ?line 4713 = F(2), + exit(completed) end), - {P2, M2} = spawn_monitor(fun () -> - [{my_fun,F}] = ets:lookup(T, my_fun), - 4714 = F(3), - exit(completed) + ?line {P3, M3} = spawn_monitor(fun () -> + ?line [{my_fun,F}] = ets:lookup(T, my_fun), + ?line 4714 = F(3), + exit(completed) end), - wait_until(fun () -> - {status, suspended} - == process_info(P1, status) - end), - wait_until(fun () -> - {status, suspended} - == process_info(P2, status) - end), + ?line ok = wait_until(fun () -> + {status, suspended} + == process_info(P2, status) + end), + ?line ok = wait_until(fun () -> + {status, suspended} + == process_info(P3, status) + end), - {current_function, - {erts_code_purger, - pending_purge_lambda, - 3}} = process_info(P0, current_function), - {current_function, + ?line {current_function, {erts_code_purger, pending_purge_lambda, 3}} = process_info(P1, current_function), - {current_function, + ?line {current_function, {erts_code_purger, pending_purge_lambda, 3}} = process_info(P2, current_function), + ?line {current_function, + {erts_code_purger, + pending_purge_lambda, + 3}} = process_info(P3, current_function), case Type of code_there -> - false = complete_purge(Purge); + ?line false = complete_purge(Purge), + P0 ! {self(), "wake up"}, + ?line completed = wait_for_down(P0,M0); _ -> - {true, true} = complete_purge(Purge) + ?line {true, true} = complete_purge(Purge), + ?line killed = wait_for_down(P0,M0) end, case Type of code_gone -> - receive - {'DOWN', M0, process, P0, Reason0} -> - {undef, _} = Reason0 - end, - receive - {'DOWN', M1, process, P1, Reason1} -> - {undef, _} = Reason1 - end, - receive - {'DOWN', M2, process, P2, Reason2} -> - {undef, _} = Reason2 - end; + ?line {undef, _} = wait_for_down(P1,M1), + ?line {undef, _} = wait_for_down(P2,M2), + ?line {undef, _} = wait_for_down(P3,M3); _ -> - receive - {'DOWN', M0, process, P0, Reason0} -> - completed = Reason0 - end, - receive - {'DOWN', M1, process, P1, Reason1} -> - completed = Reason1 - end, - receive - {'DOWN', M2, process, P2, Reason2} -> - completed = Reason2 - end, + ?line completed = wait_for_down(P1,M1), + ?line completed = wait_for_down(P2,M2), + ?line completed = wait_for_down(P3,M3), catch erlang:purge_module(my_code_test2), catch erlang:delete_module(my_code_test2), catch erlang:purge_module(my_code_test2) end, ok. +wait_for_down(P,M) -> + receive + {'DOWN', M, process, P, Reason} -> + Reason + after 1000 -> + timeout + end. + wait_until(Fun) -> - ok = wait_until(Fun, 20). + wait_until(Fun, 20). wait_until(Fun, N) -> case {Fun(),N} of diff --git a/erts/emulator/test/dirty_bif_SUITE.erl b/erts/emulator/test/dirty_bif_SUITE.erl new file mode 100644 index 0000000000..308323594d --- /dev/null +++ b/erts/emulator/test/dirty_bif_SUITE.erl @@ -0,0 +1,583 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2014. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(dirty_bif_SUITE). + +%%-define(line_trace,true). +-define(CHECK(Exp,Got), check(Exp,Got,?LINE)). +%%-define(CHECK(Exp,Got), Exp = Got). + +-include_lib("common_test/include/ct.hrl"). + +-export([all/0, suite/0, + init_per_suite/1, end_per_suite/1, + init_per_testcase/2, end_per_testcase/2, + dirty_bif/1, dirty_bif_exception/1, + dirty_bif_multischedule/1, + dirty_bif_multischedule_exception/1, + dirty_scheduler_exit/1, + dirty_call_while_terminated/1, + dirty_heap_access/1, + dirty_process_info/1, + dirty_process_register/1, + dirty_process_trace/1, + code_purge/1]). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +%% +%% All these tests utilize the debug BIFs: +%% - erts_debug:dirty_cpu/2 - Statically determined +%% to (begin to) execute on a dirty CPU scheduler. +%% - erts_debug:dirty_io/2 - Statically determined +%% to (begin to) execute on a dirty IO scheduler. +%% - erts_debug:dirty/3 +%% Their implementations are located in +%% $ERL_TOP/erts/emulator/beam/beam_debug.c +%% + +all() -> + [dirty_bif, + dirty_bif_multischedule, + dirty_bif_exception, + dirty_bif_multischedule_exception, + dirty_scheduler_exit, + dirty_call_while_terminated, + dirty_heap_access, + dirty_process_info, + dirty_process_register, + dirty_process_trace, + code_purge]. + +init_per_suite(Config) -> + case erlang:system_info(dirty_cpu_schedulers) of + N when N > 0 -> + Config; + _ -> + {skipped, "No dirty scheduler support"} + end. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(Case, Config) -> + [{testcase, Case} | Config]. + +end_per_testcase(_Case, _Config) -> + ok. + +dirty_bif(Config) when is_list(Config) -> + dirty_cpu = erts_debug:dirty_cpu(scheduler,type), + dirty_io = erts_debug:dirty_io(scheduler,type), + normal = erts_debug:dirty(normal,scheduler,type), + dirty_cpu = erts_debug:dirty(dirty_cpu,scheduler,type), + dirty_io = erts_debug:dirty(dirty_io,scheduler,type), + ok. + +dirty_bif_multischedule(Config) when is_list(Config) -> + ok = erts_debug:dirty_cpu(reschedule,1000), + ok = erts_debug:dirty_io(reschedule,1000), + ok = erts_debug:dirty(normal,reschedule,1000), + ok. + + +dirty_bif_exception(Config) when is_list(Config) -> + lists:foreach(fun (Error) -> + ErrorType = case Error of + _ when is_atom(Error) -> Error; + _ -> badarg + end, + try + erts_debug:dirty_cpu(error, Error), + ct:fail(expected_exception) + catch + error:ErrorType -> + [{erts_debug,dirty_cpu,[error, Error],_}|_] + = erlang:get_stacktrace(), + ok + end, + try + apply(erts_debug,dirty_cpu,[error, Error]), + ct:fail(expected_exception) + catch + error:ErrorType -> + [{erts_debug,dirty_cpu,[error, Error],_}|_] + = erlang:get_stacktrace(), + ok + end, + try + erts_debug:dirty_io(error, Error), + ct:fail(expected_exception) + catch + error:ErrorType -> + [{erts_debug,dirty_io,[error, Error],_}|_] + = erlang:get_stacktrace(), + ok + end, + try + apply(erts_debug,dirty_io,[error, Error]), + ct:fail(expected_exception) + catch + error:ErrorType -> + [{erts_debug,dirty_io,[error, Error],_}|_] + = erlang:get_stacktrace(), + ok + end, + try + erts_debug:dirty(normal, error, Error), + ct:fail(expected_exception) + catch + error:ErrorType -> + [{erts_debug,dirty,[normal, error, Error],_}|_] + = erlang:get_stacktrace(), + ok + end, + try + apply(erts_debug,dirty,[normal, error, Error]), + ct:fail(expected_exception) + catch + error:ErrorType -> + [{erts_debug,dirty,[normal, error, Error],_}|_] + = erlang:get_stacktrace(), + ok + end, + try + erts_debug:dirty(dirty_cpu, error, Error), + ct:fail(expected_exception) + catch + error:ErrorType -> + [{erts_debug,dirty,[dirty_cpu, error, Error],_}|_] + = erlang:get_stacktrace(), + ok + end, + try + apply(erts_debug,dirty,[dirty_cpu, error, Error]), + ct:fail(expected_exception) + catch + error:ErrorType -> + [{erts_debug,dirty,[dirty_cpu, error, Error],_}|_] + = erlang:get_stacktrace(), + ok + end, + try + erts_debug:dirty(dirty_io, error, Error), + ct:fail(expected_exception) + catch + error:ErrorType -> + [{erts_debug,dirty,[dirty_io, error, Error],_}|_] + = erlang:get_stacktrace(), + ok + end, + try + apply(erts_debug,dirty,[dirty_io, error, Error]), + ct:fail(expected_exception) + catch + error:ErrorType -> + [{erts_debug,dirty,[dirty_io, error, Error],_}|_] + = erlang:get_stacktrace(), + ok + end + end, + [badarg, undef, badarith, system_limit, noproc, + make_ref(), {another, "heap", term_to_binary("term")}]), + ok. + + +dirty_bif_multischedule_exception(Config) when is_list(Config) -> + try + erts_debug:dirty_cpu(reschedule,1001) + catch + error:badarg -> + [{erts_debug,dirty_cpu,[reschedule, 1001],_}|_] + = erlang:get_stacktrace(), + ok + end, + try + erts_debug:dirty_io(reschedule,1001) + catch + error:badarg -> + [{erts_debug,dirty_io,[reschedule, 1001],_}|_] + = erlang:get_stacktrace(), + ok + end, + try + erts_debug:dirty(normal,reschedule,1001) + catch + error:badarg -> + [{erts_debug,dirty,[normal,reschedule,1001],_}|_] + = erlang:get_stacktrace(), + ok + end. + +dirty_scheduler_exit(Config) when is_list(Config) -> + {ok, Node} = start_node(Config, "+SDio 1"), + [ok] = mcall(Node, + [fun() -> + Start = erlang:monotonic_time(millisecond), + ok = test_dirty_scheduler_exit(), + End = erlang:monotonic_time(millisecond), + io:format("Time=~p ms~n", [End-Start]), + ok + end]), + stop_node(Node), + ok. + +test_dirty_scheduler_exit() -> + process_flag(trap_exit,true), + test_dse(10,[]). +test_dse(0,Pids) -> + timer:sleep(100), + kill_dse(Pids,[]); +test_dse(N,Pids) -> + Pid = spawn_link(fun () -> erts_debug:dirty_io(wait, 5000) end), + test_dse(N-1,[Pid|Pids]). + +kill_dse([],Killed) -> + wait_dse(Killed); +kill_dse([Pid|Pids],AlreadyKilled) -> + exit(Pid,kill), + kill_dse(Pids,[Pid|AlreadyKilled]). + +wait_dse([]) -> + ok; +wait_dse([Pid|Pids]) -> + receive + {'EXIT',Pid,Reason} -> + killed = Reason + end, + wait_dse(Pids). + +dirty_call_while_terminated(Config) when is_list(Config) -> + Me = self(), + Bin = list_to_binary(lists:duplicate(4711, $r)), + {value, {BinAddr, 4711, 1}} = lists:keysearch(4711, 2, + element(2, + process_info(self(), + binary))), + {Dirty, DM} = spawn_opt(fun () -> + erts_debug:dirty_cpu(alive_waitexiting, Me), + blipp:blupp(Bin) + end, + [monitor,link]), + receive {alive, Dirty} -> ok end, + {value, {BinAddr, 4711, 2}} = lists:keysearch(4711, 2, + element(2, + process_info(self(), + binary))), + Reason = die_dirty_process, + OT = process_flag(trap_exit, true), + exit(Dirty, Reason), + receive + {'DOWN', DM, process, Dirty, R0} -> + R0 = Reason + end, + receive + {'EXIT', Dirty, R1} -> + R1 = Reason + end, + undefined = process_info(Dirty), + undefined = process_info(Dirty, status), + false = erlang:is_process_alive(Dirty), + false = lists:member(Dirty, processes()), + %% Binary still refered by Dirty process not yet cleaned up + %% since the dirty bif has not yet returned... + {value, {BinAddr, 4711, 2}} = lists:keysearch(4711, 2, + element(2, + process_info(self(), + binary))), + receive after 2000 -> ok end, + receive + Msg -> + ct:fail({unexpected_message, Msg}) + after + 0 -> + ok + end, + {value, {BinAddr, 4711, 1}} = lists:keysearch(4711, 2, + element(2, + process_info(self(), + binary))), + process_flag(trap_exit, OT), + try + blipp:blupp(Bin) + catch + _ : _ -> ok + end. + +dirty_heap_access(Config) when is_list(Config) -> + {ok, Node} = start_node(Config), + Me = self(), + RGL = rpc:call(Node,erlang,whereis,[init]), + Ref = rpc:call(Node,erlang,make_ref,[]), + Dirty = spawn_link(fun () -> + Res = erts_debug:dirty_cpu(copy, Ref), + garbage_collect(), + Me ! {self(), Res}, + receive after infinity -> ok end + end), + {N, R} = access_dirty_heap(Dirty, RGL, 0, 0), + receive + {_Pid, Res} -> + 1000 = length(Res), + lists:foreach(fun (X) -> Ref = X end, Res) + end, + unlink(Dirty), + exit(Dirty, kill), + stop_node(Node), + {comment, integer_to_list(N) ++ " GL change loops; " + ++ integer_to_list(R) ++ " while running dirty"}. + +access_dirty_heap(Dirty, RGL, N, R) -> + case process_info(Dirty, status) of + {status, waiting} -> + {N, R}; + {status, Status} -> + {group_leader, GL} = process_info(Dirty, group_leader), + true = group_leader(RGL, Dirty), + {group_leader, RGL} = process_info(Dirty, group_leader), + true = group_leader(GL, Dirty), + {group_leader, GL} = process_info(Dirty, group_leader), + access_dirty_heap(Dirty, RGL, N+1, case Status of + running -> + R+1; + _ -> + R + end) + end. + +%% These tests verify that processes that access a process executing a +%% dirty BIF where the main lock is needed for that access do not get +%% blocked. Each test passes its pid to dirty_sleeper, which sends an +%% 'alive' message when it's running on a dirty scheduler and just before +%% it starts a 6 second sleep. When it receives the message, it verifies +%% that access to the dirty process is as it expects. After the dirty +%% process finishes its 6 second sleep but before it returns from the dirty +%% scheduler, it sends a 'done' message. If the tester already received +%% that message, the test fails because it means attempting to access the +%% dirty process waited for that process to return to a regular scheduler, +%% so verify that we haven't received that message, and also verify that +%% the dirty process is still alive immediately after accessing it. +dirty_process_info(Config) when is_list(Config) -> + access_dirty_process( + Config, + fun() -> ok end, + fun(BifPid) -> + PI = process_info(BifPid), + {current_function,{erts_debug,dirty_io,2}} = + lists:keyfind(current_function, 1, PI), + ok + end, + fun(_) -> ok end). + +dirty_process_register(Config) when is_list(Config) -> + access_dirty_process( + Config, + fun() -> ok end, + fun(BifPid) -> + register(test_dirty_process_register, BifPid), + BifPid = whereis(test_dirty_process_register), + unregister(test_dirty_process_register), + false = lists:member(test_dirty_process_register, + registered()), + ok + end, + fun(_) -> ok end). + +dirty_process_trace(Config) when is_list(Config) -> + access_dirty_process( + Config, + fun() -> + erlang:trace_pattern({erts_debug,dirty_io,2}, + [{'_',[],[{return_trace}]}], + [local,meta]), + ok + end, + fun(BifPid) -> + erlang:trace(BifPid, true, [call,timestamp]), + ok + end, + fun(BifPid) -> + receive + {done, BifPid} -> + receive + {trace_ts,BifPid,call,{erts_debug,dirty_io,_},_} -> + ok + after + 0 -> + error(missing_trace_call_message) + end %%, + %% receive + %% {trace_ts,BifPid,return_from,{erts_debug,dirty_io,2}, + %% ok,_} -> + %% ok + %% after + %% 100 -> + %% error(missing_trace_return_message) + %% end + after + 6500 -> + error(missing_done_message) + end, + ok + end). + +dirty_code_test_code() -> + " +-module(dirty_code_test). + +-export([func/1]). + +func(Fun) -> + Fun(), + blipp:blapp(). + +". + +code_purge(Config) when is_list(Config) -> + Path = ?config(data_dir, Config), + File = filename:join(Path, "dirty_code_test.erl"), + ok = file:write_file(File, dirty_code_test_code()), + {ok, dirty_code_test, Bin} = compile:file(File, [binary]), + {module, dirty_code_test} = erlang:load_module(dirty_code_test, Bin), + Start = erlang:monotonic_time(), + {Pid1, Mon1} = spawn_monitor(fun () -> + dirty_code_test:func(fun () -> + %% Sleep for 6 seconds + %% in dirty bif... + erts_debug:dirty_io(wait,6000) + end) + end), + {module, dirty_code_test} = erlang:load_module(dirty_code_test, Bin), + {Pid2, Mon2} = spawn_monitor(fun () -> + dirty_code_test:func(fun () -> + %% Sleep for 6 seconds + %% in dirty bif... + erts_debug:dirty_io(wait,6000) + end) + end), + receive + {'DOWN', Mon1, process, Pid1, _} -> + ct:fail(premature_death) + after 100 -> + ok + end, + true = erlang:purge_module(dirty_code_test), + receive + {'DOWN', Mon1, process, Pid1, Reason1} -> + killed = Reason1 + end, + receive + {'DOWN', Mon2, process, Pid2, _} -> + ct:fail(premature_death) + after 100 -> + ok + end, + true = erlang:delete_module(dirty_code_test), + receive + {'DOWN', Mon2, process, Pid2, _} -> + ct:fail(premature_death) + after 100 -> + ok + end, + true = erlang:purge_module(dirty_code_test), + receive + {'DOWN', Mon2, process, Pid2, Reason2} -> + killed = Reason2 + end, + End = erlang:monotonic_time(), + Time = erlang:convert_time_unit(End-Start, native, milli_seconds), + io:format("Time=~p~n", [Time]), + true = Time =< 1000, + ok. + +%% +%% Internal... +%% + +access_dirty_process(Config, Start, Test, Finish) -> + {ok, Node} = start_node(Config, ""), + [ok] = mcall(Node, + [fun() -> + ok = test_dirty_process_access(Start, Test, Finish) + end]), + stop_node(Node), + ok. + +test_dirty_process_access(Start, Test, Finish) -> + ok = Start(), + Self = self(), + BifPid = spawn_link(fun() -> + ok = erts_debug:dirty_io(ready_wait6_done, Self) + end), + ok = receive + {ready, BifPid} -> + ok = Test(BifPid), + receive + {done, BifPid} -> + error(dirty_process_info_blocked) + after + 0 -> + true = erlang:is_process_alive(BifPid), + ok + end + after + 3000 -> + error(timeout) + end, + ok = Finish(BifPid). + +receive_any() -> + receive M -> M end. + +start_node(Config) -> + start_node(Config, ""). + +start_node(Config, Args) when is_list(Config) -> + Pa = filename:dirname(code:which(?MODULE)), + Name = list_to_atom(atom_to_list(?MODULE) + ++ "-" + ++ atom_to_list(proplists:get_value(testcase, Config)) + ++ "-" + ++ integer_to_list(erlang:system_time(second)) + ++ "-" + ++ integer_to_list(erlang:unique_integer([positive]))), + test_server:start_node(Name, slave, [{args, "-pa "++Pa++" "++Args}]). + +stop_node(Node) -> + test_server:stop_node(Node). + +mcall(Node, Funs) -> + Parent = self(), + Refs = lists:map(fun (Fun) -> + Ref = make_ref(), + spawn_link(Node, + fun () -> + Res = Fun(), + unlink(Parent), + Parent ! {Ref, Res} + end), + Ref + end, Funs), + lists:map(fun (Ref) -> + receive + {Ref, Res} -> + Res + end + end, Refs). diff --git a/erts/emulator/test/dirty_bif_SUITE_data/.gitignore b/erts/emulator/test/dirty_bif_SUITE_data/.gitignore new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/erts/emulator/test/dirty_bif_SUITE_data/.gitignore diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl index 658bdc41b6..991ba0acc8 100644 --- a/erts/emulator/test/dirty_nif_SUITE.erl +++ b/erts/emulator/test/dirty_nif_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010-2014. All Rights Reserved. +%% Copyright Ericsson AB 2010-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -54,8 +54,8 @@ all() -> dirty_nif_send_traced]. init_per_suite(Config) -> - try erlang:system_info(dirty_cpu_schedulers) of - N when is_integer(N), N > 0 -> + case erlang:system_info(dirty_cpu_schedulers) of + N when N > 0 -> case lib_loaded() of false -> ok = erlang:load_nif( @@ -64,8 +64,8 @@ init_per_suite(Config) -> true -> ok end, - Config - catch _:_ -> + Config; + _ -> {skipped, "No dirty scheduler support"} end. diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c index daaff955bc..caf99c952f 100644 --- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c +++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2014. All Rights Reserved. + * Copyright Ericsson AB 2009-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c index 614b68e865..8e5e81665c 100644 --- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c +++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c @@ -1397,10 +1397,18 @@ static void assert_print(char* str, int line) static void assert_failed(ErlDrvPort port, char* str, int line) { char buf[30]; + size_t bufsz = sizeof(buf); + assert_print(str,line); - snprintf(buf,sizeof(buf),"failed_at_line_%d",line); - driver_failure_atom(port,buf); - /*abort();*/ + + if (erl_drv_getenv("ERL_ABORT_ON_FAILURE", buf, &bufsz) == 0 + && (strcmp("true", buf) == 0 || strcmp("yes", buf) == 0)) { + abort(); + } + else { + snprintf(buf,sizeof(buf),"failed_at_line_%d",line); + driver_failure_atom(port,buf); + } } #define my_driver_select(PORT,FD,MODE,ON) \ diff --git a/erts/emulator/test/emulator_smoke.spec b/erts/emulator/test/emulator_smoke.spec index 3219aeb823..b2d0de8835 100644 --- a/erts/emulator/test/emulator_smoke.spec +++ b/erts/emulator/test/emulator_smoke.spec @@ -1,3 +1,9 @@ -{suites,"../emulator_test",[smoke_test_SUITE,time_SUITE]}. -{cases,"../emulator_test",crypto_SUITE,[t_md5]}. -{cases,"../emulator_test",float_SUITE,[fpe,cmp_integer]}.
\ No newline at end of file +{define,'Dir',"../emulator_test"}. +{suites,'Dir',[smoke_test_SUITE]}. +{suites,'Dir',[time_SUITE]}. +{skip_cases,'Dir',time_SUITE, + [univ_to_local,local_to_univ],"Depends on CET timezone"}. +{skip_cases,'Dir',time_SUITE, + [consistency],"Not reliable in October and March"}. +{cases,'Dir',crypto_SUITE,[t_md5]}. +{cases,'Dir',float_SUITE,[fpe,cmp_integer]}. diff --git a/erts/emulator/test/lttng_SUITE.erl b/erts/emulator/test/lttng_SUITE.erl index 6b7ad836f5..c12f63706a 100644 --- a/erts/emulator/test/lttng_SUITE.erl +++ b/erts/emulator/test/lttng_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2011. All Rights Reserved. +%% Copyright Ericsson AB 1999-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 7739a0bc22..27276e6646 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -1814,7 +1814,7 @@ consume_timeslice(Config) when is_list(Config) -> consume_timeslice_test(Config) when is_list(Config) -> ensure_lib_loaded(Config), - CONTEXT_REDS = 2000, + CONTEXT_REDS = 4000, Me = self(), Go = make_ref(), RedDiff = make_ref(), diff --git a/erts/emulator/test/old_scheduler_SUITE.erl b/erts/emulator/test/old_scheduler_SUITE.erl index ffe7d40139..8515a87df8 100644 --- a/erts/emulator/test/old_scheduler_SUITE.erl +++ b/erts/emulator/test/old_scheduler_SUITE.erl @@ -64,11 +64,11 @@ all() -> init_per_testcase(_Case, Config) -> %% main test process needs max prio Prio = process_flag(priority, max), - MS = erlang:system_flag(multi_scheduling, block), + MS = erlang:system_flag(multi_scheduling, block_normal), [{prio,Prio},{multi_scheduling, MS}|Config]. end_per_testcase(_Case, Config) -> - erlang:system_flag(multi_scheduling, unblock), + erlang:system_flag(multi_scheduling, unblock_normal), Prio=proplists:get_value(prio, Config), process_flag(priority, Prio), ok. diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl index d4e77d634a..2a13b2d2f4 100644 --- a/erts/emulator/test/port_SUITE.erl +++ b/erts/emulator/test/port_SUITE.erl @@ -2066,13 +2066,13 @@ exit_status_msb_test(Config, SleepSecs) when is_list(Config) -> StartedTime = (erlang:monotonic_time(microsecond) - Start)/1000000, io:format("StartedTime = ~p~n", [StartedTime]), true = StartedTime < SleepSecs, - erlang:system_flag(multi_scheduling, block), + erlang:system_flag(multi_scheduling, block_normal), lists:foreach(fun (P) -> receive {P, done} -> ok end end, Procs), DoneTime = (erlang:monotonic_time(microsecond) - Start)/1000000, io:format("DoneTime = ~p~n", [DoneTime]), true = DoneTime > SleepSecs, ok = verify_multi_scheduling_blocked(), - erlang:system_flag(multi_scheduling, unblock), + erlang:system_flag(multi_scheduling, unblock_normal), case {length(lists:usort(lists:flatten(SIds))), NoSchedsOnln} of {N, N} -> ok; @@ -2292,7 +2292,7 @@ maybe_to_list(List) -> List. format({Eol,List}) -> - io_lib:format("tuple<~w,~s>",[Eol, maybe_to_list(List)]); + io_lib:format("tuple<~w,~w>",[Eol, maybe_to_list(List)]); format(List) when is_list(List) -> case list_at_least(50, List) of true -> diff --git a/erts/emulator/test/port_SUITE_data/Makefile.src b/erts/emulator/test/port_SUITE_data/Makefile.src index fb7685c4b6..3a343e6d17 100644 --- a/erts/emulator/test/port_SUITE_data/Makefile.src +++ b/erts/emulator/test/port_SUITE_data/Makefile.src @@ -20,6 +20,12 @@ echo_args@exe@: echo_args@obj@ echo_args@obj@: echo_args.c $(CC) -c -o echo_args@obj@ $(CFLAGS) echo_args.c +dead_port@exe@: dead_port@obj@ + $(LD) $(CROSSLDFLAGS) -o dead_port dead_port@obj@ @LIBS@ + +dead_port@obj@: dead_port.c + $(CC) -c -o dead_port@obj@ $(CFLAGS) dead_port.c + port_test.@EMULATOR@: port_test.erl @erl_name@ -compile port_test diff --git a/erts/emulator/test/port_SUITE_data/port_test.c b/erts/emulator/test/port_SUITE_data/port_test.c index cc3ebdf0f8..e199a0fc13 100644 --- a/erts/emulator/test/port_SUITE_data/port_test.c +++ b/erts/emulator/test/port_SUITE_data/port_test.c @@ -10,6 +10,7 @@ #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> +#include <ctype.h> #ifndef __WIN32__ #include <unistd.h> @@ -33,7 +34,7 @@ exit(1); \ } -#define MAIN(argc, argv) main(argc, argv) +#define ASSERT(e) ((void) ((e) ? 1 : abort())) extern int errno; @@ -105,9 +106,7 @@ int err; #endif -MAIN(argc, argv) -int argc; -char *argv[]; +int main(int argc, char *argv[]) { int ret, fd_count; if((port_data = (PORT_TEST_DATA *) malloc(sizeof(PORT_TEST_DATA))) == NULL) { @@ -377,9 +376,11 @@ write_reply(buf, size) int size; /* Size of buffer to send. */ { int n; /* Temporary to hold size. */ + int rv; if (port_data->slow_writes <= 0) { /* Normal, "fast", write. */ - write(port_data->fd_to_erl, buf, size); + rv = write(port_data->fd_to_erl, buf, size); + ASSERT(rv == size); } else { /* * Write chunks with delays in between. @@ -387,7 +388,8 @@ write_reply(buf, size) while (size > 0) { n = size > port_data->slow_writes ? port_data->slow_writes : size; - write(port_data->fd_to_erl, buf, n); + rv = write(port_data->fd_to_erl, buf, n); + ASSERT(rv == n); size -= n; buf += n; if (size) @@ -558,7 +560,7 @@ char* spec; /* Specification for reply. */ buf = (char *) malloc(total_size); if (buf == NULL) { fprintf(stderr, "%s: insufficent memory for reply buffer of size %d\n", - port_data->progname, total_size); + port_data->progname, (int)total_size); exit(1); } diff --git a/erts/emulator/test/port_trace_SUITE.erl b/erts/emulator/test/port_trace_SUITE.erl index 5d9a75bcd3..03efdc15db 100644 --- a/erts/emulator/test/port_trace_SUITE.erl +++ b/erts/emulator/test/port_trace_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2012. All Rights Reserved. +%% Copyright Ericsson AB 1999-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index e035fc64fe..e14185e881 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -437,11 +437,22 @@ t_process_info(Config) when is_list(Config) -> verify_loc(Line2, Res2), pi_stacktrace([{?MODULE,t_process_info,1,?LINE}]), + verify_stacktrace_depth(), + Gleader = group_leader(), {group_leader, Gleader} = process_info(self(), group_leader), {'EXIT',{badarg,_Info}} = (catch process_info('not_a_pid')), ok. +verify_stacktrace_depth() -> + CS = current_stacktrace, + OldDepth = erlang:system_flag(backtrace_depth, 0), + {CS,[]} = erlang:process_info(self(), CS), + _ = erlang:system_flag(backtrace_depth, 8), + {CS,[{?MODULE,verify_stacktrace_depth,0,_},_|_]} = + erlang:process_info(self(), CS), + _ = erlang:system_flag(backtrace_depth, OldDepth). + pi_stacktrace(Expected0) -> {Line,Res} = {?LINE,erlang:process_info(self(), current_stacktrace)}, {current_stacktrace,Stack} = Res, @@ -1017,9 +1028,9 @@ low_prio(Config) when is_list(Config) -> 1 -> ok = low_prio_test(Config); _ -> - erlang:system_flag(multi_scheduling, block), + erlang:system_flag(multi_scheduling, block_normal), ok = low_prio_test(Config), - erlang:system_flag(multi_scheduling, unblock), + erlang:system_flag(multi_scheduling, unblock_normal), {comment, "Test not written for SMP runtime system. " "Multi scheduling blocked during test."} @@ -1086,9 +1097,9 @@ yield(Config) when is_list(Config) -> ++ ") is enabled. Testcase gets messed up by modfied " "timing."}; _ -> - MS = erlang:system_flag(multi_scheduling, block), + MS = erlang:system_flag(multi_scheduling, block_normal), yield_test(), - erlang:system_flag(multi_scheduling, unblock), + erlang:system_flag(multi_scheduling, unblock_normal), case MS of blocked -> {comment, @@ -1611,6 +1622,7 @@ spawn_initial_hangarounds(_Cleaner, NP, Max, Len, HAs) when NP > Max -> {Len, HAs}; spawn_initial_hangarounds(Cleaner, NP, Max, Len, HAs) -> Skip = 30, + wait_for_proc_slots(Skip+3), HA1 = spawn_opt(?MODULE, hangaround, [Cleaner, initial_hangaround], [{priority, low}]), HA2 = spawn_opt(?MODULE, hangaround, [Cleaner, initial_hangaround], @@ -1620,6 +1632,15 @@ spawn_initial_hangarounds(Cleaner, NP, Max, Len, HAs) -> spawn_drop(Skip), spawn_initial_hangarounds(Cleaner, NP+Skip, Max, Len+3, [HA1,HA2,HA3|HAs]). +wait_for_proc_slots(MinFreeSlots) -> + case erlang:system_info(process_limit) - erlang:system_info(process_count) of + FreeSlots when FreeSlots < MinFreeSlots -> + receive after 10 -> ok end, + wait_for_proc_slots(MinFreeSlots); + _FreeSlots -> + ok + end. + spawn_drop(N) when N =< 0 -> ok; spawn_drop(N) -> @@ -1658,7 +1679,7 @@ processes_bif_test() -> true -> %% Do it again with a process suspended while %% in the processes/0 bif. - erlang:system_flag(multi_scheduling, block), + erlang:system_flag(multi_scheduling, block_normal), Suspendee = spawn_link(fun () -> Tester ! {suspend_me, self()}, Tester ! {self(), @@ -1671,7 +1692,7 @@ processes_bif_test() -> end), receive {suspend_me, Suspendee} -> ok end, erlang:suspend_process(Suspendee), - erlang:system_flag(multi_scheduling, unblock), + erlang:system_flag(multi_scheduling, unblock_normal), [{status,suspended},{current_function,{erlang,ptab_list_continue,2}}] = process_info(Suspendee, [status, current_function]), @@ -1711,10 +1732,10 @@ do_processes_bif_test(WantReds, DieTest, Processes) -> Splt = NoTestProcs div 10, {TP1, TP23} = lists:split(Splt, TestProcs), {TP2, TP3} = lists:split(Splt, TP23), - erlang:system_flag(multi_scheduling, block), + erlang:system_flag(multi_scheduling, block_normal), Tester ! DoIt, receive GetGoing -> ok end, - erlang:system_flag(multi_scheduling, unblock), + erlang:system_flag(multi_scheduling, unblock_normal), SpawnProcesses(high), lists:foreach( fun (P) -> SpawnHangAround(), @@ -1923,7 +1944,7 @@ processes_gc_trap(Config) when is_list(Config) -> processes() end, - erlang:system_flag(multi_scheduling, block), + erlang:system_flag(multi_scheduling, block_normal), Suspendee = spawn_link(fun () -> Tester ! {suspend_me, self()}, Tester ! {self(), @@ -1933,7 +1954,7 @@ processes_gc_trap(Config) when is_list(Config) -> end), receive {suspend_me, Suspendee} -> ok end, erlang:suspend_process(Suspendee), - erlang:system_flag(multi_scheduling, unblock), + erlang:system_flag(multi_scheduling, unblock_normal), [{status,suspended}, {current_function,{erlang,ptab_list_continue,2}}] = process_info(Suspendee, [status, current_function]), @@ -2140,7 +2161,7 @@ processes_term_proc_list_test(MustChk) -> end) end, SpawnSuspendProcessesProc = fun () -> - erlang:system_flag(multi_scheduling, block), + erlang:system_flag(multi_scheduling, block_normal), P = spawn_link(fun () -> Tester ! {suspend_me, self()}, Tester ! {self(), @@ -2150,7 +2171,7 @@ processes_term_proc_list_test(MustChk) -> end), receive {suspend_me, P} -> ok end, erlang:suspend_process(P), - erlang:system_flag(multi_scheduling, unblock), + erlang:system_flag(multi_scheduling, unblock_normal), [{status,suspended}, {current_function,{erlang,ptab_list_continue,2}}] = process_info(P, [status, current_function]), @@ -2211,7 +2232,7 @@ processes_term_proc_list_test(MustChk) -> S8 = SpawnSuspendProcessesProc(), ?CHK_TERM_PROC_LIST(MustChk, 7), - erlang:system_flag(multi_scheduling, block), + erlang:system_flag(multi_scheduling, block_normal), Exit(S8), ?CHK_TERM_PROC_LIST(MustChk, 7), Exit(S5), @@ -2220,7 +2241,7 @@ processes_term_proc_list_test(MustChk) -> ?CHK_TERM_PROC_LIST(MustChk, 6), Exit(S6), ?CHK_TERM_PROC_LIST(MustChk, 0), - erlang:system_flag(multi_scheduling, unblock), + erlang:system_flag(multi_scheduling, unblock_normal), as_expected. diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl index 3aee15a8fc..b178dede5b 100644 --- a/erts/emulator/test/scheduler_SUITE.erl +++ b/erts/emulator/test/scheduler_SUITE.erl @@ -1095,16 +1095,13 @@ scheduler_threads(Config) when is_list(Config) -> end. dirty_scheduler_threads(Config) when is_list(Config) -> - SmpSupport = erlang:system_info(smp_support), - try - erlang:system_info(dirty_cpu_schedulers), - dirty_scheduler_threads_test(Config, SmpSupport) - catch - error:badarg -> - {skipped, "No dirty scheduler support"} + case erlang:system_info(dirty_cpu_schedulers) of + 0 -> {skipped, "No dirty scheduler support"}; + _ -> dirty_scheduler_threads_test(Config) end. -dirty_scheduler_threads_test(Config, SmpSupport) -> +dirty_scheduler_threads_test(Config) -> + SmpSupport = erlang:system_info(smp_support), {Sched, SchedOnln, _} = get_dsstate(Config, ""), {HalfSched, HalfSchedOnln} = case SmpSupport of false -> {1,1}; @@ -1374,12 +1371,9 @@ sst2_loop(N) -> sst2_loop(N-1). sst3_loop(S, N) -> - try erlang:system_info(dirty_cpu_schedulers) of - DS -> - sst3_loop_with_dirty_schedulers(S, DS, N) - catch - error:badarg -> - sst3_loop_normal_schedulers_only(S, N) + case erlang:system_info(dirty_cpu_schedulers) of + 0 -> sst3_loop_normal_schedulers_only(S, N); + DS -> sst3_loop_with_dirty_schedulers(S, DS, N) end. sst3_loop_normal_schedulers_only(_S, 0) -> diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl index a1f12ba93c..f51244485b 100644 --- a/erts/emulator/test/statistics_SUITE.erl +++ b/erts/emulator/test/statistics_SUITE.erl @@ -28,6 +28,8 @@ runtime_update/1, runtime_diff/1, run_queue_one/1, scheduler_wall_time/1, + scheduler_wall_time_all/1, + msb_scheduler_wall_time/1, reductions/1, reductions_big/1, garbage_collection/1, io/1, badarg/1, run_queues_lengths_active_tasks/1, msacc/1]). @@ -43,7 +45,9 @@ suite() -> all() -> [{group, wall_clock}, {group, runtime}, reductions, - reductions_big, {group, run_queue}, scheduler_wall_time, + reductions_big, {group, run_queue}, + scheduler_wall_time, scheduler_wall_time_all, + msb_scheduler_wall_time, garbage_collection, io, badarg, run_queues_lengths_active_tasks, msacc]. @@ -271,35 +275,64 @@ hog_iter(0, Mon) -> %% Tests that statistics(scheduler_wall_time) works as intended scheduler_wall_time(Config) when is_list(Config) -> + scheduler_wall_time_test(scheduler_wall_time). + +%% Tests that statistics(scheduler_wall_time_all) works as intended +scheduler_wall_time_all(Config) when is_list(Config) -> + scheduler_wall_time_test(scheduler_wall_time_all). + +scheduler_wall_time_test(Type) -> %% Should return undefined if system_flag is not turned on yet - undefined = statistics(scheduler_wall_time), + undefined = statistics(Type), %% Turn on statistics false = erlang:system_flag(scheduler_wall_time, true), try Schedulers = erlang:system_info(schedulers_online), + DirtyCPUSchedulers = erlang:system_info(dirty_cpu_schedulers_online), + DirtyIOSchedulers = erlang:system_info(dirty_io_schedulers), + TotLoadSchedulers = case Type of + scheduler_wall_time_all -> + Schedulers + DirtyCPUSchedulers + DirtyIOSchedulers; + scheduler_wall_time -> + Schedulers + DirtyCPUSchedulers + end, + %% Let testserver and everyone else finish their work timer:sleep(1500), %% Empty load - EmptyLoad = get_load(), + EmptyLoad = get_load(Type), {false, _} = {lists:any(fun(Load) -> Load > 50 end, EmptyLoad),EmptyLoad}, MeMySelfAndI = self(), StartHog = fun() -> - Pid = spawn(?MODULE, hog, [self()]), + Pid = spawn_link(?MODULE, hog, [self()]), receive hog_started -> MeMySelfAndI ! go end, Pid end, + StartDirtyHog = fun(Func) -> + F = fun () -> + erts_debug:Func(alive_waitexiting, + MeMySelfAndI) + end, + Pid = spawn_link(F), + receive {alive, Pid} -> ok end, + Pid + end, P1 = StartHog(), %% Max on one, the other schedulers empty (hopefully) %% Be generous the process can jump between schedulers %% which is ok and we don't want the test to fail for wrong reasons - _L1 = [S1Load|EmptyScheds1] = get_load(), + _L1 = [S1Load|EmptyScheds1] = get_load(Type), {true,_} = {S1Load > 50,S1Load}, {false,_} = {lists:any(fun(Load) -> Load > 50 end, EmptyScheds1),EmptyScheds1}, {true,_} = {lists:sum(EmptyScheds1) < 60,EmptyScheds1}, %% 50% load HalfHogs = [StartHog() || _ <- lists:seq(1, (Schedulers-1) div 2)], - HalfLoad = lists:sum(get_load()) div Schedulers, + HalfDirtyCPUHogs = [StartDirtyHog(dirty_cpu) + || _ <- lists:seq(1, DirtyCPUSchedulers div 2)], + HalfDirtyIOHogs = [StartDirtyHog(dirty_io) + || _ <- lists:seq(1, DirtyIOSchedulers div 2)], + HalfLoad = lists:sum(get_load(Type)) div TotLoadSchedulers, if Schedulers < 2, HalfLoad > 80 -> ok; %% Ok only one scheduler online and one hog %% We want roughly 50% load HalfLoad > 40, HalfLoad < 60 -> ok; @@ -308,23 +341,30 @@ scheduler_wall_time(Config) when is_list(Config) -> %% 100% load LastHogs = [StartHog() || _ <- lists:seq(1, Schedulers div 2)], - FullScheds = get_load(), + LastDirtyCPUHogs = [StartDirtyHog(dirty_cpu) + || _ <- lists:seq(1, DirtyCPUSchedulers div 2)], + LastDirtyIOHogs = [StartDirtyHog(dirty_io) + || _ <- lists:seq(1, DirtyIOSchedulers div 2)], + FullScheds = get_load(Type), {false,_} = {lists:any(fun(Load) -> Load < 80 end, FullScheds),FullScheds}, - FullLoad = lists:sum(FullScheds) div Schedulers, + FullLoad = lists:sum(FullScheds) div TotLoadSchedulers, if FullLoad > 90 -> ok; true -> exit({fullload, FullLoad}) end, KillHog = fun (HP) -> HPM = erlang:monitor(process, HP), + unlink(HP), exit(HP, kill), receive {'DOWN', HPM, process, HP, killed} -> ok end end, - [KillHog(Pid) || Pid <- [P1|HalfHogs++LastHogs]], - AfterLoad = get_load(), + [KillHog(Pid) || Pid <- [P1|HalfHogs++HalfDirtyCPUHogs++HalfDirtyIOHogs + ++LastHogs++LastDirtyCPUHogs++LastDirtyIOHogs]], + receive after 2000 -> ok end, %% Give dirty schedulers time to complete... + AfterLoad = get_load(Type), io:format("AfterLoad=~p~n", [AfterLoad]), {false,_} = {lists:any(fun(Load) -> Load > 25 end, AfterLoad),AfterLoad}, true = erlang:system_flag(scheduler_wall_time, false) @@ -332,16 +372,81 @@ scheduler_wall_time(Config) when is_list(Config) -> erlang:system_flag(scheduler_wall_time, false) end. -get_load() -> - Start = erlang:statistics(scheduler_wall_time), +get_load(Type) -> + Start = erlang:statistics(Type), timer:sleep(1500), - End = erlang:statistics(scheduler_wall_time), + End = erlang:statistics(Type), lists:reverse(lists:sort(load_percentage(lists:sort(Start),lists:sort(End)))). load_percentage([{Id, WN, TN}|Ss], [{Id, WP, TP}|Ps]) -> [100*(WN-WP) div (TN-TP)|load_percentage(Ss, Ps)]; load_percentage([], []) -> []. +count(0) -> + ok; +count(N) -> + count(N-1). + +msb_swt_hog(true) -> + count(1000000), + erts_debug:dirty_cpu(wait, 10), + erts_debug:dirty_io(wait, 10), + msb_swt_hog(true); +msb_swt_hog(false) -> + count(1000000), + msb_swt_hog(false). + +msb_scheduler_wall_time(Config) -> + erlang:system_flag(scheduler_wall_time, true), + Dirty = erlang:system_info(dirty_cpu_schedulers) /= 0, + Hogs = lists:map(fun (_) -> + spawn_opt(fun () -> + msb_swt_hog(Dirty) + end, [{priority,low}, link, monitor]) + end, lists:seq(1,10)), + erlang:system_flag(multi_scheduling, block), + try + SWT1 = lists:sort(statistics(scheduler_wall_time_all)), + %% io:format("SWT1 = ~p~n", [SWT1]), + receive after 4000 -> ok end, + SWT2 = lists:sort(statistics(scheduler_wall_time_all)), + %% io:format("SWT2 = ~p~n", [SWT2]), + SWT = lists:zip(SWT1, SWT2), + io:format("SU = ~p~n", [lists:map(fun({{I, A0, T0}, {I, A1, T1}}) -> + {I, (A1 - A0)/(T1 - T0)} end, + SWT)]), + {A, T} = lists:foldl(fun({{_, A0, T0}, {_, A1, T1}}, {Ai,Ti}) -> + {Ai + (A1 - A0), Ti + (T1 - T0)} + end, + {0, 0}, + SWT), + TSU = A/T, + WSU = ((TSU * (erlang:system_info(schedulers) + + erlang:system_info(dirty_cpu_schedulers) + + erlang:system_info(dirty_io_schedulers))) + / 1), + %% Weighted scheduler utilization should be + %% very close to 1.0, i.e., we execute the + %% same time as one thread executing all + %% the time... + io:format("WSU = ~p~n", [WSU]), + true = 0.9 < WSU andalso WSU < 1.1, + ok + after + erlang:system_flag(multi_scheduling, unblock), + erlang:system_flag(scheduler_wall_time, false), + lists:foreach(fun ({HP, _HM}) -> + unlink(HP), + exit(HP, kill) + end, Hogs), + lists:foreach(fun ({HP, HM}) -> + receive + {'DOWN', HM, process, HP, _} -> + ok + end + end, Hogs), + ok + end. %% Tests that statistics(garbage_collection) is callable. %% It is not clear how to test anything more. diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl index 3d9e74472b..6a772bf7c9 100644 --- a/erts/emulator/test/system_info_SUITE.erl +++ b/erts/emulator/test/system_info_SUITE.erl @@ -36,7 +36,8 @@ -export([all/0, suite/0]). -export([process_count/1, system_version/1, misc_smoke_tests/1, - heap_size/1, wordsize/1, memory/1, ets_limit/1, atom_limit/1]). + heap_size/1, wordsize/1, memory/1, ets_limit/1, atom_limit/1, + atom_count/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -44,7 +45,7 @@ suite() -> all() -> [process_count, system_version, misc_smoke_tests, - heap_size, wordsize, memory, ets_limit, atom_limit]. + heap_size, wordsize, memory, ets_limit, atom_limit, atom_count]. %%% %%% The test cases ------------------------------------------------------------- @@ -550,3 +551,13 @@ get_atom_limit(Config, AtomsMax) -> end, stop_node(Node), Res. + +%% Verify that system_info(atom_count) works. +atom_count(Config) when is_list(Config) -> + Limit = erlang:system_info(atom_limit), + Count1 = erlang:system_info(atom_count), + list_to_atom(integer_to_list(erlang:unique_integer())), + Count2 = erlang:system_info(atom_count), + true = Limit >= Count2, + true = Count2 > Count1, + ok. diff --git a/erts/emulator/test/tracer_SUITE.erl b/erts/emulator/test/tracer_SUITE.erl index 9eb55c9af3..730c43d8c2 100644 --- a/erts/emulator/test/tracer_SUITE.erl +++ b/erts/emulator/test/tracer_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/erts/emulator/test/tracer_SUITE_data/tracer_test.c b/erts/emulator/test/tracer_SUITE_data/tracer_test.c index b68e480215..1555a95d9a 100644 --- a/erts/emulator/test/tracer_SUITE_data/tracer_test.c +++ b/erts/emulator/test/tracer_SUITE_data/tracer_test.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2014. All Rights Reserved. + * Copyright Ericsson AB 2009-2016. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/erts/emulator/test/tracer_test.erl b/erts/emulator/test/tracer_test.erl index 1da80bfe31..a82fd04d2e 100644 --- a/erts/emulator/test/tracer_test.erl +++ b/erts/emulator/test/tracer_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2013. All Rights Reserved. +%% Copyright Ericsson AB 1997-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl index ab56018373..d663cc548c 100644 --- a/erts/emulator/test/z_SUITE.erl +++ b/erts/emulator/test/z_SUITE.erl @@ -68,8 +68,8 @@ schedulers_alive(Config) when is_list(Config) -> enabled -> io:format("Testing blocking process exit~n"), BF = fun () -> - blocked = erlang:system_flag(multi_scheduling, - block), + blocked_normal = erlang:system_flag(multi_scheduling, + block_normal), Master ! {self(), blocking}, receive after infinity -> ok end end, @@ -77,21 +77,21 @@ schedulers_alive(Config) when is_list(Config) -> Mon = erlang:monitor(process, Blocker), receive {Blocker, blocking} -> ok end, [Blocker] - = erlang:system_info(multi_scheduling_blockers), + = erlang:system_info(normal_multi_scheduling_blockers), unlink(Blocker), exit(Blocker, kill), receive {'DOWN', Mon, _, _, _} -> ok end, enabled = erlang:system_info(multi_scheduling), - [] = erlang:system_info(multi_scheduling_blockers), + [] = erlang:system_info(normal_multi_scheduling_blockers), ok end, io:format("Testing blocked~n"), - erlang:system_flag(multi_scheduling, block), + erlang:system_flag(multi_scheduling, block_normal), case erlang:system_info(multi_scheduling) of enabled -> ct:fail(multi_scheduling_enabled); - blocked -> - [Master] = erlang:system_info(multi_scheduling_blockers); + blocked_normal -> + [Master] = erlang:system_info(normal_multi_scheduling_blockers); disabled -> ok end, Ps = lists:map( @@ -109,8 +109,8 @@ schedulers_alive(Config) when is_list(Config) -> unlink(P), exit(P, bang) end, Ps), - case erlang:system_flag(multi_scheduling, unblock) of - blocked -> ct:fail(multi_scheduling_blocked); + case erlang:system_flag(multi_scheduling, unblock_normal) of + blocked_normal -> ct:fail(multi_scheduling_blocked); disabled -> ok; enabled -> ok end, diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables index 27f9dcc878..47e1528958 100755 --- a/erts/emulator/utils/make_tables +++ b/erts/emulator/utils/make_tables @@ -36,6 +36,10 @@ use File::Basename; # <-src>/erl_am.c # <-src>/erl_bif_table.c # <-src>/erl_bif_wrap.c +# <-src>/erl_dirty_bif_wrap.c +# <-src>/erl_guard_bifs.c +# <-src>/hipe_nbif_impl.c +# <-include>/hipe_nbif_impl.h # <-include>/erl_atom_table.h # <-include>/erl_bif_table.h # @@ -51,9 +55,13 @@ my %atom; my %atom_alias; my %aliases; my $auto_alias_num = 0; +my %dirty_bif_tab; my @bif; -my @bif_type; +my @bif_info; +my $dirty_schedulers = 'no'; +my $dirty_schedulers_test = 'no'; +my $hipe = 'no'; while (@ARGV && $ARGV[0] =~ /^-(\w+)/) { my $opt = shift; @@ -65,6 +73,18 @@ while (@ARGV && $ARGV[0] =~ /^-(\w+)/) { $include = shift; die "No directory for -include argument specified" unless defined $include; + } elsif($opt eq '-ds') { + $dirty_schedulers = shift; + die "No -ds argument specified" + unless defined $dirty_schedulers; + } elsif($opt eq '-dst') { + $dirty_schedulers_test = shift; + die "No -dst argument specified" + unless defined $dirty_schedulers_test; + } elsif($opt eq '-hipe') { + $hipe = shift; + die "No -hipe argument specified" + unless defined $hipe; } else { usage("bad option: $opt"); } @@ -84,12 +104,31 @@ while (<>) { my($bif,$alias) = (@args); $bif =~ m@^([a-z_.'0-9]+):(.*)/(\d)$@ or error("invalid BIF"); my($mod,$name,$arity) = ($1,$2,$3); + my $mfa = "$mod:$name/$arity"; save_atoms($mod, $name); unless (defined $alias) { $alias = ""; $alias = "${mod}_" unless $mod eq 'erlang'; $alias .= "${name}_$arity"; } + my $sched_type; + my $alias3 = $alias; + + $sched_type = $dirty_bif_tab{$mfa}; + + if (!$sched_type or ($type eq 'ubif')) { + $sched_type = 'normal'; + } + elsif ($sched_type eq 'dirty_cpu') { + $alias3 = "schedule_dirty_cpu_$alias" + } + elsif ($sched_type eq 'dirty_io') { + $alias3 = "schedule_dirty_io_$alias" + } + else { + error("invalid sched_type: $sched_type"); + } + my $wrapper; if ($type eq 'bif') { $wrapper = "wrap_$alias"; @@ -97,8 +136,25 @@ while (<>) { $wrapper = $alias; } push(@bif, ["am_$atom_alias{$mod}","am_$atom_alias{$name}",$arity, - $alias,$wrapper]); - push(@bif_type, $type); + $alias3,$wrapper,$alias]); + push(@bif_info, [$type, $sched_type, $alias3, $alias]); + } elsif ($type eq 'dirty-cpu' or $type eq 'dirty-io' + or $type eq 'dirty-cpu-test' or $type eq 'dirty-io-test') { + if ($dirty_schedulers eq 'yes') { + my($bif,$other) = (@args); + $bif =~ m@^([a-z_.'0-9]+):(.*)/(\d)$@ or error("invalid BIF"); + my($mod,$name,$arity) = ($1,$2,$3); + my $mfa = "$mod:$name/$arity"; + if (($type eq 'dirty-cpu') + or (($dirty_schedulers_test eq 'yes') + and ($type eq 'dirty-cpu-test'))) { + $dirty_bif_tab{$mfa} = 'dirty_cpu'; + } elsif (($type eq 'dirty-io') + or (($dirty_schedulers_test eq 'yes') + and ($type eq 'dirty-io-test'))) { + $dirty_bif_tab{$mfa} = 'dirty_io'; + } + } } else { error("invalid line"); } @@ -147,7 +203,7 @@ open_file("$include/erl_bif_list.h"); my $i; for ($i = 0; $i < @bif; $i++) { # module atom, function atom, arity, C function, table index - print "BIF_LIST($bif[$i]->[0],$bif[$i]->[1],$bif[$i]->[2],$bif[$i]->[3],$i)\n"; + print "BIF_LIST($bif[$i]->[0],$bif[$i]->[1],$bif[$i]->[2],$bif[$i]->[3],$bif[$i]->[5],$i)\n"; } # @@ -167,16 +223,24 @@ typedef struct bif_entry { int arity; BifFunction f; BifFunction traced; + BifFunction impl; } BifEntry; typedef struct erts_gc_bif { BifFunction bif; BifFunction gc_bif; + int exp_ix; } ErtsGcBif; +typedef struct erts_u_bif { + BifFunction bif; + int exp_ix; +} ErtsUBif; + extern BifEntry bif_table[]; extern Export* bif_export[]; extern const ErtsGcBif erts_gc_bifs[]; +extern const ErtsUBif erts_u_bifs[]; #define BIF_SIZE $bif_size @@ -184,21 +248,28 @@ EOF my $i; for ($i = 0; $i < @bif; $i++) { - print "#define BIF_$bif[$i]->[3] $i\n"; + print "#define BIF_$bif_info[$i]->[3] $i\n"; } print "\n"; for ($i = 0; $i < @bif; $i++) { - my $args = join(', ', 'Process*', 'Eterm*'); - my $name = $bif[$i]->[3]; + my $args = join(', ', 'Process*', 'Eterm*', 'UWord*'); + my $name = $bif_info[$i]->[3]; print "Eterm $name($args);\n"; - print "Eterm wrap_$name($args, UWord *I);\n"; + print "Eterm wrap_$name($args);\n"; print "Eterm erts_gc_$name(Process* p, Eterm* reg, Uint live);\n" - if $bif_type[$i] eq 'gcbif'; + if $bif_info[$i]->[0] eq 'gcbif'; + print "Eterm $bif_info[$i]->[2]($args);\n" + unless $bif_info[$i]->[1] eq 'normal'; print "\n"; } -print "#endif\n"; + +if ($hipe eq 'yes') { + print "\n#include \"hipe_nbif_impl.h\"\n"; +} + +print "\n#endif\n"; # # Generate the bif table file. @@ -229,7 +300,7 @@ includes("export.h", "sys.h", "erl_vm.h", "global.h", "erl_process.h", "bif.h", for ($i = 0; $i < @bif; $i++) { next if $bif[$i]->[3] eq $bif[$i]->[4]; # Skip unwrapped bifs my $arity = $bif[$i]->[2]; - my $func = $bif[$i]->[3]; + my $func = $bif_info[$i]->[3]; print "Eterm\n"; print "wrap_$func(Process* p, Eterm* args, UWord* I)\n"; print "{\n"; @@ -241,21 +312,101 @@ for ($i = 0; $i < @bif; $i++) { # Generate erl_gc_bifs.c. # -open_file("$src/erl_gc_bifs.c"); +open_file("$src/erl_guard_bifs.c"); my $i; includes("export.h", "sys.h", "erl_vm.h", "global.h", "erl_process.h", "bif.h", "erl_bif_table.h"); print "const ErtsGcBif erts_gc_bifs[] = {\n"; for ($i = 0; $i < @bif; $i++) { - next unless $bif_type[$i] eq 'gcbif'; - my $arity = $bif[$i]->[2]; - my $func = $bif[$i]->[3]; - print " {$func, erts_gc_$func},\n"; + next unless $bif_info[$i]->[0] eq 'gcbif'; + print " {$bif[$i]->[3], erts_gc_$bif[$i]->[3], BIF_$bif[$i]->[5]},\n"; +} +print " {NULL, NULL, -1}\n"; +print "};\n"; + +print "const ErtsUBif erts_u_bifs[] = {\n"; +for ($i = 0; $i < @bif; $i++) { + next unless $bif_info[$i]->[0] eq 'ubif'; + print " {$bif[$i]->[3], BIF_$bif[$i]->[5]},\n"; } -print " {0, 0}\n"; +print " {NULL, -1}\n"; print "};\n"; # +# Generate the dirty bif wrappers file. +# + +open_file("$src/erl_dirty_bif_wrap.c"); +my $i; +includes("erl_process.h", "erl_nfunc_sched.h", "erl_bif_table.h", "erl_atom_table.h"); +for ($i = 0; $i < @bif_info; $i++) { + next if $bif_info[$i]->[1] eq 'normal'; + my $dtype; + if ($bif_info[$i]->[1] eq 'dirty_cpu') { + $dtype = "ERTS_SCHED_DIRTY_CPU"; + } + else { + $dtype = "ERTS_SCHED_DIRTY_IO"; + } +print <<EOF; +Eterm $bif_info[$i]->[2](Process *c_p, Eterm *regs, BeamInstr *I) +{ + return erts_reschedule_bif(c_p, regs, I, $bif_info[$i]->[3], $dtype); +} + +EOF + +} + +if ($hipe eq 'yes') { + + # + # Generate the nbif_impl bif wrappers file. + # + + open_file("$src/hipe_nbif_impl.h"); + print <<EOF; + +#ifndef HIPE_NBIF_IMPL_H__ +#define HIPE_NBIF_IMPL_H__ + +EOF + + my $i; + for ($i = 0; $i < @bif; $i++) { + print <<EOF; +Eterm nbif_impl_$bif[$i]->[5](Process *c_p, Eterm *regs); +EOF + } + + print <<EOF; + +#endif /* ERL_HIPE_NBIF_IMPL_H__ */ + +EOF + + # + # Generate the nbif_impl bif wrappers file. + # + + open_file("$src/hipe_nbif_impl.c"); + my $i; + includes("erl_process.h", "erl_nfunc_sched.h", "erl_bif_table.h", "erl_atom_table.h"); + for ($i = 0; $i < @bif; $i++) { + + print <<EOF; +Eterm nbif_impl_$bif[$i]->[5](Process *c_p, Eterm *regs) +{ + return $bif[$i]->[3](c_p, regs, (UWord *) bif_export\[BIF_$bif[$i]->[5]\]); +} + +EOF + + } + +} # hipe + +# # Utilities follow. # |