diff options
Diffstat (limited to 'erts/emulator')
52 files changed, 2607 insertions, 1054 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index fb486c917f..2212aed5e0 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -50,6 +50,8 @@ LDFLAGS=@LDFLAGS@ ARFLAGS=rc OMIT_OMIT_FP=no +DIRTY_SCHEDULER_SUPPORT=@DIRTY_SCHEDULER_SUPPORT@ + ifeq ($(TYPE),debug) PURIFY = TYPEMARKER = .debug @@ -174,6 +176,10 @@ FLAVOR_MARKER=.smp FLAVOR_FLAGS=-DERTS_SMP ENABLE_ALLOC_TYPE_VARS += smp nofrag M4FLAGS += -DERTS_SMP=1 +ifeq ($(DIRTY_SCHEDULER_SUPPORT),yes) +THR_DEFS += -DERTS_DIRTY_SCHEDULERS +endif + else # If flavor isn't one of the above, it *is* plain flavor... @@ -182,7 +188,6 @@ FLAVOR_MARKER= FLAVOR_FLAGS= ENABLE_ALLOC_TYPE_VARS += nofrag M4FLAGS += - endif TF_MARKER=$(TYPEMARKER)$(FLAVOR_MARKER) diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 3022c0a99a..8f65e71531 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -275,6 +275,7 @@ atom garbage_collection_info atom gc_end atom gc_major_end atom gc_major_start +atom gc_max_heap_size atom gc_minor_end atom gc_minor_start atom gc_start @@ -366,6 +367,7 @@ atom match_spec atom match_spec_result atom max atom maximum +atom max_heap_size atom max_tables max_processes atom mbuf_size atom md5 diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 1e30e8d8d1..8489897d3a 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -82,7 +82,7 @@ erts_smp_atomic32_t erts_staging_bp_index; static ERTS_INLINE ErtsMonotonicTime get_mtime(Process *c_p) { - return erts_get_monotonic_time(ERTS_PROC_GET_SCHDATA(c_p)); + return erts_get_monotonic_time(erts_proc_sched_data(c_p)); } /* ************************************************************************* @@ -655,8 +655,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) erts_smp_atomic_inc_nob(&bp->count->acount); } - if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE - && ERTS_TRACER_PROC_IS_ENABLED(c_p)) { + if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) { Eterm w; erts_trace_time_call(c_p, I, bp->time); w = (BeamInstr) *c_p->cp; @@ -753,8 +752,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } } if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && - IS_TRACED_FL(p, F_TRACE_CALLS) && - ERTS_TRACER_PROC_IS_ENABLED(p)) { + IS_TRACED_FL(p, F_TRACE_CALLS)) { BeamInstr *pc = (BeamInstr *)ep->code+3; erts_trace_time_call(p, pc, bp->time); } @@ -976,7 +974,8 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) BpDataTime *pbdt = NULL; ASSERT(c_p); - ASSERT(erts_smp_atomic32_read_acqb(&c_p->state) & ERTS_PSFLG_RUNNING); + ASSERT(erts_smp_atomic32_read_acqb(&c_p->state) & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING)); /* get previous timestamp and breakpoint * from the process psd */ @@ -1053,7 +1052,8 @@ erts_trace_time_return(Process *p, BeamInstr *pc) BpDataTime *pbdt = NULL; ASSERT(p); - ASSERT(erts_smp_atomic32_read_acqb(&p->state) & ERTS_PSFLG_RUNNING); + ASSERT(erts_smp_atomic32_read_acqb(&p->state) & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING)); /* get previous timestamp and breakpoint * from the process psd */ diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 9c2fc007a2..541af77211 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -80,7 +80,7 @@ typedef struct generic_bp { #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2) #ifdef ERTS_SMP -#define bp_sched2ix_proc(p) ((p)->scheduler_data->no - 1) +#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->no - 1) #else #define bp_sched2ix_proc(p) (0) #endif diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 99eb26bec5..f8f2e29c95 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -64,18 +64,21 @@ # ifdef ERTS_SMP # define PROCESS_MAIN_CHK_LOCKS(P) \ do { \ - if ((P)) { \ + if ((P)) \ erts_proc_lc_chk_only_proc_main((P)); \ - } \ - else \ - erts_lc_check_exact(NULL, 0); \ - ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); \ + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); \ +} while (0) +# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ +do { \ + if ((P)) \ + erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN, \ + __FILE__, __LINE__); \ +} while (0) +# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ +do { \ + if ((P)) \ + erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN); \ } while (0) -# define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) \ - if ((P)) erts_proc_lc_require_lock((P), ERTS_PROC_LOCK_MAIN,\ - __FILE__, __LINE__) -# define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) \ - if ((P)) erts_proc_lc_unrequire_lock((P), ERTS_PROC_LOCK_MAIN) # else # define ERTS_SMP_REQ_PROC_MAIN_LOCK(P) # define ERTS_SMP_UNREQ_PROC_MAIN_LOCK(P) @@ -1202,12 +1205,12 @@ init_emulator(void) do { \ if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) { \ ASSERT(FC <= 0); \ - ASSERT(ERTS_PROC_GET_SCHDATA(c_p)->virtual_reds \ + ASSERT(erts_proc_sched_data(c_p)->virtual_reds \ <= 0 - (FC)); \ } \ else { \ ASSERT(FC <= CONTEXT_REDS); \ - ASSERT(ERTS_PROC_GET_SCHDATA(c_p)->virtual_reds \ + ASSERT(erts_proc_sched_data(c_p)->virtual_reds \ <= CONTEXT_REDS - (FC)); \ } \ } while (0) @@ -1321,8 +1324,8 @@ void process_main(void) if (start_time != 0) { Sint64 diff = erts_timestamp_millis() - start_time; if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule -#ifdef ERTS_DIRTY_SCHEDULERS - && !ERTS_SCHEDULER_IS_DIRTY(c_p->scheduler_data) +#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS) + && !ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p)) #endif ) { BeamInstr *inptr = find_function_from_pc(start_time_i); @@ -1351,8 +1354,8 @@ void process_main(void) start_time_i = c_p->i; } - reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; - freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; + reg = erts_proc_sched_data(c_p)->x_reg_array; + freg = erts_proc_sched_data(c_p)->f_reg_array; ERL_BITS_RELOAD_STATEP(c_p); { int reds; @@ -1704,6 +1707,14 @@ void process_main(void) BeamInstr *next; Eterm result; + if (!(FCALLS > 0 || FCALLS > neg_o_reds)) { + /* If we have run out of reductions, we do a context + switch before calling the bif */ + c_p->arity = 2; + c_p->current = NULL; + goto context_switch3; + } + PRE_BIF_SWAPOUT(c_p); c_p->fcalls = FCALLS - 1; result = erl_send(c_p, r(0), x(1)); @@ -2810,6 +2821,15 @@ do { \ BeamInstr *next; ErlHeapFragment *live_hf_end; + + if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { + /* If we have run out of reductions, we do a context + switch before calling the bif */ + c_p->arity = ((Export *)Arg(0))->code[2]; + c_p->current = ((Export *)Arg(0))->code; + goto context_switch3; + } + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { if (GET_BIF_MODULE(Arg(0)) == am_ets) { ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS); @@ -3338,10 +3358,19 @@ do { \ context_switch2: /* Entry for fun calls. */ c_p->current = I-3; /* Pointer to Mod, Func, Arity */ + context_switch3: + { Eterm* argp; int i; + if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_EXITING) { + c_p->i = beam_exit; + c_p->arity = 0; + c_p->current = NULL; + goto do_schedule; + } + /* * Make sure that there is enough room for the argument registers to be saved. */ @@ -3509,6 +3538,12 @@ do { \ BifFunction vbf; ErlHeapFragment *live_hf_end; + if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { + /* If we have run out of reductions, we do a context + switch before calling the nif */ + goto context_switch; + } + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); @@ -3524,18 +3559,27 @@ do { \ typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); NifF* fp = vbf = (NifF*) I[1]; struct enif_environment_t env; +#ifdef ERTS_DIRTY_SCHEDULERS + if (!c_p->scheduler_data) + live_hf_end = ERTS_INVALID_HFRAG_PTR; /* On dirty scheduler */ + else +#endif + live_hf_end = c_p->mbuf; erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL); - live_hf_end = c_p->mbuf; nif_bif_result = (*fp)(&env, bif_nif_arity, reg); if (env.exception_thrown) nif_bif_result = THE_NON_VALUE; erts_post_nif(&env); - } - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); - PROCESS_MAIN_CHK_LOCKS(c_p); - ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); - ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); + if (env.exiting) { + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + goto do_schedule; + } + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + } DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); goto apply_bif_or_nif_epilogue; @@ -3551,6 +3595,13 @@ do { \ * code[3]: &&apply_bif * code[4]: Function pointer to BIF function */ + + if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { + /* If we have run out of reductions, we do a context + switch before calling the bif */ + goto context_switch; + } + if (ERTS_MSACC_IS_ENABLED_CACHED_X()) { if ((Eterm)I[-3] == am_ets) { ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS); @@ -4887,8 +4938,8 @@ do { \ #ifdef DEBUG pid = c_p->common.id; /* may have switched process... */ #endif - reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; - freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; + reg = erts_proc_sched_data(c_p)->x_reg_array; + freg = erts_proc_sched_data(c_p)->f_reg_array; ERL_BITS_RELOAD_STATEP(c_p); /* XXX: this abuse of def_arg_reg[] is horrid! */ neg_o_reds = -c_p->def_arg_reg[4]; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index eed0702688..2a3bd4afe5 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -44,6 +44,7 @@ #include "erl_ptab.h" #include "erl_bits.h" #include "erl_bif_unique.h" +#include "erl_map.h" #include "erl_msacc.h" Export *erts_await_result; @@ -880,6 +881,8 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) so.flags = erts_default_spo_flags|SPO_USE_ARGS; so.min_heap_size = H_MIN_SIZE; so.min_vheap_size = BIN_VH_MIN_SIZE; + so.max_heap_size = H_MAX_SIZE; + so.max_heap_flags = H_MAX_FLAGS; so.priority = PRIORITY_NORMAL; so.max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); so.scheduler = 0; @@ -937,6 +940,9 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) } else { so.min_heap_size = erts_next_heap_size(min_heap_size, 0); } + } else if (arg == am_max_heap_size) { + if (!erts_max_heap_size(val, &so.max_heap_size, &so.max_heap_flags)) + goto error; } else if (arg == am_min_bin_vheap_size && is_small(val)) { Sint min_vheap_size = signed_val(val); if (min_vheap_size < 0) { @@ -970,6 +976,10 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) goto error; } + if (so.max_heap_size != 0 && so.max_heap_size < so.min_heap_size) { + goto error; + } + /* * Spawn the process. */ @@ -1686,7 +1696,7 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) ERTS_PSFLG_BOUND); } - curr = ERTS_GET_SCHEDULER_DATA_FROM_PROC(BIF_P)->run_queue; + curr = erts_proc_sched_data(BIF_P)->run_queue; old = (ERTS_PSFLG_BOUND & state) ? curr : NULL; ASSERT(!old || old == curr); @@ -1731,6 +1741,23 @@ BIF_RETTYPE process_flag_2(BIF_ALIST_2) } BIF_RET(old_value); } + else if (BIF_ARG_1 == am_max_heap_size) { + Eterm *hp; + Uint sz = 0, max_heap_size, max_heap_flags; + + if (!erts_max_heap_size(BIF_ARG_2, &max_heap_size, &max_heap_flags)) + goto error; + + if ((max_heap_size < MIN_HEAP_SIZE(BIF_P) && max_heap_size != 0)) + goto error; + + erts_max_heap_size_map(MAX_HEAP_SIZE_GET(BIF_P), MAX_HEAP_SIZE_FLAGS_GET(BIF_P), NULL, &sz); + hp = HAlloc(BIF_P, sz); + old_value = erts_max_heap_size_map(MAX_HEAP_SIZE_GET(BIF_P), MAX_HEAP_SIZE_FLAGS_GET(BIF_P), &hp, NULL); + MAX_HEAP_SIZE_SET(BIF_P, max_heap_size); + MAX_HEAP_SIZE_FLAGS_SET(BIF_P, max_heap_flags); + BIF_RET(old_value); + } else if (BIF_ARG_1 == am_message_queue_data) { old_value = erts_change_message_queue_management(BIF_P, BIF_ARG_2); if (is_non_value(old_value)) @@ -4198,8 +4225,28 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2) else { locks &= ~ERTS_PROC_LOCK_STATUS; erts_smp_proc_unlock(new_member, ERTS_PROC_LOCK_STATUS); - new_member->group_leader = STORE_NC_IN_PROC(new_member, - BIF_ARG_1); + if (erts_smp_atomic32_read_nob(&new_member->state) + & !(ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + new_member->group_leader = STORE_NC_IN_PROC(new_member, + BIF_ARG_1); + } + else { + ErlHeapFragment *bp; + Eterm *hp; + /* + * Other process executing on a dirty scheduler, + * so we are not allowed to write to its heap. + * Store in heap fragment. + */ + + bp = new_message_buffer(NC_HEAP_SIZE(BIF_ARG_1)); + hp = bp->mem; + new_member->group_leader = STORE_NC(&hp, + &new_member->off_heap, + BIF_ARG_1); + bp->next = new_member->mbuf; + new_member->mbuf = bp; + } } } @@ -4350,6 +4397,31 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); BIF_RET(make_small(oval)); + } else if (BIF_ARG_1 == am_max_heap_size) { + + Eterm *hp, old_value; + Uint sz = 0, max_heap_size, max_heap_flags; + + if (!erts_max_heap_size(BIF_ARG_2, &max_heap_size, &max_heap_flags)) + goto error; + + if (max_heap_size < H_MIN_SIZE && max_heap_size != 0) + goto error; + + erts_max_heap_size_map(H_MAX_SIZE, H_MAX_FLAGS, NULL, &sz); + hp = HAlloc(BIF_P, sz); + old_value = erts_max_heap_size_map(H_MAX_SIZE, H_MAX_FLAGS, &hp, NULL); + + erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_block(); + + H_MAX_SIZE = max_heap_size; + H_MAX_FLAGS = max_heap_flags; + + erts_smp_thr_progress_unblock(); + erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + + BIF_RET(old_value); } else if (BIF_ARG_1 == am_display_items) { int oval = display_items; if (!is_small(BIF_ARG_2) || (n = signed_val(BIF_ARG_2)) < 0) { diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 5d751dd67d..2203182a0d 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -59,12 +59,12 @@ extern Export *erts_convert_time_unit_trap; do { \ if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) { \ if ((fcalls) > 0) \ - ERTS_PROC_GET_SCHDATA((p))->virtual_reds += (fcalls); \ + erts_proc_sched_data((p))->virtual_reds += (fcalls); \ (fcalls) = 0; \ } \ else { \ if ((fcalls) > -CONTEXT_REDS) \ - ERTS_PROC_GET_SCHDATA((p))->virtual_reds \ + erts_proc_sched_data((p))->virtual_reds \ += ((fcalls) - (-CONTEXT_REDS)); \ (fcalls) = -CONTEXT_REDS; \ } \ @@ -91,22 +91,22 @@ do { \ if (!ERTS_PROC_GET_SAVED_CALLS_BUF((p))) { \ if ((p)->fcalls >= reds) { \ (p)->fcalls -= reds; \ - ERTS_PROC_GET_SCHDATA((p))->virtual_reds += reds; \ + erts_proc_sched_data((p))->virtual_reds += reds; \ } \ else { \ if ((p)->fcalls > 0) \ - ERTS_PROC_GET_SCHDATA((p))->virtual_reds += (p)->fcalls;\ + erts_proc_sched_data((p))->virtual_reds += (p)->fcalls; \ (p)->fcalls = 0; \ } \ } \ else { \ if ((p)->fcalls >= reds - CONTEXT_REDS) { \ (p)->fcalls -= reds; \ - ERTS_PROC_GET_SCHDATA((p))->virtual_reds += reds; \ + erts_proc_sched_data((p))->virtual_reds += reds; \ } \ else { \ if ((p)->fcalls > -CONTEXT_REDS) \ - ERTS_PROC_GET_SCHDATA((p))->virtual_reds \ + erts_proc_sched_data((p))->virtual_reds \ += (p)->fcalls - (-CONTEXT_REDS); \ (p)->fcalls = -CONTEXT_REDS; \ } \ @@ -118,14 +118,14 @@ do { \ if (ERTS_PROC_GET_SAVED_CALLS_BUF((P))) { \ int nreds__ = ((int)(Reds)) - CONTEXT_REDS; \ if ((FCalls) > nreds__) { \ - ERTS_PROC_GET_SCHDATA((P))->virtual_reds \ + erts_proc_sched_data((P))->virtual_reds \ += (FCalls) - nreds__; \ (FCalls) = nreds__; \ } \ } \ else { \ if ((FCalls) > (Reds)) { \ - ERTS_PROC_GET_SCHDATA((P))->virtual_reds \ + erts_proc_sched_data((P))->virtual_reds \ += (FCalls) - (Reds); \ (FCalls) = (Reds); \ } \ @@ -165,7 +165,7 @@ do { \ #define ERTS_BIF_ERROR_TRAPPED1(Proc, Reason, Bif, A0) \ do { \ - Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ (Proc)->current = (Bif)->code; \ reg[0] = (Eterm) (A0); \ @@ -174,7 +174,7 @@ do { \ #define ERTS_BIF_ERROR_TRAPPED2(Proc, Reason, Bif, A0, A1) \ do { \ - Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ (Proc)->current = (Bif)->code; \ reg[0] = (Eterm) (A0); \ @@ -184,7 +184,7 @@ do { \ #define ERTS_BIF_ERROR_TRAPPED3(Proc, Reason, Bif, A0, A1, A2) \ do { \ - Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ (Proc)->current = (Bif)->code; \ reg[0] = (Eterm) (A0); \ @@ -208,7 +208,7 @@ do { \ #define ERTS_BIF_PREP_ERROR_TRAPPED1(Ret, Proc, Reason, Bif, A0) \ do { \ - Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ (Proc)->current = (Bif)->code; \ reg[0] = (Eterm) (A0); \ @@ -217,7 +217,7 @@ do { \ #define ERTS_BIF_PREP_ERROR_TRAPPED2(Ret, Proc, Reason, Bif, A0, A1) \ do { \ - Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ (Proc)->current = (Bif)->code; \ reg[0] = (Eterm) (A0); \ @@ -227,7 +227,7 @@ do { \ #define ERTS_BIF_PREP_ERROR_TRAPPED3(Ret, Proc, Reason, Bif, A0, A1, A2) \ do { \ - Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ (Proc)->current = (Bif)->code; \ reg[0] = (Eterm) (A0); \ @@ -246,7 +246,7 @@ do { \ #define ERTS_BIF_PREP_TRAP1(Ret, Trap, Proc, A0) \ do { \ - Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->arity = 1; \ reg[0] = (Eterm) (A0); \ (Proc)->i = (BeamInstr*) ((Trap)->addressv[erts_active_code_ix()]); \ @@ -256,7 +256,7 @@ do { \ #define ERTS_BIF_PREP_TRAP2(Ret, Trap, Proc, A0, A1) \ do { \ - Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->arity = 2; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ @@ -267,7 +267,7 @@ do { \ #define ERTS_BIF_PREP_TRAP3(Ret, Trap, Proc, A0, A1, A2) \ do { \ - Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->arity = 3; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ @@ -279,7 +279,7 @@ do { \ #define ERTS_BIF_PREP_TRAP3_NO_RET(Trap, Proc, A0, A1, A2)\ do { \ - Eterm* reg = ERTS_PROC_GET_SCHDATA((Proc))->x_reg_array; \ + Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->arity = 3; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ @@ -296,7 +296,7 @@ do { \ } while(0) #define BIF_TRAP1(Trap_, p, A0) do { \ - Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array; \ + Eterm* reg = erts_proc_sched_data((p))->x_reg_array; \ (p)->arity = 1; \ reg[0] = (A0); \ (p)->i = (BeamInstr*) ((Trap_)->addressv[erts_active_code_ix()]); \ @@ -305,7 +305,7 @@ do { \ } while(0) #define BIF_TRAP2(Trap_, p, A0, A1) do { \ - Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array; \ + Eterm* reg = erts_proc_sched_data((p))->x_reg_array; \ (p)->arity = 2; \ reg[0] = (A0); \ reg[1] = (A1); \ @@ -315,7 +315,7 @@ do { \ } while(0) #define BIF_TRAP3(Trap_, p, A0, A1, A2) do { \ - Eterm* reg = ERTS_PROC_GET_SCHDATA((p))->x_reg_array; \ + Eterm* reg = erts_proc_sched_data((p))->x_reg_array; \ (p)->arity = 3; \ reg[0] = (A0); \ reg[1] = (A1); \ diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index d02c6828f9..3c19e82b66 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -118,7 +118,9 @@ process_killer(void) | ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_IN_RUNQ | ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS)) { + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { erts_printf("Can only kill WAITING processes this way\n"); } else { @@ -214,7 +216,8 @@ print_process_info(int to, void *to_arg, Process *p) if (state & ERTS_PSFLG_GC) { garbing = 1; running = 1; - } else if (state & ERTS_PSFLG_RUNNING) + } else if (state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING)) running = 1; /* diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index ba216c7eb4..227fedfb69 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -278,6 +278,7 @@ type IOB_REQ SHORT_LIVED SYSTEM io_bytes_request type TRACER_NIF LONG_LIVED SYSTEM tracer_nif type TRACE_MSG_QUEUE SHORT_LIVED SYSTEM trace_message_queue type SCHED_ASYNC_JOB SHORT_LIVED SYSTEM async_calls +type DIRTY_START STANDARD PROCESSES dirty_start +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 85cadafebc..2e195db0ee 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -45,6 +45,7 @@ #include "erl_async.h" #include "erl_thr_progress.h" #include "erl_bif_unique.h" +#include "erl_map.h" #define ERTS_PTAB_WANT_DEBUG_FUNCS__ #include "erl_ptab.h" #ifdef HIPE @@ -594,6 +595,7 @@ static Eterm pi_args[] = { am_suspending, am_min_heap_size, am_min_bin_vheap_size, + am_max_heap_size, am_current_location, am_current_stacktrace, am_message_queue_data, @@ -643,10 +645,11 @@ pi_arg2ix(Eterm arg) case am_suspending: return 26; case am_min_heap_size: return 27; case am_min_bin_vheap_size: return 28; - case am_current_location: return 29; - case am_current_stacktrace: return 30; - case am_message_queue_data: return 31; - case am_garbage_collection_info: return 32; + case am_max_heap_size: return 29; + case am_current_location: return 30; + case am_current_stacktrace: return 31; + case am_message_queue_data: return 32; + case am_garbage_collection_info: return 33; default: return -1; } } @@ -1107,7 +1110,7 @@ process_info_aux(Process *BIF_P, break; case am_status: - res = erts_process_status(BIF_P, ERTS_PROC_LOCK_MAIN, rp, rpid); + res = erts_process_status(rp, rpid); ASSERT(res != am_undefined); hp = HAlloc(BIF_P, 3); break; @@ -1348,6 +1351,18 @@ process_info_aux(Process *BIF_P, break; } + case am_max_heap_size: { + Uint hsz = 3; + (void) erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp), + MAX_HEAP_SIZE_FLAGS_GET(rp), + NULL, &hsz); + hp = HAlloc(BIF_P, hsz); + res = erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp), + MAX_HEAP_SIZE_FLAGS_GET(rp), + &hp, NULL); + break; + } + case am_total_heap_size: { ErtsMessage *mp; Uint total_heap_size; @@ -1391,8 +1406,12 @@ process_info_aux(Process *BIF_P, case am_garbage_collection: { DECL_AM(minor_gcs); Eterm t; + Uint map_sz = 0; + + erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp), MAX_HEAP_SIZE_FLAGS_GET(rp), NULL, &map_sz); - hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2 + 3+2 + 3); /* last "3" is for outside tuple */ + hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2 + 3+2 + 3+2 + map_sz + 3); + /* last "3" is for outside tuple */ t = TUPLE2(hp, AM_minor_gcs, make_small(GEN_GCS(rp))); hp += 3; res = CONS(hp, t, NIL); hp += 2; @@ -1403,6 +1422,11 @@ process_info_aux(Process *BIF_P, res = CONS(hp, t, res); hp += 2; t = TUPLE2(hp, am_min_bin_vheap_size, make_small(MIN_VHEAP_SIZE(rp))); hp += 3; res = CONS(hp, t, res); hp += 2; + + t = erts_max_heap_size_map(MAX_HEAP_SIZE_GET(rp), MAX_HEAP_SIZE_FLAGS_GET(rp), &hp, NULL); + + t = TUPLE2(hp, am_max_heap_size, t); hp += 3; + res = CONS(hp, t, res); hp += 2; break; } @@ -1412,12 +1436,12 @@ process_info_aux(Process *BIF_P, if (rp == BIF_P) { sz += ERTS_PROCESS_GC_INFO_MAX_SIZE; } else { - erts_process_gc_info(rp, &sz, NULL); + erts_process_gc_info(rp, &sz, NULL, 0, 0); sz += 3; } hp = HAlloc(BIF_P, sz); - res = erts_process_gc_info(rp, &actual_sz, &hp); + res = erts_process_gc_info(rp, &actual_sz, &hp, 0, 0); /* We may have some extra space, fill with 0 tuples */ if (actual_sz <= sz - 3) { @@ -2035,12 +2059,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) Uint arity = *tp++; return info_1_tuple(BIF_P, tp, arityval(arity)); } else if (BIF_ARG_1 == am_scheduler_id) { -#ifdef ERTS_SMP - ASSERT(BIF_P->scheduler_data); - BIF_RET(make_small(BIF_P->scheduler_data->no)); -#else - BIF_RET(make_small(1)); -#endif + ErtsSchedulerData *esdp = erts_proc_sched_data(BIF_P); + BIF_RET(make_small(esdp->no)); } else if (BIF_ARG_1 == am_compat_rel) { ASSERT(erts_compat_rel > 0); BIF_RET(make_small(erts_compat_rel)); @@ -2173,7 +2193,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (BIF_ARG_1 == am_garbage_collection){ Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); Eterm tup; - hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2); + hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2 + 3+2); tup = TUPLE2(hp, am_fullsweep_after, make_small(val)); hp += 3; res = CONS(hp, tup, NIL); hp += 2; @@ -2184,6 +2204,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) tup = TUPLE2(hp, am_min_bin_vheap_size, make_small(BIN_VH_MIN_SIZE)); hp += 3; res = CONS(hp, tup, res); hp += 2; + tup = TUPLE2(hp, am_max_heap_size, make_small(H_MAX_SIZE)); hp += 3; + res = CONS(hp, tup, res); hp += 2; + BIF_RET(res); } else if (BIF_ARG_1 == am_fullsweep_after){ Uint val = (Uint) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); @@ -2194,6 +2217,12 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_min_heap_size,make_small(H_MIN_SIZE)); BIF_RET(res); + } else if (BIF_ARG_1 == am_max_heap_size) { + Uint sz = 0; + erts_max_heap_size_map(H_MAX_SIZE, H_MAX_FLAGS, NULL, &sz); + hp = HAlloc(BIF_P, sz); + res = erts_max_heap_size_map(H_MAX_SIZE, H_MAX_FLAGS, &hp, NULL); + BIF_RET(res); } else if (BIF_ARG_1 == am_min_bin_vheap_size) { hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_min_bin_vheap_size,make_small(BIN_VH_MIN_SIZE)); @@ -3589,10 +3618,7 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) /* Used by timer process_SUITE, timer_bif_SUITE, and node_container_SUITE (emulator) */ if (is_internal_pid(tp[2])) { - BIF_RET(erts_process_status(BIF_P, - ERTS_PROC_LOCK_MAIN, - NULL, - tp[2])); + BIF_RET(erts_process_status(NULL, tp[2])); } } else if (ERTS_IS_ATOM_STR("link_list", tp[1])) { diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index b65c0e303f..66e5146da0 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -512,8 +512,7 @@ start_trace(Process *c_p, ErtsTracer tracer, && !ERTS_TRACER_COMPARE(ERTS_TRACER(port), tracer)) { /* This tracee is already being traced, and not by the * tracer to be */ - if (erts_is_tracer_proc_enabled(c_p, ERTS_PROC_LOCKS_ALL, - common, am_trace_status)) { + if (erts_is_tracer_enabled(tracer, common)) { /* The tracer is still in use */ return 1; } @@ -856,7 +855,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) return am_undefined; if (!ERTS_TRACER_IS_NIL(ERTS_TRACER(tracee))) - erts_is_tracer_proc_enabled(NULL, 0, &tracee->common, am_trace_status); + erts_is_tracer_proc_enabled(NULL, 0, &tracee->common); tracer = erts_tracer_to_term(p, ERTS_TRACER(tracee)); trace_flags = ERTS_TRACE_FLAGS(tracee); @@ -864,22 +863,24 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) erts_port_release(tracee); } else if (is_internal_pid(pid_spec)) { - Process *tracee; - tracee = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, - pid_spec, ERTS_PROC_LOCK_MAIN); + Process *tracee = erts_pid2proc_not_running(p, ERTS_PROC_LOCK_MAIN, + pid_spec, ERTS_PROC_LOCK_MAIN); + + if (tracee == ERTS_PROC_LOCK_BUSY) + ERTS_BIF_YIELD2(bif_export[BIF_trace_info_2], p, pid_spec, key); if (!tracee) return am_undefined; if (!ERTS_TRACER_IS_NIL(ERTS_TRACER(tracee))) erts_is_tracer_proc_enabled(tracee, ERTS_PROC_LOCK_MAIN, - &tracee->common, am_trace_status); + &tracee->common); tracer = erts_tracer_to_term(p, ERTS_TRACER(tracee)); trace_flags = ERTS_TRACE_FLAGS(tracee); - if (tracee != p) - erts_smp_proc_unlock(tracee, ERTS_PROC_LOCK_MAIN); + if (tracee != p) + erts_smp_proc_unlock(tracee, ERTS_PROC_LOCK_MAIN); } else if (is_external_pid(pid_spec) && external_pid_dist_entry(pid_spec) == erts_this_dist_entry) { return am_undefined; diff --git a/erts/emulator/beam/erl_bif_unique.c b/erts/emulator/beam/erl_bif_unique.c index 1e57e9fa53..7c70217d8d 100644 --- a/erts/emulator/beam/erl_bif_unique.c +++ b/erts/emulator/beam/erl_bif_unique.c @@ -257,7 +257,7 @@ static ERTS_INLINE Eterm unique_integer_bif(Process *c_p, int positive) Uint hsz; Eterm *hp; - esdp = ERTS_PROC_GET_SCHDATA(c_p); + esdp = erts_proc_sched_data(c_p); thr_id = (Uint64) esdp->thr_id; unique = esdp->unique++; bld_unique_integer_term(NULL, &hsz, thr_id, unique, positive); @@ -515,7 +515,7 @@ BIF_RETTYPE make_ref_0(BIF_ALIST_0) hp = HAlloc(BIF_P, REF_THING_SIZE); - res = erts_sched_make_ref_in_buffer(ERTS_PROC_GET_SCHDATA(BIF_P), hp); + res = erts_sched_make_ref_in_buffer(erts_proc_sched_data(BIF_P), hp); BIF_RET(res); } diff --git a/erts/emulator/beam/erl_bits.h b/erts/emulator/beam/erl_bits.h index 1c2a090f07..4bd5b24157 100644 --- a/erts/emulator/beam/erl_bits.h +++ b/erts/emulator/beam/erl_bits.h @@ -83,8 +83,8 @@ typedef struct erl_bin_match_struct{ #ifdef ERTS_SMP /* the state resides in the current process' scheduler data */ #define ERL_BITS_DECLARE_STATEP struct erl_bits_state *EBS -#define ERL_BITS_RELOAD_STATEP(P) do{EBS = &(P)->scheduler_data->erl_bits_state;}while(0) -#define ERL_BITS_DEFINE_STATEP(P) struct erl_bits_state *EBS = &(P)->scheduler_data->erl_bits_state +#define ERL_BITS_RELOAD_STATEP(P) do{EBS = &erts_proc_sched_data((P))->erl_bits_state;}while(0) +#define ERL_BITS_DEFINE_STATEP(P) struct erl_bits_state *EBS = &erts_proc_sched_data((P))->erl_bits_state #else /* reentrant API but with a hidden single global state, for testing only */ extern struct erl_bits_state ErlBitsState_; diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index acaca54e9a..bad34211a5 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -3462,7 +3462,7 @@ static void fix_table_locked(Process* p, DbTable* tb) fix = tb->common.fixations; if (fix == NULL) { tb->common.time.monotonic - = erts_get_monotonic_time(ERTS_PROC_GET_SCHDATA(p)); + = erts_get_monotonic_time(erts_proc_sched_data(p)); tb->common.time.offset = erts_get_time_offset(); } else { diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 95b1cd0148..6732b708a8 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -174,7 +174,8 @@ set_match_trace(Process *tracee_p, Eterm fail_term, ErtsTracer tracer, ERTS_PROC_LOCKS_ALL == erts_proc_lc_my_proc_locks(tracee_p) || erts_thr_progress_is_blocking()); - if (ERTS_TRACER_IS_NIL(tracer) || erts_is_tracer_enabled(tracee_p, tracer)) + if (ERTS_TRACER_IS_NIL(tracer) + || erts_is_tracer_enabled(tracer, &tracee_p->common)) return set_tracee_flags(tracee_p, tracer, d_flags, e_flags); return fail_term; } @@ -1801,7 +1802,7 @@ Eterm db_prog_match(Process *c_p, /* We need to lure the scheduler into believing in the pseudo process, because of floating point exceptions. Do *after* mpsp is set!!! */ - esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(psp); + esdp = erts_get_scheduler_data(); if (esdp) current_scheduled = esdp->current_process; /* SMP: psp->scheduler_data is set by get_match_pseudo_process */ diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index 2700b62854..6ec5fbb895 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -43,12 +43,11 @@ typedef struct { int suggested_stack_size; } ErlDrvThreadOpts; -#if defined(ERL_DRV_DIRTY_SCHEDULER_SUPPORT) || defined(ERL_NIF_DIRTY_SCHEDULER_SUPPORT) + typedef enum { - ERL_DRV_DIRTY_JOB_CPU_BOUND = 1, - ERL_DRV_DIRTY_JOB_IO_BOUND = 2 -} ErlDrvDirtyJobFlags; -#endif + ERL_DIRTY_JOB_CPU_BOUND = 1, + ERL_DIRTY_JOB_IO_BOUND = 2 +} ErlDirtyJobFlags; #ifdef SIZEOF_CHAR # define SIZEOF_CHAR_SAVED__ SIZEOF_CHAR diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index bed7e668d7..d740b2baec 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -41,6 +41,7 @@ #endif #include "dtrace-wrapper.h" #include "erl_bif_unique.h" +#include "dist.h" #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20 @@ -146,7 +147,8 @@ static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size, static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size); static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size); static void move_msgq_to_heap(Process *p); - +static int reached_max_heap_size(Process *p, Uint total_heap_size, + Uint extra_heap_size, Uint extra_old_heap_size); static void init_gc_info(ErtsGCInfo *gcip); #ifdef HARDDEBUG @@ -389,7 +391,7 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end, if (p->freason == TRAP) { #if HIPE if (regs == NULL) { - regs = ERTS_PROC_GET_SCHDATA(p)->x_reg_array; + regs = erts_proc_sched_data(p)->x_reg_array; } #endif cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls); @@ -404,6 +406,7 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end, result = val[0]; } BUMP_REDS(p, cost); + return result; } @@ -507,14 +510,14 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int /* Make sure that we do a proper GC as soon as possible... */ p->flags |= F_FORCE_GC; reds_left = ERTS_REDS_LEFT(p, fcalls); - ASSERT(CONTEXT_REDS - reds_left >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); + ASSERT(CONTEXT_REDS - reds_left >= erts_proc_sched_data(p)->virtual_reds); if (reds_left > ERTS_ABANDON_HEAP_COST) { int vreds = reds_left - ERTS_ABANDON_HEAP_COST; - ERTS_PROC_GET_SCHDATA((p))->virtual_reds += vreds; + erts_proc_sched_data((p))->virtual_reds += vreds; } - ASSERT(CONTEXT_REDS >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); + ASSERT(CONTEXT_REDS >= erts_proc_sched_data(p)->virtual_reds); return reds_left; } @@ -577,18 +580,22 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, int need, Eterm* objv, int nobj, int fcalls) { Uint reclaimed_now = 0; + Eterm gc_trace_end_tag; int reds; ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */ ErtsSchedulerData *esdp; + erts_aint32_t state; ERTS_MSACC_PUSH_STATE_M(); #ifdef USE_VM_PROBES DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); #endif ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) - >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); + >= erts_proc_sched_data(p)->virtual_reds); - if (p->flags & (F_DISABLE_GC|F_DELAY_GC)) + state = erts_smp_atomic32_read_nob(&p->state); + + if (p->flags & (F_DISABLE_GC|F_DELAY_GC) || state & ERTS_PSFLG_EXITING) return delay_garbage_collection(p, live_hf_end, need, fcalls); if (p->abandoned_heap) @@ -623,28 +630,28 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, if (GEN_GCS(p) < MAX_GEN_GCS(p) && !(FLAGS(p) & F_NEED_FULLSWEEP)) { if (IS_TRACED_FL(p, F_TRACE_GC)) { - trace_gc(p, am_gc_minor_start, need); + trace_gc(p, am_gc_minor_start, need, THE_NON_VALUE); } DTRACE2(gc_minor_start, pidbuf, need); reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); DTRACE2(gc_minor_end, pidbuf, reclaimed_now); - if (IS_TRACED_FL(p, F_TRACE_GC)) { - trace_gc(p, am_gc_minor_end, reclaimed_now); - } - if (reds < 0) + if (reds == -1) { + if (IS_TRACED_FL(p, F_TRACE_GC)) { + trace_gc(p, am_gc_minor_end, reclaimed_now, THE_NON_VALUE); + } goto do_major_collection; + } + gc_trace_end_tag = am_gc_minor_end; } else { do_major_collection: ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC_FULL); if (IS_TRACED_FL(p, F_TRACE_GC)) { - trace_gc(p, am_gc_major_start, need); + trace_gc(p, am_gc_major_start, need, THE_NON_VALUE); } DTRACE2(gc_major_start, pidbuf, need); reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); DTRACE2(gc_major_end, pidbuf, reclaimed_now); - if (IS_TRACED_FL(p, F_TRACE_GC)) { - trace_gc(p, am_gc_major_end, reclaimed_now); - } + gc_trace_end_tag = am_gc_major_end; ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC); } @@ -658,8 +665,33 @@ do_major_collection: ErtsGcQuickSanityCheck(p); + /* Max heap size has been reached and the process was configured + to be killed, so we kill it and set it in a delayed garbage + collecting state. There should be no gc_end trace or + long_gc/large_gc triggers when this happens as process was + killed before a GC could be done. */ + if (reds == -2) { + ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL; + + erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); + erts_send_exit_signal(p, p->common.id, p, &locks, + am_kill, NIL, NULL, 0); + erts_smp_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR); + + /* erts_send_exit_signal looks for ERTS_PSFLG_GC, so + we have to remove it after the signal is sent */ + erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + + /* We have to make sure that we have space for need on the heap */ + return delay_garbage_collection(p, live_hf_end, need, fcalls); + } + erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + if (IS_TRACED_FL(p, F_TRACE_GC)) { + trace_gc(p, gc_trace_end_tag, reclaimed_now, THE_NON_VALUE); + } + if (erts_system_monitor_long_gc != 0) { ErtsMonotonicTime end_time; Uint gc_time; @@ -716,7 +748,7 @@ erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fca int reds_left = ERTS_REDS_LEFT(p, fcalls); if (reds > reds_left) reds = reds_left; - ASSERT(CONTEXT_REDS - (reds_left - reds) >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); + ASSERT(CONTEXT_REDS - (reds_left - reds) >= erts_proc_sched_data(p)->virtual_reds); return reds; } @@ -726,7 +758,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls); BUMP_REDS(p, reds); ASSERT(CONTEXT_REDS - ERTS_BIF_REDS_LEFT(p) - >= ERTS_PROC_GET_SCHDATA(p)->virtual_reds); + >= erts_proc_sched_data(p)->virtual_reds); } /* @@ -761,6 +793,7 @@ erts_garbage_collect_hibernate(Process* p) heap_size = p->heap_sz + (p->old_htop - p->old_heap) + p->mbuf_sz; + heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_TMP_HEAP, sizeof(Eterm)*heap_size); htop = heap; @@ -1031,6 +1064,34 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, Uint size_before = young_gen_usage(p); /* + * Check if we have gone past the max heap size limit + */ + + if (MAX_HEAP_SIZE_GET(p)) { + Uint heap_size = size_before, + /* Note that we also count the un-allocated area + in between the stack and heap */ + stack_size = HEAP_END(p) - HEAP_TOP(p), + extra_heap_size, + extra_old_heap_size = 0; + + /* Add potential old heap size */ + if (OLD_HEAP(p) == NULL && mature_size != 0) { + extra_old_heap_size = erts_next_heap_size(size_before, 1); + heap_size += extra_old_heap_size; + } else if (OLD_HEAP(p)) + heap_size += OLD_HEND(p) - OLD_HEAP(p); + + /* Add potential new young heap size */ + extra_heap_size = next_heap_size(p, stack_size + size_before, 0); + heap_size += extra_heap_size; + + if (heap_size > MAX_HEAP_SIZE_GET(p)) + if (reached_max_heap_size(p, heap_size, extra_heap_size, extra_old_heap_size)) + return -2; + } + + /* * Allocate an old heap if we don't have one and if we'll need one. */ @@ -1139,6 +1200,16 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); ASSERT(MBUF(p) == NULL); + /* The heap usage during GC should be larger than what we end up + after a GC, even if we grow it. If this assertion is not true + we have to check size in grow_new_heap and potentially kill the + process from there */ + ASSERT(!MAX_HEAP_SIZE_GET(p) || + !(MAX_HEAP_SIZE_FLAGS_GET(p) & MAX_HEAP_SIZE_KILL) || + MAX_HEAP_SIZE_GET(p) > (young_gen_usage(p) + + (OLD_HEND(p) - OLD_HEAP(p)) + + (HEAP_END(p) - HEAP_TOP(p)))); + return gc_cost(size_after, adjust_size); } @@ -1457,6 +1528,25 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, if (new_sz == HEAP_SIZE(p) && FLAGS(p) & F_HEAP_GROW) { new_sz = next_heap_size(p, HEAP_SIZE(p), 1); } + + + if (MAX_HEAP_SIZE_GET(p)) { + Uint heap_size = size_before; + + /* Add unused space in old heap */ + heap_size += OLD_HEND(p) - OLD_HTOP(p); + + /* Add stack + unused space in young heap */ + heap_size += HEAP_END(p) - HEAP_TOP(p); + + /* Add size of new young heap */ + heap_size += new_sz; + + if (MAX_HEAP_SIZE_GET(p) < heap_size) + if (reached_max_heap_size(p, heap_size, new_sz, 0)) + return -2; + } + FLAGS(p) &= ~(F_HEAP_GROW|F_NEED_FULLSWEEP); n_htop = n_heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*new_sz); @@ -2955,7 +3045,7 @@ reply_gc_info(void *vgcirp) Eterm erts_gc_info_request(Process *c_p) { - ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); Eterm ref; ErtsGCInfoReq *gcirp; Eterm *hp; @@ -2986,7 +3076,9 @@ erts_gc_info_request(Process *c_p) } Eterm -erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp) +erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp, + Uint extra_heap_block, + Uint extra_old_heap_block_size) { ERTS_DECL_AM(bin_vheap_size); ERTS_DECL_AM(bin_vheap_block_size); @@ -3009,8 +3101,9 @@ erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp) AM_bin_old_vheap_block_size }; UWord values[] = { - OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) : 0, - HEAP_SIZE(p), + OLD_HEAP(p) ? OLD_HEND(p) - OLD_HEAP(p) + extra_old_heap_block_size + : extra_old_heap_block_size, + HEAP_SIZE(p) + extra_heap_block, MBUF_SIZE(p), HIGH_WATER(p) - HEAP_START(p), STACK_START(p) - p->stop, @@ -3056,6 +3149,130 @@ erts_process_gc_info(Process *p, Uint *sizep, Eterm **hpp) return res; } +static int +reached_max_heap_size(Process *p, Uint total_heap_size, + Uint extra_heap_size, Uint extra_old_heap_size) +{ + Uint max_heap_flags = MAX_HEAP_SIZE_FLAGS_GET(p); + if (IS_TRACED_FL(p, F_TRACE_GC) || + max_heap_flags & MAX_HEAP_SIZE_LOG) { + Eterm msg; + Uint size = 0; + Eterm *o_hp , *hp; + erts_process_gc_info(p, &size, NULL, extra_heap_size, + extra_old_heap_size); + o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, size * sizeof(Eterm)); + msg = erts_process_gc_info(p, NULL, &hp, extra_heap_size, + extra_old_heap_size); + + if (max_heap_flags & MAX_HEAP_SIZE_LOG) { + int alive = erts_is_alive; + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + Eterm *o_hp, *hp, args = NIL; + + /* Build the format message */ + erts_dsprintf(dsbufp, " Process: ~p "); + if (alive) + erts_dsprintf(dsbufp, "on node ~p"); + erts_dsprintf(dsbufp, "~n Context: maximum heap size reached~n"); + erts_dsprintf(dsbufp, " Max Heap Size: ~p~n"); + erts_dsprintf(dsbufp, " Total Heap Size: ~p~n"); + erts_dsprintf(dsbufp, " Kill: ~p~n"); + erts_dsprintf(dsbufp, " Error Logger: ~p~n"); + erts_dsprintf(dsbufp, " GC Info: ~p~n"); + + /* Build the args in reverse order */ + o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, 2*(alive ? 7 : 6) * sizeof(Eterm)); + args = CONS(hp, msg, args); hp += 2; + args = CONS(hp, am_true, args); hp += 2; + args = CONS(hp, (max_heap_flags & MAX_HEAP_SIZE_KILL ? am_true : am_false), args); hp += 2; + args = CONS(hp, make_small(total_heap_size), args); hp += 2; + args = CONS(hp, make_small(MAX_HEAP_SIZE_GET(p)), args); hp += 2; + if (alive) { + args = CONS(hp, erts_this_node->sysname, args); hp += 2; + } + args = CONS(hp, p->common.id, args); hp += 2; + + erts_send_error_term_to_logger(p->group_leader, dsbufp, args); + erts_free(ERTS_ALC_T_TMP, o_hp); + } + + if (IS_TRACED_FL(p, F_TRACE_GC)) + trace_gc(p, am_gc_max_heap_size, 0, msg); + + erts_free(ERTS_ALC_T_TMP, o_hp); + } + /* returns true if we should kill the process */ + return max_heap_flags & MAX_HEAP_SIZE_KILL; +} + +Eterm +erts_max_heap_size_map(Sint max_heap_size, Uint max_heap_flags, + Eterm **hpp, Uint *sz) +{ + if (!hpp) { + *sz += (2*3 + 1 + MAP_HEADER_FLATMAP_SZ); + return THE_NON_VALUE; + } else { + Eterm *hp = *hpp; + Eterm keys = TUPLE3(hp, am_error_logger, am_kill, am_size); + flatmap_t *mp; + hp += 4; + mp = (flatmap_t*) hp; + mp->thing_word = MAP_HEADER_FLATMAP; + mp->size = 3; + mp->keys = keys; + hp += MAP_HEADER_FLATMAP_SZ; + *hp++ = max_heap_flags & MAX_HEAP_SIZE_LOG ? am_true : am_false; + *hp++ = max_heap_flags & MAX_HEAP_SIZE_KILL ? am_true : am_false; + *hp++ = make_small(max_heap_size); + *hpp = hp; + return make_flatmap(mp); + } +} + +int +erts_max_heap_size(Eterm arg, Uint *max_heap_size, Uint *max_heap_flags) +{ + Sint sz; + *max_heap_flags = H_MAX_FLAGS; + if (is_small(arg)) { + sz = signed_val(arg); + *max_heap_flags = H_MAX_FLAGS; + } else if (is_map(arg)) { + const Eterm *size = erts_maps_get(am_size, arg); + const Eterm *kill = erts_maps_get(am_kill, arg); + const Eterm *log = erts_maps_get(am_error_logger, arg); + if (size && is_small(*size)) { + sz = signed_val(*size); + } else { + /* size is mandatory */ + return 0; + } + if (kill) { + if (*kill == am_true) + *max_heap_flags |= MAX_HEAP_SIZE_KILL; + else if (*kill == am_false) + *max_heap_flags &= ~MAX_HEAP_SIZE_KILL; + else + return 0; + } + if (log) { + if (*log == am_true) + *max_heap_flags |= MAX_HEAP_SIZE_LOG; + else if (*log == am_false) + *max_heap_flags &= ~MAX_HEAP_SIZE_LOG; + else + return 0; + } + } else + return 0; + if (sz < 0) + return 0; + *max_heap_size = sz; + return 1; +} + #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG) static int diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index 8a6ff99990..54ea9ca3c0 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -135,7 +135,7 @@ typedef struct { #define ERTS_PROCESS_GC_INFO_MAX_TERMS (11) /* number of elements in process_gc_info*/ #define ERTS_PROCESS_GC_INFO_MAX_SIZE \ (ERTS_PROCESS_GC_INFO_MAX_TERMS * (2/*cons*/ + 3/*2-tuple*/ + BIG_UINT_HEAP_SIZE)) -Eterm erts_process_gc_info(struct process*, Uint *, Eterm **); +Eterm erts_process_gc_info(struct process*, Uint *, Eterm **, Uint, Uint); void erts_gc_info(ErtsGCInfo *gcip); void erts_init_gc(void); @@ -155,5 +155,7 @@ void erts_offset_off_heap(struct erl_off_heap*, Sint, Eterm*, Eterm*); void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*); void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*); void erts_free_heap_frags(struct process* p); +Eterm erts_max_heap_size_map(Sint, Uint, Eterm **, Uint *); +int erts_max_heap_size(Eterm, Uint *, Uint *); #endif /* __ERL_GC_H__ */ diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index c418762578..ebeff51aac 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -1766,7 +1766,7 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, if (is_not_internal_pid(rcvr) && is_not_atom(rcvr)) goto badarg; - esdp = ERTS_PROC_GET_SCHDATA(c_p); + esdp = erts_proc_sched_data(c_p); hp = HAlloc(c_p, REF_THING_SIZE); ref = erts_sched_make_ref_in_buffer(esdp, hp); @@ -1871,7 +1871,7 @@ access_sched_local_btm(Process *c_p, Eterm pid, if (!c_p) esdp = erts_get_scheduler_data(); else { - esdp = ERTS_PROC_GET_SCHDATA(c_p); + esdp = erts_proc_sched_data(c_p); ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data()); } @@ -2138,7 +2138,7 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) goto no_timer; } - esdp = ERTS_PROC_GET_SCHDATA(c_p); + esdp = erts_proc_sched_data(c_p); trefn = internal_ref_numbers(tref); sid = erts_get_ref_numbers_thr_id(trefn); @@ -2363,7 +2363,7 @@ typedef struct { int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) { - ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(p); + ErtsSchedulerData *esdp = erts_proc_sched_data(p); ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}}; ErtsBifTimerYieldState *ysp; int res; @@ -2409,7 +2409,7 @@ detach_bif_timer(ErtsHLTimer *tmr, void *vesdp) int erts_detach_accessor_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) { - ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(p); + ErtsSchedulerData *esdp = erts_proc_sched_data(p); ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}}; ErtsBifTimerYieldState *ysp; int res; @@ -2516,7 +2516,7 @@ BIF_RETTYPE send_after_3(BIF_ALIST_3) ErtsMonotonicTime timeout_pos; int short_time, tres; - tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL, + tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, 0, &timeout_pos, &short_time); if (tres != 0) BIF_ERROR(BIF_P, BADARG); @@ -2534,7 +2534,7 @@ BIF_RETTYPE send_after_4(BIF_ALIST_4) if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor)) BIF_ERROR(BIF_P, BADARG); - tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL, + tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, abs, &timeout_pos, &short_time); if (tres != 0) BIF_ERROR(BIF_P, BADARG); @@ -2548,7 +2548,7 @@ BIF_RETTYPE start_timer_3(BIF_ALIST_3) ErtsMonotonicTime timeout_pos; int short_time, tres; - tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL, + tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, 0, &timeout_pos, &short_time); if (tres != 0) BIF_ERROR(BIF_P, BADARG); @@ -2566,7 +2566,7 @@ BIF_RETTYPE start_timer_4(BIF_ALIST_4) if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor)) BIF_ERROR(BIF_P, BADARG); - tres = parse_timeout_pos(ERTS_PROC_GET_SCHDATA(BIF_P), BIF_ARG_1, NULL, + tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, abs, &timeout_pos, &short_time); if (tres != 0) BIF_ERROR(BIF_P, BADARG); @@ -2720,7 +2720,7 @@ set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo, int erts_set_proc_timer_term(Process *c_p, Eterm etmo) { - ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); ErtsMonotonicTime tmo, timeout_pos; int short_time, tres; @@ -2742,7 +2742,7 @@ erts_set_proc_timer_term(Process *c_p, Eterm etmo) void erts_set_proc_timer_uword(Process *c_p, UWord tmo) { - ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); ERTS_HLT_ASSERT(erts_smp_atomic_read_nob(&c_p->common.timer) == ERTS_PTMR_NONE); @@ -2776,7 +2776,7 @@ erts_cancel_proc_timer(Process *c_p) erts_smp_atomic_set_nob(&c_p->common.timer, ERTS_PTMR_NONE); return; } - continue_cancel_ptimer(ERTS_PROC_GET_SCHDATA(c_p), + continue_cancel_ptimer(erts_proc_sched_data(c_p), (ErtsTimer *) tval); } diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index fcd2739ac3..0649fb68de 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -164,6 +164,8 @@ int erts_use_sender_punish; Uint display_items; /* no of items to display in traces etc */ int H_MIN_SIZE; /* The minimum heap grain */ int BIN_VH_MIN_SIZE; /* The minimum binary virtual*/ +int H_MAX_SIZE; /* The maximum heap size */ +int H_MAX_FLAGS; /* The maximum heap flags */ Uint32 erts_debug_flags; /* Debug flags. */ int erts_backtrace_depth; /* How many functions to show in a backtrace @@ -576,6 +578,10 @@ void erts_usage(void) H_DEFAULT_SIZE); erts_fprintf(stderr, "-hmbs size set minimum binary virtual heap size in words (default %d)\n", VH_DEFAULT_SIZE); + erts_fprintf(stderr, "-hmax size set maximum heap size in words (default %d)\n", + H_DEFAULT_MAX_SIZE); + erts_fprintf(stderr, "-hmaxk bool enable or disable kill at max heap size (default true)\n"); + erts_fprintf(stderr, "-hmaxel bool enable or disable error_logger report at max heap size (default true)\n"); erts_fprintf(stderr, "-hpds size initial process dictionary size (default %d)\n", erts_pd_initial_size); erts_fprintf(stderr, "-hmqd val set default message queue data flag for processes,\n"); @@ -759,6 +765,8 @@ early_init(int *argc, char **argv) /* erts_async_thread_suggested_stack_size = ERTS_ASYNC_THREAD_MIN_STACK_SIZE; H_MIN_SIZE = H_DEFAULT_SIZE; BIN_VH_MIN_SIZE = VH_DEFAULT_SIZE; + H_MAX_SIZE = H_DEFAULT_MAX_SIZE; + H_MAX_FLAGS = MAX_HEAP_SIZE_KILL|MAX_HEAP_SIZE_LOG; erts_initialized = 0; @@ -1484,10 +1492,13 @@ erl_start(int argc, char **argv) char *sub_param = argv[i]+2; /* set default heap size * - * h|ms - min_heap_size - * h|mbs - min_bin_vheap_size - * h|pds - erts_pd_initial_size - * h|mqd - message_queue_data + * h|ms - min_heap_size + * h|mbs - min_bin_vheap_size + * h|pds - erts_pd_initial_size + * h|mqd - message_queue_data + * h|max - max_heap_size + * h|maxk - max_heap_kill + * h|maxel - max_heap_error_logger * */ if (has_prefix("mbs", sub_param)) { @@ -1530,6 +1541,41 @@ erl_start(int argc, char **argv) "Invalid message_queue_data flag: %s\n", arg); erts_usage(); } + } else if (has_prefix("maxk", sub_param)) { + arg = get_arg(sub_param+4, argv[i+1], &i); + if (strcmp(arg,"true") == 0) { + H_MAX_FLAGS |= MAX_HEAP_SIZE_KILL; + } else if (strcmp(arg,"false") == 0) { + H_MAX_FLAGS &= ~MAX_HEAP_SIZE_KILL; + } else { + erts_fprintf(stderr, "bad max heap kill %s\n", arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, ("using max heap kill %d\n", H_MAX_FLAGS)); + } else if (has_prefix("maxel", sub_param)) { + arg = get_arg(sub_param+5, argv[i+1], &i); + if (strcmp(arg,"true") == 0) { + H_MAX_FLAGS |= MAX_HEAP_SIZE_LOG; + } else if (strcmp(arg,"false") == 0) { + H_MAX_FLAGS &= ~MAX_HEAP_SIZE_LOG; + } else { + erts_fprintf(stderr, "bad max heap error logger %s\n", arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, ("using max heap log %d\n", H_MAX_FLAGS)); + } else if (has_prefix("max", sub_param)) { + arg = get_arg(sub_param+3, argv[i+1], &i); + if ((H_MAX_SIZE = atoi(arg)) < 0) { + erts_fprintf(stderr, "bad max heap size %s\n", arg); + erts_usage(); + } + if (H_MAX_SIZE < H_MIN_SIZE && H_MAX_SIZE) { + erts_fprintf(stderr, "max heap size (%s) is not allowed to be " + "smaller than min heap size (%d)\n", + arg, H_MIN_SIZE); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, ("using max heap size %d\n", H_MAX_SIZE)); } else { /* backward compatibility */ arg = get_arg(argv[i]+2, argv[i+1], &i); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 9bb6e40a11..579f6e427d 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -492,7 +492,6 @@ queue_messages(Process* receiver, erts_proc_notify_new_message(receiver, receiver_locks); #else erts_proc_notify_new_message(receiver, 0); - ERTS_HOLE_CHECK(receiver); #endif return res; } @@ -601,7 +600,9 @@ erts_try_alloc_message_on_heap(Process *pp, ASSERT(!(*psp & ERTS_PSFLG_OFF_HEAP_MSGQ)); - if ( + if ((*psp) & ERTS_PSFLGS_VOLATILE_HEAP) + goto in_message_fragment; + else if ( #if defined(ERTS_SMP) *plp & ERTS_PROC_LOCK_MAIN #else @@ -611,7 +612,7 @@ erts_try_alloc_message_on_heap(Process *pp, #ifdef ERTS_SMP try_on_heap: #endif - if ((*psp & (ERTS_PSFLG_EXITING|ERTS_PSFLG_PENDING_EXIT)) + if (((*psp) & ERTS_PSFLGS_VOLATILE_HEAP) || (pp->flags & F_DISABLE_GC) || HEAP_LIMIT(pp) - HEAP_TOP(pp) <= sz) { /* diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c index 0e625f213b..544bc8b983 100644 --- a/erts/emulator/beam/erl_msacc.c +++ b/erts/emulator/beam/erl_msacc.c @@ -338,7 +338,7 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads) { #ifdef ERTS_ENABLE_MSACC ErtsMsAcc *msacc = ERTS_MSACC_TSD_GET(); - ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); Eterm ref; ErtsMSAccReq *msaccrp; Eterm *hp; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index a695a028ba..8a3007d52a 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -20,6 +20,23 @@ /* Erlang Native InterFace */ +/* + * Environment contains a pointer to currently executing process. + * In the dirty case this pointer do however not point to the + * actual process structure of the executing process, but instead + * a "shadow process structure". This in order to be able to handle + * heap allocation without the need to acquire the main lock on + * the process. + * + * The dirty process is allowed to allocate on the heap without + * the main lock, i.e., incrementing htop, but is not allowed to + * modify mbuf, offheap, etc without the main lock. The dirty + * process moves mbuf list and offheap list of the shadow process + * structure into the real structure when the dirty nif call + * completes. + */ + + #ifdef HAVE_CONFIG_H # include "config.h" #endif @@ -81,6 +98,43 @@ void dtrace_nifenv_str(ErlNifEnv *, char *); #define MIN_HEAP_FRAG_SZ 200 static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp); +static ERTS_INLINE int +is_scheduler(void) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (!esdp) + return 0; + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + return -1; + return 1; +} + +static ERTS_INLINE void +execution_state(ErlNifEnv *env, Process **c_pp, int *schedp) +{ + if (schedp) + *schedp = is_scheduler(); + if (c_pp) { + if (!env || env->proc->common.id == ERTS_INVALID_PID) + *c_pp = NULL; + else { + Process *c_p = env->proc; + + if (!(c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC)) + ASSERT(is_scheduler() > 0); + else { + c_p = env->proc->next; + ASSERT(is_scheduler() < 0); + ASSERT(c_p && env->proc->common.id == c_p->common.id); + } + + *c_pp = c_p; + + ASSERT(!(c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC)); + } + } +} + static ERTS_INLINE Eterm* alloc_heap(ErlNifEnv* env, unsigned need) { Eterm* hp = env->hp; @@ -124,6 +178,9 @@ static ERTS_INLINE void ensure_heap(ErlNifEnv* env, unsigned may_need) void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, Process* tracee) { +#ifdef ERTS_DIRTY_SCHEDULERS + ErtsSchedulerData *esdp; +#endif env->mod_nif = mod_nif; env->proc = p; env->hp = HEAP_TOP(p); @@ -133,6 +190,61 @@ void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif, env->tmp_obj_list = NULL; env->exception_thrown = 0; env->tracee = tracee; + + ASSERT(p->common.id != ERTS_INVALID_PID); + +#ifdef ERTS_DIRTY_SCHEDULERS + esdp = erts_get_scheduler_data(); + ASSERT(esdp); + + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { +#ifdef DEBUG + erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + + ASSERT(p->scheduler_data == esdp); + ASSERT((state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS)) + && !(state & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS))); +#endif + + } + else { + Process *sproc; +#ifdef DEBUG + erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state); + + ASSERT(!p->scheduler_data); + ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING) + && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))); +#endif + + sproc = esdp->dirty_shadow_process; + ASSERT(sproc); + ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC); + ASSERT(erts_smp_atomic32_read_nob(&sproc->state) + == (ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_PROXY)); + + sproc->next = p; + sproc->common.id = p->common.id; + sproc->htop = p->htop; + sproc->stop = p->stop; + sproc->hend = p->hend; + sproc->heap = p->heap; + sproc->abandoned_heap = p->abandoned_heap; + sproc->heap_sz = p->heap_sz; + sproc->high_water = p->high_water; + sproc->old_hend = p->old_hend; + sproc->old_htop = p->old_htop; + sproc->old_heap = p->old_heap; + sproc->mbuf = NULL; + sproc->mbuf_sz = 0; + ERTS_INIT_OFF_HEAP(&sproc->off_heap); + env->proc = sproc; + } +#endif } /* Temporary object header, auto-deallocated when NIF returns @@ -157,18 +269,75 @@ static ERTS_INLINE void free_tmp_objs(ErlNifEnv* env) void erts_post_nif(ErlNifEnv* env) { erts_unblock_fpe(env->fpe_was_unmasked); - if (env->heap_frag == NULL) { - ASSERT(env->hp_end == HEAP_LIMIT(env->proc)); - ASSERT(env->hp >= HEAP_TOP(env->proc)); - ASSERT(env->hp <= HEAP_LIMIT(env->proc)); - HEAP_TOP(env->proc) = env->hp; + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!(env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)) +#endif + { + ASSERT(is_scheduler() > 0); + if (env->heap_frag == NULL) { + ASSERT(env->hp_end == HEAP_LIMIT(env->proc)); + ASSERT(env->hp >= HEAP_TOP(env->proc)); + ASSERT(env->hp <= HEAP_LIMIT(env->proc)); + HEAP_TOP(env->proc) = env->hp; + } + else { + ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); + ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); + env->heap_frag->used_size = env->hp - env->heap_frag->mem; + ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); + } + env->exiting = ERTS_PROC_IS_EXITING(env->proc); } - else { - ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); - env->heap_frag->used_size = env->hp - env->heap_frag->mem; - ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); +#ifdef ERTS_DIRTY_SCHEDULERS + else { /* Dirty nif call using shadow process struct */ + Process *c_p = env->proc->next; + + ASSERT(is_scheduler() < 0); + ASSERT(env->proc->common.id == c_p->common.id); + + if (!env->heap_frag) { + ASSERT(env->hp_end == HEAP_LIMIT(c_p)); + ASSERT(env->hp >= HEAP_TOP(c_p)); + ASSERT(env->hp <= HEAP_LIMIT(c_p)); + HEAP_TOP(c_p) = env->hp; + } + else { + ASSERT(env->hp_end != HEAP_LIMIT(c_p)); + ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); + + HEAP_TOP(c_p) = HEAP_TOP(env->proc); + env->heap_frag->used_size = env->hp - env->heap_frag->mem; + + ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); + + if (c_p->mbuf) { + ErlHeapFragment *bp; + for (bp = env->proc->mbuf; bp->next; bp = bp->next) + ; + bp->next = c_p->mbuf; + } + + c_p->mbuf = env->proc->mbuf; + c_p->mbuf_sz += env->proc->mbuf_sz; + + } + + if (!c_p->off_heap.first) + c_p->off_heap.first = env->proc->off_heap.first; + else if (env->proc->off_heap.first) { + struct erl_off_heap_header *ohhp; + for (ohhp = env->proc->off_heap.first; ohhp->next; ohhp = ohhp->next) + ; + ohhp->next = c_p->off_heap.first; + c_p->off_heap.first = env->proc->off_heap.first; + } + c_p->off_heap.overhead += env->proc->off_heap.overhead; + + env->exiting = ERTS_PROC_IS_EXITING(c_p); + BUMP_ALL_REDS(c_p); } +#endif free_tmp_objs(env); } @@ -400,9 +569,8 @@ error: #endif -int -enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, - ErlNifEnv* msg_env, ERL_NIF_TERM msg) +int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, + ErlNifEnv* msg_env, ERL_NIF_TERM msg) { struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env; ErtsProcLocks rp_locks = 0; @@ -413,34 +581,32 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, Process* c_p; ErtsMessage *mp; Eterm receiver = to_pid->pid; - int flush_me = 0; - ErtsSchedulerData *esdp = erts_get_scheduler_data(); - int scheduler = esdp ? esdp->no : 0; + int scheduler; - if (env != NULL) { - c_p = env->proc; - if (receiver == c_p->common.id) { + execution_state(env, &c_p, &scheduler); + +#ifndef ERTS_SMP + if (!scheduler) { + erts_exit(ERTS_ABORT_EXIT, + "enif_send: called from non-scheduler thread on non-SMP VM"); + return 0; + } +#endif + + if (scheduler > 0) { /* Normal scheduler */ + rp = erts_proc_lookup(receiver); + if (c_p == rp) rp_locks = ERTS_PROC_LOCK_MAIN; - flush_me = 1; - } } else { -#ifdef ERTS_SMP - c_p = NULL; -#else - erts_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM"); -#endif + if (c_p && ERTS_PROC_IS_EXITING(c_p)) + return 0; + rp = erts_pid2proc_opt(c_p, 0, receiver, rp_locks, + ERTS_P2P_FLG_INC_REFC); } - - rp = (scheduler - ? erts_proc_lookup(receiver) - : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, - receiver, rp_locks, ERTS_P2P_FLG_INC_REFC)); - - if (rp == NULL) { - ASSERT(env == NULL || receiver != c_p->common.id); + if (rp == NULL) return 0; - } + if (menv) { flush_env(msg_env); mp = erts_alloc_message(0, NULL); @@ -465,10 +631,6 @@ enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ERL_MESSAGE_TERM(mp) = msg; - if (flush_me) { - flush_env(env); /* Needed for ERTS_HOLE_CHECK */ - } - if (!env || !env->tracee) { if (c_p && IS_TRACED_FL(c_p, F_TRACE_SEND)) @@ -546,11 +708,9 @@ done: if (rp_locks & ~lc_locks) erts_smp_proc_unlock(rp, rp_locks & ~lc_locks); #endif - if (!scheduler) + if (scheduler <= 0) erts_proc_dec_refc(rp); - if (flush_me) { - cache_env(env); - } + return 1; } @@ -558,26 +718,52 @@ int enif_port_command(ErlNifEnv *env, const ErlNifPort* to_port, ErlNifEnv *msg_env, ERL_NIF_TERM msg) { - - ErtsSchedulerData *esdp = erts_get_scheduler_data(); - int scheduler = esdp ? esdp->no : 0; + int iflags = (erts_port_synchronous_ops + ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + : ERTS_PORT_SFLGS_INVALID_LOOKUP); + int scheduler; + Process *c_p; Port *prt; + int res; - if (scheduler == 0 || !env) - return 0; + if (!env) + erts_exit(ERTS_ABORT_EXIT, "enif_port_command: env == NULL"); + + execution_state(env, &c_p, &scheduler); + + if (!c_p) + c_p = env->proc; - prt = erts_port_lookup(to_port->port_id, - (erts_port_synchronous_ops - ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP - : ERTS_PORT_SFLGS_INVALID_LOOKUP)); + if (scheduler > 0) + prt = erts_port_lookup(to_port->port_id, iflags); +#ifdef ERTS_DIRTY_SCHEDULERS + else if (scheduler < 0) { + if (ERTS_PROC_IS_EXITING(c_p)) + return 0; + prt = erts_thr_port_lookup(to_port->port_id, iflags); + } +#endif + else { + erts_exit(ERTS_ABORT_EXIT, "enif_port_command: " + "called from non-scheduler thread"); + } if (!prt) - return 0; + res = 0; + else { + + if (IS_TRACED_FL(prt, F_TRACE_RECEIVE)) + trace_port_receive(prt, c_p->common.id, am_command, msg); - if (IS_TRACED_FL(prt, F_TRACE_RECEIVE)) - trace_port_receive(prt, env->proc->common.id, am_command, msg); + res = erts_port_output_async(prt, c_p->common.id, msg); + } + +#ifdef ERTS_DIRTY_SCHEDULERS + if (scheduler < 0) + erts_port_dec_refc(prt); +#endif - return erts_port_output_async(prt, env->proc->common.id, msg); + return res; } ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) @@ -1039,15 +1225,21 @@ Eterm enif_make_badarg(ErlNifEnv* env) Eterm enif_raise_exception(ErlNifEnv* env, ERL_NIF_TERM reason) { + Process *c_p; + + execution_state(env, &c_p, NULL); + env->exception_thrown = 1; - env->proc->fvalue = reason; - BIF_ERROR(env->proc, EXC_ERROR); + c_p->fvalue = reason; + BIF_ERROR(c_p, EXC_ERROR); } int enif_has_pending_exception(ErlNifEnv* env, ERL_NIF_TERM* reason) { if (env->exception_thrown && reason != NULL) { - *reason = env->proc->fvalue; + Process *c_p; + execution_state(env, &c_p, NULL); + *reason = c_p->fvalue; } return env->exception_thrown; } @@ -1441,56 +1633,71 @@ int enif_make_reverse_list(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM *list return 1; } +int enif_is_current_process_alive(ErlNifEnv* env) +{ + Process *c_p; + int scheduler; + + execution_state(env, &c_p, &scheduler); + + if (!c_p) + erts_exit(ERTS_ABORT_EXIT, + "enif_is_current_process_alive: " + "Invalid environment"); + + if (!scheduler) + erts_exit(ERTS_ABORT_EXIT, "enif_is_current_process_alive: " + "called from non-scheduler thread"); + + return !ERTS_PROC_IS_EXITING(c_p); +} + int enif_is_process_alive(ErlNifEnv* env, ErlNifPid *proc) { - ErtsProcLocks rp_locks = 0; /* We don't need any locks, - just to check if it is alive */ - Eterm target = proc->pid; - Process* rp; - Process* c_p; - int scheduler = erts_get_scheduler_id() != 0; + int scheduler; - if (env != NULL) { - c_p = env->proc; - if (target == c_p->common.id) { - /* We are alive! */ - return 1; - } - } + execution_state(env, NULL, &scheduler); + + if (scheduler > 0) + return !!erts_proc_lookup(proc->pid); else { #ifdef ERTS_SMP - c_p = NULL; + Process* rp = erts_pid2proc_opt(NULL, 0, proc->pid, 0, + ERTS_P2P_FLG_INC_REFC); + if (rp) + erts_proc_dec_refc(rp); + return !!rp; #else - erts_exit(ERTS_ABORT_EXIT,"enif_is_process_alive: " - "env==NULL on non-SMP VM"); -#endif - } - - rp = (scheduler - ? erts_proc_lookup(target) - : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, - target, rp_locks, ERTS_P2P_FLG_INC_REFC)); - if (rp == NULL) { - ASSERT(env == NULL || target != c_p->common.id); + erts_exit(ERTS_ABORT_EXIT, "enif_is_process_alive: " + "called from non-scheduler thread"); return 0; - } else { - if (!scheduler) - erts_proc_dec_refc(rp); - return 1; +#endif } } int enif_is_port_alive(ErlNifEnv *env, ErlNifPort *port) { - /* only allowed if called from scheduler */ - if (erts_get_scheduler_id() == 0) - erts_exit(ERTS_ABORT_EXIT,"enif_is_port_alive: called from non-scheduler"); + int scheduler; + Uint32 iflags = (erts_port_synchronous_ops + ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + : ERTS_PORT_SFLGS_INVALID_LOOKUP); - return erts_port_lookup( - port->port_id, - (erts_port_synchronous_ops - ? ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP - : ERTS_PORT_SFLGS_INVALID_LOOKUP)) != NULL; + execution_state(env, NULL, &scheduler); + + if (scheduler > 0) + return !!erts_port_lookup(port->port_id, iflags); + else { +#ifdef ERTS_SMP + Port *prt = erts_thr_port_lookup(port->port_id, iflags); + if (prt) + erts_port_dec_refc(prt); + return !!prt; +#else + erts_exit(ERTS_ABORT_EXIT, "enif_is_port_alive: " + "called from non-scheduler thread"); + return 0; +#endif + } } ERL_NIF_TERM @@ -1957,16 +2164,19 @@ void* enif_dlsym(void* handle, const char* symbol, int enif_consume_timeslice(ErlNifEnv* env, int percent) { + Process *proc; Sint reds; + execution_state(env, &proc, NULL); + ASSERT(is_proc_bound(env) && percent >= 1 && percent <= 100); if (percent < 1) percent = 1; else if (percent > 100) percent = 100; reds = ((CONTEXT_REDS+99) / 100) * percent; ASSERT(reds > 0 && reds <= CONTEXT_REDS); - BUMP_REDS(env->proc, reds); - return ERTS_BIF_REDS_LEFT(env->proc) == 0; + BUMP_REDS(proc, reds); + return ERTS_BIF_REDS_LEFT(proc) == 0; } /* @@ -2061,10 +2271,19 @@ static ERL_NIF_TERM init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp, int need_save, int argc, const ERL_NIF_TERM argv[]) { - Process* proc = env->proc; - Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array; + Process* proc; + Eterm* reg; NifExport* ep; - int i; + int i, scheduler; + + execution_state(env, &proc, &scheduler); + + ASSERT(scheduler); + + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc) + & ERTS_PROC_LOCK_MAIN); + + reg = erts_proc_sched_data(proc)->x_reg_array; ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); if (!ep) @@ -2076,12 +2295,13 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec } if (env->exception_thrown) { ep->exception_thrown = 1; - ep->rootset[0] = env->proc->fvalue; + ep->rootset[0] = proc->fvalue; } else { ep->exception_thrown = 0; ep->rootset[0] = NIL; } - ERTS_VBUMP_ALL_REDS(proc); + if (scheduler > 0) + ERTS_VBUMP_ALL_REDS(proc); for (i = 0; i < argc; i++) { if (need_save) ep->rootset[i+1] = reg[i]; @@ -2113,7 +2333,12 @@ static void restore_nif_mfa(Process* proc, NifExport* ep, int exception) { int i; - Eterm* reg = ERTS_PROC_GET_SCHDATA(proc)->x_reg_array; + Eterm* reg = erts_proc_sched_data(proc)->x_reg_array; + + ERTS_SMP_LC_ASSERT(!(proc->static_flags + & ERTS_STC_FLG_SHADOW_PROC)); + ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc) + & ERTS_PROC_LOCK_MAIN); proc->current[0] = ep->saved_mfa[0]; proc->current[1] = ep->saved_mfa[1]; @@ -2138,11 +2363,13 @@ restore_nif_mfa(Process* proc, NifExport* ep, int exception) static ERL_NIF_TERM dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - Process* proc = env->proc; + Process* proc; NifExport* ep; + execution_state(env, &proc, NULL); + ASSERT(argc == 1); - ASSERT(!ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data)); + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); ASSERT(!ep->exception_thrown); @@ -2157,10 +2384,12 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) static ERL_NIF_TERM dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - Process* proc = env->proc; + Process* proc; NifExport* ep; - ASSERT(!ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data)); + execution_state(env, &proc, NULL); + + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); ASSERT(ep->exception_thrown); @@ -2177,23 +2406,32 @@ dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) static ERL_NIF_TERM execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - Process* proc = env->proc; - NativeFunPtr fp = (NativeFunPtr) proc->current[6]; + Process* proc; + NativeFunPtr fp; NifExport* ep; ERL_NIF_TERM result; - ASSERT(ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data)); + execution_state(env, &proc, NULL); + + fp = (NativeFunPtr) proc->current[6]; + + ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); /* * Set ep->fp to NULL before the native call so we know later whether it scheduled another NIF for execution */ ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); - ASSERT(ep); + ASSERT(ep && fp); ep->fp = NULL; erts_smp_atomic32_read_band_mb(&proc->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC | ERTS_PSFLG_DIRTY_IO_PROC)); + + erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); + result = (*fp)(env, argc, argv); + erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); + if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0 && env->mod_nif->mod == NULL) close_lib(env->mod_nif); /* @@ -2230,29 +2468,49 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) static ERTS_INLINE ERL_NIF_TERM schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[]) { - erts_aint32_t state, n, a; - Process* proc = env->proc; - NativeFunPtr fp = (NativeFunPtr) proc->current[6]; + ERL_NIF_TERM result; + erts_aint32_t act, dirty_flag; + Process* proc; + NativeFunPtr fp; NifExport* ep; - int need_save; + int need_save, scheduler; + + execution_state(env, &proc, &scheduler); + if (scheduler <= 0) { + ASSERT(scheduler < 0); + erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); + } + + fp = (NativeFunPtr) proc->current[6]; + + ASSERT(fp); ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND); - a = erts_smp_atomic32_read_acqb(&proc->state); - while (1) { - n = state = a; + if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND) + dirty_flag = ERTS_PSFLG_DIRTY_CPU_PROC; + else + dirty_flag = ERTS_PSFLG_DIRTY_IO_PROC; + + act = erts_smp_atomic32_read_bor_nob(&proc->state, dirty_flag); + if (!(act & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC))) + erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1); + else if ((act & (ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_IO_PROC)) & ~dirty_flag) { + /* clear other flag... */ if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND) - n |= ERTS_PSFLG_DIRTY_CPU_PROC; + dirty_flag = ERTS_PSFLG_DIRTY_IO_PROC; else - n |= ERTS_PSFLG_DIRTY_IO_PROC; - a = erts_smp_atomic32_cmpxchg_mb(&proc->state, n, state); - if (a == state) - break; + dirty_flag = ERTS_PSFLG_DIRTY_CPU_PROC; + erts_smp_atomic32_read_band_nob(&proc->state, ~dirty_flag); } - erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1); + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); need_save = (ep == NULL || is_non_value(ep->saved_mfa[0])); - return init_nif_sched_data(env, execute_dirty_nif, fp, need_save, argc, argv); + result = init_nif_sched_data(env, execute_dirty_nif, fp, need_save, argc, argv); + if (scheduler <= 0) + erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); + return result; } static ERL_NIF_TERM @@ -2277,11 +2535,14 @@ schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) static ERL_NIF_TERM execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - Process* proc = env->proc; - NativeFunPtr fp = (NativeFunPtr) proc->current[6]; + Process* proc; + NativeFunPtr fp; NifExport* ep; ERL_NIF_TERM result; + execution_state(env, &proc, NULL); + fp = (NativeFunPtr) proc->current[6]; + ASSERT(!env->exception_thrown); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); @@ -2304,10 +2565,10 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, ERL_NIF_TERM (*fp)(ErlNifEnv*, int, const ERL_NIF_TERM[]), int argc, const ERL_NIF_TERM argv[]) { - Process* proc = env->proc; + Process* proc; NifExport* ep; ERL_NIF_TERM fun_name_atom, result; - int need_save; + int need_save, scheduler; if (argc > MAX_ARG) return enif_make_badarg(env); @@ -2315,6 +2576,13 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, if (enif_is_exception(env, fun_name_atom)) return fun_name_atom; + execution_state(env, &proc, &scheduler); + if (scheduler <= 0) { + if (scheduler == 0) + enif_make_badarg(env); + erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); + } + ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); need_save = (ep == NULL || is_non_value(ep->saved_mfa[0])); @@ -2326,12 +2594,15 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, sched_fun = schedule_dirty_io_nif; else if (chkflgs == ERL_NIF_DIRTY_JOB_CPU_BOUND) sched_fun = schedule_dirty_cpu_nif; - else - return enif_make_badarg(env); + else { + result = enif_make_badarg(env); + goto done; + } result = init_nif_sched_data(env, sched_fun, fp, need_save, argc, argv); #else - return enif_make_badarg(env); + result = enif_make_badarg(env); #endif + goto done; } else result = init_nif_sched_data(env, execute_nif, fp, need_save, argc, argv); @@ -2339,18 +2610,28 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); ep->exp.code[1] = (BeamInstr) fun_name_atom; + +done: + if (scheduler < 0) + erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN); + return result; } -#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT - int enif_is_on_dirty_scheduler(ErlNifEnv* env) { - return ERTS_SCHEDULER_IS_DIRTY(env->proc->scheduler_data); -} + int scheduler; + Process *c_p; + + execution_state(env, &c_p, &scheduler); -#endif /* ERL_NIF_DIRTY_SCHEDULER_SUPPORT */ + if (!c_p || !scheduler) + erts_exit(ERTS_ABORT_EXIT, "enif_is_on_dirty_scheduler: " + "Invalid env"); + + return scheduler < 0; +} /* Maps */ @@ -3051,16 +3332,16 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) (BeamInstr) BeamOp(op_i_generic_breakpoint)); g->orig_instr = (BeamInstr) BeamOp(op_call_nif); } +#ifdef ERTS_DIRTY_SCHEDULERS if ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) && (entry->options & ERL_NIF_DIRTY_NIF_OPTION) && f->flags) { -#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT code_ptr[5+3] = (BeamInstr) f->fptr; code_ptr[5+1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? (BeamInstr) schedule_dirty_io_nif : (BeamInstr) schedule_dirty_cpu_nif; -#endif } else +#endif code_ptr[5+1] = (BeamInstr) f->fptr; code_ptr[5+2] = (BeamInstr) lib; f = next_func(entry, &incr, f); diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index 3964f7f679..02c82415fd 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -28,7 +28,6 @@ # include "config.h" #endif -#include "erl_native_features_config.h" #include "erl_drv_nif.h" /* Version history: @@ -167,13 +166,11 @@ typedef int ErlNifTSDKey; typedef ErlDrvThreadOpts ErlNifThreadOpts; -#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT typedef enum { - ERL_NIF_DIRTY_JOB_CPU_BOUND = ERL_DRV_DIRTY_JOB_CPU_BOUND, - ERL_NIF_DIRTY_JOB_IO_BOUND = ERL_DRV_DIRTY_JOB_IO_BOUND + ERL_NIF_DIRTY_JOB_CPU_BOUND = ERL_DIRTY_JOB_CPU_BOUND, + ERL_NIF_DIRTY_JOB_IO_BOUND = ERL_DIRTY_JOB_IO_BOUND }ErlNifDirtyTaskFlags; -#endif typedef struct /* All fields all internal and may change */ { @@ -257,11 +254,7 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; # define ERL_NIF_INIT_DECL(MODNAME) ERL_NIF_INIT_EXPORT ErlNifEntry* nif_init(ERL_NIF_INIT_ARGS) #endif -#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT -# define ERL_NIF_ENTRY_OPTIONS ERL_NIF_DIRTY_NIF_OPTION -#else -# define ERL_NIF_ENTRY_OPTIONS 0 -#endif +#define ERL_NIF_ENTRY_OPTIONS ERL_NIF_DIRTY_NIF_OPTION #ifdef __cplusplus } diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index a5acd86551..1bdac51d1f 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -166,26 +166,18 @@ ERL_NIF_API_FUNC_DECL(ErlNifTime, enif_convert_time_unit, (ErlNifTime, ErlNifTim ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_now_time, (ErlNifEnv *env)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_cpu_time, (ErlNifEnv *env)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_unique_integer, (ErlNifEnv *env, ErlNifUniqueInteger properties)); +ERL_NIF_API_FUNC_DECL(int, enif_is_current_process_alive, (ErlNifEnv *env)); ERL_NIF_API_FUNC_DECL(int, enif_is_process_alive, (ErlNifEnv *env, ErlNifPid *pid)); ERL_NIF_API_FUNC_DECL(int, enif_is_port_alive, (ErlNifEnv *env, ErlNifPort *port_id)); ERL_NIF_API_FUNC_DECL(int, enif_get_local_port, (ErlNifEnv* env, ERL_NIF_TERM, ErlNifPort* port_id)); ERL_NIF_API_FUNC_DECL(int, enif_term_to_binary, (ErlNifEnv *env, ERL_NIF_TERM term, ErlNifBinary *bin)); ERL_NIF_API_FUNC_DECL(size_t, enif_binary_to_term, (ErlNifEnv *env, const unsigned char* data, size_t sz, ERL_NIF_TERM *term, unsigned int opts)); ERL_NIF_API_FUNC_DECL(int, enif_port_command, (ErlNifEnv *env, const ErlNifPort* to_port, ErlNifEnv *msg_env, ERL_NIF_TERM msg)); +ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*)); /* ** ADD NEW ENTRIES HERE (before this comment) !!! */ - - -/* - * Conditional EXPERIMENTAL stuff always last. - * Must be moved up and made unconditional to support binary backward - * compatibility on Windows. - */ -#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT -ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*)); -#endif #endif /* ERL_NIF_API_FUNC_DECL */ /* @@ -330,12 +322,14 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*)); # define enif_now_time ERL_NIF_API_FUNC_MACRO(enif_now_time) # define enif_cpu_time ERL_NIF_API_FUNC_MACRO(enif_cpu_time) # define enif_make_unique_integer ERL_NIF_API_FUNC_MACRO(enif_make_unique_integer) +# define enif_is_current_process_alive ERL_NIF_API_FUNC_MACRO(enif_is_current_process_alive) # define enif_is_process_alive ERL_NIF_API_FUNC_MACRO(enif_is_process_alive) # define enif_is_port_alive ERL_NIF_API_FUNC_MACRO(enif_is_port_alive) # define enif_get_local_port ERL_NIF_API_FUNC_MACRO(enif_get_local_port) # define enif_term_to_binary ERL_NIF_API_FUNC_MACRO(enif_term_to_binary) # define enif_binary_to_term ERL_NIF_API_FUNC_MACRO(enif_binary_to_term) # define enif_port_command ERL_NIF_API_FUNC_MACRO(enif_port_command) +# define enif_is_on_dirty_scheduler ERL_NIF_API_FUNC_MACRO(enif_is_on_dirty_scheduler) /* ** ADD NEW ENTRIES HERE (before this comment) @@ -346,9 +340,6 @@ ERL_NIF_API_FUNC_DECL(int,enif_is_on_dirty_scheduler,(ErlNifEnv*)); * Must be moved up and made unconditional to support binary backward * compatibility on Windows. */ -#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT -# define enif_is_on_dirty_scheduler ERL_NIF_API_FUNC_MACRO(enif_is_on_dirty_scheduler) -#endif #endif /* ERL_NIF_API_FUNC_MACRO */ diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index c2588e718d..f0075ca2b9 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -487,6 +487,7 @@ ERTS_GLB_INLINE Port*erts_id2port(Eterm id); ERTS_GLB_INLINE Port *erts_id2port_sflgs(Eterm, Process *, ErtsProcLocks, Uint32); ERTS_GLB_INLINE void erts_port_release(Port *); #ifdef ERTS_SMP +ERTS_GLB_INLINE Port *erts_thr_port_lookup(Eterm id, Uint32 invalid_sflgs); ERTS_GLB_INLINE Port *erts_thr_id2port_sflgs(Eterm id, Uint32 invalid_sflgs); ERTS_GLB_INLINE void erts_thr_port_release(Port *prt); #endif @@ -626,6 +627,44 @@ erts_port_release(Port *prt) } #ifdef ERTS_SMP +/* + * erts_thr_id2port_sflgs() and erts_port_dec_refc(prt) can + * be used by unmanaged threads in the SMP case. + */ +ERTS_GLB_INLINE Port * +erts_thr_port_lookup(Eterm id, Uint32 invalid_sflgs) +{ + Port *prt; + ErtsThrPrgrDelayHandle dhndl; + + if (is_not_internal_port(id)) + return NULL; + + dhndl = erts_thr_progress_unmanaged_delay(); + + prt = (Port *) erts_ptab_pix2intptr_ddrb(&erts_port, + internal_port_index(id)); + + if (!prt || prt->common.id != id) { + erts_thr_progress_unmanaged_continue(dhndl); + return NULL; + } + else { + erts_aint32_t state; + erts_port_inc_refc(prt); + + if (dhndl != ERTS_THR_PRGR_DHANDLE_MANAGED) + erts_thr_progress_unmanaged_continue(dhndl); + + state = erts_atomic32_read_acqb(&prt->state); + if (state & invalid_sflgs) { + erts_port_dec_refc(prt); + return NULL; + } + + return prt; + } +} /* * erts_thr_id2port_sflgs() and erts_thr_port_release() can diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 36b0af0c1a..a853ec585b 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -409,6 +409,10 @@ ErtsAlignedSchedulerData *erts_aligned_scheduler_data; #ifdef ERTS_DIRTY_SCHEDULERS ErtsAlignedSchedulerData *erts_aligned_dirty_cpu_scheduler_data; ErtsAlignedSchedulerData *erts_aligned_dirty_io_scheduler_data; +typedef union { + Process dsp; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(Process))]; +} ErtsAlignedDirtyShadowProcess; #endif typedef union { @@ -589,6 +593,7 @@ dbg_chk_aux_work_val(erts_aint32_t value) valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS; valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP; + valid |= ERTS_SSI_AUX_WORK_PENDING_EXITERS; #endif #if HAVE_ERTS_MSEG valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; @@ -611,7 +616,7 @@ dbg_chk_aux_work_val(erts_aint32_t value) #endif #ifdef ERTS_SMP -static void handle_pending_exiters(ErtsProcList *); +static void do_handle_pending_exiters(ErtsProcList *); static void wake_scheduler(ErtsRunQueue *rq); #endif @@ -679,6 +684,8 @@ erts_pre_init_process(void) = "MISC_THR_PRGR"; erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_IX] = "MISC"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_PENDING_EXITERS_IX] + = "PENDING_EXITERS"; erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_SET_TMO_IX] = "SET_TMO"; erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX] @@ -1156,7 +1163,7 @@ reply_sched_wall_time(void *vswtrp) Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable) { - ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); Eterm ref; ErtsSchedWallTimeReq *swtrp; Eterm *hp; @@ -1234,7 +1241,7 @@ reply_system_check(void *vscrp) Eterm erts_system_check_request(Process *c_p) { - ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); Eterm ref; ErtsSystemCheckReq *scrp; Eterm *hp; @@ -2336,6 +2343,30 @@ handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiti #endif +#ifdef ERTS_SMP + +static ERTS_INLINE erts_aint32_t +handle_pending_exiters(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) +{ + ErtsProcList *pnd_xtrs; + ErtsRunQueue *rq; + + rq = awdp->esdp->run_queue; + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_PENDING_EXITERS); + + erts_smp_runq_lock(rq); + pnd_xtrs = rq->procs.pending_exiters; + rq->procs.pending_exiters = NULL; + erts_smp_runq_unlock(rq); + + if (erts_proclist_fetch(&pnd_xtrs, NULL)) + do_handle_pending_exiters(pnd_xtrs); + + return aux_work & ~ERTS_SSI_AUX_WORK_PENDING_EXITERS; +} + +#endif + static ERTS_INLINE erts_aint32_t handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { @@ -2427,6 +2458,10 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC, handle_misc_aux_work); +#ifdef ERTS_SMP + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_PENDING_EXITERS, + handle_pending_exiters); +#endif HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO, handle_setup_aux_work_timer); @@ -3979,6 +4014,33 @@ schedule_bound_processes(ErtsRunQueue *rq, } } +#ifdef ERTS_DIRTY_SCHEDULERS + +static ERTS_INLINE void +clear_proc_dirty_queue_bit(Process *p, ErtsRunQueue *rq, int prio_bit) +{ +#ifdef DEBUG + erts_aint32_t old; +#endif + erts_aint32_t qb = prio_bit; + if (rq == ERTS_DIRTY_CPU_RUNQ) + qb <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET; + else { + ASSERT(rq == ERTS_DIRTY_IO_RUNQ); + qb <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET; + } +#ifdef DEBUG + old = (int) +#else + (void) +#endif + erts_smp_atomic32_read_band_mb(&p->dirty_state, ~qb); + ASSERT(old & qb); +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ + + static void evacuate_run_queue(ErtsRunQueue *rq, ErtsStuckBoundProcesses *sbpp) @@ -4141,29 +4203,8 @@ evacuate_run_queue(ErtsRunQueue *rq, } #ifdef ERTS_DIRTY_SCHEDULERS - - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { - erts_aint32_t dqbit = qbit; -#ifdef DEBUG - erts_aint32_t old_dqbit; -#endif - - if (rq == ERTS_DIRTY_CPU_RUNQ) - dqbit <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET; - else { - ASSERT(rq == ERTS_DIRTY_IO_RUNQ); - dqbit <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET; - } - -#ifdef DEBUG - old_dqbit = (int) -#else - (void) -#endif - erts_smp_atomic32_read_band_mb(&real_proc->dirty_state, - ~dqbit); - ASSERT(old_dqbit & dqbit); - } + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) + clear_proc_dirty_queue_bit(real_proc, rq, qbit); #endif if (ERTS_PSFLG_BOUND & real_state) { @@ -5653,7 +5694,8 @@ static void init_scheduler_data(ErtsSchedulerData* esdp, int num, ErtsSchedulerSleepInfo* ssi, ErtsRunQueue* runq, - char** daww_ptr, size_t daww_sz) + char** daww_ptr, size_t daww_sz, + Process *shadow_proc) { esdp->timer_wheel = NULL; #ifdef ERTS_SMP @@ -5677,6 +5719,15 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->no = (Uint) num; ERTS_DIRTY_SCHEDULER_NO(esdp) = 0; } + esdp->dirty_shadow_process = shadow_proc; + if (shadow_proc) { + erts_init_empty_process(shadow_proc); + erts_smp_atomic32_init_nob(&shadow_proc->state, + (ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_PROXY)); + shadow_proc->static_flags = ERTS_STC_FLG_SHADOW_PROC; + } #else esdp->no = (Uint) num; #endif @@ -5928,31 +5979,41 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online for (ix = 0; ix < n; ix++) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); init_scheduler_data(esdp, ix+1, ERTS_SCHED_SLEEP_INFO_IX(ix), - ERTS_RUNQ_IX(ix), &daww_ptr, daww_sz); + ERTS_RUNQ_IX(ix), &daww_ptr, daww_sz, + NULL); } #ifdef ERTS_DIRTY_SCHEDULERS -#ifdef ERTS_SMP - erts_aligned_dirty_cpu_scheduler_data = - erts_alloc_permanent_cache_aligned( - ERTS_ALC_T_SCHDLR_DATA, - no_dirty_cpu_schedulers*sizeof(ErtsAlignedSchedulerData)); - for (ix = 0; ix < no_dirty_cpu_schedulers; ix++) { - ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); - init_scheduler_data(esdp, ix+1, ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix), - ERTS_DIRTY_CPU_RUNQ, NULL, 0); - } - erts_aligned_dirty_io_scheduler_data = - erts_alloc_permanent_cache_aligned( - ERTS_ALC_T_SCHDLR_DATA, - no_dirty_io_schedulers*sizeof(ErtsAlignedSchedulerData)); - for (ix = 0; ix < no_dirty_io_schedulers; ix++) { - ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); - init_scheduler_data(esdp, ix+1, ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix), - ERTS_DIRTY_IO_RUNQ, NULL, 0); + { + int dirty_scheds = no_dirty_cpu_schedulers + no_dirty_io_schedulers; + int adspix = 0; + ErtsAlignedDirtyShadowProcess *adsp = + erts_alloc_permanent_cache_aligned( + ERTS_ALC_T_SCHDLR_DATA, + dirty_scheds * sizeof(ErtsAlignedDirtyShadowProcess)); + + erts_aligned_dirty_cpu_scheduler_data = + erts_alloc_permanent_cache_aligned( + ERTS_ALC_T_SCHDLR_DATA, + dirty_scheds * sizeof(ErtsAlignedSchedulerData)); + + erts_aligned_dirty_io_scheduler_data = + &erts_aligned_dirty_cpu_scheduler_data[no_dirty_cpu_schedulers]; + + for (ix = 0; ix < no_dirty_cpu_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); + init_scheduler_data(esdp, ix+1, ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix), + ERTS_DIRTY_CPU_RUNQ, NULL, 0, + &adsp[adspix++].dsp); + } + for (ix = 0; ix < no_dirty_io_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); + init_scheduler_data(esdp, ix+1, ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix), + ERTS_DIRTY_IO_RUNQ, NULL, 0, + &adsp[adspix++].dsp); + } } #endif -#endif init_misc_aux_work(); init_swtreq_alloc(); @@ -6167,7 +6228,7 @@ check_dirty_enqueue_in_prio_queue(Process *c_p, erts_aint32_t dact, max_qbit; /* Termination should be done on an ordinary scheduler */ - if (actual & ERTS_PSFLG_EXITING) { + if ((*newp) & ERTS_PSFLG_EXITING) { *newp &= ~ERTS_PSFLGS_DIRTY_WORK; return ERTS_ENQUEUE_NORMAL_QUEUE; } @@ -6176,7 +6237,7 @@ check_dirty_enqueue_in_prio_queue(Process *c_p, * If we have system tasks, we enqueue on ordinary run-queue * and take care of those system tasks first. */ - if (actual & ERTS_PSFLG_ACTIVE_SYS) + if ((*newp) & ERTS_PSFLG_ACTIVE_SYS) return ERTS_ENQUEUE_NORMAL_QUEUE; dact = erts_smp_atomic32_read_mb(&c_p->dirty_state); @@ -6356,23 +6417,29 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st * schedule_out_process() return with c_rq locked. */ static ERTS_INLINE int -schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Process *proxy) +schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, + Process *proxy, int is_normal_sched) { - erts_aint32_t a, e, n, enq_prio = -1; + erts_aint32_t a, e, n, enq_prio = -1, running_flgs; int enqueue; /* < 0 -> use proxy */ ErtsRunQueue* runq; + if (is_normal_sched) + running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS; + else + running_flgs = ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS; + a = state; while (1) { n = e = a; - ASSERT(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)); + ASSERT(a & running_flgs); enqueue = ERTS_ENQUEUE_NOT; - n &= ~(ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS); - if (a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS) + n &= ~running_flgs; + if ((a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS)) || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) { enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a); } @@ -6485,8 +6552,9 @@ change_proc_schedule_state(Process *p, ErtsProcLocks locks) { /* - * NOTE: ERTS_PSFLG_RUNNING, ERTS_PSFLG_RUNNING_SYS and - * ERTS_PSFLG_ACTIVE_SYS are not allowed to be + * NOTE: ERTS_PSFLG_RUNNING, ERTS_PSFLG_RUNNING_SYS, + * ERTS_PSFLG_DIRTY_RUNNING, ERTS_PSFLG_DIRTY_RUNNING_SYS + * and ERTS_PSFLG_ACTIVE_SYS are not allowed to be * altered by this function! */ erts_aint32_t a = *statep, n; @@ -6500,9 +6568,13 @@ change_proc_schedule_state(Process *p, ASSERT(!(a & ERTS_PSFLG_PROXY)); ASSERT((clear_state_flags & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS | ERTS_PSFLG_ACTIVE_SYS)) == 0); ASSERT((set_state_flags & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS | ERTS_PSFLG_ACTIVE_SYS)) == 0); if (lock_status) @@ -6526,8 +6598,16 @@ change_proc_schedule_state(Process *p, if ((n & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS | ERTS_PSFLG_IN_RUNQ - | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE) { + | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE +#ifdef ERTS_DIRTY_SCHEDULERS + || (n & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_EXITING)) == ERTS_PSFLG_EXITING +#endif + ) { /* * Active and seemingly need to be enqueued, but * process may be in a run queue via proxy, need @@ -6551,7 +6631,9 @@ change_proc_schedule_state(Process *p, | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE) && (!(a & (ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS) + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS) && (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))))) { /* We activated a prevously inactive process */ @@ -6693,7 +6775,10 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) enqueue = ERTS_ENQUEUE_NOT; n |= ERTS_PSFLG_ACTIVE_SYS; - if (!(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) + if (!(a & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS))) enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a); a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); if (a == e) @@ -6706,7 +6791,9 @@ schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) if (!(a & (ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS)) + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) && (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))) { /* We activated a prevously inactive process */ profile_runnable_proc(p, am_active); @@ -6746,11 +6833,16 @@ suspend_process(Process *c_p, Process *p) if (c_p == p) { state = erts_smp_atomic32_read_bor_relb(&p->state, ERTS_PSFLG_SUSPENDED); - ASSERT(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)); + ASSERT(state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)); suspended = (state & ERTS_PSFLG_SUSPENDED) ? -1: 1; } else { - while (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_EXITING))) { + while (!(state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_EXITING))) { erts_aint32_t n, e; n = e = state; @@ -6776,8 +6868,11 @@ suspend_process(Process *c_p, Process *p) if ((state & (ERTS_PSFLG_ACTIVE | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS | ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS | ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) { /* We made process inactive */ profile_runnable_proc(p, am_inactive); @@ -7759,8 +7854,10 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal plp = proclist_create(p); erts_proclist_store_last(&msbp->blckrs, plp); p->flags |= have_blckd_flg; - ASSERT(schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)); - ASSERT(p->scheduler_data->no == 1); + ASSERT(normal + ? 1 == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, ERTS_SCHED_NORMAL) + : schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)); + ASSERT(erts_proc_sched_data(p)->no == 1); if (schdlr_sspnd.msb.ongoing) res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; else @@ -7780,7 +7877,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal if (schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0) || (normal && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, ERTS_SCHED_NORMAL) == 1)) { - ASSERT(p->scheduler_data->no == 1); + ASSERT(erts_proc_sched_data(p)->no == 1); plp = proclist_create(p); erts_proclist_store_last(&msbp->blckrs, plp); if (schdlr_sspnd.msb.ongoing) @@ -7830,7 +7927,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal else res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED; } - ASSERT(p->scheduler_data); + ASSERT(erts_proc_sched_data(p)); } } else if (!msbp->ongoing) { @@ -8420,9 +8517,23 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, if (!suspend_process(c_p, rp)) { /* Other process running */ - ASSERT(ERTS_PSFLG_RUNNING + ASSERT((ERTS_PSFLG_RUNNING | ERTS_PSFLG_DIRTY_RUNNING) & erts_smp_atomic32_read_nob(&rp->state)); +#ifdef ERTS_DIRTY_SCHEDULERS + if (!suspend + && (erts_smp_atomic32_read_nob(&rp->state) + & ERTS_PSFLG_DIRTY_RUNNING)) { + ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS; + if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, + pid, pid_locks|ERTS_PROC_LOCK_STATUS); + } + goto done; + } +#endif + running: /* @@ -8447,7 +8558,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, else { ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS; if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { - if (ERTS_PSFLG_RUNNING_SYS + if ((ERTS_PSFLG_RUNNING_SYS|ERTS_PSFLG_DIRTY_RUNNING_SYS) & erts_smp_atomic32_read_nob(&rp->state)) { /* Executing system task... */ resume_process(rp, ERTS_PROC_LOCK_STATUS); @@ -8474,7 +8585,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, * from being selected for normal execution regardless * of locks held or not held on it... */ - ASSERT(!(ERTS_PSFLG_RUNNING + ASSERT(!((ERTS_PSFLG_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS) & erts_smp_atomic32_read_nob(&rp->state))); if (!suspend) @@ -9015,28 +9126,43 @@ erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched) } Eterm -erts_process_status(Process *c_p, ErtsProcLocks c_p_locks, - Process *rp, Eterm rpid) +erts_process_state2status(erts_aint32_t state) +{ + if (state & ERTS_PSFLG_FREE) + return am_free; + + if (state & ERTS_PSFLG_EXITING) + return am_exiting; + + if (state & ERTS_PSFLG_GC) + return am_garbage_collecting; + + if (state & ERTS_PSFLG_SUSPENDED) + return am_suspended; + + if (state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) + return am_running; + + if (state & (ERTS_PSFLG_ACTIVE + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) + return am_runnable; + + return am_waiting; +} + +Eterm +erts_process_status(Process *rp, Eterm rpid) { Eterm res = am_undefined; Process *p = rp ? rp : erts_proc_lookup_raw(rpid); if (p) { erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); - if (state & ERTS_PSFLG_FREE) - res = am_free; - else if (state & ERTS_PSFLG_EXITING) - res = am_exiting; - else if (state & ERTS_PSFLG_GC) - res = am_garbage_collecting; - else if (state & ERTS_PSFLG_SUSPENDED) - res = am_suspended; - else if (state & ERTS_PSFLG_RUNNING) - res = am_running; - else if (state & ERTS_PSFLG_ACTIVE) - res = am_runnable; - else - res = am_waiting; + res = erts_process_state2status(state); } #ifdef ERTS_SMP else { @@ -9251,7 +9377,76 @@ scheduler_gc_proc(Process *c_p, int reds_left) return reds; } +static ERTS_INLINE void +clean_dirty_start(Process *p) +{ +#if defined(ERTS_DIRTY_SCHEDULERS) && !defined(ARCH_64) + void *ptr = ERTS_PROC_SET_DIRTY_CPU_START(p, NULL); + if (ptr) + erts_free(ERTS_ALC_T_DIRTY_START, ptr); +#endif +} +static ERTS_INLINE void +save_dirty_start(ErtsSchedulerData *esdp, Process *c_p) +{ +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) { + ErtsMonotonicTime time = erts_get_monotonic_time(esdp); +#ifdef ARCH_64 + ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) time); +#else + ErtsMonotonicTime *stimep; + + stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p); + if (!stimep) { + stimep = erts_alloc(ERTS_ALC_T_DIRTY_START, + sizeof(ErtsMonotonicTime)); + ERTS_PROC_SET_DIRTY_CPU_START(c_p, (void *) stimep); + } + *stimep = time; +#endif + } +#endif +} + +static ERTS_INLINE int +get_dirty_reds(ErtsSchedulerData *esdp, Process *c_p) +{ + +#ifndef ERTS_DIRTY_SCHEDULERS + return -1; +#else + ErtsMonotonicTime stime, time; + + if (!ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) + return 1; + +#ifdef ARCH_64 + stime = (ErtsMonotonicTime) ERTS_PROC_GET_DIRTY_CPU_START(c_p); +#else + { + ErtsMonotonicTime *stimep; + stimep = (ErtsMonotonicTime *) ERTS_PROC_GET_DIRTY_CPU_START(c_p); + ASSERT(stimep); + stime = *stimep; + } +#endif + + time = erts_get_monotonic_time(esdp); + + ASSERT(stime && stime < time); + + time -= stime; + time = ERTS_MONOTONIC_TO_USEC(time); + time *= 2; + + if (time > INT_MAX) + return INT_MAX; + return (int) time; +#endif + +} /* * schedule() is called from BEAM (process_main()) or HiPE @@ -9283,6 +9478,7 @@ Process *schedule(Process *p, int calls) int reds; Uint32 flags; erts_aint32_t state = 0; /* Supress warning... */ + int is_normal_sched; ERTS_MSACC_DECLARE_CACHE(); @@ -9312,25 +9508,44 @@ Process *schedule(Process *p, int calls) */ if (!p) { /* NULL in the very first schedule() call */ esdp = erts_get_scheduler_data(); + is_normal_sched = !ERTS_SCHEDULER_IS_DIRTY(esdp); rq = erts_get_runq_current(esdp); ASSERT(esdp); fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls); actual_reds = reds = 0; erts_smp_runq_lock(rq); } else { - sched_out_proc: - #ifdef ERTS_SMP - ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); +#ifdef ERTS_DIRTY_SCHEDULERS + esdp = p->scheduler_data; + is_normal_sched = esdp != NULL; + if (is_normal_sched) + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); + else { + esdp = erts_get_scheduler_data(); + ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); + } +#else esdp = p->scheduler_data; + is_normal_sched = 1; +#endif ASSERT(esdp->current_process == p || esdp->free_process == p); #else esdp = erts_scheduler_data; ASSERT(esdp->current_process == p); + is_normal_sched = 1; #endif - reds = actual_reds = calls - esdp->virtual_reds; + sched_out_proc: + + ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); + + if (is_normal_sched) + reds = actual_reds = calls - esdp->virtual_reds; + else + reds = actual_reds = get_dirty_reds(esdp, p); + ASSERT(actual_reds >= 0); if (reds < ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST) reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST; @@ -9377,7 +9592,7 @@ Process *schedule(Process *p, int calls) state = erts_smp_atomic32_read_nob(&p->state); #ifdef ERTS_SMP - if (state & ERTS_PSFLG_PENDING_EXIT) + if (is_normal_sched && (state & ERTS_PSFLG_PENDING_EXIT)) erts_handle_pending_exit(p, (ERTS_PROC_LOCK_MAIN | ERTS_PROC_LOCK_STATUS)); if (p->pending_suspenders) @@ -9387,7 +9602,8 @@ Process *schedule(Process *p, int calls) esdp->reductions += reds; - schedule_out_process(rq, state, p, proxy_p); /* Returns with rq locked! */ + /* schedule_out_process() returns with rq locked! */ + schedule_out_process(rq, state, p, proxy_p, is_normal_sched); proxy_p = NULL; ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq, @@ -9405,13 +9621,20 @@ Process *schedule(Process *p, int calls) ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); if (state & ERTS_PSFLG_FREE) { + if (!is_normal_sched) { + ASSERT(p->flags & F_DELAYED_DEL_PROC); + erts_proc_dec_refc(p); + } + else { #ifdef ERTS_SMP - ASSERT(esdp->free_process == p); - esdp->free_process = NULL; + ASSERT(esdp->free_process == p); + esdp->free_process = NULL; #else - erts_proc_dec_refc(p); + erts_proc_dec_refc(p); #endif + } } + #ifdef ERTS_SMP ASSERT(!esdp->free_process); #endif @@ -9419,7 +9642,7 @@ Process *schedule(Process *p, int calls) ERTS_SMP_CHK_NO_PROC_LOCKS; - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (is_normal_sched) { if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS) (void) erts_get_monotonic_time(esdp); @@ -9433,23 +9656,15 @@ Process *schedule(Process *p, int calls) } - ERTS_SMP_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp) - || !erts_thr_progress_is_blocking()); + ERTS_SMP_LC_ASSERT(!is_normal_sched || !erts_thr_progress_is_blocking()); check_activities_to_run: { + erts_aint32_t psflg_running, psflg_running_sys; #ifdef ERTS_SMP ErtsMigrationPaths *mps; ErtsMigrationPath *mp; - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; - if (erts_proclist_fetch(&pnd_xtrs, NULL)) { - rq->procs.pending_exiters = NULL; - erts_smp_runq_unlock(rq); - handle_pending_exiters(pnd_xtrs); - erts_smp_runq_lock(rq); - } - + if (is_normal_sched) { if (rq->check_balance_reds <= 0) check_balance(rq); @@ -9466,32 +9681,35 @@ Process *schedule(Process *p, int calls) continue_check_activities_to_run: flags = ERTS_RUNQ_FLGS_GET_NOB(rq); continue_check_activities_to_run_known_flags: - ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp) - || flags & ERTS_RUNQ_FLG_NONEMPTY); + ASSERT(!is_normal_sched || (flags & ERTS_RUNQ_FLG_NONEMPTY)); - if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) { - if (flags & ERTS_RUNQ_FLG_SUSPENDED) { - (void) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_EXEC); + if (!is_normal_sched) { + if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED) { suspend_scheduler(esdp); - flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); - flags |= ERTS_RUNQ_FLG_EXEC; - } - if (flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) { - flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND); - flags &= ~ ERTS_RUNQ_FLG_CHK_CPU_BIND; - erts_sched_check_cpu_bind(esdp); } } -#ifdef ERTS_DIRTY_SCHEDULERS - else if (ERTS_SCHEDULER_IS_DIRTY(esdp) - && (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) - & ERTS_SSI_FLG_SUSPENDED)) - suspend_scheduler(esdp); -#endif - - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + else { erts_aint32_t aux_work; - int leader_update = erts_thr_progress_update(esdp); + int leader_update; + + ASSERT(is_normal_sched); + + if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) { + if (flags & ERTS_RUNQ_FLG_SUSPENDED) { + (void) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_EXEC); + suspend_scheduler(esdp); + flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); + flags |= ERTS_RUNQ_FLG_EXEC; + } + if (flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) { + flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND); + flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND; + erts_sched_check_cpu_bind(esdp); + } + } + + leader_update = erts_thr_progress_update(esdp); aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); if (aux_work | leader_update) { erts_smp_runq_unlock(rq); @@ -9517,19 +9735,13 @@ Process *schedule(Process *p, int calls) flags = ERTS_RUNQ_FLGS_GET_NOB(rq); -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && rq->halt_in_progress) { - /* - * TODO: if halt in progress, need to put the dirty scheduler - * to sleep somewhere around here to prevent it from picking up - * new work - */ + if (!is_normal_sched && rq->halt_in_progress) { + /* Wait for emulator to terminate... */ + while (1) + erts_milli_sleep(1000*1000); } - else -#endif - - if ((!(flags & ERTS_RUNQ_FLGS_QMASK) && !rq->misc.start) - || (rq->halt_in_progress && ERTS_EMPTY_RUNQ_PORTS(rq))) { + else if ((!(flags & ERTS_RUNQ_FLGS_QMASK) && !rq->misc.start) + || (rq->halt_in_progress && ERTS_EMPTY_RUNQ_PORTS(rq))) { /* Prepare for scheduler wait */ #ifdef ERTS_SMP ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -9543,7 +9755,7 @@ Process *schedule(Process *p, int calls) if (flags & ERTS_RUNQ_FLG_INACTIVE) empty_runq(rq); else { - if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && try_steal_task(rq)) + if (is_normal_sched && try_steal_task(rq)) goto continue_check_activities_to_run; empty_runq(rq); @@ -9572,9 +9784,9 @@ Process *schedule(Process *p, int calls) goto check_activities_to_run; } - else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && - (fcalls > input_reductions && - prepare_for_sys_schedule(!0))) { + else if (is_normal_sched + && (fcalls > input_reductions + && prepare_for_sys_schedule(!0))) { ErtsMonotonicTime current_time; /* * Schedule system-level activities. @@ -9688,11 +9900,17 @@ Process *schedule(Process *p, int calls) ASSERT(p); /* Wrong qmask in rq->flags? */ - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) - psflg_band_mask = ~((erts_aint32_t) 0); - else + if (is_normal_sched) { + psflg_running = ERTS_PSFLG_RUNNING; + psflg_running_sys = ERTS_PSFLG_RUNNING_SYS; psflg_band_mask = ~(((erts_aint32_t) 1) << (ERTS_PSFLGS_GET_PRQ_PRIO(state) + ERTS_PSFLGS_IN_PRQ_MASK_OFFSET)); + } + else { + psflg_running = ERTS_PSFLG_DIRTY_RUNNING; + psflg_running_sys = ERTS_PSFLG_DIRTY_RUNNING_SYS; + psflg_band_mask = ~((erts_aint32_t) 0); + } if (!(state & ERTS_PSFLG_PROXY)) psflg_band_mask &= ~ERTS_PSFLG_IN_RUNQ; @@ -9707,34 +9925,53 @@ Process *schedule(Process *p, int calls) state = erts_smp_atomic32_read_nob(&p->state); } +#ifdef ERTS_DIRTY_SCHEDULERS + if (!is_normal_sched) + clear_proc_dirty_queue_bit(p, rq, qbit); +#endif + while (1) { - erts_aint32_t exp, new, tmp; - tmp = new = exp = state; + erts_aint32_t exp, new; + int run_process; + new = exp = state; new &= psflg_band_mask; - if (!(state & (ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS))) { - tmp = state & (ERTS_PSFLG_SUSPENDED - | ERTS_PSFLG_PENDING_EXIT - | ERTS_PSFLG_ACTIVE_SYS - | ERTS_PSFLG_DIRTY_ACTIVE_SYS); - if (tmp != ERTS_PSFLG_SUSPENDED) { - if (state & (ERTS_PSFLG_ACTIVE_SYS - | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) - new |= ERTS_PSFLG_RUNNING_SYS; - else - new |= ERTS_PSFLG_RUNNING; - } + /* + * Run process if not already running (or free) + * or exiting and not running on a normal + * scheduler, and not suspended (and not in a + * state where suspend should be ignored). + */ + run_process = (((!(state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS + | ERTS_PSFLG_FREE))) +#ifdef ERTS_DIRTY_SCHEDULERS + | (((state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_FREE + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_EXITING)) + == ERTS_PSFLG_EXITING) + & (!!is_normal_sched)) +#endif + ) + & ((state & (ERTS_PSFLG_SUSPENDED + | ERTS_PSFLG_EXITING + | ERTS_PSFLG_FREE + | ERTS_PSFLG_PENDING_EXIT + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) + != ERTS_PSFLG_SUSPENDED)); + if (run_process) { + if (state & (ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) + new |= psflg_running_sys; + else + new |= psflg_running; } state = erts_smp_atomic32_cmpxchg_relb(&p->state, new, exp); if (state == exp) { - if ((state & (ERTS_PSFLG_RUNNING - | ERTS_PSFLG_RUNNING_SYS - | ERTS_PSFLG_FREE)) - || ((state & (ERTS_PSFLG_SUSPENDED - | ERTS_PSFLG_PENDING_EXIT - | ERTS_PSFLG_ACTIVE_SYS - | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) - == ERTS_PSFLG_SUSPENDED)) { + if (!run_process) { if (proxy_p) { free_proxy_proc(proxy_p); proxy_p = NULL; @@ -9761,34 +9998,13 @@ Process *schedule(Process *p, int calls) erts_smp_runq_unlock(rq); -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { -#ifdef DEBUG - int old_dqbit; -#endif - int dqbit = qbit; - - if (rq == ERTS_DIRTY_CPU_RUNQ) - dqbit <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET; - else { - ASSERT(rq == ERTS_DIRTY_IO_RUNQ); - dqbit <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET; - } - -#ifdef DEBUG - old_dqbit = (int) -#else - (void) -#endif - erts_smp_atomic32_read_band_mb(&p->dirty_state, ~dqbit); - ASSERT(old_dqbit & dqbit); - } -#endif /* ERTS_DIRTY_SCHEDULERS */ - #endif /* ERTS_SMP */ } + if (!is_normal_sched) + save_dirty_start(esdp, p); + #ifdef ERTS_SMP if (flags & ERTS_RUNQ_FLG_PROTECTED) @@ -9805,9 +10021,7 @@ Process *schedule(Process *p, int calls) UWord old = ERTS_PROC_SCHED_ID(p, (UWord) esdp->no); int migrated = old && old != esdp->no; -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); -#endif + ASSERT(is_normal_sched); prio = (int) ERTS_PSFLGS_GET_USR_PRIO(state); @@ -9821,20 +10035,21 @@ Process *schedule(Process *p, int calls) erts_smp_spin_unlock(&erts_sched_stat.lock); } - ASSERT(!p->scheduler_data); - p->scheduler_data = esdp; - state = erts_smp_atomic32_read_nob(&p->state); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (!!(state & ERTS_PSFLGS_DIRTY_WORK) - & !(state & ERTS_PSFLG_ACTIVE_SYS)) { + ASSERT(!p->scheduler_data); +#ifndef ERTS_DIRTY_SCHEDULERS + p->scheduler_data = esdp; +#else /* ERTS_DIRTY_SCHEDULERS */ + if (is_normal_sched) { + if ((!!(state & ERTS_PSFLGS_DIRTY_WORK)) + & (!(state & ERTS_PSFLG_ACTIVE_SYS))) { /* Migrate to dirty scheduler... */ sunlock_sched_out_proc: erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); goto sched_out_proc; } + p->scheduler_data = esdp; } else { if (state & (ERTS_PSFLG_ACTIVE_SYS @@ -9870,7 +10085,10 @@ Process *schedule(Process *p, int calls) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - if (IS_TRACED(p)) { + /* Clear tracer if it has been removed */ + if (IS_TRACED(p) && erts_is_tracer_proc_enabled( + p, ERTS_PROC_LOCK_MAIN, &p->common)) { + if (state & ERTS_PSFLG_EXITING) { if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) trace_sched(p, ERTS_PROC_LOCK_MAIN, am_in_exiting); @@ -9885,13 +10103,8 @@ Process *schedule(Process *p, int calls) } } - -#ifdef ERTS_SMP - /* Clears tracer if it has been removed */ - (void)ERTS_TRACER_PROC_IS_ENABLED(p); -#endif - - if (state & ERTS_PSFLG_RUNNING_SYS) { + if (state & (ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { /* * GC is normally never delayed when a process * is scheduled out, but might be when executing @@ -9905,7 +10118,7 @@ Process *schedule(Process *p, int calls) reds -= cost; if (reds <= 0 #ifdef ERTS_DIRTY_SCHEDULERS - || ERTS_SCHEDULER_IS_DIRTY(esdp) + || !is_normal_sched || (state & ERTS_PSFLGS_DIRTY_WORK) #endif ) { @@ -9913,8 +10126,8 @@ Process *schedule(Process *p, int calls) } } - ASSERT(state & ERTS_PSFLG_RUNNING_SYS); - ASSERT(!(state & ERTS_PSFLG_RUNNING)); + ASSERT(state & psflg_running_sys); + ASSERT(!(state & psflg_running)); while (1) { erts_aint32_t n, e; @@ -9926,8 +10139,8 @@ Process *schedule(Process *p, int calls) } n = e = state; - n &= ~ERTS_PSFLG_RUNNING_SYS; - n |= ERTS_PSFLG_RUNNING; + n &= ~psflg_running_sys; + n |= psflg_running; state = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); if (state == e) { @@ -9935,8 +10148,8 @@ Process *schedule(Process *p, int calls) break; } - ASSERT(state & ERTS_PSFLG_RUNNING_SYS); - ASSERT(!(state & ERTS_PSFLG_RUNNING)); + ASSERT(state & psflg_running_sys); + ASSERT(!(state & psflg_running)); } } @@ -10525,7 +10738,10 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) } state = erts_smp_atomic32_read_nob(&c_p->state); - ASSERT((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS) & state); + ASSERT((ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS) & state); while (!(state & ERTS_PSFLG_DELAYED_SYS) || prio < ERTS_PSFLGS_GET_ACT_PRIO(state)) { @@ -10850,6 +11066,8 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp) erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } +static void delete_process(Process* p); + void erts_free_proc(Process *p) { @@ -10858,6 +11076,8 @@ erts_free_proc(Process *p) #endif ASSERT(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_FREE); ASSERT(0 == erts_proc_read_refc(p)); + if (p->flags & F_DELAYED_DEL_PROC) + delete_process(p); erts_free(ERTS_ALC_T_PROC, (void *) p); } @@ -11021,9 +11241,13 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->min_heap_size = so->min_heap_size; p->min_vheap_size = so->min_vheap_size; p->max_gen_gcs = so->max_gen_gcs; + MAX_HEAP_SIZE_SET(p, so->max_heap_size); + MAX_HEAP_SIZE_FLAGS_SET(p, so->max_heap_flags); } else { p->min_heap_size = H_MIN_SIZE; p->min_vheap_size = BIN_VH_MIN_SIZE; + MAX_HEAP_SIZE_SET(p, H_MAX_SIZE); + MAX_HEAP_SIZE_FLAGS_SET(p, H_MAX_FLAGS); p->max_gen_gcs = (Uint16) erts_smp_atomic32_read_nob(&erts_max_gen_gcs); } p->schedule_count = 0; @@ -11486,18 +11710,36 @@ erts_cleanup_empty_process(Process* p) #endif } -/* - * p must be the currently executing process. - */ static void delete_process(Process* p) { Eterm *heap; ErtsPSD *psd; + struct saved_calls *scb; + process_breakpoint_time_t *pbt; + void *nif_export; + VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id)); VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id, HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p))); + scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL); + + if (scb) { + p->fcalls += CONTEXT_REDS; /* Reduction counting depends on this... */ + erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb); + } + + pbt = ERTS_PROC_SET_CALL_TIME(p, NULL); + if (pbt) + erts_free(ERTS_ALC_T_BPD, (void *) pbt); + + nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL); + if (nif_export) + erts_destroy_nif_export(nif_export); + + clean_dirty_start(p); + /* Cleanup psd */ psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); @@ -11600,7 +11842,8 @@ set_proc_exiting(Process *p, p->i = (BeamInstr *) beam_exit; #ifndef ERTS_SMP - if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)) { + if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS) + && !(state & ERTS_PSFLG_GC)) { /* * I non smp case: * @@ -11629,7 +11872,10 @@ set_proc_self_exiting(Process *c_p) ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCKS_ALL); state = erts_smp_atomic32_read_nob(&c_p->state); - ASSERT(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)); + ASSERT(state & (ERTS_PSFLG_RUNNING + |ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)); #ifdef DEBUG enqueue = @@ -11679,51 +11925,73 @@ erts_handle_pending_exit(Process *c_p, ErtsProcLocks locks) erts_smp_proc_unlock(c_p, xlocks); } +static void save_pending_exiter(Process *p, ErtsProcList *plp); + static void -handle_pending_exiters(ErtsProcList *pnd_xtrs) +do_handle_pending_exiters(ErtsProcList *pnd_xtrs) { /* 'list' is expected to have been fetched (i.e. not a ring anymore) */ ErtsProcList *plp = pnd_xtrs; while (plp) { - ErtsProcList *free_plp; - Process *p = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCKS_ALL); + ErtsProcList *next_plp = plp->next; + Process *p = erts_proc_lookup(plp->pid); if (p) { - if (erts_proclist_same(plp, p)) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&p->state); - if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) { - ASSERT(state & ERTS_PSFLG_PENDING_EXIT); - erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL); + erts_aint32_t state; + /* + * If the process is running on a normal scheduler, the + * pending exit will soon be detected and handled by the + * scheduler running the process (at schedule in/out). + */ + if (erts_smp_proc_trylock(p, ERTS_PROC_LOCKS_ALL) != EBUSY) { + if (erts_proclist_same(plp, p)) { + state = erts_smp_atomic32_read_acqb(&p->state); + if (!(state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_EXITING))) { + ASSERT(state & ERTS_PSFLG_PENDING_EXIT); + erts_handle_pending_exit(p, ERTS_PROC_LOCKS_ALL); + } + } + erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); + } + else { + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + if (erts_proclist_same(plp, p)) { + state = erts_smp_atomic32_read_acqb(&p->state); + if (!(state & (ERTS_PSFLG_RUNNING + | ERTS_PSFLG_RUNNING_SYS + | ERTS_PSFLG_EXITING))) { + /* + * Save process and try to acquire all + * locks at a later time... + */ + save_pending_exiter(p, plp); + plp = NULL; + } } + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } - erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); } - free_plp = plp; - plp = plp->next; - proclist_destroy(free_plp); + if (plp) + proclist_destroy(plp); + plp = next_plp; } } static void -save_pending_exiter(Process *p) +save_pending_exiter(Process *p, ErtsProcList *plp) { - ErtsProcList *plp; + ErtsSchedulerSleepInfo *ssi; ErtsRunQueue *rq; - ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - if (!esdp) - rq = RUNQ_READ_RQ(&p->run_queue); - else - rq = esdp->run_queue; - -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) - rq = ERTS_RUNQ_IX(0); /* Handle on ordinary scheduler */ -#endif + rq = RUNQ_READ_RQ(&p->run_queue); + ASSERT(rq && !ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); - plp = proclist_create(p); + if (!plp) + plp = proclist_create(p); erts_smp_runq_lock(rq); @@ -11731,9 +11999,11 @@ save_pending_exiter(Process *p) non_empty_runq(rq); + ssi = rq->scheduler->ssi; + erts_smp_runq_unlock(rq); - wake_scheduler(rq); + set_aux_work_flags_wakeup_nob(ssi, ERTS_SSI_AUX_WORK_PENDING_EXITERS); } #endif @@ -11933,7 +12203,7 @@ send_exit_signal(Process *c_p, /* current process if and only if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { /* ... but we havn't got all locks on it ... */ - save_pending_exiter(rp); + save_pending_exiter(rp, NULL); /* * The pending exit will be discovered when next * process is scheduled in @@ -12401,10 +12671,8 @@ erts_continue_exit_process(Process *p) ErtsProcLocks curr_locks = ERTS_PROC_LOCK_MAIN; Eterm reason = p->fvalue; DistEntry *dep; - struct saved_calls *scb; - process_breakpoint_time_t *pbt; erts_aint32_t state; - void *nif_export; + int delay_del_proc = 0; #ifdef DEBUG int yield_allowed = 1; @@ -12547,7 +12815,7 @@ erts_continue_exit_process(Process *p) { /* Do *not* use erts_get_runq_proc() */ ErtsRunQueue *rq; - rq = erts_get_runq_current(ERTS_GET_SCHEDULER_DATA_FROM_PROC(p)); + rq = erts_get_runq_current(erts_proc_sched_data(p)); erts_smp_runq_lock(rq); @@ -12593,16 +12861,24 @@ erts_continue_exit_process(Process *p) break; } +#ifdef ERTS_DIRTY_SCHEDULERS + if (a & (ERTS_PSFLG_DIRTY_RUNNING + | ERTS_PSFLG_DIRTY_RUNNING_SYS)) { + p->flags |= F_DELAYED_DEL_PROC; + delay_del_proc = 1; + /* + * The dirty scheduler will also decrease + * refc when done... + */ + erts_proc_inc_refc(p); + } +#endif + if (refc_inced && !(n & ERTS_PSFLG_IN_RUNQ)) erts_proc_dec_refc(p); } dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL; - scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL); - if (scb) - p->fcalls += CONTEXT_REDS; /* Reduction counting depends on this... */ - pbt = ERTS_PROC_SET_CALL_TIME(p, NULL); - nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); #ifdef BM_COUNTERS @@ -12642,22 +12918,14 @@ erts_continue_exit_process(Process *p) have none here */ } - if (scb) - erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb); - - if (pbt) - erts_free(ERTS_ALC_T_BPD, (void *) pbt); - - if (nif_export) - erts_destroy_nif_export(nif_export); - #ifdef ERTS_SMP erts_flush_trace_messages(p, 0); #endif ERTS_TRACER_CLEAR(&ERTS_TRACER(p)); - delete_process(p); + if (!delay_del_proc) + delete_process(p); #ifdef ERTS_SMP erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); @@ -12687,6 +12955,7 @@ erts_continue_exit_process(Process *p) ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN == erts_proc_lc_my_proc_locks(p)); + BUMP_ALL_REDS(p); } /* @@ -13006,11 +13275,13 @@ void erts_halt(int code) int erts_dbg_check_halloc_lock(Process *p) { + ErtsSchedulerData *esdp; if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)) return 1; if (p->common.id == ERTS_INVALID_PID) return 1; - if (p->scheduler_data && p == p->scheduler_data->match_pseudo_process) + esdp = erts_proc_sched_data(p); + if (esdp && p == esdp->match_pseudo_process) return 1; if (erts_thr_progress_is_blocking()) return 1; diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 59da9c1779..2801947613 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -304,6 +304,7 @@ typedef enum { ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX, ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX, ERTS_SSI_AUX_WORK_MISC_IX, + ERTS_SSI_AUX_WORK_PENDING_EXITERS_IX, ERTS_SSI_AUX_WORK_SET_TMO_IX, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX, ERTS_SSI_AUX_WORK_REAP_PORTS_IX, @@ -336,6 +337,8 @@ typedef enum { (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX) #define ERTS_SSI_AUX_WORK_MISC \ (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX) +#define ERTS_SSI_AUX_WORK_PENDING_EXITERS \ + (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_PENDING_EXITERS_IX) #define ERTS_SSI_AUX_WORK_SET_TMO \ (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX) #define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \ @@ -645,6 +648,7 @@ struct ErtsSchedulerData_ { Uint no; /* Scheduler number for normal schedulers */ #ifdef ERTS_DIRTY_SCHEDULERS ErtsDirtySchedId dirty_no; /* Scheduler number for dirty schedulers */ + Process *dirty_shadow_process; #endif Port *current_port; ErtsRunQueue *run_queue; @@ -805,14 +809,26 @@ erts_smp_reset_max_len(ErtsRunQueue *rq, ErtsRunQueueInfo *rqi) #define ERTS_PSD_CALL_TIME_BP 3 #define ERTS_PSD_DELAYED_GC_TASK_QS 4 #define ERTS_PSD_NIF_TRAP_EXPORT 5 -#ifdef HIPE #define ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF 6 -#endif - -#ifdef HIPE -#define ERTS_PSD_SIZE 7 -#else -#define ERTS_PSD_SIZE 6 +#define ERTS_PSD_DIRTY_CPU_START 7 + +#define ERTS_PSD_SIZE 8 + +#if !defined(HIPE) && !defined(ERTS_DIRTY_SCHEDULERS) +# undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF +# undef ERTS_PSD_DIRTY_CPU_START +# undef ERTS_PSD_SIZE +# define ERTS_PSD_SIZE 6 +#elif !defined(HIPE) +# undef ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF +# undef ERTS_PSD_DIRTY_CPU_START +# undef ERTS_PSD_SIZE +# define ERTS_PSD_DIRTY_CPU_START 6 +# define ERTS_PSD_SIZE 7 +#elif !defined(ERTS_DIRTY_SCHEDULERS) +# undef ERTS_PSD_DIRTY_CPU_START +# undef ERTS_PSD_SIZE +# define ERTS_PSD_SIZE 7 #endif typedef struct { @@ -918,6 +934,15 @@ struct ErtsPendingSuspend_ { # define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz # define BIN_OLD_VHEAP(p) (p)->bin_old_vheap +# define MAX_HEAP_SIZE_GET(p) ((p)->max_heap_size >> 2) +# define MAX_HEAP_SIZE_SET(p, sz) ((p)->max_heap_size = ((sz) << 2) | \ + MAX_HEAP_SIZE_FLAGS_GET(p)) +# define MAX_HEAP_SIZE_FLAGS_GET(p) ((p)->max_heap_size & 0x3) +# define MAX_HEAP_SIZE_FLAGS_SET(p, flags) ((p)->max_heap_size = flags | \ + ((p)->max_heap_size & ~0x3)) +# define MAX_HEAP_SIZE_KILL 1 +# define MAX_HEAP_SIZE_LOG 2 + struct process { ErtsPTabElementCommon common; /* *Need* to be first in struct */ @@ -935,6 +960,7 @@ struct process { Uint heap_sz; /* Size of heap in words */ Uint min_heap_size; /* Minimum size of heap (in words). */ Uint min_vheap_size; /* Minimum size of virtual heap (in words). */ + Uint max_heap_size; /* Maximum size of heap (in words). */ #if !defined(NO_FPE_SIGNALS) || defined(HIPE) volatile unsigned long fp_exception; @@ -1183,7 +1209,10 @@ void erts_check_for_holes(Process* p); #define ERTS_PSFLG_DIRTY_CPU_PROC ERTS_PSFLG_BIT(20) #define ERTS_PSFLG_DIRTY_IO_PROC ERTS_PSFLG_BIT(21) #define ERTS_PSFLG_DIRTY_ACTIVE_SYS ERTS_PSFLG_BIT(22) -#define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 22) +#define ERTS_PSFLG_DIRTY_RUNNING ERTS_PSFLG_BIT(23) +#define ERTS_PSFLG_DIRTY_RUNNING_SYS ERTS_PSFLG_BIT(24) + +#define ERTS_PSFLG_MAX (ERTS_PSFLGS_ZERO_BIT_OFFSET + 24) #define ERTS_PSFLGS_DIRTY_WORK (ERTS_PSFLG_DIRTY_CPU_PROC \ | ERTS_PSFLG_DIRTY_IO_PROC \ @@ -1194,6 +1223,11 @@ void erts_check_for_holes(Process* p); | ERTS_PSFLG_IN_PRQ_NORMAL \ | ERTS_PSFLG_IN_PRQ_LOW) +#define ERTS_PSFLGS_VOLATILE_HEAP (ERTS_PSFLG_EXITING \ + | ERTS_PSFLG_PENDING_EXIT \ + | ERTS_PSFLG_DIRTY_RUNNING \ + | ERTS_PSFLG_DIRTY_RUNNING_SYS) + #define ERTS_PSFLGS_GET_ACT_PRIO(PSFLGS) \ (((PSFLGS) >> ERTS_PSFLGS_ACT_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK) #define ERTS_PSFLGS_GET_USR_PRIO(PSFLGS) \ @@ -1235,6 +1269,7 @@ void erts_check_for_holes(Process* p); * Static flags that do not change after process creation. */ #define ERTS_STC_FLG_SYSTEM_PROC (((Uint32) 1) << 0) +#define ERTS_STC_FLG_SHADOW_PROC (((Uint32) 1) << 1) /* The sequential tracing token is a tuple of size 5: * @@ -1285,6 +1320,8 @@ typedef struct { Uint min_vheap_size; /* Minimum virtual heap size */ int priority; /* Priority for process. */ Uint16 max_gen_gcs; /* Maximum number of gen GCs before fullsweep. */ + Uint max_heap_size; /* Maximum heap size in words */ + Uint max_heap_flags; /* Maximum heap flags (kill | log) */ int scheduler; } ErlSpawnOpts; @@ -1363,6 +1400,7 @@ extern int erts_system_profile_ts_type; #define F_SCHDLR_ONLN_WAITQ (1 << 17) /* Process enqueued waiting to change schedulers online */ #define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */ #define F_HIPE_MODE (1 << 19) +#define F_DELAYED_DEL_PROC (1 << 20) /* Delay delete process (dirty proc exit case) */ /* * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent @@ -1783,7 +1821,8 @@ erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int); void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); void erts_init_process(int, int, int); -Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm); +Eterm erts_process_state2status(erts_aint32_t); +Eterm erts_process_status(Process *, Eterm); Uint erts_run_queues_len(Uint *, int, int); void erts_add_to_runq(Process *); Eterm erts_bound_schedulers_term(Process *c_p); @@ -1860,19 +1899,11 @@ int erts_debug_wait_completed(Process *c_p, int flags); Uint erts_process_memory(Process *c_p, int incl_msg_inq); -#ifdef ERTS_SMP -# define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data) -# define ERTS_PROC_GET_SCHDATA(PROC) ((PROC)->scheduler_data) -#else -# define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) (erts_scheduler_data) -# define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data) -#endif - #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC # define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P) \ do { \ ErtsSchedulerData *esdp__ = ((P) \ - ? ERTS_PROC_GET_SCHDATA((Process *) (P)) \ + ? erts_proc_sched_data((Process *) (P)) \ : erts_get_scheduler_data()); \ if (esdp__ && !ERTS_SCHEDULER_IS_DIRTY(esdp__)) \ esdp__->verify_unused_temp_alloc( \ @@ -1965,12 +1996,15 @@ erts_psd_set(Process *p, int ix, void *data) ErtsPSD *psd; #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ErtsProcLocks locks = erts_proc_lc_my_proc_locks(p); - if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks) - ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking()); - else { - locks &= erts_psd_required_locks[ix].set_locks; - ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks - || erts_thr_progress_is_blocking()); + erts_aint32_t state = state = erts_smp_atomic32_read_nob(&p->state); + if (!(state & ERTS_PSFLG_FREE)) { + if (ERTS_LC_PSD_ANY_LOCK == erts_psd_required_locks[ix].set_locks) + ERTS_SMP_LC_ASSERT(locks || erts_thr_progress_is_blocking()); + else { + locks &= erts_psd_required_locks[ix].set_locks; + ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks == locks + || erts_thr_progress_is_blocking()); + } } #endif psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); @@ -2027,6 +2061,13 @@ erts_psd_set(Process *p, int ix, void *data) ((struct saved_calls *) erts_psd_set((P), ERTS_PSD_SUSPENDED_SAVED_CALLS_BUF, (void *) (SCB))) #endif +#ifdef ERTS_DIRTY_SCHEDULERS +#define ERTS_PROC_GET_DIRTY_CPU_START(P) \ + ((void *) erts_psd_get((P), ERTS_PSD_DIRTY_CPU_START)) +#define ERTS_PROC_SET_DIRTY_CPU_START(P, DCS) \ + ((void *) erts_psd_set((P), ERTS_PSD_DIRTY_CPU_START, (void *) (DCS))) +#endif + ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p); ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, Eterm handler); @@ -2169,6 +2210,7 @@ erts_check_emigration_need(ErtsRunQueue *c_rq, int prio) #endif +ERTS_GLB_INLINE ErtsSchedulerData *erts_proc_sched_data(Process *c_p); ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp); ERTS_GLB_INLINE Process *erts_get_current_process(void); ERTS_GLB_INLINE Eterm erts_get_current_pid(void); @@ -2202,6 +2244,31 @@ ERTS_GLB_INLINE void erts_shrink_message_heap(ErtsMessage **msgpp, Process *pp, #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE +ErtsSchedulerData *erts_proc_sched_data(Process *c_p) +{ + ErtsSchedulerData *esdp; + ASSERT(c_p); +#if !defined(ERTS_SMP) + esdp = erts_get_scheduler_data(); +#else + esdp = c_p->scheduler_data; +# if defined(ERTS_DIRTY_SCHEDULERS) + if (esdp) { + ASSERT(esdp == erts_get_scheduler_data()); + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); + } + else { + esdp = erts_get_scheduler_data(); + ASSERT(esdp); + ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp)); + } +# endif +#endif + ASSERT(esdp); + return esdp; +} + +ERTS_GLB_INLINE int erts_is_scheduler_bound(ErtsSchedulerData *esdp) { if (!esdp) @@ -2416,7 +2483,7 @@ ERTS_GLB_INLINE ErtsAtomCacheMap * erts_get_atom_cache_map(Process *c_p) { ErtsSchedulerData *esdp = (c_p - ? ERTS_PROC_GET_SCHDATA(c_p) + ? erts_proc_sched_data(c_p) : erts_get_scheduler_data()); ASSERT(esdp); return &esdp->atom_cache_map; diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index fa76773cac..eeaa9a569c 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -568,23 +568,21 @@ dump_externally(int to, void *to_arg, Eterm term) } } -void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg) { - if (psflg & ERTS_PSFLG_FREE) - erts_print(to, to_arg, "Non Existing\n"); /* Should never happen */ - else if (psflg & ERTS_PSFLG_EXITING) - erts_print(to, to_arg, "Exiting\n"); - else if (psflg & ERTS_PSFLG_GC) { - erts_print(to, to_arg, "Garbing\n"); - } - else if (psflg & ERTS_PSFLG_SUSPENDED) - erts_print(to, to_arg, "Suspended\n"); - else if (psflg & ERTS_PSFLG_RUNNING) { - erts_print(to, to_arg, "Running\n"); +void erts_dump_process_state(int to, void *to_arg, erts_aint32_t psflg) +{ + char *s; + switch (erts_process_state2status(psflg)) { + case am_free: s = "Non Existing"; break; /* Should never happen */ + case am_exiting: s = "Exiting"; break; + case am_garbage_collecting: s = "Garbing"; break; + case am_suspended: s = "Suspended"; break; + case am_running: s = "Running"; break; + case am_runnable: s = "Scheduled"; break; + case am_waiting: s = "Waiting"; break; + default: s = "Undefined"; break; /* Should never happen */ } - else if (psflg & ERTS_PSFLG_ACTIVE) - erts_print(to, to_arg, "Scheduled\n"); - else - erts_print(to, to_arg, "Waiting\n"); + + erts_print(to, to_arg, "%s\n", s); } void @@ -668,6 +666,10 @@ erts_dump_extended_process_state(int to, void *to_arg, erts_aint32_t psflg) { erts_print(to, to_arg, "DIRTY_IO_PROC"); break; case ERTS_PSFLG_DIRTY_ACTIVE_SYS: erts_print(to, to_arg, "DIRTY_ACTIVE_SYS"); break; + case ERTS_PSFLG_DIRTY_RUNNING: + erts_print(to, to_arg, "DIRTY_RUNNING"); break; + case ERTS_PSFLG_DIRTY_RUNNING_SYS: + erts_print(to, to_arg, "DIRTY_RUNNING_SYS"); break; default: erts_print(to, to_arg, "UNKNOWN(%d)", chk); break; } diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index fadbd704bd..9e37106b88 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -2348,7 +2348,7 @@ erts_napi_convert_time_unit(ErtsMonotonicTime val, int from, int to) BIF_RETTYPE monotonic_time_0(BIF_ALIST_0) { ErtsMonotonicTime mtime = time_sup.r.o.get_time(); - update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + update_last_mtime(erts_proc_sched_data(BIF_P), mtime); mtime += ERTS_MONOTONIC_OFFSET_NATIVE; BIF_RET(make_time_val(BIF_P, mtime)); } @@ -2356,7 +2356,7 @@ BIF_RETTYPE monotonic_time_0(BIF_ALIST_0) BIF_RETTYPE monotonic_time_1(BIF_ALIST_1) { ErtsMonotonicTime mtime = time_sup.r.o.get_time(); - update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + update_last_mtime(erts_proc_sched_data(BIF_P), mtime); BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, mtime, 1)); } @@ -2365,7 +2365,7 @@ BIF_RETTYPE system_time_0(BIF_ALIST_0) ErtsMonotonicTime mtime, offset; mtime = time_sup.r.o.get_time(); offset = get_time_offset(); - update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + update_last_mtime(erts_proc_sched_data(BIF_P), mtime); BIF_RET(make_time_val(BIF_P, mtime + offset)); } @@ -2374,7 +2374,7 @@ BIF_RETTYPE system_time_1(BIF_ALIST_0) ErtsMonotonicTime mtime, offset; mtime = time_sup.r.o.get_time(); offset = get_time_offset(); - update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + update_last_mtime(erts_proc_sched_data(BIF_P), mtime); BIF_RET(time_unit_conversion(BIF_P, BIF_ARG_1, mtime + offset, 0)); } @@ -2404,7 +2404,7 @@ BIF_RETTYPE timestamp_0(BIF_ALIST_0) mtime = time_sup.r.o.get_time(); offset = get_time_offset(); - update_last_mtime(ERTS_PROC_GET_SCHDATA(BIF_P), mtime); + update_last_mtime(erts_proc_sched_data(BIF_P), mtime); make_timestamp_value(&mega_sec, &sec, µ_sec, mtime, offset); diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 1c0fc0a11f..ca001fc156 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -396,7 +396,7 @@ send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p, Eterm tag, Eterm msg, Eterm extra, Eterm pam_result); static ERTS_INLINE Eterm -call_enabled_tracer(Process *c_p, const ErtsTracer tracer, +call_enabled_tracer(const ErtsTracer tracer, ErtsTracerNif **tnif_ref, enum ErtsTracerOpt topt, Eterm tag, Eterm t_p_id); @@ -459,8 +459,7 @@ erts_set_system_seq_tracer(Process *c_p, ErtsProcLocks c_p_locks, ErtsTracer new if (!ERTS_TRACER_IS_NIL(new)) { Eterm nif_result = call_enabled_tracer( - NULL, new, NULL, - TRACE_FUN_ENABLED, am_trace_status, am_undefined); + new, NULL, TRACE_FUN_ENABLED, am_trace_status, am_undefined); switch (nif_result) { case am_trace: break; default: @@ -492,7 +491,7 @@ erts_get_system_seq_tracer(void) erts_smp_rwmtx_runlock(&sys_trace_rwmtx); if (st != erts_tracer_nil && - call_enabled_tracer(NULL, st, NULL, TRACE_FUN_ENABLED, + call_enabled_tracer(st, NULL, TRACE_FUN_ENABLED, am_trace_status, am_undefined) == am_remove) { erts_set_system_seq_tracer(NULL, 0, erts_tracer_nil); st = erts_tracer_nil; @@ -513,7 +512,7 @@ get_default_tracing(Uint *flagsp, ErtsTracer *tracerp, *default_trace_flags &= ~TRACEE_FLAGS; } else { Eterm nif_res; - nif_res = call_enabled_tracer(NULL, *default_tracer, + nif_res = call_enabled_tracer(*default_tracer, NULL, TRACE_FUN_ENABLED, am_trace_status, am_undefined); switch (nif_res) { @@ -915,8 +914,8 @@ seq_trace_update_send(Process *p) ASSERT((is_tuple(SEQ_TRACE_TOKEN(p)) || is_nil(SEQ_TRACE_TOKEN(p)))); if (have_no_seqtrace(SEQ_TRACE_TOKEN(p)) || (seq_tracer != NIL && - call_enabled_tracer(NULL, seq_tracer, NULL, - TRACE_FUN_ENABLED, am_trace_status, + call_enabled_tracer(seq_tracer, NULL, + TRACE_FUN_ENABLED, am_seq_trace, p ? p->common.id : am_undefined) != am_trace) #ifdef USE_VM_PROBES || (SEQ_TRACE_TOKEN(p) == am_have_dt_utag) @@ -963,9 +962,9 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, ASSERT(is_tuple(token) || is_nil(token)); if (token == NIL || (process && ERTS_TRACE_FLAGS(process) & F_SENSITIVE) || ERTS_TRACER_IS_NIL(seq_tracer) || - call_enabled_tracer(NULL, seq_tracer, + call_enabled_tracer(seq_tracer, NULL, TRACE_FUN_ENABLED, - am_trace_status, + am_seq_trace, process ? process->common.id : am_undefined) != am_trace) { return; } @@ -1186,7 +1185,11 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, * use process flags */ tracee_flags = &ERTS_TRACE_FLAGS(p); + /* Is is not ideal at all to call this check twice, + it should be optimized so that only one call is made. */ if (!is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, + TRACE_FUN_ENABLED, am_trace_status) + || !is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, TRACE_FUN_E_CALL, am_call)) { return 0; } @@ -1202,13 +1205,21 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, } meta_flags = F_TRACE_CALLS | F_NOW_TS; tracee_flags = &meta_flags; - switch (call_enabled_tracer(p, *tracer, - &tnif, TRACE_FUN_T_CALL, - am_call, p->common.id)) { + switch (call_enabled_tracer(*tracer, + &tnif, TRACE_FUN_ENABLED, + am_trace_status, p->common.id)) { default: case am_remove: *tracer = erts_tracer_nil; case am_discard: return 0; - case am_trace: break; + case am_trace: + switch (call_enabled_tracer(*tracer, + &tnif, TRACE_FUN_T_CALL, + am_call, p->common.id)) { + default: + case am_discard: return 0; + case am_trace: break; + } + break; } } @@ -1346,9 +1357,9 @@ trace_proc(Process *c_p, ErtsProcLocks c_p_locks, Process *t_p, Eterm what, Eterm data) { ErtsTracerNif *tnif = NULL; - if (is_tracer_enabled(c_p, c_p_locks, &t_p->common, &tnif, + if (is_tracer_enabled(NULL, 0, &t_p->common, &tnif, TRACE_FUN_E_PROCS, what)) - send_to_tracer_nif(c_p, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_PROCS, + send_to_tracer_nif(NULL, &t_p->common, t_p->common.id, tnif, TRACE_FUN_T_PROCS, what, data, THE_NON_VALUE, am_true); } @@ -1365,16 +1376,15 @@ trace_proc_spawn(Process *p, Eterm what, Eterm pid, Eterm mod, Eterm func, Eterm args) { ErtsTracerNif *tnif = NULL; - if (is_tracer_enabled(p, ERTS_PROC_LOCKS_ALL & - ~(ERTS_PROC_LOCK_STATUS|ERTS_PROC_LOCK_TRACE), - &p->common, &tnif, TRACE_FUN_E_PROCS, what)) { + if (is_tracer_enabled(NULL, 0, + &p->common, &tnif, TRACE_FUN_E_PROCS, what)) { Eterm mfa; Eterm* hp; hp = HAlloc(p, 4); mfa = TUPLE3(hp, mod, func, args); hp += 4; - send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_PROCS, + send_to_tracer_nif(NULL, &p->common, p->common.id, tnif, TRACE_FUN_T_PROCS, what, pid, mfa, am_true); } } @@ -1409,25 +1419,28 @@ void save_calls(Process *p, Export *e) * are all small (atomic) integers. */ void -trace_gc(Process *p, Eterm what, Uint size) +trace_gc(Process *p, Eterm what, Uint size, Eterm msg) { ErtsTracerNif *tnif = NULL; Eterm* hp; - Eterm msg = NIL; Uint sz = 0; Eterm tup; - if (is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, TRACE_FUN_E_GC, what)) { + if (is_tracer_enabled(p, ERTS_PROC_LOCK_MAIN, &p->common, &tnif, + TRACE_FUN_E_GC, what)) { + + if (is_non_value(msg)) { - (void) erts_process_gc_info(p, &sz, NULL); - hp = HAlloc(p, sz + 3 + 2); + (void) erts_process_gc_info(p, &sz, NULL, 0, 0); + hp = HAlloc(p, sz + 3 + 2); - msg = erts_process_gc_info(p, NULL, &hp); - tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3; - msg = CONS(hp, tup, msg); hp += 2; + msg = erts_process_gc_info(p, NULL, &hp, 0, 0); + tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3; + msg = CONS(hp, tup, msg); hp += 2; + } send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_GC, - what, msg, am_undefined, am_true); + what, msg, THE_NON_VALUE, am_true); } } @@ -2896,23 +2909,28 @@ send_to_tracer_nif(Process *c_p, ErtsPTabElementCommon *t_p, } static ERTS_INLINE Eterm -call_enabled_tracer(Process *c_p, const ErtsTracer tracer, +call_enabled_tracer(const ErtsTracer tracer, ErtsTracerNif **tnif_ret, enum ErtsTracerOpt topt, Eterm tag, Eterm t_p_id) { ErtsTracerNif *tnif = lookup_tracer_nif(tracer); if (tnif) { - Eterm argv[] = {tag, ERTS_TRACER_STATE(tracer), t_p_id}; + Eterm argv[] = {tag, ERTS_TRACER_STATE(tracer), t_p_id}, + ret; topt = (tnif->tracers[topt].cb) ? topt : TRACE_FUN_ENABLED; ASSERT(topt < NIF_TRACER_TYPES); ASSERT(tnif->tracers[topt].cb != NULL); if (tnif_ret) *tnif_ret = tnif; - return erts_nif_call_function(c_p, NULL, tnif->nif_mod, - tnif->tracers[topt].cb, - tnif->tracers[topt].arity, - argv); + ret = erts_nif_call_function(NULL, NULL, tnif->nif_mod, + tnif->tracers[topt].cb, + tnif->tracers[topt].arity, + argv); + if (tag == am_trace_status && ret != am_remove) + return am_trace; + ASSERT(tag == am_trace_status || ret != am_remove); + return ret; } - return am_remove; + return tag == am_trace_status ? am_remove : am_discard; } static int @@ -2937,12 +2955,12 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks, } #endif - nif_result = call_enabled_tracer(c_p, t_p->tracer, tnif_ret, topt, tag, t_p->id); + nif_result = call_enabled_tracer(t_p->tracer, tnif_ret, topt, tag, t_p->id); switch (nif_result) { case am_discard: return 0; case am_trace: return 1; case THE_NON_VALUE: - case am_remove: break; + case am_remove: ASSERT(tag == am_trace_status); break; default: /* only am_remove should be returned, but if something else is returned we fall-through @@ -2973,19 +2991,14 @@ is_tracer_enabled(Process* c_p, ErtsProcLocks c_p_locks, return 0; } -int erts_is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks, - ErtsPTabElementCommon *t_p, Eterm type) -{ - return is_tracer_enabled(c_p, c_p_locks, t_p, NULL, TRACE_FUN_ENABLED, am_trace_status); -} - -int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer) +int erts_is_tracer_enabled(const ErtsTracer tracer, ErtsPTabElementCommon *t_p) { ErtsTracerNif *tnif = lookup_tracer_nif(tracer); if (tnif) { - Eterm nif_result = call_enabled_tracer(c_p, tracer, &tnif, - TRACE_FUN_ENABLED, am_trace_status, - c_p->common.id); + Eterm nif_result = call_enabled_tracer(tracer, &tnif, + TRACE_FUN_ENABLED, + am_trace_status, + t_p->id); switch (nif_result) { case am_discard: case am_trace: return 1; @@ -2996,6 +3009,20 @@ int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer) return 0; } +int erts_is_tracer_proc_enabled(Process* c_p, ErtsProcLocks c_p_locks, + ErtsPTabElementCommon *t_p) +{ + return is_tracer_enabled(c_p, c_p_locks, t_p, NULL, TRACE_FUN_ENABLED, + am_trace_status); +} + +int erts_is_tracer_proc_enabled_send(Process* c_p, ErtsProcLocks c_p_locks, + ErtsPTabElementCommon *t_p) +{ + return is_tracer_enabled(c_p, c_p_locks, t_p, NULL, TRACE_FUN_T_SEND, am_send); +} + + void erts_tracer_replace(ErtsPTabElementCommon *t_p, const ErtsTracer tracer) { #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index de4beadb7e..0095d4386b 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -112,7 +112,7 @@ void trace_sched(Process*, ErtsProcLocks, Eterm); void trace_proc(Process*, ErtsProcLocks, Process*, Eterm, Eterm); void trace_proc_spawn(Process*, Eterm what, Eterm pid, Eterm mod, Eterm func, Eterm args); void save_calls(Process *p, Export *); -void trace_gc(Process *p, Eterm what, Uint size); +void trace_gc(Process *p, Eterm what, Uint size, Eterm msg); /* port tracing */ void trace_virtual_sched(Process*, ErtsProcLocks, Eterm); void trace_sched_ports(Port *pp, Eterm); @@ -193,8 +193,10 @@ int erts_finish_breakpointing(void); /* Nif tracer functions */ int erts_is_tracer_proc_enabled(Process *c_p, ErtsProcLocks c_p_locks, - ErtsPTabElementCommon *t_p, Eterm type); -int erts_is_tracer_enabled(Process *c_p, const ErtsTracer tracer); + ErtsPTabElementCommon *t_p); +int erts_is_tracer_proc_enabled_send(Process* c_p, ErtsProcLocks c_p_locks, + ErtsPTabElementCommon *t_p); +int erts_is_tracer_enabled(const ErtsTracer tracer, ErtsPTabElementCommon *t_p); Eterm erts_tracer_to_term(Process *p, ErtsTracer tracer); ErtsTracer erts_term_to_tracer(Eterm prefix, Eterm term); void erts_tracer_replace(ErtsPTabElementCommon *t_p, @@ -224,9 +226,4 @@ ERTS_DECLARE_DUMMY(erts_tracer_nil) = NIL; #define ERTS_TRACER_FROM_ETERM(termp) \ ((ErtsTracer*)(termp)) -#define ERTS_TRACER_PROC_IS_ENABLED(PROC) \ - (!ERTS_TRACER_IS_NIL(ERTS_TRACER(PROC)) \ - && erts_is_tracer_proc_enabled(PROC, ERTS_PROC_LOCK_MAIN, \ - &(PROC)->common, am_trace_status)) - #endif /* ERL_TRACE_H__ */ diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index f3c54de214..f97716d030 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -50,6 +50,7 @@ #define H_DEFAULT_SIZE 233 /* default (heap + stack) min size */ #define VH_DEFAULT_SIZE 32768 /* default virtual (bin) heap min size (words) */ +#define H_DEFAULT_MAX_SIZE 0 /* default max heap size is off */ #define CP_SIZE 1 @@ -160,6 +161,8 @@ extern int num_instructions; /* Number of instruction in opc[]. */ extern int H_MIN_SIZE; /* minimum (heap + stack) */ extern int BIN_VH_MIN_SIZE; /* minimum virtual (bin) heap */ +extern int H_MAX_SIZE; /* maximum (heap + stack) */ +extern int H_MAX_FLAGS; /* maximum heap flags */ extern int erts_atom_table_size;/* Atom table size */ extern int erts_pd_initial_size;/* Initial Process dictionary table size */ diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 550db95ba8..1abcc6cbf4 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -57,6 +57,7 @@ struct enif_environment_t /* ErlNifEnv */ struct enif_tmp_obj_t* tmp_obj_list; int exception_thrown; /* boolean */ Process *tracee; + int exiting; /* boolean (dirty nifs might return in exiting state) */ }; extern void erts_pre_nif(struct enif_environment_t*, Process*, struct erl_module_nif*, Process* tracee); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index b9f6ab04c6..0377f6cb5e 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -2512,7 +2512,7 @@ erts_port_output(Process *c_p, sigdp->flags &= ~ERTS_P2P_SIG_DATA_FLG_NOSUSPEND; else if (async_nosuspend) { ErtsSchedulerData *esdp = (c_p - ? ERTS_PROC_GET_SCHDATA(c_p) + ? erts_proc_sched_data(c_p) : erts_get_scheduler_data()); ASSERT(esdp); ns_pthp = &esdp->nosuspend_port_task_handle; @@ -5140,7 +5140,7 @@ erts_request_io_bytes(Process *c_p) Uint *hp; Eterm ref; Uint32 *refn; - ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + ErtsSchedulerData *esdp = erts_proc_sched_data(c_p); ErtsIOBytesReq *req = erts_alloc(ERTS_ALC_T_IOB_REQ, sizeof(ErtsIOBytesReq)); @@ -5941,7 +5941,7 @@ driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len) if (!rp) { if (!prt || !IS_TRACED_FL(prt, F_TRACE_SEND)) goto done; - if (!erts_is_tracer_proc_enabled(NULL, 0, &prt->common, am_send)) + if (!erts_is_tracer_proc_enabled_send(NULL, 0, &prt->common)) goto done; res = -2; @@ -7881,11 +7881,13 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size) * (driver version 3.1, NIF version 2.7) */ if (si_size >= ERL_DRV_SYS_INFO_SIZE(dirty_scheduler_support)) { -#if defined(ERL_NIF_DIRTY_SCHEDULER_SUPPORT) && defined(USE_THREADS) - sip->dirty_scheduler_support = 1; + sip->dirty_scheduler_support = +#ifdef ERTS_DIRTY_SCHEDULERS + 1 #else - sip->dirty_scheduler_support = 0; + 0 #endif + ; } } diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index dda3ba565c..f303d4f167 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -34,6 +34,10 @@ (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL))) #endif +#if defined(ERTS_DIRTY_SCHEDULERS) && !defined(ERTS_SMP) +# error "Dirty schedulers not supported without smp support" +#endif + #ifdef ERTS_INLINE # ifndef ERTS_CAN_INLINE # define ERTS_CAN_INLINE 1 diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index 9ad44b25ac..f532d3151f 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -59,6 +59,7 @@ * TODO: check PCB consistency at native BIF calls */ int hipe_modeswitch_debug = 0; +extern BeamInstr beam_exit[]; #define HIPE_DEBUG 0 @@ -509,6 +510,10 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) if (scb) ERTS_PROC_SET_SAVED_CALLS_BUF(p, scb); + /* The process may have died while it was executing, + if so we return out from native code to the interpreter */ + if (erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_EXITING) + p->i = beam_exit; #ifdef DEBUG ASSERT(p->debug_reds_in == reds_in); #endif diff --git a/erts/emulator/nifs/common/erl_tracer_nif.c b/erts/emulator/nifs/common/erl_tracer_nif.c index 8a9a1bf16c..6dddc80607 100644 --- a/erts/emulator/nifs/common/erl_tracer_nif.c +++ b/erts/emulator/nifs/common/erl_tracer_nif.c @@ -70,6 +70,7 @@ ERL_NIF_INIT(erl_tracer, nif_funcs, load, NULL, upgrade, unload) ATOM_DECL(strict_monotonic); \ ATOM_DECL(timestamp); \ ATOM_DECL(trace); \ + ATOM_DECL(trace_status); \ ATOM_DECL(trace_ts); \ ATOM_DECL(true); \ ATOM_DECL(gc_minor_start); \ @@ -118,19 +119,22 @@ static ERL_NIF_TERM enabled(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ErlNifPid to_pid; ErlNifPort to_port; + ERL_NIF_TERM ret = enif_is_identical(argv[0], atom_trace_status) ? + atom_remove : atom_discard; + ASSERT(argc == 3); if (enif_get_local_pid(env, argv[1], &to_pid)) { if (!enif_is_process_alive(env, &to_pid)) /* tracer is dead so we should remove this trace point */ - return atom_remove; + return ret; } else if (enif_get_local_port(env, argv[1], &to_port)) { if (!enif_is_port_alive(env, &to_port)) /* tracer is dead so we should remove this trace point */ - return atom_remove; + return ret; } else { /* The state was not a pid or a port */ - return atom_remove; + return ret; } /* Only generate trace for when tracer != tracee */ diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index de395dfb97..b580211eff 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -53,6 +53,7 @@ MODULES= \ crypto_SUITE \ ddll_SUITE \ decode_packet_SUITE \ + dirty_nif_SUITE \ distribution_SUITE \ driver_SUITE \ efile_SUITE \ diff --git a/erts/emulator/test/dirty_nif_SUITE.erl b/erts/emulator/test/dirty_nif_SUITE.erl new file mode 100644 index 0000000000..c3afbc0803 --- /dev/null +++ b/erts/emulator/test/dirty_nif_SUITE.erl @@ -0,0 +1,327 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2014. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(dirty_nif_SUITE). + +%%-define(line_trace,true). +-define(CHECK(Exp,Got), check(Exp,Got,?LINE)). +%%-define(CHECK(Exp,Got), Exp = Got). + +-include_lib("common_test/include/ct.hrl"). + +-export([all/0, suite/0, + init_per_suite/1, end_per_suite/1, + init_per_testcase/2, end_per_testcase/2, + dirty_nif/1, dirty_nif_send/1, + dirty_nif_exception/1, call_dirty_nif_exception/1, + dirty_scheduler_exit/1, dirty_call_while_terminated/1, + dirty_heap_access/1]). + +-define(nif_stub,nif_stub_error(?LINE)). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [dirty_nif, + dirty_nif_send, + dirty_nif_exception, + dirty_scheduler_exit, + dirty_call_while_terminated, + dirty_heap_access]. + +init_per_suite(Config) -> + try erlang:system_info(dirty_cpu_schedulers) of + N when is_integer(N), N > 0 -> + case lib_loaded() of + false -> + ok = erlang:load_nif( + filename:join(?config(data_dir, Config), + "dirty_nif_SUITE"), []); + true -> + ok + end, + Config + catch _:_ -> + {skipped, "No dirty scheduler support"} + end. + +end_per_suite(_Config) -> + ok. + +init_per_testcase(Case, Config) -> + [{testcase, Case} | Config]. + +end_per_testcase(_Case, _Config) -> + ok. + +dirty_nif(Config) when is_list(Config) -> + Val1 = 42, + Val2 = "Erlang", + Val3 = list_to_binary([Val2, 0]), + {Val1, Val2, Val3} = call_dirty_nif(Val1, Val2, Val3), + LargeArray = lists:duplicate(1000, ok), + LargeArray = call_dirty_nif_zero_args(), + ok. + +dirty_nif_send(Config) when is_list(Config) -> + Parent = self(), + Pid = spawn_link(fun() -> + Self = self(), + {ok, Self} = receive_any(), + Parent ! {ok, Self} + end), + {ok, Pid} = send_from_dirty_nif(Pid), + {ok, Pid} = receive_any(), + ok. + +dirty_nif_exception(Config) when is_list(Config) -> + try + %% this checks that the expected exception occurs when the + %% dirty NIF returns the result of enif_make_badarg + %% directly + call_dirty_nif_exception(1), + ct:fail(expected_badarg) + catch + error:badarg -> + [{?MODULE,call_dirty_nif_exception,[1],_}|_] = + erlang:get_stacktrace(), + ok + end, + try + %% this checks that the expected exception occurs when the + %% dirty NIF calls enif_make_badarg at some point but then + %% returns a value that isn't an exception + call_dirty_nif_exception(0), + ct:fail(expected_badarg) + catch + error:badarg -> + [{?MODULE,call_dirty_nif_exception,[0],_}|_] = + erlang:get_stacktrace(), + ok + end, + %% this checks that a dirty NIF can raise various terms as + %% exceptions + ok = nif_raise_exceptions(call_dirty_nif_exception). + +nif_raise_exceptions(NifFunc) -> + ExcTerms = [{error, test}, "a string", <<"a binary">>, + 42, [1,2,3,4,5], [{p,1},{p,2},{p,3}]], + lists:foldl(fun(Term, ok) -> + try + erlang:apply(?MODULE,NifFunc,[Term]), + ct:fail({expected,Term}) + catch + error:Term -> + [{?MODULE,NifFunc,[Term],_}|_] = erlang:get_stacktrace(), + ok + end + end, ok, ExcTerms). + +dirty_scheduler_exit(Config) when is_list(Config) -> + {ok, Node} = start_node(Config, "+SDio 1"), + Path = proplists:get_value(data_dir, Config), + NifLib = filename:join(Path, atom_to_list(?MODULE)), + [ok] = mcall(Node, + [fun() -> + ok = erlang:load_nif(NifLib, []), + Start = erlang:monotonic_time(milli_seconds), + ok = test_dirty_scheduler_exit(), + End = erlang:monotonic_time(milli_seconds), + io:format("Time=~p ms~n", [End-Start]), + ok + end]), + stop_node(Node), + ok. + +test_dirty_scheduler_exit() -> + process_flag(trap_exit,true), + test_dse(10,[]). +test_dse(0,Pids) -> + timer:sleep(100), + kill_dse(Pids,[]); +test_dse(N,Pids) -> + Pid = spawn_link(fun dirty_sleeper/0), + test_dse(N-1,[Pid|Pids]). + +kill_dse([],Killed) -> + wait_dse(Killed); +kill_dse([Pid|Pids],AlreadyKilled) -> + exit(Pid,kill), + kill_dse(Pids,[Pid|AlreadyKilled]). + +wait_dse([]) -> + ok; +wait_dse([Pid|Pids]) -> + receive + {'EXIT',Pid,Reason} -> + killed = Reason + end, + wait_dse(Pids). + +dirty_call_while_terminated(Config) when is_list(Config) -> + Me = self(), + Bin = list_to_binary(lists:duplicate(4711, $r)), + {value, {BinAddr, 4711, 1}} = lists:keysearch(4711, 2, + element(2, + process_info(self(), + binary))), + {Dirty, DM} = spawn_opt(fun () -> + dirty_call_while_terminated_nif(Me), + blipp:blupp(Bin) + end, + [monitor,link]), + receive {dirty_alive, Pid} -> ok end, + {value, {BinAddr, 4711, 2}} = lists:keysearch(4711, 2, + element(2, + process_info(self(), + binary))), + Reason = die_dirty_process, + OT = process_flag(trap_exit, true), + exit(Dirty, Reason), + receive + {'DOWN', DM, process, Dirty, R0} -> + R0 = Reason + end, + receive + {'EXIT', Dirty, R1} -> + R1 = Reason + end, + undefined = process_info(Dirty), + undefined = process_info(Dirty, status), + false = erlang:is_process_alive(Dirty), + false = lists:member(Dirty, processes()), + %% Binary still refered by Dirty process not yet cleaned up + %% since the dirty nif has not yet returned... + {value, {BinAddr, 4711, 2}} = lists:keysearch(4711, 2, + element(2, + process_info(self(), + binary))), + receive after 2000 -> ok end, + receive + Msg -> + ct:fail({unexpected_message, Msg}) + after + 0 -> + ok + end, + {value, {BinAddr, 4711, 1}} = lists:keysearch(4711, 2, + element(2, + process_info(self(), + binary))), + process_flag(trap_exit, OT), + ok. + +dirty_heap_access(Config) when is_list(Config) -> + {ok, Node} = start_node(Config), + Me = self(), + RGL = rpc:call(Node,erlang,whereis,[init]), + Ref = rpc:call(Node,erlang,make_ref,[]), + Dirty = spawn_link(fun () -> + Res = dirty_heap_access_nif(Ref), + garbage_collect(), + Me ! {self(), Res}, + receive after infinity -> ok end + end), + {N, R} = access_dirty_heap(Dirty, RGL, 0, 0), + receive + {Pid, Res} -> + 1000 = length(Res), + lists:foreach(fun (X) -> Ref = X end, Res) + end, + unlink(Dirty), + exit(Dirty, kill), + stop_node(Node), + {comment, integer_to_list(N) ++ " GL change loops; " + ++ integer_to_list(R) ++ " while running dirty"}. + +access_dirty_heap(Dirty, RGL, N, R) -> + case process_info(Dirty, status) of + {status, waiting} -> + {N, R}; + {status, Status} -> + {group_leader, GL} = process_info(Dirty, group_leader), + true = group_leader(RGL, Dirty), + {group_leader, RGL} = process_info(Dirty, group_leader), + true = group_leader(GL, Dirty), + {group_leader, GL} = process_info(Dirty, group_leader), + access_dirty_heap(Dirty, RGL, N+1, case Status of + running -> + R+1; + _ -> + R + end) + end. + +%% +%% Internal... +%% + +receive_any() -> + receive M -> M end. + +start_node(Config) -> + start_node(Config, ""). + +start_node(Config, Args) when is_list(Config) -> + Pa = filename:dirname(code:which(?MODULE)), + Name = list_to_atom(atom_to_list(?MODULE) + ++ "-" + ++ atom_to_list(proplists:get_value(testcase, Config)) + ++ "-" + ++ integer_to_list(erlang:system_time(seconds)) + ++ "-" + ++ integer_to_list(erlang:unique_integer([positive]))), + test_server:start_node(Name, slave, [{args, "-pa "++Pa++" "++Args}]). + +stop_node(Node) -> + test_server:stop_node(Node). + +mcall(Node, Funs) -> + Parent = self(), + Refs = lists:map(fun (Fun) -> + Ref = make_ref(), + spawn_link(Node, + fun () -> + Res = Fun(), + unlink(Parent), + Parent ! {Ref, Res} + end), + Ref + end, Funs), + lists:map(fun (Ref) -> + receive + {Ref, Res} -> + Res + end + end, Refs). + +%% The NIFs: +lib_loaded() -> false. +call_nif_schedule(_,_) -> ?nif_stub. +call_dirty_nif(_,_,_) -> ?nif_stub. +send_from_dirty_nif(_) -> ?nif_stub. +call_dirty_nif_exception(_) -> ?nif_stub. +call_dirty_nif_zero_args() -> ?nif_stub. +dirty_call_while_terminated_nif(_) -> ?nif_stub. +dirty_sleeper() -> ?nif_stub. +dirty_heap_access_nif(_) -> ?nif_stub. + +nif_stub_error(Line) -> + exit({nif_not_loaded,module,?MODULE,line,Line}). diff --git a/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src b/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src new file mode 100644 index 0000000000..e9301753b0 --- /dev/null +++ b/erts/emulator/test/dirty_nif_SUITE_data/Makefile.src @@ -0,0 +1,6 @@ + +NIF_LIBS = dirty_nif_SUITE@dll@ + +all: $(NIF_LIBS) + +@SHLIB_RULES@ diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c new file mode 100644 index 0000000000..2013c88167 --- /dev/null +++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c @@ -0,0 +1,223 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2014. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +#include "erl_nif.h" +#include <assert.h> +#ifndef __WIN32__ +#include <unistd.h> +#endif + +static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +{ + return 0; +} + +static ERL_NIF_TERM lib_loaded(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + return enif_make_atom(env, "true"); +} + +static int have_dirty_schedulers(void) +{ + ErlNifSysInfo si; + enif_system_info(&si, sizeof(si)); + return si.dirty_scheduler_support; +} + +static ERL_NIF_TERM dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + int n; + char s[10]; + ErlNifBinary b; + if (have_dirty_schedulers()) { + assert(enif_is_on_dirty_scheduler(env)); + } + assert(argc == 3); + enif_get_int(env, argv[0], &n); + enif_get_string(env, argv[1], s, sizeof s, ERL_NIF_LATIN1); + enif_inspect_binary(env, argv[2], &b); + return enif_make_tuple3(env, + enif_make_int(env, n), + enif_make_string(env, s, ERL_NIF_LATIN1), + enif_make_binary(env, &b)); +} + +static ERL_NIF_TERM call_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + int n; + char s[10]; + ErlNifBinary b; + assert(!enif_is_on_dirty_scheduler(env)); + if (argc != 3) + return enif_make_badarg(env); + if (have_dirty_schedulers()) { + if (enif_get_int(env, argv[0], &n) && + enif_get_string(env, argv[1], s, sizeof s, ERL_NIF_LATIN1) && + enif_inspect_binary(env, argv[2], &b)) + return enif_schedule_nif(env, "call_dirty_nif", ERL_NIF_DIRTY_JOB_CPU_BOUND, dirty_nif, argc, argv); + else + return enif_make_badarg(env); + } else { + return dirty_nif(env, argc, argv); + } +} + +static ERL_NIF_TERM send_from_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ERL_NIF_TERM result; + ErlNifPid pid; + ErlNifEnv* menv; + int res; + + if (!enif_get_local_pid(env, argv[0], &pid)) + return enif_make_badarg(env); + result = enif_make_tuple2(env, enif_make_atom(env, "ok"), enif_make_pid(env, &pid)); + menv = enif_alloc_env(); + res = enif_send(env, &pid, menv, result); + enif_free_env(menv); + if (!res) + return enif_make_badarg(env); + else + return result; +} + +static ERL_NIF_TERM call_dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + switch (argc) { + case 1: { + int arg; + if (enif_get_int(env, argv[0], &arg) && arg < 2) { + ERL_NIF_TERM args[255]; + int i; + args[0] = argv[0]; + for (i = 1; i < 255; i++) + args[i] = enif_make_int(env, i); + return enif_schedule_nif(env, "call_dirty_nif_exception", ERL_NIF_DIRTY_JOB_CPU_BOUND, + call_dirty_nif_exception, 255, args); + } else { + return enif_raise_exception(env, argv[0]); + } + } + case 2: { + int return_badarg_directly; + enif_get_int(env, argv[0], &return_badarg_directly); + assert(return_badarg_directly == 1 || return_badarg_directly == 0); + if (return_badarg_directly) + return enif_make_badarg(env); + else { + /* ignore return value */ enif_make_badarg(env); + return enif_make_atom(env, "ok"); + } + } + default: + return enif_schedule_nif(env, "call_dirty_nif_exception", ERL_NIF_DIRTY_JOB_CPU_BOUND, + call_dirty_nif_exception, argc-1, argv); + } +} + +static ERL_NIF_TERM call_dirty_nif_zero_args(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + int i; + ERL_NIF_TERM result[1000]; + ERL_NIF_TERM ok = enif_make_atom(env, "ok"); + assert(argc == 0); + for (i = 0; i < sizeof(result)/sizeof(*result); i++) { + result[i] = ok; + } + return enif_make_list_from_array(env, result, i); +} + +static ERL_NIF_TERM +dirty_sleeper(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + assert(enif_is_on_dirty_scheduler(env)); +#ifdef __WIN32__ + Sleep(6000); +#else + sleep(6); +#endif + return enif_make_atom(env, "ok"); +} + +static ERL_NIF_TERM dirty_call_while_terminated_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ErlNifPid self; + ERL_NIF_TERM result, self_term; + ErlNifPid to; + ErlNifEnv* menv; + int res; + + if (!enif_get_local_pid(env, argv[0], &to)) + return enif_make_badarg(env); + + if (!enif_self(env, &self)) + return enif_make_badarg(env); + + self_term = enif_make_pid(env, &self); + + result = enif_make_tuple2(env, enif_make_atom(env, "dirty_alive"), self_term); + menv = enif_alloc_env(); + res = enif_send(env, &to, menv, result); + enif_free_env(menv); + if (!res) + return enif_make_badarg(env); + + /* Wait until we have been killed */ + while (enif_is_process_alive(env, &self)) + ; + + result = enif_make_tuple2(env, enif_make_atom(env, "dirty_dead"), self_term); + menv = enif_alloc_env(); + res = enif_send(env, &to, menv, result); + enif_free_env(menv); + +#ifdef __WIN32__ + Sleep(1000); +#else + sleep(1); +#endif + + return enif_make_atom(env, "ok"); +} + +static ERL_NIF_TERM dirty_heap_access_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ERL_NIF_TERM res = enif_make_list(env, 0); + int i; + assert(enif_is_on_dirty_scheduler(env)); + for (i = 0; i < 1000; i++) + res = enif_make_list_cell(env, enif_make_copy(env, argv[0]), res); + + return res; +} + + +static ErlNifFunc nif_funcs[] = +{ + {"lib_loaded", 0, lib_loaded}, + {"call_dirty_nif", 3, call_dirty_nif}, + {"send_from_dirty_nif", 1, send_from_dirty_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"call_dirty_nif_exception", 1, call_dirty_nif_exception, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"call_dirty_nif_zero_args", 0, call_dirty_nif_zero_args, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"dirty_sleeper", 0, dirty_sleeper, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"dirty_call_while_terminated_nif", 1, dirty_call_while_terminated_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"dirty_heap_access_nif", 1, dirty_heap_access_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND} +}; + +ERL_NIF_INIT(dirty_nif_SUITE,nif_funcs,load,NULL,NULL,NULL) diff --git a/erts/emulator/test/gc_SUITE.erl b/erts/emulator/test/gc_SUITE.erl index 79c229a34d..8a600b7d9f 100644 --- a/erts/emulator/test/gc_SUITE.erl +++ b/erts/emulator/test/gc_SUITE.erl @@ -25,13 +25,13 @@ -include_lib("common_test/include/ct.hrl"). -export([all/0, suite/0]). --export([grow_heap/1, grow_stack/1, grow_stack_heap/1]). +-export([grow_heap/1, grow_stack/1, grow_stack_heap/1, max_heap_size/1]). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [grow_heap, grow_stack, grow_stack_heap]. + [grow_heap, grow_stack, grow_stack_heap, max_heap_size]. %% Produce a growing list of elements, @@ -163,3 +163,30 @@ show_heap(String) -> {stack_size, SSize}=process_info(self(), stack_size), io:format("Heap/Stack "++String++"~p/~p", [HSize, SSize]). +%% Test that doing a remote GC that triggers the max heap size +%% kills the process. +max_heap_size(_Config) -> + + Pid = spawn_opt(fun long_receive/0,[{max_heap_size, 1024}, + {message_queue_data, on_heap}]), + [Pid ! lists:duplicate(I,I) || I <- lists:seq(1,100)], + Ref = erlang:monitor(process, Pid), + + %% Force messages to be viewed as part of heap + erlang:process_info(Pid, messages), + + %% Do the GC that triggers max heap + erlang:garbage_collect(Pid), + + %% Verify that max heap was triggered + receive + {'DOWN', Ref, process, Pid, killed} -> ok + after 5000 -> + ct:fail({process_did_not_die, Pid, erlang:process_info(Pid)}) + end. + +long_receive() -> + receive + after 10000 -> + ok + end. diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index a7767132ee..a1e1495480 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -38,8 +38,7 @@ is_checks/1, get_length/1, make_atom/1, make_string/1, reverse_list_test/1, otp_9828/1, - otp_9668/1, consume_timeslice/1, dirty_nif/1, dirty_nif_send/1, - dirty_nif_exception/1, call_dirty_nif_exception/1, nif_schedule/1, + otp_9668/1, consume_timeslice/1, nif_schedule/1, nif_exception/1, call_nif_exception/1, nif_nan_and_inf/1, nif_atom_too_long/1, nif_monotonic_time/1, nif_time_offset/1, nif_convert_time_unit/1, @@ -75,8 +74,7 @@ all() -> make_string,reverse_list_test, otp_9828, otp_9668, consume_timeslice, - nif_schedule, dirty_nif, dirty_nif_send, dirty_nif_exception, - nif_exception, nif_nan_and_inf, nif_atom_too_long, + nif_schedule, nif_exception, nif_nan_and_inf, nif_atom_too_long, nif_monotonic_time, nif_time_offset, nif_convert_time_unit, nif_now_time, nif_cpu_time, nif_unique_integer, nif_is_process_alive, nif_is_port_alive, @@ -1541,76 +1539,6 @@ nif_schedule(Config) when is_list(Config) -> end, ok. -dirty_nif(Config) when is_list(Config) -> - try erlang:system_info(dirty_cpu_schedulers) of - N when is_integer(N) -> - ensure_lib_loaded(Config), - Val1 = 42, - Val2 = "Erlang", - Val3 = list_to_binary([Val2, 0]), - {Val1, Val2, Val3} = call_dirty_nif(Val1, Val2, Val3), - LargeArray = lists:duplicate(1000, ok), - LargeArray = call_dirty_nif_zero_args(), - ok - catch - error:badarg -> - {skipped,"No dirty scheduler support"} - end. - -dirty_nif_send(Config) when is_list(Config) -> - try erlang:system_info(dirty_cpu_schedulers) of - N when is_integer(N) -> - ensure_lib_loaded(Config), - Parent = self(), - Pid = spawn_link(fun() -> - Self = self(), - {ok, Self} = receive_any(), - Parent ! {ok, Self} - end), - {ok, Pid} = send_from_dirty_nif(Pid), - {ok, Pid} = receive_any(), - ok - catch - error:badarg -> - {skipped,"No dirty scheduler support"} - end. - -dirty_nif_exception(Config) when is_list(Config) -> - try erlang:system_info(dirty_cpu_schedulers) of - N when is_integer(N) -> - ensure_lib_loaded(Config), - try - %% this checks that the expected exception occurs when the - %% dirty NIF returns the result of enif_make_badarg - %% directly - call_dirty_nif_exception(1), - ct:fail(expected_badarg) - catch - error:badarg -> - [{?MODULE,call_dirty_nif_exception,[1],_}|_] = - erlang:get_stacktrace(), - ok - end, - try - %% this checks that the expected exception occurs when the - %% dirty NIF calls enif_make_badarg at some point but then - %% returns a value that isn't an exception - call_dirty_nif_exception(0), - ct:fail(expected_badarg) - catch - error:badarg -> - [{?MODULE,call_dirty_nif_exception,[0],_}|_] = - erlang:get_stacktrace(), - ok - end, - %% this checks that a dirty NIF can raise various terms as - %% exceptions - ok = nif_raise_exceptions(call_dirty_nif_exception) - catch - error:badarg -> - {skipped,"No dirty scheduler support"} - end. - nif_exception(Config) when is_list(Config) -> ensure_lib_loaded(Config), try @@ -2078,10 +2006,6 @@ otp_9668_nif(_) -> ?nif_stub. otp_9828_nif(_) -> ?nif_stub. consume_timeslice_nif(_,_) -> ?nif_stub. call_nif_schedule(_,_) -> ?nif_stub. -call_dirty_nif(_,_,_) -> ?nif_stub. -send_from_dirty_nif(_) -> ?nif_stub. -call_dirty_nif_exception(_) -> ?nif_stub. -call_dirty_nif_zero_args() -> ?nif_stub. call_nif_exception(_) -> ?nif_stub. call_nif_nan_or_inf(_) -> ?nif_stub. call_nif_atom_too_long(_) -> ?nif_stub. diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index 11e5dab58e..73073ad59f 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -23,6 +23,9 @@ #include <string.h> #include <assert.h> #include <limits.h> +#ifndef __WIN32__ +#include <unistd.h> +#endif #include "nif_mod.h" @@ -1574,120 +1577,6 @@ static ERL_NIF_TERM call_nif_schedule(ErlNifEnv* env, int argc, const ERL_NIF_TE return result; } -#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT - -static int have_dirty_schedulers(void) -{ - ErlNifSysInfo si; - enif_system_info(&si, sizeof(si)); - return si.dirty_scheduler_support; -} - -static ERL_NIF_TERM dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - int n; - char s[10]; - ErlNifBinary b; - ERL_NIF_TERM result; - if (have_dirty_schedulers()) { - assert(enif_is_on_dirty_scheduler(env)); - } - assert(argc == 3); - enif_get_int(env, argv[0], &n); - enif_get_string(env, argv[1], s, sizeof s, ERL_NIF_LATIN1); - enif_inspect_binary(env, argv[2], &b); - return enif_make_tuple3(env, - enif_make_int(env, n), - enif_make_string(env, s, ERL_NIF_LATIN1), - enif_make_binary(env, &b)); -} - -static ERL_NIF_TERM call_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - int n; - char s[10]; - ErlNifBinary b; - assert(!enif_is_on_dirty_scheduler(env)); - if (argc != 3) - return enif_make_badarg(env); - if (have_dirty_schedulers()) { - if (enif_get_int(env, argv[0], &n) && - enif_get_string(env, argv[1], s, sizeof s, ERL_NIF_LATIN1) && - enif_inspect_binary(env, argv[2], &b)) - return enif_schedule_nif(env, "call_dirty_nif", ERL_NIF_DIRTY_JOB_CPU_BOUND, dirty_nif, argc, argv); - else - return enif_make_badarg(env); - } else { - return dirty_nif(env, argc, argv); - } -} - -static ERL_NIF_TERM send_from_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - ERL_NIF_TERM result; - ErlNifPid pid; - ErlNifEnv* menv; - int res; - - if (!enif_get_local_pid(env, argv[0], &pid)) - return enif_make_badarg(env); - result = enif_make_tuple2(env, enif_make_atom(env, "ok"), enif_make_pid(env, &pid)); - menv = enif_alloc_env(); - res = enif_send(env, &pid, menv, result); - enif_free_env(menv); - if (!res) - return enif_make_badarg(env); - else - return result; -} - -static ERL_NIF_TERM call_dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - switch (argc) { - case 1: { - int arg; - if (enif_get_int(env, argv[0], &arg) && arg < 2) { - ERL_NIF_TERM args[255]; - int i; - args[0] = argv[0]; - for (i = 1; i < 255; i++) - args[i] = enif_make_int(env, i); - return enif_schedule_nif(env, "call_dirty_nif_exception", ERL_NIF_DIRTY_JOB_CPU_BOUND, - call_dirty_nif_exception, 255, args); - } else { - return enif_raise_exception(env, argv[0]); - } - } - case 2: { - int return_badarg_directly; - enif_get_int(env, argv[0], &return_badarg_directly); - assert(return_badarg_directly == 1 || return_badarg_directly == 0); - if (return_badarg_directly) - return enif_make_badarg(env); - else { - /* ignore return value */ enif_make_badarg(env); - return enif_make_atom(env, "ok"); - } - } - default: - return enif_schedule_nif(env, "call_dirty_nif_exception", ERL_NIF_DIRTY_JOB_CPU_BOUND, - call_dirty_nif_exception, argc-1, argv); - } -} - -static ERL_NIF_TERM call_dirty_nif_zero_args(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ - int i; - ERL_NIF_TERM result[1000]; - ERL_NIF_TERM ok = enif_make_atom(env, "ok"); - assert(argc == 0); - for (i = 0; i < sizeof(result)/sizeof(*result); i++) { - result[i] = ok; - } - return enif_make_list_from_array(env, result, i); -} -#endif - /* * If argv[0] is the integer 0, call enif_make_badarg, but don't return its * return value. Instead, return ok. Result should still be a badarg @@ -2162,12 +2051,6 @@ static ErlNifFunc nif_funcs[] = {"otp_9828_nif", 1, otp_9828_nif}, {"consume_timeslice_nif", 2, consume_timeslice_nif}, {"call_nif_schedule", 2, call_nif_schedule}, -#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT - {"call_dirty_nif", 3, call_dirty_nif}, - {"send_from_dirty_nif", 1, send_from_dirty_nif, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"call_dirty_nif_exception", 1, call_dirty_nif_exception, ERL_NIF_DIRTY_JOB_IO_BOUND}, - {"call_dirty_nif_zero_args", 0, call_dirty_nif_zero_args, ERL_NIF_DIRTY_JOB_CPU_BOUND}, -#endif {"call_nif_exception", 1, call_nif_exception}, {"call_nif_nan_or_inf", 1, call_nif_nan_or_inf}, {"call_nif_atom_too_long", 1, call_nif_atom_too_long}, diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 61a68f9759..eaa4026a8a 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -46,7 +46,7 @@ process_status_exiting/1, otp_4725/1, bad_register/1, garbage_collect/1, otp_6237/1, process_info_messages/1, process_flag_badarg/1, process_flag_heap_size/1, - spawn_opt_heap_size/1, + spawn_opt_heap_size/1, spawn_opt_max_heap_size/1, processes_large_tab/1, processes_default_tab/1, processes_small_tab/1, processes_this_tab/1, processes_apply_trap/1, processes_last_call_trap/1, processes_gc_trap/1, @@ -60,7 +60,7 @@ system_task_on_suspended/1, gc_request_when_gc_disabled/1, gc_request_blast_when_gc_disabled/1]). --export([prio_server/2, prio_client/2]). +-export([prio_server/2, prio_client/2, init/1, handle_event/2]). -export([init_per_testcase/2, end_per_testcase/2]). @@ -84,7 +84,8 @@ all() -> bump_reductions, low_prio, yield, yield2, otp_4725, bad_register, garbage_collect, process_info_messages, process_flag_badarg, process_flag_heap_size, - spawn_opt_heap_size, otp_6237, {group, processes_bif}, + spawn_opt_heap_size, spawn_opt_max_heap_size, otp_6237, + {group, processes_bif}, {group, otp_7738}, garb_other_running, {group, system_task}]. @@ -425,6 +426,8 @@ t_process_info(Config) when is_list(Config) -> {status, running} = process_info(self(), status), {min_heap_size, 233} = process_info(self(), min_heap_size), {min_bin_vheap_size,46422} = process_info(self(), min_bin_vheap_size), + {max_heap_size, #{ size := 0, kill := true, error_logger := true}} = + process_info(self(), max_heap_size), {current_function,{?MODULE,t_process_info,1}} = process_info(self(), current_function), {current_function,{?MODULE,t_process_info,1}} = @@ -564,6 +567,8 @@ process_info_other_msg(Config) when is_list(Config) -> {min_heap_size, 233} = process_info(Pid, min_heap_size), {min_bin_vheap_size, 46422} = process_info(Pid, min_bin_vheap_size), + {max_heap_size, #{ size := 0, kill := true, error_logger := true}} = + process_info(self(), max_heap_size), Pid ! stop, ok. @@ -914,10 +919,14 @@ process_info_garbage_collection(_Config) -> Parent = self(), Pid = spawn_link( fun() -> + %% We set mqd to off_heap and send an tuple + %% to process in order to force mbuf_size + %% to be used + process_flag(message_queue_data, off_heap), receive go -> ok end, (fun F(0) -> Parent ! deep, - receive ok -> ok end, + receive {ok,_} -> ok end, []; F(N) -> timer:sleep(1), @@ -926,31 +935,52 @@ process_info_garbage_collection(_Config) -> Parent ! shallow, receive done -> ok end end), - {garbage_collection_info, Before} = - erlang:process_info(Pid, garbage_collection_info), + [{garbage_collection_info, Before},{total_heap_size, THSBefore}] = + erlang:process_info(Pid, [garbage_collection_info, total_heap_size]), Pid ! go, receive deep -> ok end, - {_, Deep} = erlang:process_info(Pid, garbage_collection_info), - Pid ! ok, receive shallow -> ok end, - {_, After} = erlang:process_info(Pid, garbage_collection_info), + [{_, Deep},{_,THSDeep}] = + erlang:process_info(Pid, [garbage_collection_info, total_heap_size]), + Pid ! {ok, make_ref()}, receive shallow -> ok end, + [{_, After},{_, THSAfter}] = + erlang:process_info(Pid, [garbage_collection_info, total_heap_size]), Pid ! done, %% Do some general checks to see if everything seems to be roughly correct ct:log("Before: ~p",[Before]), ct:log("Deep: ~p",[Deep]), ct:log("After: ~p",[After]), + ct:log("Before THS: ~p",[THSBefore]), + ct:log("Deep THS: ~p",[THSDeep]), + ct:log("After THS: ~p",[THSAfter]), %% Check stack_size - true = proplists:get_value(stack_size, Before) < proplists:get_value(stack_size, Deep), - true = proplists:get_value(stack_size, After) < proplists:get_value(stack_size, Deep), + true = gv(stack_size, Before) < gv(stack_size, Deep), + true = gv(stack_size, After) < gv(stack_size, Deep), %% Check used heap size - true = proplists:get_value(heap_size, Before) + proplists:get_value(old_heap_size, Before) - < proplists:get_value(heap_size, Deep) + proplists:get_value(old_heap_size, Deep), - true = proplists:get_value(heap_size, Before) + proplists:get_value(old_heap_size, Before) - < proplists:get_value(heap_size, After) + proplists:get_value(old_heap_size, After), + true = gv(heap_size, Before) + gv(old_heap_size, Before) + < gv(heap_size, Deep) + gv(old_heap_size, Deep), + true = gv(heap_size, Before) + gv(old_heap_size, Before) + < gv(heap_size, After) + gv(old_heap_size, After), + + %% Check that total_heap_size == heap_block_size + old_heap_block_size + mbuf_size + THSBefore = gv(heap_block_size, Before) + + gv(old_heap_block_size, Before) + + gv(mbuf_size, Before), + + THSDeep = gv(heap_block_size, Deep) + + gv(old_heap_block_size, Deep) + + gv(mbuf_size, Deep), + + THSAfter = gv(heap_block_size, After) + + gv(old_heap_block_size, After) + + gv(mbuf_size, After), ok. +gv(Key,List) -> + proplists:get_value(Key,List). + %% Tests erlang:bump_reductions/1. bump_reductions(Config) when is_list(Config) -> erlang:garbage_collect(), @@ -1323,6 +1353,28 @@ process_flag_badarg(Config) when is_list(Config) -> chk_badarg(fun () -> process_flag(min_heap_size, gurka) end), chk_badarg(fun () -> process_flag(min_bin_vheap_size, gurka) end), chk_badarg(fun () -> process_flag(min_bin_vheap_size, -1) end), + + chk_badarg(fun () -> process_flag(max_heap_size, gurka) end), + chk_badarg(fun () -> process_flag(max_heap_size, -1) end), + chk_badarg(fun () -> + {_,Min} = process_info(self(), min_heap_size), + process_flag(max_heap_size, Min - 1) + end), + chk_badarg(fun () -> + {_,Min} = process_info(self(), min_heap_size), + process_flag(max_heap_size, #{size => Min - 1}) + end), + chk_badarg(fun () -> process_flag(max_heap_size, #{}) end), + chk_badarg(fun () -> process_flag(max_heap_size, #{ kill => true }) end), + chk_badarg(fun () -> process_flag(max_heap_size, #{ size => 233, + kill => gurka }) end), + chk_badarg(fun () -> process_flag(max_heap_size, #{ size => 233, + error_logger => gurka }) end), + chk_badarg(fun () -> process_flag(max_heap_size, #{ size => 233, + kill => true, + error_logger => gurka }) end), + chk_badarg(fun () -> process_flag(max_heap_size, #{ size => 1 bsl 64 }) end), + chk_badarg(fun () -> process_flag(priority, 4711) end), chk_badarg(fun () -> process_flag(save_calls, hmmm) end), P= spawn_link(fun () -> receive die -> ok end end), @@ -1923,6 +1975,110 @@ spawn_opt_heap_size(Config) when is_list(Config) -> Pid ! stop, ok. +spawn_opt_max_heap_size(_Config) -> + + error_logger:add_report_handler(?MODULE, self()), + + %% Test that numerical limit works + max_heap_size_test(1024, 1024, true, true), + + %% Test that map limit works + max_heap_size_test(#{ size => 1024 }, 1024, true, true), + + %% Test that no kill is sent + max_heap_size_test(#{ size => 1024, kill => false }, 1024, false, true), + + %% Test that no error_logger report is sent + max_heap_size_test(#{ size => 1024, error_logger => false }, 1024, true, false), + + %% Test that system_flag works + erlang:system_flag(max_heap_size, #{ size => 0, kill => false, + error_logger => true}), + max_heap_size_test(#{ size => 1024 }, 1024, false, true), + max_heap_size_test(#{ size => 1024, kill => true }, 1024, true, true), + + erlang:system_flag(max_heap_size, #{ size => 0, kill => true, + error_logger => false}), + max_heap_size_test(#{ size => 1024 }, 1024, true, false), + max_heap_size_test(#{ size => 1024, error_logger => true }, 1024, true, true), + + erlang:system_flag(max_heap_size, #{ size => 1 bsl 20, kill => true, + error_logger => true}), + max_heap_size_test(#{ }, 1 bsl 20, true, true), + + erlang:system_flag(max_heap_size, #{ size => 0, kill => true, + error_logger => true}), + + %% Test that ordinary case works as expected again + max_heap_size_test(1024, 1024, true, true), + + ok. + +max_heap_size_test(Option, Size, Kill, ErrorLogger) + when map_size(Option) == 0 -> + max_heap_size_test([], Size, Kill, ErrorLogger); +max_heap_size_test(Option, Size, Kill, ErrorLogger) + when is_map(Option); is_integer(Option) -> + max_heap_size_test([{max_heap_size, Option}], Size, Kill, ErrorLogger); +max_heap_size_test(Option, Size, Kill, ErrorLogger) -> + OomFun = fun F() -> timer:sleep(5),[lists:seq(1,1000)|F()] end, + Pid = spawn_opt(OomFun, Option), + {max_heap_size, MHSz} = erlang:process_info(Pid, max_heap_size), + ct:log("Default: ~p~nOption: ~p~nProc: ~p~n", + [erlang:system_info(max_heap_size), Option, MHSz]), + + #{ size := Size} = MHSz, + + Ref = erlang:monitor(process, Pid), + if Kill -> + receive + {'DOWN', Ref, process, Pid, killed} -> + ok + end; + true -> + ok + end, + if ErrorLogger -> + receive + {error, _, {emulator, _, [Pid|_]}} -> + ok + end; + true -> + ok + end, + if not Kill -> + exit(Pid, die), + receive + {'DOWN', Ref, process, Pid, die} -> + ok + end, + flush(); + true -> + ok + end, + receive + M -> + ct:fail({unexpected_message, M}) + after 10 -> + ok + end. + +flush() -> + receive + _M -> + flush() + after 1000 -> + ok + end. + +%% error_logger report handler proxy +init(Pid) -> + {ok, Pid}. + +handle_event(Event, Pid) -> + Pid ! Event, + {ok, Pid}. + processes_term_proc_list(Config) when is_list(Config) -> Tester = self(), as_expected = processes_term_proc_list_test(false), diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl index 6b49b68ec8..f18d79d770 100644 --- a/erts/emulator/test/scheduler_SUITE.erl +++ b/erts/emulator/test/scheduler_SUITE.erl @@ -57,7 +57,6 @@ scheduler_suspend_basic/1, scheduler_suspend/1, dirty_scheduler_threads/1, - dirty_scheduler_exit/1, reader_groups/1]). suite() -> @@ -72,7 +71,7 @@ all() -> bound_process, {group, scheduler_bind}, scheduler_threads, scheduler_suspend_basic, scheduler_suspend, - dirty_scheduler_threads, dirty_scheduler_exit, + dirty_scheduler_threads, reader_groups]. groups() -> @@ -1162,53 +1161,6 @@ get_dsstate(Config, Cmd) -> stop_node(Node), {DSCPU, DSCPUOnln, DSIO}. -dirty_scheduler_exit(Config) when is_list(Config) -> - try - erlang:system_info(dirty_cpu_schedulers), - dirty_scheduler_exit_test(Config) - catch - error:badarg -> - {skipped, "No dirty scheduler support"} - end. - -dirty_scheduler_exit_test(Config) -> - {ok, Node} = start_node(Config, "+SDio 1"), - [ok] = mcall(Node, - [fun() -> - Path = proplists:get_value(data_dir, Config), - Lib = atom_to_list(?MODULE), - ok = erlang:load_nif(filename:join(Path,Lib), []), - ok = test_dirty_scheduler_exit() - end]), - stop_node(Node), - ok. - -test_dirty_scheduler_exit() -> - process_flag(trap_exit,true), - test_dse(10,[]). -test_dse(0,Pids) -> - timer:sleep(100), - kill_dse(Pids,[]); -test_dse(N,Pids) -> - Pid = spawn_link(fun dirty_sleeper/0), - test_dse(N-1,[Pid|Pids]). -kill_dse([],Killed) -> - wait_dse(Killed); -kill_dse([Pid|Pids],AlreadyKilled) -> - exit(Pid,kill), - kill_dse(Pids,[Pid|AlreadyKilled]). -wait_dse([]) -> - ok; -wait_dse([Pid|Pids]) -> - receive - {'EXIT',Pid,killed} -> - ok - end, - wait_dse(Pids). - -dirty_sleeper() -> - erlang:nif_error({error,?MODULE}). - scheduler_suspend_basic(Config) when is_list(Config) -> case erlang:system_info(multi_scheduling) of disabled -> diff --git a/erts/emulator/test/scheduler_SUITE_data/Makefile.src b/erts/emulator/test/scheduler_SUITE_data/Makefile.src deleted file mode 100644 index 859112cf19..0000000000 --- a/erts/emulator/test/scheduler_SUITE_data/Makefile.src +++ /dev/null @@ -1,8 +0,0 @@ - -SCHEDULER_LIBS = scheduler_SUITE@dll@ - -all: $(SCHEDULER_LIBS) - -@SHLIB_RULES@ - -$(SCHEDULER_LIBS): scheduler_SUITE.c diff --git a/erts/emulator/test/scheduler_SUITE_data/scheduler_SUITE.c b/erts/emulator/test/scheduler_SUITE_data/scheduler_SUITE.c deleted file mode 100644 index ab4863337f..0000000000 --- a/erts/emulator/test/scheduler_SUITE_data/scheduler_SUITE.c +++ /dev/null @@ -1,37 +0,0 @@ -#ifndef __WIN32__ -#include <unistd.h> -#endif -#include "erl_nif.h" - -static int -load(ErlNifEnv* env, void** priv, ERL_NIF_TERM info) -{ - ErlNifSysInfo sys_info; - enif_system_info(&sys_info, sizeof(ErlNifSysInfo)); - if (!sys_info.smp_support || !sys_info.dirty_scheduler_support) - return 1; - return 0; -} - -static ERL_NIF_TERM -dirty_sleeper(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) -{ -#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT -#ifdef __WIN32__ - Sleep(3000); -#else - sleep(3); -#endif -#endif - return enif_make_atom(env, "ok"); -} - -static ErlNifFunc funcs[] = { -#ifdef ERL_NIF_DIRTY_SCHEDULER_SUPPORT - {"dirty_sleeper", 0, dirty_sleeper, ERL_NIF_DIRTY_JOB_IO_BOUND} -#else - {"dirty_sleeper", 0, dirty_sleeper, 0} -#endif -}; - -ERL_NIF_INIT(scheduler_SUITE, funcs, &load, NULL, NULL, NULL); diff --git a/erts/emulator/test/trace_bif_SUITE.erl b/erts/emulator/test/trace_bif_SUITE.erl index 8c3ffccc45..491b37ae46 100644 --- a/erts/emulator/test/trace_bif_SUITE.erl +++ b/erts/emulator/test/trace_bif_SUITE.erl @@ -232,6 +232,7 @@ do_trace_bif_return(TsType, TsFlags) -> {?MODULE, bif_process,0}}, Ts11, TsType), check_ts(TsType, Ts12, make_ts(TsType)), + erlang:trace_pattern({erlang,'_','_'}, false, [local]), ok. diff --git a/erts/emulator/test/tracer_SUITE.erl b/erts/emulator/test/tracer_SUITE.erl index de44d6656a..20fb7e475e 100644 --- a/erts/emulator/test/tracer_SUITE.erl +++ b/erts/emulator/test/tracer_SUITE.erl @@ -44,6 +44,8 @@ groups() -> gc_start, gc_end]}]. init_per_suite(Config) -> + erlang:trace_pattern({'_','_','_'}, false, [local]), + erlang:trace_pattern({'_','_','_'}, false, []), purge(), Config. @@ -119,23 +121,24 @@ unload(_Config) -> end, 1 = erlang:trace(Pid, true, [{tracer, tracer_test, - {#{ call => trace}, self(), []}}, + {#{ call => trace }, self(), []}}, call]), 1 = erlang:trace_pattern({?MODULE, all, 0}, [], []), Tc(1), - receive _ -> ok after 0 -> ct:fail(timeout) end, + receive _M -> ok after 0 -> ct:fail(timeout) end, + receive M0 -> ct:fail({unexpected_message0, M0}) after 0 -> ok end, code:purge(tracer_test), code:delete(tracer_test), Tc(1), - receive M1 -> ct:fail({unexpected_message, M1}) after 0 -> ok end, + receive M1 -> ct:fail({unexpected_message1, M1}) after 0 -> ok end, code:purge(tracer_test), Tc(1), - receive M2 -> ct:fail({unexpected_message, M2}) after 0 -> ok end, + receive M2 -> ct:fail({unexpected_message2, M2}) after 0 -> ok end, ok. @@ -167,13 +170,19 @@ reload(_Config) -> false = code:purge(tracer_test), true = code:delete(tracer_test), - false = code:purge(tracer_test) + false = code:purge(tracer_test), + timer:sleep(10) end || _ <- lists:seq(1,15)], ok. reload_loop() -> ?MODULE:all(), + ?MODULE:all(), + ?MODULE:all(), + ?MODULE:all(), + ?MODULE:all(), + timer:sleep(1), reload_loop(). invalid_tracers(_Config) -> @@ -214,7 +223,7 @@ send(_Config) -> Expect = fun(Pid, State, EOpts) -> receive Msg -> - {Pid, send, State, Pid, ok, Self, Opts} = Msg, + {send, State, Pid, ok, Self, Opts} = Msg, check_opts(EOpts, Opts) end end, @@ -230,7 +239,7 @@ recv(_Config) -> Expect = fun(Pid, State, EOpts) -> receive Msg -> - {undefined, 'receive', State, Pid, ok, undefined, Opts} = Msg, + {'receive', State, Pid, ok, undefined, Opts} = Msg, check_opts(EOpts, Opts) end end, @@ -247,14 +256,14 @@ spawn(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {Pid, spawn, State, Pid, NewPid, + {spawn, State, Pid, NewPid, {lists,seq,[1,10]}, Opts} = Msg, check_opts(EOpts, Opts), true = is_pid(NewPid) andalso NewPid /= Pid end end, - test(spawn, procs, Tc, Expect, true). + test(spawn, procs, Tc, Expect, false). exit(_Config) -> Tc = fun(Pid) -> @@ -265,7 +274,7 @@ exit(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {Pid, exit, State, Pid, normal, undefined, Opts} = Msg, + {exit, State, Pid, normal, undefined, Opts} = Msg, check_opts(EOpts, Opts) end end, @@ -286,13 +295,13 @@ link(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {Pid, link, State, Pid, NewPid, undefined, Opts} = Msg, + {link, State, Pid, NewPid, undefined, Opts} = Msg, check_opts(EOpts, Opts), true = is_pid(NewPid) andalso NewPid /= Pid end end, - test(link, procs, Tc, Expect, true). + test(link, procs, Tc, Expect, false). unlink(_Config) -> @@ -309,13 +318,13 @@ unlink(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {Pid, unlink, State, Pid, NewPid, undefined, Opts} = Msg, + {unlink, State, Pid, NewPid, undefined, Opts} = Msg, check_opts(EOpts, Opts), true = is_pid(NewPid) andalso NewPid /= Pid end end, - test(unlink, procs, Tc, Expect, true). + test(unlink, procs, Tc, Expect, false). getting_linked(_Config) -> @@ -331,7 +340,7 @@ getting_linked(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {NewPid, getting_linked, State, Pid, NewPid, undefined, Opts} = Msg, + {getting_linked, State, Pid, NewPid, undefined, Opts} = Msg, check_opts(EOpts, Opts), true = is_pid(NewPid) andalso NewPid /= Pid end @@ -355,7 +364,7 @@ getting_unlinked(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {NewPid, getting_unlinked, State, Pid, NewPid, undefined, Opts} = Msg, + {getting_unlinked, State, Pid, NewPid, undefined, Opts} = Msg, check_opts(EOpts, Opts), true = is_pid(NewPid) andalso NewPid /= Pid end @@ -377,12 +386,12 @@ register(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {Pid, register, State, Pid, ?MODULE, undefined, Opts} = Msg, + {register, State, Pid, ?MODULE, undefined, Opts} = Msg, check_opts(EOpts, Opts) end end, - test(register, procs, Tc, Expect, true). + test(register, procs, Tc, Expect, false). unregister(_Config) -> @@ -398,18 +407,18 @@ unregister(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {Pid, unregister, State, Pid, ?MODULE, undefined, Opts} = Msg, + {unregister, State, Pid, ?MODULE, undefined, Opts} = Msg, check_opts(EOpts, Opts) end end, - test(unregister, procs, Tc, Expect, true). + test(unregister, procs, Tc, Expect, false). in(_Config) -> Tc = fun(Pid) -> Self = self(), - Pid ! fun() -> receive after 1 -> Self ! ok end end, + Pid ! fun() -> receive after 10 -> Self ! ok end end, receive ok -> ok end end, @@ -418,7 +427,7 @@ in(_Config) -> N = (fun F(N) -> receive Msg -> - {Pid, in, State, Pid, _, + {in, State, Pid, _, undefined, Opts} = Msg, check_opts(EOpts, Opts), F(N+1) @@ -428,7 +437,7 @@ in(_Config) -> true = N > 0 end, - test(in, running, Tc, Expect, true). + test(in, running, Tc, Expect, false). out(_Config) -> Tc = fun(Pid) -> @@ -443,7 +452,7 @@ out(_Config) -> N = (fun F(N) -> receive Msg -> - {Pid, out, State, Pid, _, + {out, State, Pid, _, undefined, Opts} = Msg, check_opts(EOpts, Opts), F(N+1) @@ -453,7 +462,7 @@ out(_Config) -> true = N > 0 end, - test(out, running, Tc, Expect, true, true). + test(out, running, Tc, Expect, false, true). gc_start(_Config) -> @@ -468,12 +477,12 @@ gc_start(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {Pid, gc_major_start, State, Pid, _, undefined, Opts} = Msg, + {gc_major_start, State, Pid, _, undefined, Opts} = Msg, check_opts(EOpts, Opts) end end, - test(gc_major_start, garbage_collection, Tc, Expect, true). + test(gc_major_start, garbage_collection, Tc, Expect, false). gc_end(_Config) -> @@ -488,20 +497,20 @@ gc_end(_Config) -> fun(Pid, State, EOpts) -> receive Msg -> - {Pid, gc_major_end, State, Pid, _, undefined, Opts} = Msg, + {gc_major_end, State, Pid, _, undefined, Opts} = Msg, check_opts(EOpts, Opts) end end, - test(gc_major_end, garbage_collection, Tc, Expect, true). + test(gc_major_end, garbage_collection, Tc, Expect, false). test(Event, Tc, Expect) -> - test(Event, Tc, Expect, true). + test(Event, Tc, Expect, false). test(Event, Tc, Expect, Removes) -> test(Event, Event, Tc, Expect, Removes). test(Event, TraceFlag, Tc, Expect, Removes) -> test(Event, TraceFlag, Tc, Expect, Removes, false). -test(Event, TraceFlag, Tc, Expect, Removes, Dies) -> +test(Event, TraceFlag, Tc, Expect, _Removes, Dies) -> ComplexState = {fun() -> ok end, <<0:(128*8)>>}, Opts = #{ timestamp => undefined, @@ -557,25 +566,6 @@ test(Event, TraceFlag, Tc, Expect, Removes, Dies) -> ok end, - %% Test that remove works - Pid3 = start_tracee(), - State3 = {#{ Event => remove }, self(), ComplexState}, - 1 = erlang:trace(Pid3, true, [TraceFlag, {tracer, tracer_test, State3}]), - Tc(Pid3), - ok = trace_delivered(Pid3), - receive M3 -> ct:fail({unexpected, M3}) after 0 -> ok end, - if not Dies -> - if Removes -> - {flags, []} = erlang:trace_info(Pid3, flags), - {tracer, []} = erlang:trace_info(Pid3, tracer); - true -> - {flags, [TraceFlag]} = erlang:trace_info(Pid3, flags), - {tracer, {tracer_test, State3}} = erlang:trace_info(Pid3, tracer) - end, - erlang:trace(Pid3, false, [TraceFlag]); - true -> - ok - end, ok. check_opts(#{ scheduler_id := number } = E, #{ scheduler_id := N } = O) diff --git a/erts/emulator/test/tracer_SUITE_data/tracer_test.c b/erts/emulator/test/tracer_SUITE_data/tracer_test.c index 8b4be1345d..908f35da9c 100644 --- a/erts/emulator/test/tracer_SUITE_data/tracer_test.c +++ b/erts/emulator/test/tracer_SUITE_data/tracer_test.c @@ -104,16 +104,10 @@ static ERL_NIF_TERM trace(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) enif_get_tuple(env, argv[1], &state_arity, &state_tuple); - tuple = enif_alloc(sizeof(ERL_NIF_TERM)*(argc+1)); - memcpy(tuple+1,argv,sizeof(ERL_NIF_TERM)*argc); + tuple = enif_alloc(sizeof(ERL_NIF_TERM)*(argc)); + memcpy(tuple,argv,sizeof(ERL_NIF_TERM)*argc); - if (enif_self(env, &self)) { - tuple[0] = enif_make_pid(env, &self); - } else { - tuple[0] = enif_make_atom(env, "undefined"); - } - - msg = enif_make_tuple_from_array(env, tuple, argc + 1); + msg = enif_make_tuple_from_array(env, tuple, argc); enif_get_local_pid(env, state_tuple[1], &to); enif_send(env, &to, NULL, msg); enif_free(tuple); |