diff options
Diffstat (limited to 'erts/emulator/beam/erl_process.c')
| -rw-r--r-- | erts/emulator/beam/erl_process.c | 2820 |
1 files changed, 2090 insertions, 730 deletions
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 5ceb4ce9a8..0fa2def5af 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -28,7 +28,6 @@ #include "erl_vm.h" #include "global.h" #include "erl_process.h" -#include "erl_nmgc.h" #include "error.h" #include "bif.h" #include "erl_db.h" @@ -39,6 +38,10 @@ #include "erl_binary.h" #include "beam_bp.h" #include "erl_cpu_topology.h" +#include "erl_thr_progress.h" +#include "erl_thr_queue.h" +#include "erl_async.h" +#include "dtrace-wrapper.h" #define ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED (2000*CONTEXT_REDS) #define ERTS_RUNQ_CALL_CHECK_BALANCE_REDS \ @@ -48,21 +51,22 @@ #define ERTS_SCHED_SPIN_UNTIL_YIELD 100 -#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT 10 +#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG 40 +#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_LONG 1000 +#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_LONG 20 +#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_LONG 1000 +#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM 10 +#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM 1000 +#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_SHORT 10 +#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_SHORT 0 +#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_SHORT 5 +#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_SHORT 0 +#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE 0 +#define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_NONE 0 + #define ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT 1000 -#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT \ - (ERTS_SCHED_SYS_SLEEP_SPINCOUNT*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT) #define ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT 0 -#define ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH (200*CONTEXT_REDS) -#define ERTS_WAKEUP_OTHER_LIMIT_HIGH (50*CONTEXT_REDS) -#define ERTS_WAKEUP_OTHER_LIMIT_MEDIUM (10*CONTEXT_REDS) -#define ERTS_WAKEUP_OTHER_LIMIT_LOW (CONTEXT_REDS) -#define ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW (CONTEXT_REDS/10) - -#define ERTS_WAKEUP_OTHER_DEC 10 -#define ERTS_WAKEUP_OTHER_FIXED_INC (CONTEXT_REDS/10) - #if 0 || defined(DEBUG) #define ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA #endif @@ -101,6 +105,9 @@ do { \ #define ERTS_EMPTY_RUNQ(RQ) \ ((RQ)->len == 0 && (RQ)->misc.start == NULL) +#define ERTS_EMPTY_RUNQ_PORTS(RQ) \ + ((RQ)->ports.info.len == 0 && (RQ)->misc.start == NULL) + extern BeamInstr beam_apply[]; extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; @@ -111,22 +118,28 @@ static Sint p_serial; static Uint p_serial_mask; static Uint p_serial_shift; +int erts_sched_compact_load; Uint erts_no_schedulers; Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES; Uint erts_process_tab_index_mask; -static int wakeup_other_limit; - int erts_sched_thread_suggested_stack_size = -1; #ifdef ERTS_ENABLE_LOCK_CHECK ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; #endif -#ifdef ERTS_SMP +static struct { + int aux_work; + int tse; + int sys_schedule; +} sched_busy_wait; +#ifdef ERTS_SMP int erts_disable_proc_not_running_opt; +static ErtsAuxWorkData *aux_thread_aux_work_data; + #define ERTS_SCHDLR_SSPND_CHNG_WAITER (((erts_aint32_t) 1) << 0) #define ERTS_SCHDLR_SSPND_CHNG_MSB (((erts_aint32_t) 1) << 1) #define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2) @@ -191,8 +204,6 @@ do { \ erts_sched_stat_t erts_sched_stat; -ErtsRunQueue *erts_common_run_queue; - #ifdef USE_THREADS static erts_tsd_key_t sched_data_key; #endif @@ -213,8 +224,6 @@ Uint erts_no_run_queues; ErtsAlignedSchedulerData *erts_aligned_scheduler_data; -#ifdef ERTS_SMP - typedef union { ErtsSchedulerSleepInfo ssi; char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerSleepInfo))]; @@ -222,18 +231,11 @@ typedef union { static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; -#endif - -#ifndef BM_COUNTERS -static int processes_busy; -#endif - Process** process_tab; static Uint last_reductions; static Uint last_exact_reductions; Uint erts_default_process_flags; Eterm erts_system_monitor; -Eterm erts_system_monitor_msg_queue_len; Eterm erts_system_monitor_long_gc; Eterm erts_system_monitor_large_heap; struct erts_system_monitor_flags_t erts_system_monitor_flags; @@ -242,11 +244,6 @@ struct erts_system_monitor_flags_t erts_system_monitor_flags; Eterm erts_system_profile; struct erts_system_profile_flags_t erts_system_profile_flags; -#ifdef HYBRID -Uint erts_num_active_procs; -Process** erts_active_procs; -#endif - #if ERTS_MAX_PROCESSES > 0x7fffffff #error "Need to store process_count in another type" #endif @@ -285,8 +282,9 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, ERTS_ALC_T_PROC_LIST) #define ERTS_SCHED_SLEEP_INFO_IX(IX) \ - (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ - &aligned_sched_sleep_info[(IX)].ssi) + (ASSERT_EXPR(-1 <= ((int) (IX)) \ + && ((int) (IX)) < ((int) erts_no_schedulers)), \ + &aligned_sched_sleep_info[(IX)].ssi) #define ERTS_FOREACH_RUNQ(RQVAR, DO) \ do { \ @@ -339,6 +337,55 @@ static void exec_misc_ops(ErtsRunQueue *); static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg); + +static void aux_work_timeout(void *unused); +static void aux_work_timeout_early_init(int no_schedulers); +static void aux_work_timeout_late_init(void); +static void setup_aux_work_timer(void); + +#if defined(DEBUG) || 0 +#define ERTS_DBG_CHK_AUX_WORK_VAL(V) dbg_chk_aux_work_val((V)) +static void +dbg_chk_aux_work_val(erts_aint32_t value) +{ + erts_aint32_t valid = 0; + + valid |= ERTS_SSI_AUX_WORK_SET_TMO; + valid |= ERTS_SSI_AUX_WORK_MISC; + valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM; + valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; +#if ERTS_USE_ASYNC_READY_Q + valid |= ERTS_SSI_AUX_WORK_ASYNC_READY; + valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; +#endif +#ifdef ERTS_SMP + valid |= ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP; + valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; + valid |= ERTS_SSI_AUX_WORK_DD; + valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR; +#endif +#if HAVE_ERTS_MSEG + valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; +#endif +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; +#endif +#ifdef ERTS_SSI_AUX_WORK_REAP_PORTS + valid |= ERTS_SSI_AUX_WORK_REAP_PORTS; +#endif + + if (~valid & value) + erl_exit(ERTS_ABORT_EXIT, + "Invalid aux_work value found: 0x%x\n", + ~valid & value); +} +#define ERTS_DBG_CHK_SSI_AUX_WORK(SSI) \ + ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&(SSI)->aux_work)) +#else +#define ERTS_DBG_CHK_AUX_WORK_VAL(V) +#define ERTS_DBG_CHK_SSI_AUX_WORK(SSI) +#endif + #ifdef ERTS_SMP static void handle_pending_exiters(ErtsProcList *); @@ -420,12 +467,6 @@ erts_init_process(int ncpu) process_tab = (Process**) erts_alloc(ERTS_ALC_T_PROC_TABLE, erts_max_processes*sizeof(Process*)); sys_memzero(process_tab, erts_max_processes * sizeof(Process*)); -#ifdef HYBRID - erts_active_procs = (Process**) - erts_alloc(ERTS_ALC_T_ACTIVE_PROCS, - erts_max_processes * sizeof(Process*)); - erts_num_active_procs = 0; -#endif erts_smp_mtx_init(&proc_tab_mtx, "proc_tab"); p_last = -1; @@ -435,9 +476,6 @@ erts_init_process(int ncpu) p_serial_shift = erts_fit_in_bits(erts_max_processes - 1); p_serial_mask = ((~(~((Uint) 0) << proc_bits)) >> p_serial_shift); erts_process_tab_index_mask = ~(~((Uint) 0) << p_serial_shift); -#ifndef BM_COUNTERS - processes_busy = 0; -#endif last_reductions = 0; last_exact_reductions = 0; erts_default_process_flags = 0; @@ -484,6 +522,213 @@ erts_late_init_process(void) } +static void +init_sched_wall_time(ErtsSchedWallTime *swtp) +{ + swtp->enabled = 0; + swtp->start = 0; + swtp->working.total = 0; + swtp->working.start = 0; + swtp->working.currently = 0; +} + +static ERTS_INLINE Uint64 +sched_wall_time_ts(void) +{ +#ifdef HAVE_GETHRTIME + return (Uint64) sys_gethrtime(); +#else + Uint64 res; + SysTimeval tv; + sys_gettimeofday(&tv); + res = (Uint64) tv.tv_sec*1000000; + res += (Uint64) tv.tv_usec; + return res; +#endif +} + +static ERTS_INLINE void +sched_wall_time_change(ErtsSchedulerData *esdp, int working) +{ + if (esdp->sched_wall_time.enabled) { + Uint64 ts = sched_wall_time_ts(); + if (working) { +#ifdef DEBUG + ASSERT(!esdp->sched_wall_time.working.currently); + esdp->sched_wall_time.working.currently = 1; +#endif + ts -= esdp->sched_wall_time.start; + esdp->sched_wall_time.working.start = ts; + } + else { +#ifdef DEBUG + ASSERT(esdp->sched_wall_time.working.currently); + esdp->sched_wall_time.working.currently = 0; +#endif + ts -= esdp->sched_wall_time.start; + ts -= esdp->sched_wall_time.working.start; + esdp->sched_wall_time.working.total += ts; + } + } +} + +typedef struct { + int set; + int enable; + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + Uint req_sched; + erts_smp_atomic32_t refc; +} ErtsSchedWallTimeReq; + +#if !HALFWORD_HEAP +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq, + ErtsSchedWallTimeReq, + 5, + ERTS_ALC_T_SCHED_WTIME_REQ) +#else +static ERTS_INLINE ErtsSchedWallTimeReq * +swtreq_alloc(void) +{ + return erts_alloc(ERTS_ALC_T_SCHED_WTIME_REQ, + sizeof(ErtsSchedWallTimeReq)); +} + +static ERTS_INLINE void +swtreq_free(ErtsSchedWallTimeReq *ptr) +{ + erts_free(ERTS_ALC_T_SCHED_WTIME_REQ, ptr); +} +#endif + +static void +reply_sched_wall_time(void *vswtrp) +{ + Uint64 working = 0, total = 0; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsSchedWallTimeReq *swtrp = (ErtsSchedWallTimeReq *) vswtrp; + ErtsProcLocks rp_locks = (swtrp->req_sched == esdp->no + ? ERTS_PROC_LOCK_MAIN + : 0); + Process *rp = swtrp->proc; + Eterm ref_copy = NIL, msg; + Eterm *hp = NULL; + Eterm **hpp; + Uint sz, *szp; + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + + ASSERT(esdp); + + if (swtrp->set) { + if (!swtrp->enable && esdp->sched_wall_time.enabled) + esdp->sched_wall_time.enabled = 0; + else if (swtrp->enable && !esdp->sched_wall_time.enabled) { + Uint64 ts = sched_wall_time_ts(); + esdp->sched_wall_time.enabled = 1; + esdp->sched_wall_time.start = ts; + esdp->sched_wall_time.working.total = 0; + esdp->sched_wall_time.working.start = 0; + esdp->sched_wall_time.working.currently = 1; + } + } + + if (esdp->sched_wall_time.enabled) { + Uint64 ts = sched_wall_time_ts(); + ASSERT(esdp->sched_wall_time.working.currently); + ts -= esdp->sched_wall_time.start; + total = ts; + ts -= esdp->sched_wall_time.working.start; + working = esdp->sched_wall_time.working.total + ts; + } + + sz = 0; + hpp = NULL; + szp = &sz; + + while (1) { + if (hpp) + ref_copy = STORE_NC(hpp, ohp, swtrp->ref); + else + *szp += REF_THING_SIZE; + + if (swtrp->set) + msg = ref_copy; + else { + msg = (!esdp->sched_wall_time.enabled + ? am_notsup + : erts_bld_tuple(hpp, szp, 3, + make_small(esdp->no), + erts_bld_uint64(hpp, szp, working), + erts_bld_uint64(hpp, szp, total))); + + msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); + } + if (hpp) + break; + + hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + szp = NULL; + hpp = &hp; + } + + erts_queue_message(rp, &rp_locks, bp, msg, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); + + if (swtrp->req_sched == esdp->no) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + erts_smp_proc_dec_refc(rp); + + if (erts_smp_atomic32_dec_read_nob(&swtrp->refc) == 0) + swtreq_free(vswtrp); +} + +Eterm +erts_sched_wall_time_request(Process *c_p, int set, int enable) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + Eterm ref; + ErtsSchedWallTimeReq *swtrp; + Eterm *hp; + + if (!set && !esdp->sched_wall_time.enabled) + return THE_NON_VALUE; + + swtrp = swtreq_alloc(); + ref = erts_make_ref(c_p); + hp = &swtrp->ref_heap[0]; + + swtrp->set = set; + swtrp->enable = enable; + swtrp->proc = c_p; + swtrp->ref = STORE_NC(&hp, NULL, ref); + swtrp->req_sched = esdp->no; + erts_smp_atomic32_init_nob(&swtrp->refc, + (erts_aint32_t) erts_no_schedulers); + + erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_sched_wall_time, + (void *) swtrp); +#endif + + reply_sched_wall_time((void *) swtrp); + + return ref; +} + static ERTS_INLINE ErtsProcList * proclist_create(Process *p) { @@ -577,6 +822,13 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) case ERTS_SSI_FLG_POLL_SLEEPING: erts_sys_schedule_interrupt(1); break; + case ERTS_SSI_FLG_POLL_SLEEPING|ERTS_SSI_FLG_TSE_SLEEPING: + /* + * Thread progress blocking while poll sleeping; need + * to signal on both... + */ + erts_sys_schedule_interrupt(1); + /* fall through */ case ERTS_SSI_FLG_TSE_SLEEPING: erts_tse_set(ssi->event); break; @@ -589,189 +841,894 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) } } +#endif + +static ERTS_INLINE void +set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi, + erts_aint32_t flgs) +{ + erts_aint32_t old_flgs; + + ERTS_DBG_CHK_SSI_AUX_WORK(ssi); + + old_flgs = erts_atomic32_read_nob(&ssi->aux_work); + if ((old_flgs & flgs) == 0) { + + old_flgs = erts_atomic32_read_bor_nob(&ssi->aux_work, flgs); + + if ((old_flgs & flgs) == 0) { +#ifdef ERTS_SMP + erts_sched_poke(ssi); +#else + erts_sys_schedule_interrupt(1); +#endif + } + } +} + +static ERTS_INLINE void +set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi, + erts_aint32_t flgs) +{ + erts_aint32_t old_flgs; + + ERTS_DBG_CHK_SSI_AUX_WORK(ssi); + + old_flgs = erts_atomic32_read_bor_relb(&ssi->aux_work, flgs); + + if ((old_flgs & flgs) == 0) { +#ifdef ERTS_SMP + erts_sched_poke(ssi); +#else + erts_sys_schedule_interrupt(1); +#endif + } +} + +static ERTS_INLINE erts_aint32_t +set_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) +{ + return erts_atomic32_read_bor_nob(&ssi->aux_work, flgs); +} + +static ERTS_INLINE erts_aint32_t +unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) +{ + return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs); +} + +#ifdef ERTS_SMP + +static ERTS_INLINE void +haw_thr_prgr_current_reset(ErtsAuxWorkData *awdp) +{ + awdp->current_thr_prgr = ERTS_THR_PRGR_INVALID; +} + +static ERTS_INLINE ErtsThrPrgrVal +haw_thr_prgr_current(ErtsAuxWorkData *awdp) +{ + ErtsThrPrgrVal current = awdp->current_thr_prgr; + if (current == ERTS_THR_PRGR_INVALID) { + current = erts_thr_progress_current(); + awdp->current_thr_prgr = current; + } + return current; +} + +static ERTS_INLINE void +haw_thr_prgr_current_check_progress(ErtsAuxWorkData *awdp) +{ + ErtsThrPrgrVal current = awdp->current_thr_prgr; + if (current != ERTS_THR_PRGR_INVALID + && !erts_thr_progress_equal(current, erts_thr_progress_current())) { + /* + * We have used a previouly read current value that isn't the + * latest; need to poke ourselfs in order to guarantee no loss + * of wakeups. + */ + erts_sched_poke(awdp->ssi); + } +} + +static ERTS_INLINE erts_aint32_t +handle_delayed_aux_work_wakeup(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + int jix, max_jix; + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP); + + ERTS_THR_MEMORY_BARRIER; + + max_jix = awdp->delayed_wakeup.jix; + awdp->delayed_wakeup.jix = -1; + for (jix = 0; jix <= max_jix; jix++) { + int sched = awdp->delayed_wakeup.job[jix].sched; + erts_aint32_t aux_work = awdp->delayed_wakeup.job[jix].aux_work; + + ASSERT(awdp->delayed_wakeup.sched2jix[sched] == jix); + awdp->delayed_wakeup.sched2jix[sched] = -1; + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(sched-1), + aux_work); + } + return aux_work & ~ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP; +} + +static ERTS_INLINE void +schedule_aux_work_wakeup(ErtsAuxWorkData *awdp, int sched, erts_aint32_t aux_work) +{ + int jix = awdp->delayed_wakeup.sched2jix[sched]; + if (jix >= 0) { + ASSERT(awdp->delayed_wakeup.job[jix].sched == sched); + awdp->delayed_wakeup.job[jix].aux_work |= aux_work; + } + else { + jix = ++awdp->delayed_wakeup.jix; + awdp->delayed_wakeup.sched2jix[sched] = jix; + awdp->delayed_wakeup.job[jix].sched = sched; + awdp->delayed_wakeup.job[jix].aux_work = aux_work; + } + set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP); +} + +#endif + typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t; struct erts_misc_aux_work_t_ { - erts_misc_aux_work_t *next; void (*func)(void *); void *arg; }; -typedef struct { - erts_smp_mtx_t mtx; - erts_misc_aux_work_t *first; - erts_misc_aux_work_t *last; -} erts_misc_aux_work_q_t; +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_aux_work, + erts_misc_aux_work_t, + 200, + ERTS_ALC_T_MISC_AUX_WORK) typedef union { - erts_misc_aux_work_q_t data; - char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_misc_aux_work_q_t))]; + ErtsThrQ_t q; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrQ_t))]; } erts_algnd_misc_aux_work_q_t; static erts_algnd_misc_aux_work_q_t *misc_aux_work_queues; -ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_aux_work, - erts_misc_aux_work_t, - 200, - ERTS_ALC_T_MISC_AUX_WORK) +static void +notify_aux_work(void *vssi) +{ + set_aux_work_flags_wakeup_nob((ErtsSchedulerSleepInfo *) vssi, + ERTS_SSI_AUX_WORK_MISC); +} static void init_misc_aux_work(void) { int ix; + ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; + qinit.notify = notify_aux_work; init_misc_aux_work_alloc(); misc_aux_work_queues = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_MISC_AUX_WORK_Q, - erts_no_schedulers * - sizeof(erts_algnd_misc_aux_work_q_t)); + sizeof(erts_algnd_misc_aux_work_q_t) + * (erts_no_schedulers+1)); - for (ix = 0; ix < erts_no_schedulers; ix++) { - erts_smp_mtx_init_x(&misc_aux_work_queues[ix].data.mtx, - "misc_aux_work_queue", - make_small(ix + 1)); - misc_aux_work_queues[ix].data.first = NULL; - misc_aux_work_queues[ix].data.last = NULL; +#ifdef ERTS_SMP + ix = 0; /* aux_thread + schedulers */ +#else + ix = 1; /* scheduler only */ +#endif + + for (; ix <= erts_no_schedulers; ix++) { + qinit.arg = (void *) ERTS_SCHED_SLEEP_INFO_IX(ix-1); + erts_thr_q_initialize(&misc_aux_work_queues[ix].q, &qinit); } } -static void -handle_misc_aux_work(ErtsSchedulerData *esdp) -{ - int ix = (int) esdp->no - 1; - erts_misc_aux_work_t *mawp; +static erts_aint32_t +misc_aux_work_clean(ErtsThrQ_t *q, + ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) +{ + switch (erts_thr_q_clean(q)) { + case ERTS_THR_Q_DIRTY: + set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC); + return aux_work | ERTS_SSI_AUX_WORK_MISC; + case ERTS_THR_Q_NEED_THR_PRGR: +#ifdef ERTS_SMP + set_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR); + erts_thr_progress_wakeup(awdp->esdp, + erts_thr_q_need_thr_progress(q)); +#endif + case ERTS_THR_Q_CLEAN: + break; + } + return aux_work; +} - erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); - mawp = misc_aux_work_queues[ix].data.first; - misc_aux_work_queues[ix].data.first = NULL; - misc_aux_work_queues[ix].data.last = NULL; - erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); +static ERTS_INLINE erts_aint32_t +handle_misc_aux_work(ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) +{ + ErtsThrQ_t *q = &misc_aux_work_queues[awdp->sched_id].q; - while (mawp) { - erts_misc_aux_work_t *free_mawp; + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC); + while (1) { + erts_misc_aux_work_t *mawp = erts_thr_q_dequeue(q); + if (!mawp) + break; mawp->func(mawp->arg); - free_mawp = mawp; - mawp = mawp->next; - misc_aux_work_free(free_mawp); + misc_aux_work_free(mawp); } + + return misc_aux_work_clean(q, awdp, aux_work & ~ERTS_SSI_AUX_WORK_MISC); +} + +#ifdef ERTS_SMP + +static ERTS_INLINE erts_aint32_t +handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) +{ + if (!erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp), + awdp->misc.thr_prgr)) + return aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR; + + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR); + + return misc_aux_work_clean(&misc_aux_work_queues[awdp->sched_id].q, + awdp, + aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR); +} + +#endif + +static ERTS_INLINE void +schedule_misc_aux_work(int sched_id, + void (*func)(void *), + void *arg) +{ + ErtsThrQ_t *q; + erts_misc_aux_work_t *mawp; + +#ifdef ERTS_SMP + ASSERT(0 <= sched_id && sched_id <= erts_no_schedulers); +#else + ASSERT(sched_id == 1); +#endif + + q = &misc_aux_work_queues[sched_id].q; + mawp = misc_aux_work_alloc(); + mawp->func = func; + mawp->arg = arg; + erts_thr_q_enqueue(q, mawp); +} + +void +erts_schedule_misc_aux_work(int sched_id, + void (*func)(void *), + void *arg) +{ + schedule_misc_aux_work(sched_id, func, arg); } void -erts_smp_schedule_misc_aux_work(int ignore_self, - int max_sched, - void (*func)(void *), - void *arg) +erts_schedule_multi_misc_aux_work(int ignore_self, + int max_sched, + void (*func)(void *), + void *arg) { - int ix, ignore_ix = -1; + int id, self = 0; if (ignore_self) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); if (esdp) - ignore_ix = (int) esdp->no - 1; + self = (int) esdp->no; } - ASSERT(0 <= max_sched && max_sched <= erts_no_schedulers); + ASSERT(0 < max_sched && max_sched <= erts_no_schedulers); - for (ix = 0; ix < max_sched; ix++) { - erts_aint32_t aux_work; - erts_misc_aux_work_t *mawp; - ErtsSchedulerSleepInfo *ssi; - if (ix == ignore_ix) + for (id = 1; id <= max_sched; id++) { + if (id == self) continue; + schedule_misc_aux_work(id, func, arg); + } +} - mawp = misc_aux_work_alloc(); +#if ERTS_USE_ASYNC_READY_Q - mawp->func = func; - mawp->arg = arg; - mawp->next = NULL; +void +erts_notify_check_async_ready_queue(void *vno) +{ + int ix = ((int) (SWord) vno) -1; + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix), + ERTS_SSI_AUX_WORK_ASYNC_READY); +} - erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); - if (!misc_aux_work_queues[ix].data.last) - misc_aux_work_queues[ix].data.first = mawp; - else - misc_aux_work_queues[ix].data.last->next = mawp; - misc_aux_work_queues[ix].data.last = mawp; - erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); - - ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); - aux_work = erts_smp_atomic32_read_bor_nob(&ssi->aux_work, - ERTS_SSI_AUX_WORK_MISC); - if ((aux_work & ERTS_SSI_AUX_WORK_MISC) == 0) - erts_sched_poke(ssi); - } +static ERTS_INLINE erts_aint32_t +handle_async_ready(ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY); + if (erts_check_async_ready(awdp->async_ready.queue)) { + if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY) + & ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN) { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); + aux_work &= ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; + } + return aux_work; + } +#ifdef ERTS_SMP + awdp->async_ready.need_thr_prgr = 0; +#endif + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); + return ((aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY) + | ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); +} + +static ERTS_INLINE erts_aint32_t +handle_async_ready_clean(ErtsAuxWorkData *awdp, + erts_aint32_t aux_work) +{ + void *thr_prgr_p; + +#ifdef ERTS_SMP + if (awdp->async_ready.need_thr_prgr + && !erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp), + awdp->async_ready.thr_prgr)) { + return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; + } + + awdp->async_ready.need_thr_prgr = 0; + thr_prgr_p = (void *) &awdp->async_ready.thr_prgr; +#else + thr_prgr_p = NULL; +#endif + + switch (erts_async_ready_clean(awdp->async_ready.queue, thr_prgr_p)) { + case ERTS_ASYNC_READY_CLEAN: + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); + return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; +#ifdef ERTS_SMP + case ERTS_ASYNC_READY_NEED_THR_PRGR: + erts_thr_progress_wakeup(awdp->esdp, + awdp->async_ready.thr_prgr); + awdp->async_ready.need_thr_prgr = 1; + return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; +#endif + default: + return aux_work; + } +} + +#endif + +static ERTS_INLINE erts_aint32_t +handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + erts_aint32_t res; + + unset_aux_work_flags(ssi, (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)); + aux_work &= ~(ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC); + res = erts_alloc_fix_alloc_shrink(awdp->sched_id, aux_work); + if (res) { + set_aux_work_flags(ssi, res); + aux_work |= res; + } + + return aux_work; } +#ifdef ERTS_SMP + +void +erts_alloc_notify_delayed_dealloc(int ix) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp) + schedule_aux_work_wakeup(&esdp->aux_work_data, + ix, + ERTS_SSI_AUX_WORK_DD); + else + set_aux_work_flags_wakeup_relb(ERTS_SCHED_SLEEP_INFO_IX(ix-1), + ERTS_SSI_AUX_WORK_DD); +} + +static ERTS_INLINE erts_aint32_t +handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + int need_thr_progress = 0; + ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; + int more_work = 0; + + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); + erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, + &need_thr_progress, + &wakeup, + &more_work); + if (more_work) { + if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD) + & ERTS_SSI_AUX_WORK_DD_THR_PRGR) { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); + aux_work &= ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; + } + return aux_work; + } + + if (need_thr_progress) { + if (wakeup == ERTS_THR_PRGR_INVALID) + wakeup = erts_thr_progress_later(awdp->esdp); + awdp->dd.thr_prgr = wakeup; + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); + awdp->dd.thr_prgr = wakeup; + erts_thr_progress_wakeup(awdp->esdp, wakeup); + } + else if (awdp->dd.completed_callback) { + awdp->dd.completed_callback(awdp->dd.completed_arg); + awdp->dd.completed_callback = NULL; + awdp->dd.completed_arg = NULL; + } + return aux_work & ~ERTS_SSI_AUX_WORK_DD; +} + +static ERTS_INLINE erts_aint32_t +handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + ErtsSchedulerSleepInfo *ssi; + int need_thr_progress; + int more_work; + ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; + ErtsThrPrgrVal current = haw_thr_prgr_current(awdp); + + if (!erts_thr_progress_has_reached_this(current, awdp->dd.thr_prgr)) + return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; + + ssi = awdp->ssi; + need_thr_progress = 0; + more_work = 0; + + erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, + &need_thr_progress, + &wakeup, + &more_work); + if (more_work) { + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); + return ((aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR) + | ERTS_SSI_AUX_WORK_DD); + } + + if (need_thr_progress) { + if (wakeup == ERTS_THR_PRGR_INVALID) + wakeup = erts_thr_progress_later(awdp->esdp); + awdp->dd.thr_prgr = wakeup; + erts_thr_progress_wakeup(awdp->esdp, wakeup); + } + else { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); + if (awdp->dd.completed_callback) { + awdp->dd.completed_callback(awdp->dd.completed_arg); + awdp->dd.completed_callback = NULL; + awdp->dd.completed_arg = NULL; + } + } + + return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; +} + +static erts_atomic32_t completed_dealloc_count; + +static void +completed_dealloc(void *vproc) +{ + if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == 0) { + erts_resume((Process *) vproc, (ErtsProcLocks) 0); + erts_smp_proc_dec_refc((Process *) vproc); + } +} + +static void +setup_completed_dealloc(void *vproc) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsAuxWorkData *awdp = (esdp + ? &esdp->aux_work_data + : aux_thread_aux_work_data); + erts_alloc_fix_alloc_shrink(awdp->sched_id, 0); + set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_DD); + awdp->dd.completed_callback = completed_dealloc; + awdp->dd.completed_arg = vproc; +} + +static void +prep_setup_completed_dealloc(void *vproc) +{ + erts_aint32_t count = (erts_aint32_t) (erts_no_schedulers+1); + if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == count) { + /* scheduler threads */ + erts_schedule_multi_misc_aux_work(0, + erts_no_schedulers, + setup_completed_dealloc, + vproc); + /* aux_thread */ + erts_schedule_misc_aux_work(0, + setup_completed_dealloc, + vproc); + } +} + +#endif /* ERTS_SMP */ + +int +erts_debug_wait_deallocations(Process *c_p) +{ +#ifndef ERTS_SMP + erts_alloc_fix_alloc_shrink(1, 0); + return 1; +#else + /* Only one process at a time can do this */ + erts_aint32_t count = (erts_aint32_t) (2*(erts_no_schedulers+1)); + if (0 == erts_atomic32_cmpxchg_mb(&completed_dealloc_count, + count, + 0)) { + erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); + erts_smp_proc_inc_refc(c_p); + /* scheduler threads */ + erts_schedule_multi_misc_aux_work(0, + erts_no_schedulers, + prep_setup_completed_dealloc, + (void *) c_p); + /* aux_thread */ + erts_schedule_misc_aux_work(0, + prep_setup_completed_dealloc, + (void *) c_p); + return 1; + } + return 0; +#endif +} + + #ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN void erts_smp_notify_check_children_needed(void) { int i; + for (i = 0; i < erts_no_schedulers; i++) + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(i), + ERTS_SSI_AUX_WORK_CHECK_CHILDREN); +} +static ERTS_INLINE erts_aint32_t +handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + erts_check_children(); + return aux_work & ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; +} + +#endif + +static void +notify_reap_ports_relb(void) +{ + int i; for (i = 0; i < erts_no_schedulers; i++) { - erts_aint32_t aux_work; - ErtsSchedulerSleepInfo *ssi; - ssi = ERTS_SCHED_SLEEP_INFO_IX(i); - aux_work = erts_smp_atomic32_read_bor_nob(&ssi->aux_work, - ERTS_SSI_AUX_WORK_CHECK_CHILDREN); - if (!(aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN)) - erts_sched_poke(ssi); + set_aux_work_flags_wakeup_relb(ERTS_SCHED_SLEEP_INFO_IX(i), + ERTS_SSI_AUX_WORK_REAP_PORTS); } } -#endif -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK +erts_smp_atomic32_t erts_halt_progress; +int erts_halt_code; + static ERTS_INLINE erts_aint32_t -blockable_aux_work(ErtsSchedulerData *esdp, - ErtsSchedulerSleepInfo *ssi, - erts_aint32_t aux_work) +handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { - if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { - if (aux_work & ERTS_SSI_AUX_WORK_MISC) { - aux_work = erts_smp_atomic32_read_band_nob(&ssi->aux_work, - ~ERTS_SSI_AUX_WORK_MISC); - aux_work &= ~ERTS_SSI_AUX_WORK_MISC; - handle_misc_aux_work(esdp); + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS); + awdp->esdp->run_queue->halt_in_progress = 1; + if (erts_smp_atomic32_dec_read_acqb(&erts_halt_progress) == 0) { + int i; + erts_smp_atomic32_set_nob(&erts_halt_progress, 1); + for (i = 0; i < erts_max_ports; i++) { + Port *prt = &erts_port[i]; + erts_smp_port_state_lock(prt); + if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + | ERTS_PORT_SFLG_HALT))) { + erts_smp_port_state_unlock(prt); + continue; + } + /* We need to set the halt flag - get the port lock */ +#ifdef ERTS_SMP + erts_smp_atomic_inc_nob(&prt->refc); +#endif + erts_smp_port_state_unlock(prt); +#ifdef ERTS_SMP + erts_smp_mtx_lock(prt->lock); +#endif + if ((prt->status & (ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP + | ERTS_PORT_SFLG_HALT))) { + erts_port_release(prt); + continue; + } + erts_port_status_bor_set(prt, ERTS_PORT_SFLG_HALT); + erts_smp_atomic32_inc_nob(&erts_halt_progress); + if (prt->status & (ERTS_PORT_SFLG_EXITING + | ERTS_PORT_SFLG_CLOSING)) { + erts_port_release(prt); + continue; + } + erts_do_exit_port(prt, prt->id, am_killed); + erts_port_release(prt); } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { - aux_work = erts_smp_atomic32_band_nob(&ssi->aux_work, - ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN); - aux_work &= ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; - erts_check_children(); + if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) { + erl_exit_flush_async(erts_halt_code, ""); } -#endif } - return aux_work; + return aux_work & ~ERTS_SSI_AUX_WORK_REAP_PORTS; +} + +#if HAVE_ERTS_MSEG + +static ERTS_INLINE erts_aint32_t +handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK); + erts_mseg_cache_check(); + return aux_work & ~ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; } #endif -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK static ERTS_INLINE erts_aint32_t -nonblockable_aux_work(ErtsSchedulerData *esdp, - ErtsSchedulerSleepInfo *ssi, - erts_aint32_t aux_work) +handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { - if (aux_work & ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK) { + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO); + setup_aux_work_timer(); + return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO; +} +static erts_aint32_t +handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) +{ +#undef HANDLE_AUX_WORK +#define HANDLE_AUX_WORK(FLG, HNDLR) \ + ignore |= FLG; \ + if (aux_work & FLG) { \ + aux_work = HNDLR(awdp, aux_work); \ + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ + if (!(aux_work & ~ignore)) { \ + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ + return aux_work; \ + } \ } -} + + erts_aint32_t aux_work = orig_aux_work; + erts_aint32_t ignore = 0; + +#ifdef ERTS_SMP + haw_thr_prgr_current_reset(awdp); +#endif + + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + ASSERT(aux_work); + + /* + * Handlers are *only* allowed to modify flags in return value + * and ssi flags that are explicity handled by the handler. + * Handlers are, e.g., not allowed to read the ssi flag field and + * then unconditionally return that value. + * + * Flag field returned should only contain flags for work that + * can continue immediately. + */ + + /* + * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative + * eachother. Most frequent first. + */ +#ifdef ERTS_SMP + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP, + handle_delayed_aux_work_wakeup); + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD, + handle_delayed_dealloc); + /* DD must be before DD_THR_PRGR */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD_THR_PRGR, + handle_delayed_dealloc_thr_prgr); #endif + HANDLE_AUX_WORK((ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + handle_fix_alloc); + +#if ERTS_USE_ASYNC_READY_Q + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY, + handle_async_ready); + /* ASYNC_READY must be before ASYNC_READY_CLEAN */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN, + handle_async_ready_clean); +#endif + +#ifdef ERTS_SMP + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC_THR_PRGR, + handle_misc_aux_work_thr_prgr); +#endif + /* MISC_THR_PRGR must be before MISC */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC, + handle_misc_aux_work); + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CHECK_CHILDREN, + handle_check_children); +#endif + + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO, + handle_setup_aux_work_timer); + +#if HAVE_ERTS_MSEG + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK, + handle_mseg_cache_check); +#endif + + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_REAP_PORTS, + handle_reap_ports); + + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + +#ifdef ERTS_SMP + if (waiting && !aux_work) + haw_thr_prgr_current_check_progress(awdp); +#endif + + return aux_work; + +#undef HANDLE_AUX_WORK + +} + +typedef struct { + union { + ErlTimer data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErlTimer))]; + } timer; + + int initialized; + erts_atomic32_t refc; + erts_atomic32_t type[1]; +} ErtsAuxWorkTmo; + +static ErtsAuxWorkTmo *aux_work_tmo; + +static void +aux_work_timeout_early_init(int no_schedulers) +{ + int i; + UWord p; + + /* + * This is done really early. Our own allocators have + * not been started yet. + */ + + p = (UWord) malloc((sizeof(ErtsAuxWorkTmo) + + sizeof(erts_atomic32_t)*(no_schedulers+1)) + + ERTS_CACHE_LINE_SIZE-1); + if (p & ERTS_CACHE_LINE_MASK) + p = (p & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE; + ASSERT((p & ERTS_CACHE_LINE_MASK) == 0); + + aux_work_tmo = (ErtsAuxWorkTmo *) p; + aux_work_tmo->initialized = 0; + erts_atomic32_init_nob(&aux_work_tmo->refc, 0); + for (i = 0; i <= no_schedulers; i++) + erts_atomic32_init_nob(&aux_work_tmo->type[i], 0); +} + +void +aux_work_timeout_late_init(void) +{ + aux_work_tmo->initialized = 1; + if (erts_atomic32_read_nob(&aux_work_tmo->refc)) { + aux_work_tmo->timer.data.active = 0; + erts_set_timer(&aux_work_tmo->timer.data, + aux_work_timeout, + NULL, + NULL, + 1000); + } +} + static void -prepare_for_block(void *vrq) +aux_work_timeout(void *unused) { - erts_smp_runq_unlock((ErtsRunQueue *) vrq); + erts_aint32_t refc; + int i; +#ifdef ERTS_SMP + i = 0; +#else + i = 1; +#endif + + for (; i <= erts_no_schedulers; i++) { + erts_aint32_t type; + type = erts_atomic32_read_acqb(&aux_work_tmo->type[i]); + if (type) + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(i-1), + type); + } + + refc = erts_atomic32_read_nob(&aux_work_tmo->refc); + ASSERT(refc >= 1); + if (refc != 1 + || 1 != erts_atomic32_cmpxchg_relb(&aux_work_tmo->refc, 0, 1)) { + /* Setup next timeout... */ + aux_work_tmo->timer.data.active = 0; + erts_set_timer(&aux_work_tmo->timer.data, + aux_work_timeout, + NULL, + NULL, + 1000); + } } static void -resume_after_block(void *vrq) +setup_aux_work_timer(void) { - erts_smp_runq_lock((ErtsRunQueue *) vrq); +#ifndef ERTS_SMP + if (!erts_get_scheduler_data()) + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(0), + ERTS_SSI_AUX_WORK_SET_TMO); + else +#endif + { + aux_work_tmo->timer.data.active = 0; + erts_set_timer(&aux_work_tmo->timer.data, + aux_work_timeout, + NULL, + NULL, + 1000); + } } +erts_aint32_t +erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable) +{ + erts_aint32_t old, refc; + +#ifndef ERTS_SMP + ix = 1; #endif + ERTS_DBG_CHK_AUX_WORK_VAL(type); + ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix])); +// erts_fprintf(stderr, "t(%d, 0x%x, %d)\n", ix, type, enable); + + if (!enable) { + old = erts_atomic32_read_band_mb(&aux_work_tmo->type[ix], ~type); + ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix])); + if (old != 0 && (old & ~type) == 0) + erts_atomic32_dec_relb(&aux_work_tmo->refc); + return old; + } + + old = erts_atomic32_read_bor_mb(&aux_work_tmo->type[ix], type); + ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix])); + if (old == 0 && type != 0) { + refc = erts_atomic32_inc_read_acqb(&aux_work_tmo->refc); + if (refc == 1) { + erts_atomic32_inc_acqb(&aux_work_tmo->refc); + if (aux_work_tmo->initialized) + setup_aux_work_timer(); + } + } + return old; +} + + + static ERTS_INLINE void sched_waiting_sys(Uint no, ErtsRunQueue *rq) { @@ -800,8 +1757,6 @@ sched_active_sys(Uint no, ErtsRunQueue *rq) Uint erts_active_schedulers(void) { - /* RRRRRRRRR */ - Uint as = erts_no_schedulers; ERTS_ATOMIC_FOREACH_RUNQ(rq, as -= abs(rq->waiting)); @@ -988,6 +1943,10 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) if (sleep_type == ERTS_SSI_FLG_TSE_SLEEPING) erts_tse_reset(ssi->event); + else { + ASSERT(sleep_type == ERTS_SSI_FLG_POLL_SLEEPING); + erts_sys_schedule_interrupt(0); + } while (1) { oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); @@ -1006,34 +1965,137 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) (((FLGS) & (ERTS_SSI_FLG_WAITING|ERTS_SSI_FLG_SUSPENDED)) \ != ERTS_SSI_FLG_WAITING) + +static void +thr_prgr_wakeup(void *vssi) +{ + erts_sched_poke((ErtsSchedulerSleepInfo *) vssi); +} + +static void +thr_prgr_prep_wait(void *vssi) +{ + ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi; + erts_smp_atomic32_read_bor_acqb(&ssi->flags, + ERTS_SSI_FLG_SLEEPING); +} + +static void +thr_prgr_wait(void *vssi) +{ + ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi; + erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING; + + erts_tse_reset(ssi->event); + + while (1) { + erts_aint32_t aflgs, nflgs; + nflgs = xflgs | ERTS_SSI_FLG_TSE_SLEEPING; + aflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs); + if (aflgs == xflgs) { + erts_tse_wait(ssi->event); + break; + } + if ((aflgs & ERTS_SSI_FLG_SLEEPING) == 0) + break; + xflgs = aflgs; + } +} + +static void +thr_prgr_fin_wait(void *vssi) +{ + ErtsSchedulerSleepInfo *ssi = (ErtsSchedulerSleepInfo *) vssi; + erts_smp_atomic32_read_band_nob(&ssi->flags, + ~(ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING)); +} + +static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp); + +static void * +aux_thread(void *unused) +{ + ErtsAuxWorkData *awdp = aux_thread_aux_work_data; + ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(-1); + erts_aint32_t aux_work; + ErtsThrPrgrCallbacks callbacks; + int thr_prgr_active = 1; + + ssi->event = erts_tse_fetch(); + + callbacks.arg = (void *) ssi; + callbacks.wakeup = thr_prgr_wakeup; + callbacks.prepare_wait = thr_prgr_prep_wait; + callbacks.wait = thr_prgr_wait; + callbacks.finalize_wait = thr_prgr_fin_wait; + + erts_thr_progress_register_managed_thread(NULL, &callbacks, 1); + init_aux_work_data(awdp, NULL, NULL); + awdp->ssi = ssi; + + sched_prep_spin_wait(ssi); + + while (1) { + erts_aint32_t flgs; + + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work) { + if (!thr_prgr_active) + erts_thr_progress_active(NULL, thr_prgr_active = 1); + aux_work = handle_aux_work(awdp, aux_work, 1); + if (aux_work && erts_thr_progress_update(NULL)) + erts_thr_progress_leader_update(NULL); + } + + if (!aux_work) { + if (thr_prgr_active) + erts_thr_progress_active(NULL, thr_prgr_active = 0); + erts_thr_progress_prepare_wait(NULL); + + flgs = sched_spin_wait(ssi, 0); + + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } + erts_thr_progress_finalize_wait(NULL); + } + + flgs = sched_prep_spin_wait(ssi); + } + return NULL; +} + +#endif /* ERTS_SMP */ + static void scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) { + int working = 1; ErtsSchedulerSleepInfo *ssi = esdp->ssi; int spincount; + erts_aint32_t aux_work = 0; +#ifdef ERTS_SMP + int thr_prgr_active = 1; erts_aint32_t flgs; -#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ - || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) - erts_aint32_t aux_work; -#endif ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - erts_smp_spin_lock(&rq->sleepers.lock); flgs = sched_prep_spin_wait(ssi); if (flgs & ERTS_SSI_FLG_SUSPENDED) { /* Go suspend instead... */ - erts_smp_spin_unlock(&rq->sleepers.lock); return; } - ssi->prev = NULL; - ssi->next = rq->sleepers.list; - if (rq->sleepers.list) - rq->sleepers.list->prev = ssi; - rq->sleepers.list = ssi; - erts_smp_spin_unlock(&rq->sleepers.lock); - /* * If all schedulers are waiting, one of them *should* * be waiting in erl_sys_schedule() @@ -1045,38 +2107,49 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_smp_runq_unlock(rq); - spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT; + spincount = sched_busy_wait.tse; tse_wait: -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); - tse_blockable_aux_work: - aux_work = blockable_aux_work(esdp, ssi, aux_work); -#endif - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + if (thr_prgr_active != working) + sched_wall_time_change(esdp, thr_prgr_active); while (1) { -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK -#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); -#endif - nonblockable_aux_work(esdp, ssi, aux_work); -#endif + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); + } - flgs = sched_spin_wait(ssi, spincount); - if (flgs & ERTS_SSI_FLG_SLEEPING) { - ASSERT(flgs & ERTS_SSI_FLG_WAITING); - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (aux_work) + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + else { + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(esdp); + + flgs = sched_spin_wait(ssi, spincount); if (flgs & ERTS_SSI_FLG_SLEEPING) { - int res; - ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); ASSERT(flgs & ERTS_SSI_FLG_WAITING); - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } } + erts_thr_progress_finalize_wait(esdp); } if (!(flgs & ERTS_SSI_FLG_WAITING)) { @@ -1085,33 +2158,30 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) } flgs = sched_prep_cont_spin_wait(ssi); - spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT; + spincount = sched_busy_wait.aux_work; if (!(flgs & ERTS_SSI_FLG_WAITING)) { ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); break; } -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); - if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - goto tse_blockable_aux_work; - } -#endif - } - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - if (flgs & ~ERTS_SSI_FLG_SUSPENDED) erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_smp_runq_lock(rq); sched_active(esdp->no, rq); } - else { + else +#endif + { erts_aint_t dt; erts_smp_atomic32_set_relb(&function_calls, 0); @@ -1119,14 +2189,23 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sched_waiting_sys(esdp->no, rq); + erts_smp_runq_unlock(rq); - spincount = ERTS_SCHED_SYS_SLEEP_SPINCOUNT; + ASSERT(working); + sched_wall_time_change(esdp, working = 0); + + spincount = sched_busy_wait.sys_schedule; + if (spincount == 0) + goto sys_aux_work; while (spincount-- > 0) { sys_poll_aux_work: + if (working) + sched_wall_time_change(esdp, working = 0); + ASSERT(!erts_port_task_have_outstanding_io_tasks()); erl_sys_schedule(1); /* Might give us something to do */ @@ -1135,18 +2214,29 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) if (dt) erts_bump_timer(dt); sys_aux_work: - -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); - aux_work = blockable_aux_work(esdp, ssi, aux_work); +#ifndef ERTS_SMP + erts_sys_schedule_interrupt(0); #endif -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK -#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); + + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work) { + if (!working) + sched_wall_time_change(esdp, working = 1); +#ifdef ERTS_SMP + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); #endif - nonblockable_aux_work(esdp, ssi, aux_work); + aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); +#ifdef ERTS_SMP + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); #endif + } +#ifndef ERTS_SMP + if (rq->len != 0 || rq->misc.start) + goto sys_woken; +#else flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_WAITING)) { ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); @@ -1168,10 +2258,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) goto tse_wait; } } +#endif } erts_smp_runq_lock(rq); +#ifdef ERTS_SMP /* * If we got new I/O tasks we aren't allowed to * sleep in erl_sys_schedule(). @@ -1183,64 +2275,93 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to wait in erl_sys_schedule() after all... */ - if (prepare_for_sys_schedule()) - goto do_sys_schedule; - - /* - * Not allowed to wait in erl_sys_schedule; - * do tse wait instead... - */ - sched_change_waiting_sys_to_waiting(esdp->no, rq); + if (!prepare_for_sys_schedule()) { + /* + * Not allowed to wait in erl_sys_schedule; + * do tse wait instead... + */ + sched_change_waiting_sys_to_waiting(esdp->no, rq); + erts_smp_runq_unlock(rq); + spincount = 0; + goto tse_wait; + } + } +#endif + if (aux_work) { erts_smp_runq_unlock(rq); - spincount = 0; - goto tse_wait; + goto sys_poll_aux_work; } - else { - do_sys_schedule: - erts_sys_schedule_interrupt(0); - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); - if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { - if (!(flgs & ERTS_SSI_FLG_WAITING)) - goto sys_locked_woken; - erts_smp_runq_unlock(rq); - flgs = sched_prep_cont_spin_wait(ssi); - if (!(flgs & ERTS_SSI_FLG_WAITING)) { - ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); - goto sys_woken; - } - ASSERT(!erts_port_task_have_outstanding_io_tasks()); - goto sys_poll_aux_work; +#ifdef ERTS_SMP + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); + if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_locked_woken; } + erts_smp_runq_unlock(rq); + flgs = sched_prep_cont_spin_wait(ssi); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + goto sys_poll_aux_work; + } - ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); - ASSERT(flgs & ERTS_SSI_FLG_WAITING); + ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); +#endif - erts_smp_runq_unlock(rq); + erts_smp_runq_unlock(rq); - ASSERT(!erts_port_task_have_outstanding_io_tasks()); + if (working) + sched_wall_time_change(esdp, working = 0); - erl_sys_schedule(0); +#ifdef ERTS_SMP + if (thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 0); +#endif - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + ASSERT(!erts_port_task_have_outstanding_io_tasks()); - flgs = sched_prep_cont_spin_wait(ssi); - if (flgs & ERTS_SSI_FLG_WAITING) - goto sys_aux_work; + erl_sys_schedule(0); - sys_woken: + dt = erts_do_time_read_and_reset(); + if (dt) erts_bump_timer(dt); + +#ifndef ERTS_SMP + if (rq->len == 0 && !rq->misc.start) + goto sys_aux_work; + sys_woken: +#else + flgs = sched_prep_cont_spin_wait(ssi); + if (flgs & ERTS_SSI_FLG_WAITING) + goto sys_aux_work; + + sys_woken: + if (!thr_prgr_active) + erts_thr_progress_active(esdp, thr_prgr_active = 1); + erts_smp_runq_lock(rq); + sys_locked_woken: + if (!thr_prgr_active) { + erts_smp_runq_unlock(rq); + erts_thr_progress_active(esdp, thr_prgr_active = 1); erts_smp_runq_lock(rq); - sys_locked_woken: - clear_sys_scheduling(); - if (flgs & ~ERTS_SSI_FLG_SUSPENDED) - erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); - sched_active_sys(esdp->no, rq); } + clear_sys_scheduling(); + if (flgs & ~ERTS_SSI_FLG_SUSPENDED) + erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); +#endif + if (!working) + sched_wall_time_change(esdp, working = 1); + sched_active_sys(esdp->no, rq); } ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); } +#ifdef ERTS_SMP + static ERTS_INLINE erts_aint32_t ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) { @@ -1258,10 +2379,10 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) } static void -wake_scheduler(ErtsRunQueue *rq, int incq, int one) +wake_scheduler(ErtsRunQueue *rq, int incq) { ErtsSchedulerSleepInfo *ssi; - ErtsSchedulerSleepList *sl; + erts_aint32_t flgs; /* * The unlocked run queue is not strictly necessary @@ -1273,56 +2394,13 @@ wake_scheduler(ErtsRunQueue *rq, int incq, int one) */ ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)); - sl = &rq->sleepers; - - erts_smp_spin_lock(&sl->lock); - ssi = sl->list; - if (!ssi) - erts_smp_spin_unlock(&sl->lock); - else if (one) { - erts_aint32_t flgs; - if (ssi->prev) - ssi->prev->next = ssi->next; - else { - ASSERT(sl->list == ssi); - sl->list = ssi->next; - } - if (ssi->next) - ssi->next->prev = ssi->prev; + ssi = rq->scheduler->ssi; - erts_smp_spin_unlock(&sl->lock); + flgs = ssi_flags_set_wake(ssi); + erts_sched_finish_poke(ssi, flgs); - flgs = ssi_flags_set_wake(ssi); - erts_sched_finish_poke(ssi, flgs); - - if (incq && !erts_common_run_queue && (flgs & ERTS_SSI_FLG_WAITING)) - non_empty_runq(rq); - } - else { - sl->list = NULL; - erts_smp_spin_unlock(&sl->lock); - - ERTS_THR_MEMORY_BARRIER; - do { - ErtsSchedulerSleepInfo *wake_ssi = ssi; - ssi = ssi->next; - erts_sched_finish_poke(wake_ssi, ssi_flags_set_wake(wake_ssi)); - } while (ssi); - } -} - -static void -wake_all_schedulers(void) -{ - if (erts_common_run_queue) - wake_scheduler(erts_common_run_queue, 0, 0); - else { - int ix; - for (ix = 0; ix < erts_no_run_queues; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq, 0, 1); - } - } + if (incq && (flgs & ERTS_SSI_FLG_WAITING)) + non_empty_runq(rq); } #define ERTS_NO_USED_RUNQS_SHIFT 16 @@ -1415,7 +2493,7 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) erts_smp_xrunq_unlock(crq, wrq); } } - wake_scheduler(wrq, 0, 1); + wake_scheduler(wrq, 0); return 1; } return 0; @@ -1463,7 +2541,7 @@ smp_notify_inc_runq(ErtsRunQueue *runq) { #ifdef ERTS_SMP if (runq) - wake_scheduler(runq, 1, 1); + wake_scheduler(runq, 1); #endif } @@ -1478,19 +2556,12 @@ erts_sched_notify_check_cpu_bind(void) { #ifdef ERTS_SMP int ix; - if (erts_common_run_queue) { - for (ix = 0; ix < erts_no_schedulers; ix++) - erts_smp_atomic32_set_relb(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1); - wake_all_schedulers(); - } - else { - for (ix = 0; ix < erts_no_run_queues; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; - erts_smp_runq_unlock(rq); - wake_scheduler(rq, 0, 1); - }; + for (ix = 0; ix < erts_no_run_queues; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + erts_smp_runq_lock(rq); + rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; + erts_smp_runq_unlock(rq); + wake_scheduler(rq, 0); } #else erts_sched_check_cpu_bind(erts_get_scheduler_data()); @@ -1759,7 +2830,7 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) if (notify_to_rq) smp_notify_inc_runq(rq); - wake_scheduler(evac_rq, 0, 1); + wake_scheduler(evac_rq, 0); } static int @@ -1777,6 +2848,9 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq) ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp); ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked); + if (rq->halt_in_progress) + goto try_steal_port; + /* * Check for a runnable process to steal... */ @@ -1863,6 +2937,8 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq) vrq_locked = 1; } + try_steal_port: + ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, *rq_lockedp); ERTS_SMP_LC_CHK_RUNQ_LOCK(vrq, vrq_locked); @@ -1917,9 +2993,6 @@ static int try_steal_task(ErtsRunQueue *rq) { int res, rq_locked, vix, active_rqs, blnc_rqs; - - if (erts_common_run_queue) - return 0; /* * We are not allowed to steal jobs to this run queue @@ -1981,7 +3054,8 @@ try_steal_task(ErtsRunQueue *rq) erts_smp_runq_lock(rq); if (!res) - res = !ERTS_EMPTY_RUNQ(rq); + res = rq->halt_in_progress ? + !ERTS_EMPTY_RUNQ_PORTS(rq) : !ERTS_EMPTY_RUNQ(rq); return res; } @@ -2211,6 +3285,9 @@ check_balance(ErtsRunQueue *c_rq) mmax_len = run_queue_info[qix].max_len; } + if (!erts_sched_compact_load) + goto all_active; + if (!forced && half_full_scheds != blnc_no_rqs) { int min = 1; if (min < half_full_scheds) @@ -2546,49 +3623,339 @@ erts_debug_nbalance(void) #endif } +/* Wakeup other schedulers */ + +typedef enum { + ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH, + ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH, + ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM, + ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW, + ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW +} ErtsSchedWakeupOtherThreshold; + +typedef enum { + ERTS_SCHED_WAKEUP_OTHER_TYPE_PROPOSAL, + ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY +} ErtsSchedWakeupOtherType; + +/* First proposal */ + +#define ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH (200*CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_HIGH (50*CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_MEDIUM (10*CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_LOW (CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW (CONTEXT_REDS/10) + +#define ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_HIGH 3 +#define ERTS_WAKEUP_OTHER_DEC_SHIFT_HIGH 1 +#define ERTS_WAKEUP_OTHER_DEC_SHIFT_MEDIUM 0 +#define ERTS_WAKEUP_OTHER_DEC_SHIFT_LOW -2 +#define ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_LOW -5 + +#define ERTS_WAKEUP_OTHER_DEC_SHIFT 2 +#define ERTS_WAKEUP_OTHER_FIXED_INC (CONTEXT_REDS/10) + +/* To be legacy */ + +#define ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH_LEGACY (200*CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_HIGH_LEGACY (50*CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_MEDIUM_LEGACY (10*CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_LOW_LEGACY (CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW_LEGACY (CONTEXT_REDS/10) + +#define ERTS_WAKEUP_OTHER_DEC_LEGACY 10 +#define ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY (CONTEXT_REDS/10) + +#ifdef ERTS_SMP + +static struct { + ErtsSchedWakeupOtherThreshold threshold; + ErtsSchedWakeupOtherType type; + int limit; + int dec_shift; + int dec_mask; + void (*check)(ErtsRunQueue *rq); +} wakeup_other; + +static void +wakeup_other_check(ErtsRunQueue *rq) +{ + int wo_reds = rq->wakeup_other_reds; + if (wo_reds) { + int left_len = rq->len - 1; + if (left_len < 1) { + int wo_reduce = wo_reds << wakeup_other.dec_shift; + wo_reduce &= wakeup_other.dec_mask; + rq->wakeup_other -= wo_reduce; + if (rq->wakeup_other < 0) + rq->wakeup_other = 0; + } + else { + rq->wakeup_other += (left_len*wo_reds + + ERTS_WAKEUP_OTHER_FIXED_INC); + if (rq->wakeup_other > wakeup_other.limit) { + int empty_rqs = + erts_smp_atomic32_read_acqb(&no_empty_run_queues); + if (empty_rqs != 0) + wake_scheduler_on_empty_runq(rq); + rq->wakeup_other = 0; + } + } + rq->wakeup_other_reds = 0; + } +} + +static void +wakeup_other_set_limit(void) +{ + switch (wakeup_other.threshold) { + case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH: + wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH; + wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_HIGH; + break; + case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH: + wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH; + wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_HIGH; + break; + case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM: + wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM; + wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_MEDIUM; + break; + case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW: + wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_LOW; + wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_LOW; + break; + case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW: + wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW; + wakeup_other.dec_shift = ERTS_WAKEUP_OTHER_DEC_SHIFT_VERY_LOW; + break; + } + if (wakeup_other.dec_shift < 0) + wakeup_other.dec_mask = (1 << (sizeof(wakeup_other.dec_mask)*8 + + wakeup_other.dec_shift)) - 1; + else { + wakeup_other.dec_mask = 0; + wakeup_other.dec_mask = ~wakeup_other.dec_mask; + } +} + +static void +wakeup_other_check_legacy(ErtsRunQueue *rq) +{ + int wo_reds = rq->wakeup_other_reds; + if (wo_reds) { + if (rq->len < 2) { + rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC_LEGACY*wo_reds; + if (rq->wakeup_other < 0) + rq->wakeup_other = 0; + } + else if (rq->wakeup_other < wakeup_other.limit) + rq->wakeup_other += rq->len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC_LEGACY; + else { + if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) { + wake_scheduler_on_empty_runq(rq); + rq->wakeup_other = 0; + } + rq->wakeup_other = 0; + } + } + rq->wakeup_other_reds = 0; +} + +static void +wakeup_other_set_limit_legacy(void) +{ + switch (wakeup_other.threshold) { + case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH: + wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH_LEGACY; + break; + case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH: + wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH_LEGACY; + break; + case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM: + wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM_LEGACY; + break; + case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW: + wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_LOW_LEGACY; + break; + case ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW: + wakeup_other.limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW_LEGACY; + break; + } +} + +static void +set_wakeup_other_data(void) +{ + switch (wakeup_other.type) { + case ERTS_SCHED_WAKEUP_OTHER_TYPE_PROPOSAL: + wakeup_other.check = wakeup_other_check; + wakeup_other_set_limit(); + break; + case ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY: + wakeup_other.check = wakeup_other_check_legacy; + wakeup_other_set_limit_legacy(); + break; + } +} + +#endif + void -erts_early_init_scheduling(void) +erts_early_init_scheduling(int no_schedulers) { - wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM; + aux_work_timeout_early_init(no_schedulers); +#ifdef ERTS_SMP + wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM; + wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY; +#endif + sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM; + sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM + * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT); + sched_busy_wait.aux_work = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM + * ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM); } int -erts_sched_set_wakeup_limit(char *str) +erts_sched_set_wakeup_other_thresold(char *str) { + ErtsSchedWakeupOtherThreshold threshold; if (sys_strcmp(str, "very_high") == 0) - wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH; + threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_HIGH; else if (sys_strcmp(str, "high") == 0) - wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH; + threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_HIGH; else if (sys_strcmp(str, "medium") == 0) - wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM; + threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM; else if (sys_strcmp(str, "low") == 0) - wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_LOW; + threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_LOW; else if (sys_strcmp(str, "very_low") == 0) - wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW; + threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_VERY_LOW; else return EINVAL; +#ifdef ERTS_SMP + wakeup_other.threshold = threshold; + set_wakeup_other_data(); +#endif return 0; } - -void -erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) +int +erts_sched_set_wakeup_other_type(char *str) +{ + ErtsSchedWakeupOtherType type; + if (sys_strcmp(str, "proposal") == 0) + type = ERTS_SCHED_WAKEUP_OTHER_TYPE_PROPOSAL; + else if (sys_strcmp(str, "default") == 0) + type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY; + else if (sys_strcmp(str, "legacy") == 0) + type = ERTS_SCHED_WAKEUP_OTHER_TYPE_LEGACY; + else + return EINVAL; +#ifdef ERTS_SMP + wakeup_other.type = type; +#endif + return 0; +} + +int +erts_sched_set_busy_wait_threshold(char *str) { - int ix, n; + int sys_sched; + int aux_work_fact; -#ifndef ERTS_SMP - mrq = 0; + if (sys_strcmp(str, "very_long") == 0) { + sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG; + aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_LONG; + } + else if (sys_strcmp(str, "long") == 0) { + sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_LONG; + aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_LONG; + } + else if (sys_strcmp(str, "medium") == 0) { + sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM; + aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM; + } + else if (sys_strcmp(str, "short") == 0) { + sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_SHORT; + aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_SHORT; + } + else if (sys_strcmp(str, "very_short") == 0) { + sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_SHORT; + aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_SHORT; + } + else if (sys_strcmp(str, "none") == 0) { + sys_sched = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; + aux_work_fact = ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_NONE; + } + else { + return EINVAL; + } + + sched_busy_wait.sys_schedule = sys_sched; + sched_busy_wait.tse = sys_sched*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; + sched_busy_wait.aux_work = sys_sched*aux_work_fact; + + return 0; +} +static void +init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) +{ + awdp->sched_id = esdp ? (int) esdp->no : 0; + awdp->esdp = esdp; + awdp->ssi = esdp ? esdp->ssi : NULL; +#ifdef ERTS_SMP + awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; + awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; + awdp->dd.completed_callback = NULL; + awdp->dd.completed_arg = NULL; +#endif +#ifdef ERTS_USE_ASYNC_READY_Q +#ifdef ERTS_SMP + awdp->async_ready.need_thr_prgr = 0; + awdp->async_ready.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; +#endif + awdp->async_ready.queue = NULL; +#endif +#ifdef ERTS_SMP + if (!dawwp) { + awdp->delayed_wakeup.job = NULL; + awdp->delayed_wakeup.sched2jix = NULL; + awdp->delayed_wakeup.jix = -1; + } + else { + int i; + awdp->delayed_wakeup.job = (ErtsDelayedAuxWorkWakeupJob *) dawwp; + dawwp += sizeof(ErtsDelayedAuxWorkWakeupJob)*(erts_no_schedulers+1); + awdp->delayed_wakeup.sched2jix = (int *) dawwp; + awdp->delayed_wakeup.jix = -1; + for (i = 0; i <= erts_no_schedulers; i++) + awdp->delayed_wakeup.sched2jix[i] = -1; + } +#endif +} + +void +erts_init_scheduling(int no_schedulers, int no_schedulers_online) +{ + int ix, n, no_ssi; + char *daww_ptr; +#ifdef ERTS_SMP + size_t daww_sz; #endif init_misc_op_list_alloc(); +#ifdef ERTS_SMP + set_wakeup_other_data(); +#endif + ASSERT(no_schedulers_online <= no_schedulers); ASSERT(no_schedulers_online >= 1); ASSERT(no_schedulers >= 1); /* Create and initialize run queues */ - n = (int) (mrq ? no_schedulers : 1); + n = no_schedulers; erts_aligned_run_queues = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, @@ -2613,14 +3980,9 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_mtx_init_x(&rq->mtx, "run_queue", make_small(ix + 1)); erts_smp_cnd_init(&rq->cnd); -#ifdef ERTS_SMP - erts_smp_spinlock_init(&rq->sleepers.lock, "run_queue_sleep_list"); - rq->sleepers.list = NULL; -#endif - rq->waiting = 0; rq->woken = 0; - rq->flags = !mrq ? ERTS_RUNQ_FLG_SHARED_RUNQ : 0; + rq->flags = 0; rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; rq->full_reds_history_sum = 0; for (rix = 0; rix < ERTS_FULL_REDS_HISTORY_SIZE; rix++) { @@ -2632,6 +3994,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) rq->len = 0; rq->wakeup_other = 0; rq->wakeup_other_reds = 0; + rq->halt_in_progress = 0; rq->procs.len = 0; rq->procs.pending_exiters = NULL; @@ -2666,8 +4029,6 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) rq->ports.end = NULL; } - erts_common_run_queue = !mrq ? ERTS_RUNQ_IX(0) : NULL; - #ifdef ERTS_SMP if (erts_no_run_queues != 1) { @@ -2684,27 +4045,44 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) n = (int) no_schedulers; erts_no_schedulers = n; -#ifdef ERTS_SMP /* Create and initialize scheduler sleep info */ - +#ifdef ERTS_SMP + no_ssi = n+1; +#else + no_ssi = 1; +#endif aligned_sched_sleep_info = - erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_SLP_INFO, - n * sizeof(ErtsAlignedSchedulerSleepInfo)); - - for (ix = 0; ix < n; ix++) { - ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + erts_alloc_permanent_cache_aligned( + ERTS_ALC_T_SCHDLR_SLP_INFO, + no_ssi*sizeof(ErtsAlignedSchedulerSleepInfo)); + for (ix = 0; ix < no_ssi; ix++) { + ErtsSchedulerSleepInfo *ssi = &aligned_sched_sleep_info[ix].ssi; +#ifdef ERTS_SMP #if 0 /* no need to initialize these... */ ssi->next = NULL; ssi->prev = NULL; #endif erts_smp_atomic32_init_nob(&ssi->flags, 0); ssi->event = NULL; /* initialized in sched_thread_func */ - erts_smp_atomic32_init_nob(&ssi->aux_work, 0); +#endif + erts_atomic32_init_nob(&ssi->aux_work, 0); } + +#ifdef ERTS_SMP + aligned_sched_sleep_info++; #endif /* Create and initialize scheduler specific data */ +#ifdef ERTS_SMP + daww_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE((sizeof(ErtsDelayedAuxWorkWakeupJob) + + sizeof(int))*(n+1)); + daww_ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, + daww_sz*n); +#else + daww_ptr = NULL; +#endif + erts_aligned_scheduler_data = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, n*sizeof(ErtsAlignedSchedulerData)); @@ -2714,7 +4092,6 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) #ifdef ERTS_SMP erts_bits_init_state(&esdp->erl_bits_state); esdp->match_pseudo_process = NULL; - esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); esdp->free_process = NULL; #endif esdp->x_reg_array = @@ -2728,6 +4105,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) esdp->num_tmp_heap_used = 0; #endif esdp->no = (Uint) ix+1; + esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); esdp->current_process = NULL; esdp->current_port = NULL; @@ -2736,21 +4114,30 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_init_atom_cache_map(&esdp->atom_cache_map); - if (erts_common_run_queue) { - esdp->run_queue = erts_common_run_queue; - esdp->run_queue->scheduler = NULL; - } - else { - esdp->run_queue = ERTS_RUNQ_IX(ix); - esdp->run_queue->scheduler = esdp; - } + esdp->run_queue = ERTS_RUNQ_IX(ix); + esdp->run_queue->scheduler = esdp; + init_aux_work_data(&esdp->aux_work_data, esdp, daww_ptr); #ifdef ERTS_SMP - erts_smp_atomic32_init_nob(&esdp->chk_cpu_bind, 0); + daww_ptr += daww_sz; #endif + init_sched_wall_time(&esdp->sched_wall_time); } + init_misc_aux_work(); +#if !HALFWORD_HEAP + init_swtreq_alloc(); +#endif + + #ifdef ERTS_SMP + + erts_atomic32_init_nob(&completed_dealloc_count, 0); /* debug only */ + + aux_thread_aux_work_data = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, + sizeof(ErtsAuxWorkData)); + erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); erts_smp_cnd_init(&schdlr_sspnd.cnd); @@ -2760,8 +4147,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) schdlr_sspnd.msb.ongoing = 0; erts_smp_atomic32_init_nob(&schdlr_sspnd.active, no_schedulers); schdlr_sspnd.msb.procs = NULL; - init_no_runqs(no_schedulers, - erts_common_run_queue ? 1 : no_schedulers_online); + init_no_runqs(no_schedulers, no_schedulers_online); balance_info.last_active_runqs = no_schedulers; erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); balance_info.forced_check_balance = 0; @@ -2774,16 +4160,9 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) balance_info.n = 0; if (no_schedulers_online < no_schedulers) { - if (erts_common_run_queue) { - for (ix = no_schedulers_online; ix < no_schedulers; ix++) - erts_smp_atomic32_read_bor_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ERTS_SSI_FLG_SUSPENDED); - } - else { - for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) - evacuate_run_queue(ERTS_RUNQ_IX(ix), - ERTS_RUNQ_IX(ix % no_schedulers_online)); - } + for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) + evacuate_run_queue(ERTS_RUNQ_IX(ix), + ERTS_RUNQ_IX(ix % no_schedulers_online)); } schdlr_sspnd.wait_curr_online = no_schedulers_online; @@ -2812,6 +4191,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) /* init port tasks */ erts_port_task_init(); + aux_work_timeout_late_init(); + #ifndef ERTS_SMP #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC erts_scheduler_data->verify_unused_temp_alloc @@ -2820,14 +4201,15 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); #endif #endif + + erts_smp_atomic32_init_relb(&erts_halt_progress, -1); + erts_halt_code = 0; } ErtsRunQueue * erts_schedid2runq(Uint id) { int ix; - if (erts_common_run_queue) - return erts_common_run_queue; ix = (int) id - 1; ASSERT(0 <= ix && ix < erts_no_run_queues); return ERTS_RUNQ_IX(ix); @@ -2946,18 +4328,6 @@ erts_get_max_no_executing_schedulers(void) #ifdef ERTS_SMP static void -susp_sched_prep_block(void *unused) -{ - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); -} - -static void -susp_sched_resume_block(void *unused) -{ - erts_smp_mtx_lock(&schdlr_sspnd.mtx); -} - -static void scheduler_ix_resume_wake(Uint ix) { ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); @@ -3061,10 +4431,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) long active_schedulers; int curr_online = 1; int wake = 0; -#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ - || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) erts_aint32_t aux_work; -#endif + int thr_prgr_active = 1; /* * Schedulers may be suspended in two different ways: @@ -3086,6 +4454,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_inactive); + sched_wall_time_change(esdp, 0); + erts_smp_mtx_lock(&schdlr_sspnd.mtx); flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); @@ -3132,43 +4502,53 @@ suspend_scheduler(ErtsSchedulerData *esdp) wake = 0; } - flgs = erts_smp_atomic32_read_acqb(&ssi->flags); - if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) - break; + if (curr_online && !ongoing_multi_scheduling_block()) { + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) + break; + } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); - blockable_aux_work: - blockable_aux_work(esdp, ssi, aux_work); -#endif - - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); while (1) { erts_aint32_t flgs; -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK -#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read(&ssi->aux_work); -#endif - nonblockable_aux_work(esdp, ssi, aux_work); -#endif - flgs = sched_spin_suspended(ssi, - ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - flgs = sched_set_suspended_sleeptype(ssi); + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + if (aux_work) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + aux_work = handle_aux_work(&esdp->aux_work_data, + aux_work, + 1); + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); + } + + if (!aux_work) { + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(esdp); + flgs = sched_spin_suspended(ssi, + ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)) { - int res; - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_suspended_sleeptype(ssi); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } } + erts_thr_progress_finalize_wait(esdp); } flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING @@ -3178,20 +4558,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) break; - - -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); - if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - goto blockable_aux_work; - } -#endif - } - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); } @@ -3216,6 +4584,11 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_active); + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); @@ -3298,12 +4671,16 @@ erts_set_schedulers_online(Process *p, Sint new_no, Sint *old_no) { - int ix, res, no, have_unlocked_plocks; + ErtsSchedulerData *esdp; + int ix, res, no, have_unlocked_plocks, end_wait; erts_aint32_t changing; if (new_no < 1 || erts_no_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; + esdp = ERTS_PROC_GET_SCHDATA(p); + end_wait = 0; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); have_unlocked_plocks = 0; @@ -3329,10 +4706,6 @@ erts_set_schedulers_online(Process *p, for (ix = online; ix < no; ix++) erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); } - else if (erts_common_run_queue) { - for (ix = online; ix < no; ix++) - scheduler_ix_resume_wake(ix); - } else { if (plocks) { have_unlocked_plocks = 1; @@ -3380,15 +4753,6 @@ erts_set_schedulers_online(Process *p, for (ix = no; ix < online; ix++) erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); } - else if (erts_common_run_queue) { - for (ix = no; ix < online; ix++) { - ErtsSchedulerSleepInfo *ssi; - ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_all_schedulers(); - } else { if (plocks) { have_unlocked_plocks = 1; @@ -3415,21 +4779,26 @@ erts_set_schedulers_online(Process *p, erts_smp_mtx_lock(&schdlr_sspnd.mtx); for (ix = no; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq, 0, 1); + wake_scheduler(rq, 0); } } } - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); + if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (plocks && !have_unlocked_plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); + } + erts_thr_progress_active(esdp, 0); + erts_thr_progress_prepare_wait(esdp); + end_wait = 1; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + } + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); + ASSERT(res != ERTS_SCHDLR_SSPND_DONE ? (ERTS_SCHDLR_SSPND_CHNG_WAITER & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) @@ -3437,10 +4806,15 @@ erts_set_schedulers_online(Process *p, == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + } } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (end_wait) { + erts_thr_progress_finalize_wait(esdp); + erts_thr_progress_active(esdp, 1); + } if (have_unlocked_plocks) erts_smp_proc_lock(p, plocks); @@ -3498,44 +4872,58 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; schdlr_sspnd.msb.wait_active = 2; } - if (erts_common_run_queue) { - for (ix = 1; ix < online; ix++) - erts_smp_atomic32_read_bor_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ERTS_SSI_FLG_SUSPENDED); - wake_all_schedulers(); + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + erts_smp_mtx_lock(&balance_info.update_mtx); + set_no_used_runqs(1); + for (ix = 0; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + erts_smp_runq_lock(rq); + ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED)); + ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7); + erts_smp_runq_unlock(rq); } - else { + /* + * Evacuate all activities in all other run queues + * into the first run queue. Note order is important, + * online run queues has to be evacuated last. + */ + for (ix = erts_no_run_queues-1; ix >= 1; ix--) + evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(0)); + erts_smp_mtx_unlock(&balance_info.update_mtx); + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + + if (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) + != schdlr_sspnd.msb.wait_active) { + ErtsSchedulerData *esdp; + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - erts_smp_mtx_lock(&balance_info.update_mtx); - set_no_used_runqs(1); - for (ix = 0; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED)); - ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7); - erts_smp_runq_unlock(rq); + + if (plocks && !have_unlocked_plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); } - /* - * Evacuate all activities in all other run queues - * into the first run queue. Note order is important, - * online run queues has to be evacuated last. - */ - for (ix = erts_no_run_queues-1; ix >= 1; ix--) - evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(0)); - erts_smp_mtx_unlock(&balance_info.update_mtx); + + esdp = ERTS_PROC_GET_SCHDATA(p); + + erts_thr_progress_active(esdp, 0); + erts_thr_progress_prepare_wait(esdp); + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + + while (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) + != schdlr_sspnd.msb.wait_active) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, + &schdlr_sspnd.mtx); + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + + erts_thr_progress_active(esdp, 1); + erts_thr_progress_finalize_wait(esdp); + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + } - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) - != schdlr_sspnd.msb.wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED ? (ERTS_SCHDLR_SSPND_CHNG_WAITER & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) @@ -3615,12 +5003,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB); } - else if (erts_common_run_queue) { - for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic32_read_band_nob(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ~ERTS_SSI_FLG_SUSPENDED); - wake_all_schedulers(); - } else { int online = schdlr_sspnd.online; erts_smp_mtx_unlock(&schdlr_sspnd.mtx); @@ -3723,8 +5105,19 @@ erts_multi_scheduling_blockers(Process *p) static void * sched_thread_func(void *vesdp) { + ErtsThrPrgrCallbacks callbacks; + ErtsSchedulerData *esdp = vesdp; + Uint no = esdp->no; #ifdef ERTS_SMP - Uint no = ((ErtsSchedulerData *) vesdp)->no; + ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); + callbacks.arg = (void *) esdp->ssi; + callbacks.wakeup = thr_prgr_wakeup; + callbacks.prepare_wait = thr_prgr_prep_wait; + callbacks.wait = thr_prgr_wait; + callbacks.finalize_wait = thr_prgr_fin_wait; + + erts_thr_progress_register_managed_thread(esdp, &callbacks, 0); + erts_alloc_register_scheduler(vesdp); #endif #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -3733,22 +5126,30 @@ sched_thread_func(void *vesdp) erts_lc_set_thread_name(&buf[0]); } #endif - erts_alloc_reg_scheduler_id(no); erts_tsd_set(sched_data_key, vesdp); #ifdef ERTS_SMP +#if HAVE_ERTS_MSEG + erts_mseg_late_init(); +#endif +#if ERTS_USE_ASYNC_READY_Q + esdp->aux_work_data.async_ready.queue = erts_get_async_ready_queue(no); +#endif - erts_sched_init_check_cpu_bind((ErtsSchedulerData *) vesdp); + erts_sched_init_check_cpu_bind(esdp); erts_proc_lock_prepare_proc_lock_waiter(); - ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); - - #endif - erts_register_blockable_thread(); + #ifdef HIPE hipe_thread_signal_init(); #endif erts_thread_init_float(); + + if (no == 1) { + erts_thr_progress_active(esdp, 0); + erts_thr_progress_prepare_wait(esdp); + } + erts_smp_mtx_lock(&schdlr_sspnd.mtx); ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.changing) @@ -3757,41 +5158,39 @@ sched_thread_func(void *vesdp) if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) { erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - if (((ErtsSchedulerData *) vesdp)->no != 1) + if (no != 1) erts_smp_cnd_signal(&schdlr_sspnd.cnd); } - if (((ErtsSchedulerData *) vesdp)->no == 1) { - if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - } + if (no == 1) { + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (no == 1) { + erts_thr_progress_finalize_wait(esdp); + erts_thr_progress_active(esdp, 1); + } + #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC - ((ErtsSchedulerData *) vesdp)->verify_unused_temp_alloc + esdp->verify_unused_temp_alloc = erts_alloc_get_verify_unused_temp_alloc( - &((ErtsSchedulerData *) vesdp)->verify_unused_temp_alloc_data); + &esdp->verify_unused_temp_alloc_data); ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); #endif process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, "Scheduler thread number %beu terminated\n", - ((ErtsSchedulerData *) vesdp)->no); + erl_exit(ERTS_ABORT_EXIT, + "Scheduler thread number %beu terminated\n", + no); return NULL; } +static ethr_tid aux_tid; + void erts_start_schedulers(void) { @@ -3811,8 +5210,6 @@ erts_start_schedulers(void) res = ENOTSUP; } - erts_block_system(0); - while (actual < wanted) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual); actual++; @@ -3825,7 +5222,12 @@ erts_start_schedulers(void) } erts_no_schedulers = actual; - erts_release_system(); + + ERTS_THR_MEMORY_BARRIER; + + res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts); + if (res != 0) + erl_exit(1, "Failed to create aux thread\n"); if (actual < 1) erl_exit(1, @@ -4279,7 +5681,7 @@ suspend_process_2(BIF_ALIST_2) /* This is really a piece of cake without SMP support... */ if (!smon->active) { - suspend_process(erts_common_run_queue, suspendee); + suspend_process(ERTS_RUNQ_IX(0), suspendee); smon->active++; res = am_true; } @@ -4849,8 +6251,6 @@ erts_proc_migrate(Process *p, ErtsProcLocks *plcks, || from_locked); ERTS_SMP_LC_CHK_RUNQ_LOCK(from_rq, *from_locked); ERTS_SMP_LC_CHK_RUNQ_LOCK(to_rq, *to_locked); - - ASSERT(!erts_common_run_queue); /* * If we have the lock on the run queue to migrate to, @@ -5001,25 +6401,17 @@ erts_process_status(Process *c_p, ErtsProcLocks c_p_locks, int i; ErtsSchedulerData *esdp; - if (erts_common_run_queue) - erts_smp_runq_lock(erts_common_run_queue); - for (i = 0; i < erts_no_schedulers; i++) { esdp = ERTS_SCHEDULER_IX(i); - if (!erts_common_run_queue) - erts_smp_runq_lock(esdp->run_queue); + erts_smp_runq_lock(esdp->run_queue); if (esdp->free_process && esdp->free_process->id == rpid) { res = am_free; - if (!erts_common_run_queue) - erts_smp_runq_unlock(esdp->run_queue); + erts_smp_runq_unlock(esdp->run_queue); break; } - if (!erts_common_run_queue) - erts_smp_runq_unlock(esdp->run_queue); + erts_smp_runq_unlock(esdp->run_queue); } - if (erts_common_run_queue) - erts_smp_runq_unlock(erts_common_run_queue); #endif } @@ -5177,6 +6569,15 @@ Process *schedule(Process *p, int calls) int actual_reds; int reds; +#ifdef USE_VM_PROBES + if (p != NULL && DTRACE_ENABLED(process_unscheduled)) { + DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE); + + dtrace_proc_str(p, process_buf); + DTRACE1(process_unscheduled, process_buf); + } +#endif + if (ERTS_USE_MODIFIED_TIMING()) { context_reds = ERTS_MODIFIED_TIMING_CONTEXT_REDS; input_reductions = ERTS_MODIFIED_TIMING_INPUT_REDS; @@ -5186,7 +6587,7 @@ Process *schedule(Process *p, int calls) input_reductions = INPUT_REDUCTIONS; } - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); /* * Clean up after the process being scheduled out. @@ -5324,17 +6725,16 @@ Process *schedule(Process *p, int calls) } - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); + check_activities_to_run: { #ifdef ERTS_SMP - if (!(rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - && rq->check_balance_reds <= 0) { + if (rq->check_balance_reds <= 0) check_balance(rq); - } - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); if (rq->flags & ERTS_RUNQ_FLGS_IMMIGRATE_QMASK) @@ -5342,58 +6742,51 @@ Process *schedule(Process *p, int calls) continue_check_activities_to_run: - if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ - | ERTS_RUNQ_FLG_CHK_CPU_BIND + if (rq->flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND | ERTS_RUNQ_FLG_SUSPENDED)) { - if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) - & ERTS_SSI_FLG_SUSPENDED)) { + if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) { ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED); suspend_scheduler(esdp); } - if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) - || erts_smp_atomic32_read_acqb(&esdp->chk_cpu_bind)) { + if (rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) erts_sched_check_cpu_bind(esdp); - } } -#if defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) \ - || defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) { - ErtsSchedulerSleepInfo *ssi = esdp->ssi; - erts_aint32_t aux_work = erts_smp_atomic32_read_nob(&ssi->aux_work); - if (aux_work) { + erts_aint32_t aux_work; + int leader_update = erts_thr_progress_update(esdp); + aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); + if (aux_work | leader_update) { erts_smp_runq_unlock(rq); -#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = blockable_aux_work(esdp, ssi, aux_work); -#endif -#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK - nonblockable_aux_work(esdp, ssi, aux_work); -#endif + if (leader_update) + erts_thr_progress_leader_update(esdp); + if (aux_work) + handle_aux_work(&esdp->aux_work_data, aux_work, 0); erts_smp_runq_lock(rq); } } -#endif - erts_smp_chk_system_block(prepare_for_block, - resume_after_block, - (void *) rq); - - ERTS_SMP_LC_ASSERT(!ERTS_LC_IS_BLOCKING); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); -#endif +#else /* ERTS_SMP */ + { + erts_aint32_t aux_work; + aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); + if (aux_work) + handle_aux_work(&esdp->aux_work_data, aux_work, 0); + } +#endif /* ERTS_SMP */ ASSERT(rq->len == rq->procs.len + rq->ports.info.len); -#ifndef ERTS_SMP + if ((rq->len == 0 && !rq->misc.start) + || (rq->halt_in_progress + && rq->ports.info.len == 0 && !rq->misc.start)) { - if (rq->len == 0 && !rq->misc.start) - goto do_sys_schedule; +#ifdef ERTS_SMP -#else /* ERTS_SMP */ - if (rq->len == 0 && !rq->misc.start) { ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); rq->wakeup_other = 0; @@ -5401,16 +6794,11 @@ Process *schedule(Process *p, int calls) empty_runq(rq); - if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ - | ERTS_RUNQ_FLG_SUSPENDED)) { - if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags) - & ERTS_SSI_FLG_SUSPENDED)) { - ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags) - & ERTS_SSI_FLG_SUSPENDED); - non_empty_runq(rq); - goto continue_check_activities_to_run; - } + if (rq->flags & ERTS_RUNQ_FLG_SUSPENDED) { + ASSERT(erts_smp_atomic32_read_nob(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED); + non_empty_runq(rq); + goto continue_check_activities_to_run; } else if (!(rq->flags & ERTS_RUNQ_FLG_INACTIVE)) { /* @@ -5424,26 +6812,17 @@ Process *schedule(Process *p, int calls) } } +#endif + scheduler_wait(&fcalls, esdp, rq); +#ifdef ERTS_SMP non_empty_runq(rq); +#endif goto check_activities_to_run; } - else -#endif /* ERTS_SMP */ - if (fcalls > input_reductions && prepare_for_sys_schedule()) { - int runnable; - -#ifdef ERTS_SMP - runnable = 1; -#else - do_sys_schedule: - runnable = rq->len != 0; - if (!runnable) - sched_waiting_sys(esdp->no, rq); -#endif - + else if (fcalls > input_reductions && prepare_for_sys_schedule()) { /* * Schedule system-level activities. */ @@ -5453,11 +6832,11 @@ Process *schedule(Process *p, int calls) ASSERT(!erts_port_task_have_outstanding_io_tasks()); -#ifdef ERTS_SMP - /* erts_sys_schedule_interrupt(0); */ +#if 0 /* Not needed since we wont wait in sys schedule */ + erts_sys_schedule_interrupt(0); #endif erts_smp_runq_unlock(rq); - erl_sys_schedule(runnable); + erl_sys_schedule(1); dt = erts_do_time_read_and_reset(); if (dt) erts_bump_timer(dt); #ifdef ERTS_SMP @@ -5465,8 +6844,6 @@ Process *schedule(Process *p, int calls) clear_sys_scheduling(); goto continue_check_activities_to_run; #else - if (!runnable) - sched_active_sys(esdp->no, rq); goto check_activities_to_run; #endif } @@ -5475,30 +6852,7 @@ Process *schedule(Process *p, int calls) exec_misc_ops(rq); #ifdef ERTS_SMP - { - int wo_reds = rq->wakeup_other_reds; - if (wo_reds) { - if (rq->len < 2) { - rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC*wo_reds; - if (rq->wakeup_other < 0) - rq->wakeup_other = 0; - } - else if (rq->wakeup_other < wakeup_other_limit) - rq->wakeup_other += rq->len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC; - else { - if (erts_common_run_queue) { - if (erts_common_run_queue->waiting) - wake_scheduler(erts_common_run_queue, 0, 1); - } - else if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) { - wake_scheduler_on_empty_runq(rq); - rq->wakeup_other = 0; - } - rq->wakeup_other = 0; - } - } - rq->wakeup_other_reds = 0; - } + wakeup_other.check(rq); #endif /* @@ -5508,7 +6862,8 @@ Process *schedule(Process *p, int calls) if (rq->ports.info.len) { int have_outstanding_io; have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port); - if (have_outstanding_io && fcalls > 2*input_reductions) { + if ((have_outstanding_io && fcalls > 2*input_reductions) + || rq->halt_in_progress) { /* * If we have performed more than 2*INPUT_REDUCTIONS since * last call to erl_sys_schedule() and we still haven't @@ -5661,7 +7016,6 @@ Process *schedule(Process *p, int calls) #endif ASSERT(p->status != P_SUSPENDED); /* Never run a suspended process */ - ACTIVATE(p); reds = context_reds; if (IS_TRACED(p)) { @@ -5702,7 +7056,6 @@ Process *schedule(Process *p, int calls) } p->fcalls = reds; - ASSERT(IS_ACTIVE(p)); ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); return p; } @@ -5714,14 +7067,14 @@ erts_sched_stat_modify(int what) int ix; switch (what) { case ERTS_SCHED_STAT_MODIFY_ENABLE: - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_sched_stat.enabled = 1; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); break; case ERTS_SCHED_STAT_MODIFY_DISABLE: - erts_smp_block_system(0); + erts_smp_thr_progress_block(); erts_sched_stat.enabled = 1; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); break; case ERTS_SCHED_STAT_MODIFY_CLEAR: erts_smp_spin_lock(&erts_sched_stat.lock); @@ -5781,20 +7134,9 @@ erts_sched_stat_term(Process *p, int total) void erts_schedule_misc_op(void (*func)(void *), void *arg) { - ErtsRunQueue *rq; - ErtsMiscOpList *molp = misc_op_list_alloc(); ErtsSchedulerData *esdp = erts_get_scheduler_data(); - - if (esdp) { - rq = esdp->run_queue; - } else { - /* - * This can only happen when the sys msg dispatcher - * thread schedules misc ops (this happens *very* - * seldom; only when trace drivers are unloaded). - */ - rq = ERTS_RUNQ_IX(0); - } + ErtsRunQueue *rq = esdp ? esdp->run_queue : ERTS_RUNQ_IX(0); + ErtsMiscOpList *molp = misc_op_list_alloc(); erts_smp_runq_lock(rq); @@ -5887,7 +7229,7 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp) * Wait for other schedulers to schedule out their processes * and update 'reductions'. */ - erts_smp_block_system(0); + erts_smp_thr_progress_block(); for (reds = 0, ix = 0; ix < erts_no_run_queues; ix++) reds += ERTS_RUNQ_IX(ix)->procs.reductions; if (redsp) @@ -5895,7 +7237,7 @@ erts_get_exact_total_reductions(Process *c_p, Uint *redsp, Uint *diffp) if (diffp) *diffp = reds - last_exact_reductions; last_exact_reductions = reds; - erts_smp_release_system(); + erts_smp_thr_progress_unblock(); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN); } @@ -6061,9 +7403,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). ErtsRunQueue *rq, *notify_runq; Process *p; Sint arity; /* Number of arguments. */ -#ifndef HYBRID Uint arg_size; /* Size of arguments. */ -#endif Uint sz; /* Needed words on heap. */ Uint heap_need; /* Size needed on heap. */ Eterm res = THE_NON_VALUE; @@ -6072,17 +7412,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); #endif -#ifdef HYBRID - /* - * Copy the arguments to the global heap - * Since global GC might occur we want to do this before adding the - * new process to the process_tab. - */ - BM_SWAP_TIMER(system,copy); - LAZY_COPY(parent,args); - BM_SWAP_TIMER(copy,system); - heap_need = 0; -#endif /* HYBRID */ /* * Check for errors. */ @@ -6100,15 +7429,15 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). goto error; } +#ifdef BM_COUNTERS processes_busy++; +#endif BM_COUNT(processes_spawned); -#ifndef HYBRID BM_SWAP_TIMER(system,size); arg_size = size_object(args); BM_SWAP_TIMER(size,system); heap_need = arg_size; -#endif p->flags = erts_default_process_flags; @@ -6159,9 +7488,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->heap = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_HEAP, sizeof(Eterm)*sz); p->old_hend = p->old_htop = p->old_heap = NULL; p->high_water = p->heap; -#ifdef INCREMENTAL - p->scan_top = p->high_water; -#endif p->gen_gcs = 0; p->stop = p->hend = p->heap + sz; p->htop = p->heap; @@ -6187,19 +7513,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). BM_STOP_TIMER(system); BM_MESSAGE(args,p,parent); BM_START_TIMER(system); -#ifdef HYBRID - p->arg_reg[2] = args; -#ifdef INCREMENTAL - p->active = 0; - if (ptr_val(args) >= inc_fromspc && ptr_val(args) < inc_fromend) - INC_ACTIVATE(p); -#endif -#else BM_SWAP_TIMER(system,copy); p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap); BM_MESSAGE_COPIED(arg_size); BM_SWAP_TIMER(copy,system); -#endif p->arity = 3; p->fvalue = NIL; @@ -6251,14 +7568,11 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->seq_trace_lastcnt = 0; p->seq_trace_clock = 0; SEQ_TRACE_TOKEN(p) = NIL; - p->parent = parent->id == ERTS_INVALID_PID ? NIL : parent->id; - -#ifdef HYBRID - p->rrma = NULL; - p->rrsrc = NULL; - p->nrr = 0; - p->rrsz = 0; +#ifdef USE_VM_PROBES + DT_UTAG(p) = NIL; + DT_UTAG_FLAGS(p) = 0; #endif + p->parent = parent->id == ERTS_INVALID_PID ? NIL : parent->id; INIT_HOLE_CHECK(p); #ifdef DEBUG @@ -6328,15 +7642,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). so->mref = mref; } -#ifdef HYBRID - /* - * Add process to the array of active processes. - */ - ACTIVATE(p); - p->active_index = erts_num_active_procs++; - erts_active_procs[p->active_index] = p; -#endif - #ifdef ERTS_SMP p->scheduler_data = NULL; p->is_exiting = 0; @@ -6348,7 +7653,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->pending_exit.bp = NULL; #endif -#if !defined(NO_FPE_SIGNALS) +#if !defined(NO_FPE_SIGNALS) || defined(HIPE) p->fp_exception = 0; #endif @@ -6383,6 +7688,16 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->id)); +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(process_spawn)) { + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); + + dtrace_fun_decode(p, mod, func, arity, process_name, mfa); + DTRACE2(process_spawn, process_name, mfa); + } +#endif + error: erts_smp_proc_unlock(parent, ERTS_PROC_LOCKS_ALL_MINOR); @@ -6437,9 +7752,6 @@ void erts_init_empty_process(Process *p) p->reg = NULL; p->heap_sz = 0; p->high_water = NULL; -#ifdef INCREMENTAL - p->scan_top = NULL; -#endif p->old_hend = NULL; p->old_htop = NULL; p->old_heap = NULL; @@ -6491,14 +7803,6 @@ void erts_init_empty_process(Process *p) #endif #endif - ACTIVATE(p); - -#ifdef HYBRID - p->rrma = NULL; - p->rrsrc = NULL; - p->nrr = 0; - p->rrsz = 0; -#endif INIT_HOLE_CHECK(p); #ifdef DEBUG p->last_old_htop = NULL; @@ -6522,7 +7826,7 @@ void erts_init_empty_process(Process *p) p->run_queue = ERTS_RUNQ_IX(0); #endif -#if !defined(NO_FPE_SIGNALS) +#if !defined(NO_FPE_SIGNALS) || defined(HIPE) p->fp_exception = 0; #endif @@ -6546,9 +7850,6 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->reg == NULL); ASSERT(p->heap_sz == 0); ASSERT(p->high_water == NULL); -#ifdef INCREMENTAL - ASSERT(p->scan_top == NULL); -#endif ASSERT(p->old_hend == NULL); ASSERT(p->old_htop == NULL); ASSERT(p->old_heap == NULL); @@ -6696,22 +7997,6 @@ delete_process(Process* p) ASSERT(!p->suspend_monitors); p->fvalue = NIL; - -#ifdef HYBRID - erts_active_procs[p->active_index] = - erts_active_procs[--erts_num_active_procs]; - erts_active_procs[p->active_index]->active_index = p->active_index; -#ifdef INCREMENTAL - if (INC_IS_ACTIVE(p)) - INC_DEACTIVATE(p); -#endif - - if (p->rrma != NULL) { - erts_free(ERTS_ALC_T_ROOTSET,p->rrma); - erts_free(ERTS_ALC_T_ROOTSET,p->rrsrc); - } -#endif - } static ERTS_INLINE void @@ -6834,7 +8119,11 @@ static ERTS_INLINE void send_exit_message(Process *to, ErtsProcLocks *to_locksp, Eterm exit_term, Uint term_size, Eterm token) { - if (token == NIL) { + if (token == NIL +#ifdef USE_VM_PROBES + || token == am_have_dt_utag +#endif + ) { Eterm* hp; Eterm mess; ErlHeapFragment* bp; @@ -6842,7 +8131,11 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, hp = erts_alloc_message_heap(term_size, &bp, &ohp, to, to_locksp); mess = copy_struct(exit_term, term_size, &hp, ohp); - erts_queue_message(to, to_locksp, bp, mess, NIL); + erts_queue_message(to, to_locksp, bp, mess, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); } else { ErlHeapFragment* bp; Eterm* hp; @@ -6858,7 +8151,11 @@ send_exit_message(Process *to, ErtsProcLocks *to_locksp, /* the trace token must in this case be updated by the caller */ seq_trace_output(token, mess, SEQ_TRACE_SEND, to->id, NULL); temp_token = copy_struct(token, sz_token, &hp, &bp->off_heap); - erts_queue_message(to, to_locksp, bp, mess, temp_token); + erts_queue_message(to, to_locksp, bp, mess, temp_token +#ifdef USE_VM_PROBES + , NIL +#endif + ); } } @@ -6951,9 +8248,26 @@ send_exit_signal(Process *c_p, /* current process if and only ASSERT(reason != THE_NON_VALUE); +#ifdef USE_VM_PROBES + if(DTRACE_ENABLED(process_exit_signal) && is_pid(from)) { + DTRACE_CHARBUF(sender_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(receiver_str, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(reason_buf, DTRACE_TERM_BUF_SIZE); + + dtrace_pid_str(from, sender_str); + dtrace_proc_str(rp, receiver_str); + erts_snprintf(reason_buf, sizeof(reason_buf) - 1, "%T", reason); + DTRACE3(process_exit_signal, sender_str, receiver_str, reason_buf); + } +#endif + if (ERTS_PROC_IS_TRAPPING_EXITS(rp) && (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) { - if (is_not_nil(token) && token_update) + if (is_not_nil(token) +#ifdef USE_VM_PROBES + && token != am_have_dt_utag +#endif + && token_update) seq_trace_update_send(token_update); if (is_value(exit_tuple)) send_exit_message(rp, rp_locks, exit_tuple, exit_tuple_sz, token); @@ -7054,7 +8368,6 @@ send_exit_signal(Process *c_p, /* current process if and only set_proc_exiting(rp, is_immed(rsn) ? rsn : copy_object(rsn, rp), NULL); - ACTIVATE(rp); if (old_status != P_RUNABLE && old_status != P_RUNNING) erts_add_to_runq(rp); } @@ -7336,15 +8649,6 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) if (rlnk) erts_destroy_link(rlnk); erts_deref_dist_entry(dep); - } else { -#ifndef ERTS_SMP - /* XXX Is this possible? Shouldn't this link - previously have been removed if the node - had previously been disconnected. */ - ASSERT(0); -#endif - /* This is possible when smp support has been enabled, - and dist port and process exits simultaneously. */ } break; @@ -7386,7 +8690,18 @@ erts_do_exit_process(Process* p, Eterm reason) p->arity = 0; /* No live registers */ p->fvalue = reason; - + +#ifdef USE_VM_PROBES + if (DTRACE_ENABLED(process_exit)) { + DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(reason_buf, DTRACE_TERM_BUF_SIZE); + + dtrace_proc_str(p, process_buf); + erts_snprintf(reason_buf, DTRACE_TERM_BUF_SIZE - 1, "%T", reason); + DTRACE2(process_exit, process_buf, reason_buf); + } +#endif + #ifdef ERTS_SMP ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* By locking all locks (main lock is already locked) when going @@ -7623,7 +8938,9 @@ continue_exit_process(Process *p pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); +#ifdef BM_COUNTERS processes_busy--; +#endif if (dep) { erts_do_net_exits(dep, reason); @@ -8700,6 +10017,22 @@ init_processes_bif(void) * Debug stuff */ +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) +int +erts_dbg_check_halloc_lock(Process *p) +{ + if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)) + return 1; + if (p->id == ERTS_INVALID_PID) + return 1; + if (p->scheduler_data && p == p->scheduler_data->match_pseudo_process) + return 1; + if (erts_thr_progress_is_blocking()) + return 1; + return 0; +} +#endif + Eterm erts_debug_processes(Process *c_p) { @@ -8934,3 +10267,30 @@ debug_processes_assert_error(char* expr, char* file, int line) /* *\ * End of the processes/0 BIF implementation. * \* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * A nice system halt closing all open port goes as follows: + * 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS + * on all schedulers, then schedules itself out. + * 2) All shedulers detect this and set the flag halt_in_progress + * on their run queue. The last scheduler sets all non-closed ports + * ERTS_PORT_SFLG_HALT. Global atomic erts_halt_progress is used + * as refcount to determine which is last. + * 3) While the run ques has flag halt_in_progress no processes + * will be scheduled, only ports. + * 4) When the last port closes that scheduler calls erlang:halt/1. + * The same global atomic is used as refcount. + * + * A BIF that calls this should make sure to schedule out to never come back: + * erl_halt((int)(- code)); + * ERTS_BIF_YIELD1(bif_export[BIF_erlang_halt_1], BIF_P, NIL); + */ +void erl_halt(int code) +{ + if (-1 == erts_smp_atomic32_cmpxchg_acqb(&erts_halt_progress, + erts_no_schedulers, + -1)) { + erts_halt_code = code; + notify_reap_ports_relb(); + } +} |