diff options
Diffstat (limited to 'erts/emulator/beam/erl_process.c')
| -rw-r--r-- | erts/emulator/beam/erl_process.c | 5138 |
1 files changed, 2905 insertions, 2233 deletions
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 2b468a9ad9..794af60b2f 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -3,16 +3,17 @@ * * Copyright Ericsson AB 1996-2014. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -43,8 +44,11 @@ #include "erl_async.h" #include "dtrace-wrapper.h" #include "erl_ptab.h" +#include "erl_bif_unique.h" +#define ERTS_WANT_TIMER_WHEEL_API +#include "erl_time.h" - +#define ERTS_CHECK_TIME_REDS CONTEXT_REDS #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0) #define ERTS_DELAYED_WAKEUP_REDUCTIONS ((Uint64) CONTEXT_REDS/2) @@ -54,11 +58,7 @@ #define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10) -#ifndef ERTS_SCHED_MIN_SPIN #define ERTS_SCHED_SPIN_UNTIL_YIELD 100 -#else -#define ERTS_SCHED_SPIN_UNTIL_YIELD 1 -#endif #define ERTS_SCHED_SYS_SLEEP_SPINCOUNT_VERY_LONG 40 #define ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_VERY_LONG 1000 @@ -115,7 +115,7 @@ #define RUNQ_SET_RQ(X, RQ) erts_smp_atomic_set_nob((X), (erts_aint_t) (RQ)) #ifdef DEBUG -# if defined(ARCH_64) && !HALFWORD_HEAP +# if defined(ARCH_64) # define ERTS_DBG_SET_INVALID_RUNQP(RQP, N) \ (RUNQ_SET_RQ((RQP), (0xdeadbeefdead0003LL | ((N) << 4))) # define ERTS_DBG_VERIFY_VALID_RUNQP(RQP) \ @@ -148,13 +148,16 @@ extern BeamInstr beam_apply[]; extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; -int erts_sched_compact_load; -int erts_sched_balance_util = 0; -Uint erts_no_schedulers; -#ifdef ERTS_DIRTY_SCHEDULERS -Uint erts_no_dirty_cpu_schedulers; -Uint erts_no_dirty_io_schedulers; -#endif +int ERTS_WRITE_UNLIKELY(erts_default_spo_flags) = 0; +int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1; +int ERTS_WRITE_UNLIKELY(erts_sched_compact_load); +int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0; +Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers); +Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers) = 0; +Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers) = 0; + +static char *erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_NO_FLAGS] = {0}; +int erts_aux_work_no_flags = ERTS_SSI_AUX_WORK_NO_FLAGS; #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_LAZY (4*1024*1024) #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_LAZY (512*1024) @@ -183,84 +186,176 @@ int erts_disable_proc_not_running_opt; static ErtsAuxWorkData *aux_thread_aux_work_data; -#define ERTS_SCHDLR_SSPND_CHNG_WAITER (((erts_aint32_t) 1) << 0) +#define ERTS_SCHDLR_SSPND_CHNG_NMSB (((erts_aint32_t) 1) << 0) #define ERTS_SCHDLR_SSPND_CHNG_MSB (((erts_aint32_t) 1) << 1) #define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2) +#define ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN (((erts_aint32_t) 1) << 3) -#ifndef DEBUG +typedef enum { + ERTS_SCHED_NORMAL, + ERTS_SCHED_DIRTY_CPU, + ERTS_SCHED_DIRTY_IO +} ErtsSchedType; -#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ - erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, (VAL)) +typedef struct { + int ongoing; + ErtsProcList *blckrs; + ErtsProcList *chngq; +} ErtsMultiSchedulingBlock; -#ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(VAL, OLD_VAL) \ - erts_smp_atomic32_set_nob(&schdlr_sspnd.dirty_cpu_changing, (VAL)) -#define ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(VAL, OLD_VAL) \ - erts_smp_atomic32_set_nob(&schdlr_sspnd.dirty_io_changing, (VAL)) -#endif +static struct { + erts_smp_mtx_t mtx; + Uint32 online; + Uint32 curr_online; + Uint32 active; + erts_smp_atomic32_t changing; + ErtsProcList *chngq; + Eterm changer; + ErtsMultiSchedulingBlock nmsb; /* Normal multi Scheduling Block */ + ErtsMultiSchedulingBlock msb; /* Multi Scheduling Block */ +} schdlr_sspnd; -#else +#define ERTS_SCHDLR_SSPND_S_BITS 10 +#define ERTS_SCHDLR_SSPND_DCS_BITS 11 +#define ERTS_SCHDLR_SSPND_DIS_BITS 11 -#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ -do { \ - erts_aint32_t old_val__; \ - old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.changing, \ - (VAL)); \ - ASSERT(old_val__ == (OLD_VAL)); \ -} while (0) +#define ERTS_SCHDLR_SSPND_S_MASK ((1 << ERTS_SCHDLR_SSPND_S_BITS)-1) +#define ERTS_SCHDLR_SSPND_DCS_MASK ((1 << ERTS_SCHDLR_SSPND_DCS_BITS)-1) +#define ERTS_SCHDLR_SSPND_DIS_MASK ((1 << ERTS_SCHDLR_SSPND_DIS_BITS)-1) -#ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(VAL, OLD_VAL) \ -do { \ - erts_aint32_t old_val__; \ - old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.dirty_cpu_changing, \ - (VAL)); \ - ASSERT(old_val__ == (OLD_VAL)); \ -} while (0) -#define ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(VAL, OLD_VAL) \ -do { \ - erts_aint32_t old_val__; \ - old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.dirty_io_changing, \ - (VAL)); \ - ASSERT(old_val__ == (OLD_VAL)); \ -} while (0) -#endif +#define ERTS_SCHDLR_SSPND_S_SHIFT 0 +#define ERTS_SCHDLR_SSPND_DCS_SHIFT (ERTS_SCHDLR_SSPND_S_SHIFT \ + + ERTS_SCHDLR_SSPND_S_BITS) +#define ERTS_SCHDLR_SSPND_DIS_SHIFT (ERTS_SCHDLR_SSPND_DCS_SHIFT \ + + ERTS_SCHDLR_SSPND_DCS_BITS) +#if (ERTS_SCHDLR_SSPND_S_BITS \ + + ERTS_SCHDLR_SSPND_DCS_BITS \ + + ERTS_SCHDLR_SSPND_DIS_BITS) > 32 +# error Wont fit in Uint32 #endif - -static struct { - erts_smp_mtx_t mtx; - erts_smp_cnd_t cnd; - int online; - int curr_online; - int wait_curr_online; -#ifdef ERTS_DIRTY_SCHEDULERS - int dirty_cpu_online; - int dirty_cpu_curr_online; - int dirty_cpu_wait_curr_online; - int dirty_io_online; - int dirty_io_curr_online; - int dirty_io_wait_curr_online; +#if (ERTS_MAX_NO_OF_SCHEDULERS-1) > ERTS_SCHDLR_SSPND_S_MASK +# error Max no schedulers wont fit in its bit-field #endif - erts_smp_atomic32_t changing; - erts_smp_atomic32_t active; -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_atomic32_t dirty_cpu_changing; - erts_smp_atomic32_t dirty_cpu_active; - erts_smp_atomic32_t dirty_io_changing; - erts_smp_atomic32_t dirty_io_active; +#if ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS > ERTS_SCHDLR_SSPND_DCS_MASK +# error Max no dirty cpu schedulers wont fit in its bit-field #endif - struct { - int ongoing; - long wait_active; -#ifdef ERTS_DIRTY_SCHEDULERS - long dirty_cpu_wait_active; - long dirty_io_wait_active; +#if ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS > ERTS_SCHDLR_SSPND_DIS_MASK +# error Max no dirty io schedulers wont fit in its bit-field #endif - ErtsProcList *procs; - } msb; /* Multi Scheduling Block */ -} schdlr_sspnd; + +#define ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(S, DCS, DIS) \ + ((((Uint32) (((S) & ERTS_SCHDLR_SSPND_S_MASK))-1) \ + << ERTS_SCHDLR_SSPND_S_SHIFT) \ + | ((((Uint32) ((DCS) & ERTS_SCHDLR_SSPND_DCS_MASK)) \ + << ERTS_SCHDLR_SSPND_DCS_SHIFT)) \ + | ((((Uint32) ((DIS) & ERTS_SCHDLR_SSPND_DIS_MASK)) \ + << ERTS_SCHDLR_SSPND_DIS_SHIFT))) + +static void init_scheduler_suspend(void); + +static ERTS_INLINE Uint32 +schdlr_sspnd_get_nscheds(Uint32 *valp, ErtsSchedType type) +{ + Uint32 res = (Uint32) (*valp); + switch (type) { + case ERTS_SCHED_NORMAL: + res >>= ERTS_SCHDLR_SSPND_S_SHIFT; + res &= (Uint32) ERTS_SCHDLR_SSPND_S_MASK; + res++; + break; + case ERTS_SCHED_DIRTY_CPU: + res >>= ERTS_SCHDLR_SSPND_DCS_SHIFT; + res &= (Uint32) ERTS_SCHDLR_SSPND_DCS_MASK; + break; + case ERTS_SCHED_DIRTY_IO: + res >>= ERTS_SCHDLR_SSPND_DIS_SHIFT; + res &= (Uint32) ERTS_SCHDLR_SSPND_DIS_MASK; + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + return 0; + } + + return res; +} + +static ERTS_INLINE void +schdlr_sspnd_dec_nscheds(Uint32 *valp, ErtsSchedType type) +{ + ASSERT(schdlr_sspnd_get_nscheds(valp, type) > 0); + + switch (type) { + case ERTS_SCHED_NORMAL: + *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT; + break; + case ERTS_SCHED_DIRTY_CPU: + *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT; + break; + case ERTS_SCHED_DIRTY_IO: + *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT; + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + } +} + +static ERTS_INLINE void +schdlr_sspnd_inc_nscheds(Uint32 *valp, ErtsSchedType type) +{ + switch (type) { + case ERTS_SCHED_NORMAL: + ASSERT(schdlr_sspnd_get_nscheds(valp, type) + < ERTS_MAX_NO_OF_SCHEDULERS-1); + *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT; + break; + case ERTS_SCHED_DIRTY_CPU: + ASSERT(schdlr_sspnd_get_nscheds(valp, type) + < ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS); + *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT; + break; + case ERTS_SCHED_DIRTY_IO: + ASSERT(schdlr_sspnd_get_nscheds(valp, type) + < ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS); + *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT; + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + } +} + +static ERTS_INLINE void +schdlr_sspnd_set_nscheds(Uint32 *valp, ErtsSchedType type, Uint32 no) +{ + Uint32 val = *valp; + + switch (type) { + case ERTS_SCHED_NORMAL: + ASSERT(no > 0); + val &= ~(((Uint32) ERTS_SCHDLR_SSPND_S_MASK) + << ERTS_SCHDLR_SSPND_S_SHIFT); + val |= (((no-1) & ((Uint32) ERTS_SCHDLR_SSPND_S_MASK)) + << ERTS_SCHDLR_SSPND_S_SHIFT); + break; + case ERTS_SCHED_DIRTY_CPU: + val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK) + << ERTS_SCHDLR_SSPND_DCS_SHIFT); + val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK)) + << ERTS_SCHDLR_SSPND_DCS_SHIFT); + break; + case ERTS_SCHED_DIRTY_IO: + val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK) + << ERTS_SCHDLR_SSPND_DIS_SHIFT); + val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK)) + << ERTS_SCHDLR_SSPND_DIS_SHIFT); + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + } + + *valp = val; +} static struct { erts_smp_mtx_t update_mtx; @@ -290,7 +385,7 @@ do { \ erts_sched_stat_t erts_sched_stat; #ifdef USE_THREADS -static erts_tsd_key_t sched_data_key; +static erts_tsd_key_t ERTS_WRITE_UNLIKELY(sched_data_key); #endif static erts_smp_atomic32_t function_calls; @@ -330,16 +425,16 @@ static ErtsAlignedSchedulerSleepInfo *aligned_dirty_io_sched_sleep_info; static Uint last_reductions; static Uint last_exact_reductions; -Uint erts_default_process_flags; -Eterm erts_system_monitor; -Eterm erts_system_monitor_long_gc; -Uint erts_system_monitor_long_schedule; -Eterm erts_system_monitor_large_heap; +Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor); +Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor_long_gc); +Uint ERTS_WRITE_UNLIKELY(erts_system_monitor_long_schedule); +Eterm ERTS_WRITE_UNLIKELY(erts_system_monitor_large_heap); struct erts_system_monitor_flags_t erts_system_monitor_flags; /* system performance monitor */ Eterm erts_system_profile; struct erts_system_profile_flags_t erts_system_profile_flags; +int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP; #if ERTS_MAX_PROCESSES > 0x7fffffff #error "Need to store process_count in another type" @@ -347,7 +442,8 @@ struct erts_system_profile_flags_t erts_system_profile_flags; typedef enum { ERTS_PSTT_GC, /* Garbage Collect */ - ERTS_PSTT_CPC /* Check Process Code */ + ERTS_PSTT_CPC, /* Check Process Code */ + ERTS_PSTT_COHMQ /* Change off heap message queue */ } ErtsProcSysTaskType; #define ERTS_MAX_PROC_SYS_TASK_ARGS 2 @@ -420,8 +516,10 @@ do { \ do { \ ErtsRunQueue *RQVAR; \ int ix__; \ + int online__ = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, \ + ERTS_SCHED_NORMAL); \ ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&schdlr_sspnd.mtx)); \ - for (ix__ = 0; ix__ < schdlr_sspnd.online; ix__++) { \ + for (ix__ = 0; ix__ < online__; ix__++) { \ RQVAR = ERTS_RUNQ_IX(ix__); \ erts_smp_runq_lock(RQVAR); \ { DO; } \ @@ -451,13 +549,12 @@ do { \ static void exec_misc_ops(ErtsRunQueue *); static void print_function_from_pc(int to, void *to_arg, BeamInstr* x); -static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, - int yreg); +static int stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg); static void aux_work_timeout(void *unused); static void aux_work_timeout_early_init(int no_schedulers); static void aux_work_timeout_late_init(void); -static void setup_aux_work_timer(void); +static void setup_aux_work_timer(ErtsSchedulerData *esdp); static int execute_sys_tasks(Process *c_p, erts_aint32_t *statep, @@ -487,20 +584,20 @@ dbg_chk_aux_work_val(erts_aint32_t value) valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_DD; valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR; + valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS; + valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP; #endif #if HAVE_ERTS_MSEG valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; #endif -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; -#endif #ifdef ERTS_SSI_AUX_WORK_REAP_PORTS valid |= ERTS_SSI_AUX_WORK_REAP_PORTS; #endif + valid |= ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; if (~valid & value) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid aux_work value found: 0x%x\n", ~valid & value); } @@ -556,63 +653,78 @@ erts_pre_init_process(void) erts_tsd_key_create(&sched_data_key, "erts_sched_data_key"); #endif + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX] + = "DELAYED_AW_WAKEUP"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DD_IX] + = "DD"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX] + = "DD_THR_PRGR"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX] + = "FIX_ALLOC_DEALLOC"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX] + = "FIX_ALLOC_LOWER_LIM"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX] + = "THR_PRGR_LATER_OP"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX] + = "CNCLD_TMRS"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX] + = "CNCLD_TMRS_THR_PRGR"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_ASYNC_READY_IX] + = "ASYNC_READY"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX] + = "ASYNC_READY_CLEAN"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX] + = "MISC_THR_PRGR"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_IX] + = "MISC"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_SET_TMO_IX] + = "SET_TMO"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX] + = "MSEG_CACHE_CHECK"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_REAP_PORTS_IX] + = "REAP_PORTS"; + erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX] + = "DEBUG_WAIT_COMPLETED"; + #ifdef ERTS_ENABLE_LOCK_CHECK - { - int ix; - - erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].get_locks - = ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].set_locks - = ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS; - - erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].get_locks - = ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].set_locks - = ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS; - - erts_psd_required_locks[ERTS_PSD_SCHED_ID].get_locks - = ERTS_PSD_SCHED_ID_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_SCHED_ID].set_locks - = ERTS_PSD_SCHED_ID_SET_LOCKS; - - erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks - = ERTS_PSD_DIST_ENTRY_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks - = ERTS_PSD_DIST_ENTRY_SET_LOCKS; - - erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks - = ERTS_PSD_CALL_TIME_BP_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks - = ERTS_PSD_CALL_TIME_BP_SET_LOCKS; - - erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].get_locks - = ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].set_locks - = ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS; -#ifdef ERTS_DIRTY_SCHEDULERS - erts_psd_required_locks[ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT].get_locks - = ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT_GET_LOCKS; - erts_psd_required_locks[ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT].set_locks - = ERTS_PSD_DIRTY_SCHED_TRAP_EXPORT_SET_LOCKS; -#endif + erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].get_locks + = ERTS_PSD_ERROR_HANDLER_BUF_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_ERROR_HANDLER].set_locks + = ERTS_PSD_ERROR_HANDLER_BUF_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].get_locks + = ERTS_PSD_SAVED_CALLS_BUF_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_SAVED_CALLS_BUF].set_locks + = ERTS_PSD_SAVED_CALLS_BUF_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_SCHED_ID].get_locks + = ERTS_PSD_SCHED_ID_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_SCHED_ID].set_locks + = ERTS_PSD_SCHED_ID_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks + = ERTS_PSD_CALL_TIME_BP_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks + = ERTS_PSD_CALL_TIME_BP_SET_LOCKS; - /* Check that we have locks for all entries */ - for (ix = 0; ix < ERTS_PSD_SIZE; ix++) { - ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].get_locks); - ERTS_SMP_LC_ASSERT(erts_psd_required_locks[ix].set_locks); - } - } + erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].get_locks + = ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_DELAYED_GC_TASK_QS].set_locks + = ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].get_locks + = ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_NIF_TRAP_EXPORT].set_locks + = ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS; #endif } -#ifdef ERTS_SMP static void release_process(void *vproc) { - erts_smp_proc_dec_refc((Process *) vproc); + erts_proc_dec_refc((Process *) vproc); } -#endif /* initialize the scheduler */ void @@ -628,20 +740,21 @@ erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab) erts_ptab_init_table(&erts_proc, ERTS_ALC_T_PROC_TABLE, -#ifdef ERTS_SMP release_process, -#else - NULL, -#endif (ErtsPTabElementCommon *) &erts_invalid_process.common, proc_tab_size, sizeof(Process), "process_table", - legacy_proc_tab); + legacy_proc_tab, +#ifdef ERTS_SMP + 1 +#else + 0 +#endif + ); last_reductions = 0; last_exact_reductions = 0; - erts_default_process_flags = 0; } void @@ -698,8 +811,8 @@ init_sched_wall_time(ErtsSchedWallTime *swtp) static ERTS_INLINE Uint64 sched_wall_time_ts(void) { -#ifdef HAVE_GETHRTIME - return (Uint64) sys_gethrtime(); +#ifdef ERTS_HAVE_OS_MONOTONIC_TIME_SUPPORT + return (Uint64) erts_os_monotonic_time(); #else Uint64 res; SysTimeval tv; @@ -712,72 +825,24 @@ sched_wall_time_ts(void) #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT -#ifdef ARCH_64 - -static ERTS_INLINE Uint64 -aschedtime_read(ErtsAtomicSchedTime *var) -{ - return (Uint64) erts_atomic_read_nob((erts_atomic_t *) var); -} - -static ERTS_INLINE void -aschedtime_set(ErtsAtomicSchedTime *var, Uint64 val) -{ - erts_atomic_set_nob((erts_atomic_t *) var, (erts_aint_t) val); -} - -static ERTS_INLINE void -aschedtime_init(ErtsAtomicSchedTime *var) -{ - erts_atomic_init_nob((erts_atomic_t *) var, (erts_aint_t) 0); -} - -#elif defined(ARCH_32) - static ERTS_INLINE Uint64 aschedtime_read(ErtsAtomicSchedTime *var) { - erts_dw_aint_t dw; - erts_dw_atomic_read_nob((erts_dw_atomic_t *) var, &dw); -#ifdef ETHR_SU_DW_NAINT_T__ - return (Uint64) dw.dw_sint; -#else - { - Uint64 res; - res = (Uint64) ((Uint32) dw.sint[ERTS_DW_AINT_HIGH_WORD]); - res <<= 32; - res |= (Uint64) ((Uint32) dw.sint[ERTS_DW_AINT_LOW_WORD]); - return res; - } -#endif + return (Uint64) erts_atomic64_read_nob((erts_atomic64_t *) var); } static ERTS_INLINE void aschedtime_set(ErtsAtomicSchedTime *var, Uint64 val) { - erts_dw_aint_t dw; -#ifdef ETHR_SU_DW_NAINT_T__ - dw.dw_sint = (ETHR_SU_DW_NAINT_T__) val; -#else - dw.sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff); - dw.sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff); -#endif - erts_dw_atomic_set_nob((erts_dw_atomic_t *) var, &dw); + erts_atomic64_set_nob((erts_atomic64_t *) var, (erts_aint64_t) val); } static ERTS_INLINE void aschedtime_init(ErtsAtomicSchedTime *var) { - erts_dw_aint_t dw; - dw.sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) 0; - dw.sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) 0; - erts_dw_atomic_init_nob((erts_dw_atomic_t *) var, &dw); + erts_atomic64_init_nob((erts_atomic64_t *) var, (erts_aint64_t) 0); } -#else -# error :-/ -#endif - #define ERTS_GET_AVG_MAX_UNLOCKED_TRY 50 #define ERTS_SCHED_AVG_UTIL_WRITE_MARKER (~((Uint64) 0)) @@ -959,6 +1024,12 @@ sched_wall_time_change(ErtsSchedulerData *esdp, int working) } } } + if (!working) { + ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_BUSY_WAIT); + } else { + ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_OTHER); + } + } typedef struct { @@ -971,25 +1042,24 @@ typedef struct { erts_smp_atomic32_t refc; } ErtsSchedWallTimeReq; -#if !HALFWORD_HEAP +typedef struct { + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + Uint req_sched; + erts_smp_atomic32_t refc; +} ErtsSystemCheckReq; + + ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq, ErtsSchedWallTimeReq, 5, ERTS_ALC_T_SCHED_WTIME_REQ) -#else -static ERTS_INLINE ErtsSchedWallTimeReq * -swtreq_alloc(void) -{ - return erts_alloc(ERTS_ALC_T_SCHED_WTIME_REQ, - sizeof(ErtsSchedWallTimeReq)); -} -static ERTS_INLINE void -swtreq_free(ErtsSchedWallTimeReq *ptr) -{ - erts_free(ERTS_ALC_T_SCHED_WTIME_REQ, ptr); -} -#endif +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(screq, + ErtsSystemCheckReq, + 5, + ERTS_ALC_T_SYS_CHECK_REQ) static void reply_sched_wall_time(void *vswtrp) @@ -1006,7 +1076,7 @@ reply_sched_wall_time(void *vswtrp) Eterm **hpp; Uint sz, *szp; ErlOffHeap *ohp = NULL; - ErlHeapFragment *bp = NULL; + ErtsMessage *mp = NULL; ASSERT(esdp); #ifdef ERTS_DIRTY_SCHEDULERS @@ -1062,16 +1132,12 @@ reply_sched_wall_time(void *vswtrp) if (hpp) break; - hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); szp = NULL; hpp = &hp; } - erts_queue_message(rp, &rp_locks, bp, msg, NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, mp, msg, NIL); if (swtrp->req_sched == esdp->no) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -1079,7 +1145,7 @@ reply_sched_wall_time(void *vswtrp) if (rp_locks) erts_smp_proc_unlock(rp, rp_locks); - erts_smp_proc_dec_refc(rp); + erts_proc_dec_refc(rp); if (erts_smp_atomic32_dec_read_nob(&swtrp->refc) == 0) swtreq_free(vswtrp); @@ -1111,7 +1177,7 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable) erts_smp_atomic32_init_nob(&swtrp->refc, (erts_aint32_t) erts_no_schedulers); - erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers); + erts_proc_add_refc(c_p, (Sint32) erts_no_schedulers); #ifdef ERTS_SMP if (erts_no_schedulers > 1) @@ -1126,6 +1192,75 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable) return ref; } +static void +reply_system_check(void *vscrp) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsSystemCheckReq *scrp = (ErtsSystemCheckReq *) vscrp; + ErtsProcLocks rp_locks = (scrp->req_sched == esdp->no ? ERTS_PROC_LOCK_MAIN : 0); + Process *rp = scrp->proc; + Eterm msg; + Eterm *hp = NULL; + Eterm **hpp; + Uint sz; + ErlOffHeap *ohp = NULL; + ErtsMessage *mp = NULL; + + ASSERT(esdp); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); +#endif + + sz = REF_THING_SIZE; + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); + hpp = &hp; + msg = STORE_NC(hpp, ohp, scrp->ref); + + erts_queue_message(rp, &rp_locks, mp, msg, NIL); + + if (scrp->req_sched == esdp->no) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + erts_proc_dec_refc(rp); + + if (erts_smp_atomic32_dec_read_nob(&scrp->refc) == 0) + screq_free(vscrp); +} + + +Eterm erts_system_check_request(Process *c_p) { + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + Eterm ref; + ErtsSystemCheckReq *scrp; + Eterm *hp; + + scrp = screq_alloc(); + ref = erts_make_ref(c_p); + hp = &scrp->ref_heap[0]; + + scrp->proc = c_p; + scrp->ref = STORE_NC(&hp, NULL, ref); + scrp->req_sched = esdp->no; + erts_smp_atomic32_init_nob(&scrp->refc, (erts_aint32_t) erts_no_schedulers); + + erts_proc_add_refc(c_p, (Sint) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_system_check, + (void *) scrp); +#endif + + reply_system_check((void *) scrp); + + return ref; +} + static ERTS_INLINE ErtsProcList * proclist_create(Process *p) { @@ -1136,6 +1271,15 @@ proclist_create(Process *p) return plp; } +static ERTS_INLINE ErtsProcList * +proclist_copy(ErtsProcList *plp0) +{ + ErtsProcList *plp1 = proclist_alloc(); + plp1->pid = plp0->pid; + plp1->started_interval = plp0->started_interval; + return plp1; +} + static ERTS_INLINE void proclist_destroy(ErtsProcList *plp) { @@ -1143,6 +1287,12 @@ proclist_destroy(ErtsProcList *plp) } ErtsProcList * +erts_proclist_copy(ErtsProcList *plp) +{ + return proclist_copy(plp); +} + +ErtsProcList * erts_proclist_create(Process *p) { return proclist_create(p); @@ -1155,46 +1305,25 @@ erts_proclist_destroy(ErtsProcList *plp) } void * -erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) +erts_psd_set_init(Process *p, int ix, void *data) { void *old; - ErtsProcLocks xplocks; - int refc = 0; - ErtsPSD *psd = erts_alloc(ERTS_ALC_T_PSD, sizeof(ErtsPSD)); + ErtsPSD *psd, *new_psd; int i; - for (i = 0; i < ERTS_PSD_SIZE; i++) - psd->data[i] = NULL; - ERTS_SMP_LC_ASSERT(plocks); - ERTS_SMP_LC_ASSERT(plocks == erts_proc_lc_my_proc_locks(p)); + new_psd = erts_alloc(ERTS_ALC_T_PSD, sizeof(ErtsPSD)); + for (i = 0; i < ERTS_PSD_SIZE; i++) + new_psd->data[i] = NULL; - xplocks = ERTS_PROC_LOCKS_ALL; - xplocks &= ~plocks; - if (xplocks && erts_smp_proc_trylock(p, xplocks) == EBUSY) { - if (xplocks & ERTS_PROC_LOCK_MAIN) { - erts_smp_proc_inc_refc(p); - erts_smp_proc_unlock(p, plocks); - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL); - refc = 1; - } - else { - if (plocks & ERTS_PROC_LOCKS_ALL_MINOR) - erts_smp_proc_unlock(p, plocks & ERTS_PROC_LOCKS_ALL_MINOR); - erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR); - } - } - if (!p->psd) - p->psd = psd; - if (xplocks) - erts_smp_proc_unlock(p, xplocks); - if (refc) - erts_smp_proc_dec_refc(p); - ASSERT(p->psd); - if (p->psd != psd) - erts_free(ERTS_ALC_T_PSD, psd); - old = p->psd->data[ix]; - p->psd->data[ix] = data; - ERTS_SMP_LC_ASSERT(plocks == erts_proc_lc_my_proc_locks(p)); + psd = (ErtsPSD *) erts_smp_atomic_cmpxchg_mb(&p->psd, + (erts_aint_t) new_psd, + (erts_aint_t) NULL); + if (psd) + erts_free(ERTS_ALC_T_PSD, new_psd); + else + psd = new_psd; + old = psd->data[ix]; + psd->data[ix] = data; return old; } @@ -1220,7 +1349,7 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) case 0: break; default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); break; } @@ -1237,11 +1366,11 @@ set_aux_work_flags_wakeup_nob(ErtsSchedulerSleepInfo *ssi, ERTS_DBG_CHK_SSI_AUX_WORK(ssi); old_flgs = erts_atomic32_read_nob(&ssi->aux_work); - if ((old_flgs & flgs) == 0) { + if ((old_flgs & flgs) != flgs) { old_flgs = erts_atomic32_read_bor_nob(&ssi->aux_work, flgs); - if ((old_flgs & flgs) == 0) { + if ((old_flgs & flgs) != flgs) { #ifdef ERTS_SMP erts_sched_poke(ssi); #else @@ -1261,7 +1390,7 @@ set_aux_work_flags_wakeup_relb(ErtsSchedulerSleepInfo *ssi, old_flgs = erts_atomic32_read_bor_relb(&ssi->aux_work, flgs); - if ((old_flgs & flgs) == 0) { + if ((old_flgs & flgs) != flgs) { #ifdef ERTS_SMP erts_sched_poke(ssi); #else @@ -1733,15 +1862,17 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin int need_thr_progress = 0; ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; int more_work = 0; - + ERTS_MSACC_PUSH_STATE_M_X(); #ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); #endif unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); + ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ALLOC); erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, &need_thr_progress, &wakeup, &more_work); + ERTS_MSACC_POP_STATE_M_X(); if (more_work) { if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD) & ERTS_SSI_AUX_WORK_DD_THR_PRGR) { @@ -1759,11 +1890,6 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin awdp->dd.thr_prgr = wakeup; haw_thr_prgr_soft_wakeup(awdp, wakeup); } - else if (awdp->dd.completed_callback) { - awdp->dd.completed_callback(awdp->dd.completed_arg); - awdp->dd.completed_callback = NULL; - awdp->dd.completed_arg = NULL; - } return aux_work & ~ERTS_SSI_AUX_WORK_DD; } @@ -1805,17 +1931,107 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i } else { unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); - if (awdp->dd.completed_callback) { - awdp->dd.completed_callback(awdp->dd.completed_arg); - awdp->dd.completed_callback = NULL; - awdp->dd.completed_arg = NULL; - } } return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; } /* + * Canceled timers + */ + +void +erts_notify_canceled_timer(ErtsSchedulerData *esdp, int rsid) +{ + ASSERT(esdp && esdp == erts_get_scheduler_data()); + if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)) + schedule_aux_work_wakeup(&esdp->aux_work_data, + rsid, + ERTS_SSI_AUX_WORK_CNCLD_TMRS); + else + set_aux_work_flags_wakeup_relb(ERTS_SCHED_SLEEP_INFO_IX(rsid-1), + ERTS_SSI_AUX_WORK_CNCLD_TMRS); +} + +static ERTS_INLINE erts_aint32_t +handle_canceled_timers(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + int need_thr_progress = 0; + ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; + int more_work = 0; + +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS); + erts_handle_canceled_timers((void *) awdp->esdp, + &need_thr_progress, + &wakeup, + &more_work); + if (more_work) { + if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS) + & ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR) { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); + aux_work &= ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; + } + return aux_work; + } + + if (need_thr_progress) { + if (wakeup == ERTS_THR_PRGR_INVALID) + wakeup = erts_thr_progress_later(awdp->esdp); + awdp->cncld_tmrs.thr_prgr = wakeup; + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); + haw_thr_prgr_soft_wakeup(awdp, wakeup); + } + return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS; +} + +static ERTS_INLINE erts_aint32_t +handle_canceled_timers_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) +{ + ErtsSchedulerSleepInfo *ssi; + int need_thr_progress; + int more_work; + ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; + ErtsThrPrgrVal current = haw_thr_prgr_current(awdp); + +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif + if (!erts_thr_progress_has_reached_this(current, awdp->cncld_tmrs.thr_prgr)) + return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; + + ssi = awdp->ssi; + need_thr_progress = 0; + more_work = 0; + + erts_handle_canceled_timers((void *) awdp->esdp, + &need_thr_progress, + &wakeup, + &more_work); + if (more_work) { + set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS); + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); + return ((aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR) + | ERTS_SSI_AUX_WORK_CNCLD_TMRS); + } + + if (need_thr_progress) { + if (wakeup == ERTS_THR_PRGR_INVALID) + wakeup = erts_thr_progress_later(awdp->esdp); + awdp->cncld_tmrs.thr_prgr = wakeup; + haw_thr_prgr_soft_wakeup(awdp, wakeup); + } + else { + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); + } + + return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR; +} + +/* * Handle scheduled thread progress later operations. */ #define ERTS_MAX_THR_PRGR_LATER_OPS 50 @@ -1904,109 +2120,154 @@ erts_schedule_thr_prgr_later_cleanup_op(void (*later_func)(void *), #endif } -#ifdef ERTS_SMP +static ERTS_INLINE erts_aint32_t +handle_debug_wait_completed(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) +{ + ErtsSchedulerSleepInfo *ssi = awdp->ssi; + erts_aint32_t saved_aux_work, flags; + +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); +#endif + + flags = awdp->debug.wait_completed.flags; + + if (aux_work & flags) + return aux_work; -static erts_atomic32_t completed_dealloc_count; + saved_aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + + if (saved_aux_work & flags) + return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; + + awdp->debug.wait_completed.callback(awdp->debug.wait_completed.arg); + + awdp->debug.wait_completed.flags = 0; + awdp->debug.wait_completed.callback = NULL; + awdp->debug.wait_completed.arg = NULL; + + unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED); + + return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; +} + +static erts_atomic32_t debug_wait_completed_count; +static int debug_wait_completed_flags; static void -completed_dealloc(void *vproc) +thr_debug_wait_completed(void *vproc) { - if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == 0) { + if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == 0) { erts_resume((Process *) vproc, (ErtsProcLocks) 0); - erts_smp_proc_dec_refc((Process *) vproc); + erts_proc_dec_refc((Process *) vproc); } } static void -setup_completed_dealloc(void *vproc) +setup_thr_debug_wait_completed(void *vproc) { ErtsSchedulerData *esdp = erts_get_scheduler_data(); - ErtsAuxWorkData *awdp = (esdp - ? &esdp->aux_work_data - : aux_thread_aux_work_data); - erts_alloc_fix_alloc_shrink(awdp->sched_id, 0); - set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_DD); - awdp->dd.completed_callback = completed_dealloc; - awdp->dd.completed_arg = vproc; + ErtsAuxWorkData *awdp; + erts_aint32_t wait_flags, aux_work_flags; +#ifdef ERTS_SMP + awdp = esdp ? &esdp->aux_work_data : aux_thread_aux_work_data; +#else + awdp = &esdp->aux_work_data; +#endif + + wait_flags = 0; + aux_work_flags = ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; + + if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS) { + erts_alloc_fix_alloc_shrink(awdp->sched_id, 0); + wait_flags |= (ERTS_SSI_AUX_WORK_DD + | ERTS_SSI_AUX_WORK_DD_THR_PRGR); +#ifdef ERTS_SMP + aux_work_flags |= ERTS_SSI_AUX_WORK_DD; +#endif + } + + if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS) { + wait_flags |= (ERTS_SSI_AUX_WORK_CNCLD_TMRS + | ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR); +#ifdef ERTS_SMP + if (awdp->esdp && !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)) + aux_work_flags |= ERTS_SSI_AUX_WORK_CNCLD_TMRS; +#endif + } + + set_aux_work_flags_wakeup_nob(awdp->ssi, aux_work_flags); + + awdp->debug.wait_completed.flags = wait_flags; + awdp->debug.wait_completed.callback = thr_debug_wait_completed; + awdp->debug.wait_completed.arg = vproc; } -static void -prep_setup_completed_dealloc(void *vproc) +struct debug_lop { + ErtsThrPrgrLaterOp lop; + Process *proc; +}; + +static void later_thr_debug_wait_completed(void *vlop) { - erts_aint32_t count = (erts_aint32_t) (erts_no_schedulers+1); - if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == count) { - /* scheduler threads */ - erts_schedule_multi_misc_aux_work(0, - erts_no_schedulers, - setup_completed_dealloc, - vproc); - /* aux_thread */ - erts_schedule_misc_aux_work(0, - setup_completed_dealloc, - vproc); + struct debug_lop *lop = vlop; + erts_aint32_t count = (erts_aint32_t) erts_no_schedulers; +#ifdef ERTS_SMP + count += 1; /* aux thread */ +#endif + if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == count) { + /* scheduler threads */ + erts_schedule_multi_misc_aux_work(0, + erts_no_schedulers, + setup_thr_debug_wait_completed, + lop->proc); +#ifdef ERTS_SMP + /* aux_thread */ + erts_schedule_misc_aux_work(0, + setup_thr_debug_wait_completed, + lop->proc); +#endif } + erts_free(ERTS_ALC_T_DEBUG, lop); +} + + +static void +init_thr_debug_wait_completed(void *vproc) +{ + struct debug_lop* lop = erts_alloc(ERTS_ALC_T_DEBUG, + sizeof(struct debug_lop)); + lop->proc = vproc; + erts_schedule_thr_prgr_later_op(later_thr_debug_wait_completed, lop, &lop->lop); } -#endif /* ERTS_SMP */ int -erts_debug_wait_deallocations(Process *c_p) +erts_debug_wait_completed(Process *c_p, int flags) { -#ifndef ERTS_SMP - erts_alloc_fix_alloc_shrink(1, 0); - return 1; -#else /* Only one process at a time can do this */ - erts_aint32_t count = (erts_aint32_t) (2*(erts_no_schedulers+1)); - if (0 == erts_atomic32_cmpxchg_mb(&completed_dealloc_count, + erts_aint32_t count = (erts_aint32_t) (2*erts_no_schedulers); +#ifdef ERTS_SMP + count += 1; /* aux thread */ +#endif + if (0 == erts_atomic32_cmpxchg_mb(&debug_wait_completed_count, count, 0)) { + debug_wait_completed_flags = flags; erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); - erts_smp_proc_inc_refc(c_p); - /* scheduler threads */ + erts_proc_inc_refc(c_p); + + /* First flush later-ops on all scheduler threads */ erts_schedule_multi_misc_aux_work(0, erts_no_schedulers, - prep_setup_completed_dealloc, + init_thr_debug_wait_completed, (void *) c_p); - /* aux_thread */ - erts_schedule_misc_aux_work(0, - prep_setup_completed_dealloc, - (void *) c_p); return 1; } return 0; -#endif } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN -void -erts_smp_notify_check_children_needed(void) -{ - int i; - for (i = 0; i < erts_no_schedulers; i++) - set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(i), - ERTS_SSI_AUX_WORK_CHECK_CHILDREN); -#ifdef ERTS_DIRTY_SCHEDULERS - for (i = 0; i < erts_no_dirty_cpu_schedulers; i++) - set_aux_work_flags_wakeup_nob(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(i), - ERTS_SSI_AUX_WORK_CHECK_CHILDREN); - for (i = 0; i < erts_no_dirty_io_schedulers; i++) - set_aux_work_flags_wakeup_nob(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(i), - ERTS_SSI_AUX_WORK_CHECK_CHILDREN); -#endif -} - -static ERTS_INLINE erts_aint32_t -handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) -{ - unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN); - erts_check_children(); - return aux_work & ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; -} - -#endif - static void notify_reap_ports_relb(void) { @@ -2055,7 +2316,7 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) erts_port_release(prt); } if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) { - erl_exit_flush_async(erts_halt_code, ""); + erts_flush_async_exit(erts_halt_code, ""); } } return aux_work & ~ERTS_SSI_AUX_WORK_REAP_PORTS; @@ -2077,7 +2338,7 @@ static ERTS_INLINE erts_aint32_t handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO); - setup_aux_work_timer(); + setup_aux_work_timer(awdp->esdp); return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO; } @@ -2092,13 +2353,17 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ if (!(aux_work & ~ignore)) { \ ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ + ERTS_MSACC_UPDATE_CACHE(); \ + ERTS_MSACC_POP_STATE_M(); \ return aux_work; \ } \ } erts_aint32_t aux_work = orig_aux_work; erts_aint32_t ignore = 0; + ERTS_MSACC_PUSH_AND_SET_STATE_M(ERTS_MSACC_STATE_AUX); + ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp)); #ifdef ERTS_SMP haw_thr_prgr_current_reset(awdp); #endif @@ -2137,6 +2402,11 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) #ifdef ERTS_SMP HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP, handle_thr_prgr_later_op); + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CNCLD_TMRS, + handle_canceled_timers); + /* CNCLD_TMRS must be before CNCLD_TMRS_THR_PRGR */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR, + handle_canceled_timers_thr_prgr); #endif #if ERTS_USE_ASYNC_READY_Q @@ -2155,10 +2425,6 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC, handle_misc_aux_work); -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CHECK_CHILDREN, - handle_check_children); -#endif HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO, handle_setup_aux_work_timer); @@ -2171,6 +2437,14 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_REAP_PORTS, handle_reap_ports); + /* + * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED *need* to be + * the last flag checked! + */ + + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED, + handle_debug_wait_completed); + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); #ifdef ERTS_SMP @@ -2178,6 +2452,8 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) haw_thr_prgr_current_check_progress(awdp); #endif + ERTS_MSACC_UPDATE_CACHE(); + ERTS_MSACC_POP_STATE_M(); return aux_work; #undef HANDLE_AUX_WORK @@ -2186,8 +2462,8 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) typedef struct { union { - ErlTimer data; - char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErlTimer))]; + ErtsTWheelTimer data; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsTWheelTimer))]; } timer; int initialized; @@ -2197,6 +2473,22 @@ typedef struct { static ErtsAuxWorkTmo *aux_work_tmo; +static ERTS_INLINE void +start_aux_work_timer(ErtsSchedulerData *esdp) +{ + ErtsMonotonicTime tmo = erts_get_monotonic_time(esdp); + tmo = ERTS_MONOTONIC_TO_CLKTCKS(tmo-1); + tmo += ERTS_MSEC_TO_CLKTCKS(1000) + 1; + erts_twheel_init_timer(&aux_work_tmo->timer.data); + ASSERT(esdp); + erts_twheel_set_timer(esdp->timer_wheel, + &aux_work_tmo->timer.data, + aux_work_timeout, + NULL, + (void *) esdp, + tmo); +} + static void aux_work_timeout_early_init(int no_schedulers) { @@ -2211,6 +2503,9 @@ aux_work_timeout_early_init(int no_schedulers) p = (UWord) malloc((sizeof(ErtsAuxWorkTmo) + sizeof(erts_atomic32_t)*(no_schedulers+1)) + ERTS_CACHE_LINE_SIZE-1); + if (!p) { + ERTS_INTERNAL_ERROR("malloc failed to allocate memory!"); + } if (p & ERTS_CACHE_LINE_MASK) p = (p & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE; ASSERT((p & ERTS_CACHE_LINE_MASK) == 0); @@ -2226,18 +2521,12 @@ void aux_work_timeout_late_init(void) { aux_work_tmo->initialized = 1; - if (erts_atomic32_read_nob(&aux_work_tmo->refc)) { - aux_work_tmo->timer.data.active = 0; - erts_set_timer(&aux_work_tmo->timer.data, - aux_work_timeout, - NULL, - NULL, - 1000); - } + if (erts_atomic32_read_nob(&aux_work_tmo->refc)) + start_aux_work_timer(erts_get_scheduler_data()); } static void -aux_work_timeout(void *unused) +aux_work_timeout(void *vesdp) { erts_aint32_t refc; int i; @@ -2260,32 +2549,18 @@ aux_work_timeout(void *unused) if (refc != 1 || 1 != erts_atomic32_cmpxchg_relb(&aux_work_tmo->refc, 0, 1)) { /* Setup next timeout... */ - aux_work_tmo->timer.data.active = 0; - erts_set_timer(&aux_work_tmo->timer.data, - aux_work_timeout, - NULL, - NULL, - 1000); + start_aux_work_timer((ErtsSchedulerData *) vesdp); } } static void -setup_aux_work_timer(void) +setup_aux_work_timer(ErtsSchedulerData *esdp) { -#ifndef ERTS_SMP - if (!erts_get_scheduler_data()) + if (!esdp || !esdp->timer_wheel) set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(0), ERTS_SSI_AUX_WORK_SET_TMO); else -#endif - { - aux_work_tmo->timer.data.active = 0; - erts_set_timer(&aux_work_tmo->timer.data, - aux_work_timeout, - NULL, - NULL, - 1000); - } + start_aux_work_timer(esdp); } erts_aint32_t @@ -2316,7 +2591,7 @@ erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable) if (refc == 1) { erts_atomic32_inc_acqb(&aux_work_tmo->refc); if (aux_work_tmo->initialized) - setup_aux_work_timer(); + setup_aux_work_timer(erts_get_scheduler_data()); } } return old; @@ -2359,7 +2634,6 @@ erts_active_schedulers(void) ERTS_ATOMIC_FOREACH_RUNQ(rq, as -= abs(rq->waiting)); - ASSERT(as >= 0); return as; } @@ -2380,29 +2654,28 @@ try_set_sys_scheduling(void) #endif static ERTS_INLINE int -prepare_for_sys_schedule(ErtsSchedulerData *esdp) +prepare_for_sys_schedule(int non_blocking) { + if (non_blocking && erts_eager_check_io) { #ifdef ERTS_SMP - while (!erts_port_task_have_outstanding_io_tasks() - && try_set_sys_scheduling()) { -#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 - if (esdp->no != 1) { - /* If we are not scheduler 1 and ERTS_SCHED_ONLY_POLL_SCHED_1 is used - then we make sure to wake scheduler 1 */ - ErtsRunQueue *rq = ERTS_RUNQ_IX(0); - clear_sys_scheduling(); - wake_scheduler(rq); - return 0; - } -#endif - if (!erts_port_task_have_outstanding_io_tasks()) + return try_set_sys_scheduling(); +#else return 1; - clear_sys_scheduling(); +#endif } - return 0; + else { +#ifdef ERTS_SMP + while (!erts_port_task_have_outstanding_io_tasks() + && try_set_sys_scheduling()) { + if (!erts_port_task_have_outstanding_io_tasks()) + return 1; + clear_sys_scheduling(); + } + return 0; #else - return !erts_port_task_have_outstanding_io_tasks(); + return !erts_port_task_have_outstanding_io_tasks(); #endif + } } #ifdef ERTS_SMP @@ -2445,13 +2718,6 @@ sched_active(Uint no, ErtsRunQueue *rq) profile_scheduler(make_small(no), am_active); } -static int ERTS_INLINE -ongoing_multi_scheduling_block(void) -{ - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&schdlr_sspnd.mtx)); - return schdlr_sspnd.msb.ongoing; -} - static ERTS_INLINE void empty_runq_aux(ErtsRunQueue *rq, Uint32 old_flags) { @@ -2663,6 +2929,13 @@ thr_prgr_fin_wait(void *vssi) static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp); +void +erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time) +{ + /* TODO only poke when needed (based on timeout_time) */ + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(-1)); +} + static void * aux_thread(void *unused) { @@ -2672,8 +2945,17 @@ aux_thread(void *unused) ErtsThrPrgrCallbacks callbacks; int thr_prgr_active = 1; +#ifdef ERTS_ENABLE_LOCK_CHECK + { + char buf[] = "aux_thread"; + erts_lc_set_thread_name(buf); + } +#endif + ssi->event = erts_tse_fetch(); + erts_msacc_init_thread("aux", 1, 1); + callbacks.arg = (void *) ssi; callbacks.wakeup = thr_prgr_wakeup; callbacks.prepare_wait = thr_prgr_prep_wait; @@ -2684,6 +2966,7 @@ aux_thread(void *unused) init_aux_work_data(awdp, NULL, NULL); awdp->ssi = ssi; + sched_prep_spin_wait(ssi); while (1) { @@ -2703,8 +2986,6 @@ aux_thread(void *unused) erts_thr_progress_active(NULL, thr_prgr_active = 0); erts_thr_progress_prepare_wait(NULL); - ERTS_SCHED_FAIR_YIELD(); - flgs = sched_spin_wait(ssi, 0); if (flgs & ERTS_SSI_FLG_SLEEPING) { @@ -2739,6 +3020,9 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) #ifdef ERTS_SMP int thr_prgr_active = 1; erts_aint32_t flgs; +#endif + ERTS_MSACC_PUSH_STATE_M(); +#ifdef ERTS_SMP ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -2772,7 +3056,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * be waiting in erl_sys_schedule() */ - if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(esdp)) { + if (ERTS_SCHEDULER_IS_DIRTY(esdp) || !prepare_for_sys_schedule(0)) { sched_waiting(esdp->no, rq); @@ -2786,47 +3070,92 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sched_wall_time_change(esdp, thr_prgr_active); while (1) { + ErtsMonotonicTime current_time = 0; aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { + if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); } aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); - if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp) - && erts_thr_progress_update(esdp)) + ERTS_MSACC_UPDATE_CACHE(); + if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); } - if (aux_work) - flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + if (aux_work) { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + flgs = erts_smp_atomic32_read_acqb(&ssi->flags); + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_bump_timers(esdp->timer_wheel, current_time); + } + } + } else { + ErtsMonotonicTime timeout_time; + int do_timeout = 0; if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); + timeout_time = erts_check_next_timeout_time(esdp); + current_time = erts_get_monotonic_time(esdp); + do_timeout = (current_time >= timeout_time); + } else { + current_time = 0; + timeout_time = ERTS_MONOTONIC_TIME_MAX; + } + if (do_timeout) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); } - erts_thr_progress_prepare_wait(esdp); } + else { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(esdp); + } - ERTS_SCHED_FAIR_YIELD(); - - flgs = sched_spin_wait(ssi, spincount); - if (flgs & ERTS_SSI_FLG_SLEEPING) { - ASSERT(flgs & ERTS_SSI_FLG_WAITING); - flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + flgs = sched_spin_wait(ssi, spincount); if (flgs & ERTS_SSI_FLG_SLEEPING) { - int res; - ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); ASSERT(flgs & ERTS_SSI_FLG_WAITING); - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); + do { + Sint64 timeout; + if (current_time >= timeout_time) + break; + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time + - current_time + - 1) + 1; + } else + timeout = -1; + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_SLEEP); + res = erts_tse_twait(ssi->event, timeout); + ERTS_MSACC_POP_STATE_M(); + current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : + erts_get_monotonic_time(esdp); + } while (res == EINTR); + } } + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + erts_thr_progress_finalize_wait(esdp); } - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) - erts_thr_progress_finalize_wait(esdp); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time) + erts_bump_timers(esdp->timer_wheel, current_time); } if (!(flgs & ERTS_SSI_FLG_WAITING)) { @@ -2859,7 +3188,6 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) else #endif { - erts_aint_t dt; erts_smp_atomic32_set_relb(&function_calls, 0); *fcalls = 0; @@ -2867,13 +3195,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) #ifdef ERTS_DIRTY_SCHEDULERS ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); #endif - -#ifdef ERTS_SCHED_ONLY_POLL_SCHED_1 - ASSERT(esdp->no == 1); -#endif sched_waiting_sys(esdp->no, rq); - erts_smp_runq_unlock(rq); ASSERT(working); @@ -2884,17 +3207,25 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) goto sys_aux_work; while (spincount-- > 0) { + ErtsMonotonicTime current_time; sys_poll_aux_work: if (working) sched_wall_time_change(esdp, working = 0); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); + ASSERT(!erts_port_task_have_outstanding_io_tasks()); erl_sys_schedule(1); /* Might give us something to do */ - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + ERTS_MSACC_POP_STATE_M(); + + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) + erts_bump_timers(esdp->timer_wheel, current_time); + } sys_aux_work: #ifndef ERTS_SMP @@ -2902,7 +3233,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) #endif aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work) { + if (aux_work && !ERTS_SCHEDULER_IS_DIRTY(esdp)) { if (!working) sched_wall_time_change(esdp, working = 1); #ifdef ERTS_SMP @@ -2910,6 +3241,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_thr_progress_active(esdp, thr_prgr_active = 1); #endif aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); + ERTS_MSACC_UPDATE_CACHE(); #ifdef ERTS_SMP if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); @@ -2917,7 +3249,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) } #ifndef ERTS_SMP - if (rq->len != 0 || rq->misc.start) + if (erts_smp_atomic32_read_dirty(&rq->len) != 0 || rq->misc.start) goto sys_woken; #else flgs = erts_smp_atomic32_read_acqb(&ssi->flags); @@ -2936,7 +3268,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to continue checking for I/O... */ - if (!prepare_for_sys_schedule(esdp)) { + if (!prepare_for_sys_schedule(0)) { spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; goto tse_wait; } @@ -2958,7 +3290,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * Got to check that we still got I/O tasks; otherwise * we have to wait in erl_sys_schedule() after all... */ - if (!prepare_for_sys_schedule(esdp)) { + if (!prepare_for_sys_schedule(0)) { /* * Not allowed to wait in erl_sys_schedule; * do tse wait instead... @@ -3007,13 +3339,20 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) ASSERT(!erts_port_task_have_outstanding_io_tasks()); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); + erl_sys_schedule(0); - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + ERTS_MSACC_POP_STATE_M(); + + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + ErtsMonotonicTime current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) + erts_bump_timers(esdp->timer_wheel, current_time); + } #ifndef ERTS_SMP - if (rq->len == 0 && !rq->misc.start) + if (erts_smp_atomic32_read_dirty(&rq->len) == 0 && !rq->misc.start) goto sys_aux_work; sys_woken: #else @@ -3075,7 +3414,8 @@ wake_scheduler(ErtsRunQueue *rq) * so all code *should* handle this without having * the lock on the run queue. */ - ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)); + ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq) + || ERTS_RUNQ_IX_IS_DIRTY(rq->ix)); ssi = rq->scheduler->ssi; @@ -3212,11 +3552,11 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) return 0; wrq = ERTS_RUNQ_IX(ix); flags = ERTS_RUNQ_FLGS_GET(wrq); + if (activate && !(flags & ERTS_RUNQ_FLG_SUSPENDED)) { + if (try_inc_no_active_runqs(ix+1)) + (void) ERTS_RUNQ_FLGS_UNSET(wrq, ERTS_RUNQ_FLG_INACTIVE); + } if (!(flags & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_NONEMPTY))) { - if (activate) { - if (try_inc_no_active_runqs(ix+1)) - (void) ERTS_RUNQ_FLGS_UNSET(wrq, ERTS_RUNQ_FLG_INACTIVE); - } wake_scheduler(wrq); return 1; } @@ -3400,6 +3740,13 @@ check_requeue_process(ErtsRunQueue *rq, int prio_q) return 0; } +static ERTS_INLINE void +free_proxy_proc(Process *proxy) +{ + ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); + erts_free(ERTS_ALC_T_PROC, proxy); +} + #ifdef ERTS_SMP static ErtsRunQueue * @@ -3482,7 +3829,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) prio = ERTS_PORT_PRIO_LEVEL; break; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Invalid immigrate queue mask", __FILE__, __LINE__, __func__); prio = 0; @@ -3506,7 +3853,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) rq = erts_port_runq(prt); if (rq) { if (rq != c_rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error", __FILE__, __LINE__, __func__); erts_enqueue_port(c_rq, prt); @@ -3571,24 +3918,33 @@ static ERTS_INLINE void resume_run_queue(ErtsRunQueue *rq) { int pix; + Uint32 oflgs; erts_smp_runq_lock(rq); - (void) ERTS_RUNQ_FLGS_READ_BSET(rq, - (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK - | ERTS_RUNQ_FLG_SUSPENDED), - (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); + oflgs = ERTS_RUNQ_FLGS_READ_BSET(rq, + (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK + | ERTS_RUNQ_FLG_SUSPENDED), + (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); + + if (oflgs & ERTS_RUNQ_FLG_SUSPENDED) { + erts_aint32_t len; + + rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; + for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { + len = erts_smp_atomic32_read_dirty(&rq->procs.prio_info[pix].len); + rq->procs.prio_info[pix].max_len = len; + rq->procs.prio_info[pix].reds = 0; + } + len = erts_smp_atomic32_read_dirty(&rq->ports.info.len); + rq->ports.info.max_len = len; + rq->ports.info.reds = 0; + len = erts_smp_atomic32_read_dirty(&rq->len); + rq->max_len = len; - rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; - for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { - rq->procs.prio_info[pix].max_len = 0; - rq->procs.prio_info[pix].reds = 0; } - rq->ports.info.max_len = 0; - rq->ports.info.reds = 0; - rq->max_len = 0; erts_smp_runq_unlock(rq); @@ -3697,7 +4053,7 @@ evacuate_run_queue(ErtsRunQueue *rq, prt_rq = erts_port_runq(prt); if (prt_rq) { if (prt_rq != to_rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s() internal error\n", __FILE__, __LINE__, __func__); erts_enqueue_port(to_rq, prt); @@ -3714,9 +4070,6 @@ evacuate_run_queue(ErtsRunQueue *rq, erts_aint32_t state; Process *proc; int notify = 0; -#ifdef ERTS_DIRTY_SCHEDULERS - int requeue; -#endif to_rq = NULL; #ifdef ERTS_DIRTY_SCHEDULERS @@ -3731,10 +4084,85 @@ evacuate_run_queue(ErtsRunQueue *rq, proc = dequeue_process(rq, prio_q, &state); while (proc) { + Process *real_proc; + int prio; + erts_aint32_t max_qbit, qbit, real_state; + + prio = ERTS_PSFLGS_GET_PRQ_PRIO(state); + qbit = ((erts_aint32_t) 1) << prio; + + if (!(state & ERTS_PSFLG_PROXY)) { + real_proc = proc; + real_state = state; + } + else { + real_proc = erts_proc_lookup_raw(proc->common.id); + if (!real_proc) { + free_proxy_proc(proc); + goto handle_next_proc; + } + real_state = erts_smp_atomic32_read_acqb(&real_proc->state); + } + + max_qbit = (state >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET); + max_qbit &= ERTS_PSFLGS_QMASK; + max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS; + max_qbit &= -max_qbit; + + if (qbit > max_qbit) { + /* Process already queued with higher prio; drop it... */ + if (real_proc != proc) + free_proxy_proc(proc); + else { + erts_aint32_t clr_bits; +#ifdef DEBUG + erts_aint32_t old; +#endif + + clr_bits = ERTS_PSFLG_IN_RUNQ; + clr_bits |= qbit << ERTS_PSFLGS_IN_PRQ_MASK_OFFSET; + +#ifdef DEBUG + old = +#else + (void) +#endif + erts_smp_atomic32_read_band_mb(&proc->state, + ~clr_bits); + ASSERT((old & clr_bits) == clr_bits); + + } + + goto handle_next_proc; + } + #ifdef ERTS_DIRTY_SCHEDULERS - requeue = 1; + + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { + erts_aint32_t dqbit = qbit; +#ifdef DEBUG + erts_aint32_t old_dqbit; #endif - if (ERTS_PSFLG_BOUND & state) { + + if (rq == ERTS_DIRTY_CPU_RUNQ) + dqbit <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET; + else { + ASSERT(rq == ERTS_DIRTY_IO_RUNQ); + dqbit <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET; + } + +#ifdef DEBUG + old_dqbit = (int) +#else + (void) +#endif + erts_smp_atomic32_read_band_mb(&real_proc->dirty_state, + ~dqbit); + ASSERT(old_dqbit & dqbit); + } +#endif + + if (ERTS_PSFLG_BOUND & real_state) { /* Bound processes get stuck here... */ proc->next = NULL; if (sbpp->last) @@ -3742,30 +4170,8 @@ evacuate_run_queue(ErtsRunQueue *rq, else sbpp->first = proc; sbpp->last = proc; -#ifdef ERTS_DIRTY_SCHEDULERS - requeue = 0; -#endif } -#ifdef ERTS_DIRTY_SCHEDULERS - else if (state & ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q) { - erts_aint32_t old; - old = erts_smp_atomic32_read_band_nob(&proc->state, - ~(ERTS_PSFLG_DIRTY_CPU_PROC - | ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q)); - /* assert that no other dirty flags are set */ - ASSERT(!(old & (ERTS_PSFLG_DIRTY_IO_PROC|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q))); - } else if (state & ERTS_PSFLG_DIRTY_IO_PROC_IN_Q) { - erts_aint32_t old; - old = erts_smp_atomic32_read_band_nob(&proc->state, - ~(ERTS_PSFLG_DIRTY_IO_PROC - | ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)); - /* assert that no other dirty flags are set */ - ASSERT(!(old & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q))); - } - if (requeue) { -#else else { -#endif int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state); erts_smp_runq_unlock(rq); @@ -3788,6 +4194,8 @@ evacuate_run_queue(ErtsRunQueue *rq, erts_smp_runq_lock(rq); } + + handle_next_proc: proc = dequeue_process(rq, prio_q, &state); } if (notify) @@ -3890,7 +4298,7 @@ no_procs: return 0; else { if (prt_rq != rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s() internal error\n", __FILE__, __LINE__, __func__); *rq_lockedp = 1; @@ -4722,7 +5130,7 @@ erts_fprintf(stderr, "--------------------------------\n"); rq->out_of_work_count = 0; (void) ERTS_RUNQ_FLGS_READ_BSET(rq, ERTS_RUNQ_FLGS_MIGRATION_INFO, flags); - rq->max_len = rq->len; + rq->max_len = erts_smp_atomic32_read_dirty(&rq->len); for (pix = 0; pix < ERTS_NO_PRIO_LEVELS; pix++) { ErtsRunQueueInfo *rqi; rqi = (pix == ERTS_PORT_PRIO_LEVEL @@ -4880,7 +5288,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) { int wo_reds = rq->wakeup_other_reds; if (wo_reds) { - int left_len = rq->len - 1; + int left_len = erts_smp_atomic32_read_dirty(&rq->len) - 1; if (left_len < 1) { int wo_reduce = wo_reds << wakeup_other.dec_shift; wo_reduce &= wakeup_other.dec_mask; @@ -4893,9 +5301,11 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags) + ERTS_WAKEUP_OTHER_FIXED_INC); if (rq->wakeup_other > wakeup_other.limit) { #ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix) && rq->waiting) - wake_dirty_schedulers(rq, 1); - else + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) { + if (rq->waiting) { + wake_dirty_schedulers(rq, 1); + } + } else #endif { int empty_rqs = @@ -4951,7 +5361,7 @@ wakeup_other_check_legacy(ErtsRunQueue *rq, Uint32 flags) { int wo_reds = rq->wakeup_other_reds; if (wo_reds) { - erts_aint32_t len = rq->len; + erts_aint32_t len = erts_smp_atomic32_read_dirty(&rq->len); if (len < 2) { rq->wakeup_other -= ERTS_WAKEUP_OTHER_DEC_LEGACY*wo_reds; if (rq->wakeup_other < 0) @@ -5047,7 +5457,7 @@ runq_supervisor(void *unused) ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); if (ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY) { erts_smp_runq_lock(rq); - if (rq->len != 0) + if (erts_smp_atomic32_read_dirty(&rq->len) != 0) wake_scheduler_on_empty_runq(rq); /* forced wakeup... */ erts_smp_runq_unlock(rq); } @@ -5066,17 +5476,11 @@ erts_early_init_scheduling(int no_schedulers) wakeup_other.threshold = ERTS_SCHED_WAKEUP_OTHER_THRESHOLD_MEDIUM; wakeup_other.type = ERTS_SCHED_WAKEUP_OTHER_TYPE_DEFAULT; #endif -#ifndef ERTS_SCHED_MIN_SPIN sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM; sched_busy_wait.tse = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM * ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT); sched_busy_wait.aux_work = (ERTS_SCHED_SYS_SLEEP_SPINCOUNT_MEDIUM * ERTS_SCHED_AUX_WORK_SLEEP_SPINCOUNT_FACT_MEDIUM); -#else - sched_busy_wait.sys_schedule = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; - sched_busy_wait.tse = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; - sched_busy_wait.aux_work = ERTS_SCHED_SYS_SLEEP_SPINCOUNT_NONE; -#endif } int @@ -5206,8 +5610,7 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) awdp->latest_wakeup = ERTS_THR_PRGR_VAL_FIRST; awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; - awdp->dd.completed_callback = NULL; - awdp->dd.completed_arg = NULL; + awdp->cncld_tmrs.thr_prgr = ERTS_THR_PRGR_VAL_WAITING; awdp->later_op.thr_prgr = ERTS_THR_PRGR_VAL_FIRST; awdp->later_op.size = 0; awdp->later_op.first = NULL; @@ -5237,6 +5640,9 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) awdp->delayed_wakeup.sched2jix[i] = -1; } #endif + awdp->debug.wait_completed.flags = 0; + awdp->debug.wait_completed.callback = NULL; + awdp->debug.wait_completed.arg = NULL; } static void @@ -5245,6 +5651,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, ErtsRunQueue* runq, char** daww_ptr, size_t daww_sz) { + esdp->timer_wheel = NULL; #ifdef ERTS_SMP erts_bits_init_state(&esdp->erl_bits_state); esdp->match_pseudo_process = NULL; @@ -5257,9 +5664,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->f_reg_array = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER, MAX_REG * sizeof(FloatDef)); -#if !HEAP_ON_C_STACK - esdp->num_tmp_heap_used = 0; -#endif #ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) { esdp->no = 0; @@ -5272,6 +5676,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, #else esdp->no = (Uint) num; #endif + esdp->ssi = ssi; esdp->current_process = NULL; esdp->current_port = NULL; @@ -5284,6 +5689,15 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num, esdp->run_queue = runq; esdp->run_queue->scheduler = esdp; + esdp->last_monotonic_time = 0; + esdp->check_time_reds = 0; + + esdp->thr_id = (Uint32) num; + erts_sched_bif_unique_init(esdp); + + esdp->io.out = (Uint64) 0; + esdp->io.in = (Uint64) 0; + if (daww_ptr) { init_aux_work_data(&esdp->aux_work_data, esdp, *daww_ptr); #ifdef ERTS_SMP @@ -5309,6 +5723,9 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online char *daww_ptr; size_t daww_sz; size_t size_runqs; +#ifdef ERTS_SMP + erts_aint32_t set_schdlr_sspnd_change_flags; +#endif init_misc_op_list_alloc(); init_proc_sys_task_queues_alloc(); @@ -5384,7 +5801,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online } rq->out_of_work_count = 0; rq->max_len = 0; - rq->len = 0; + erts_smp_atomic32_set_nob(&rq->len, 0); rq->wakeup_other = 0; rq->wakeup_other_reds = 0; rq->halt_in_progress = 0; @@ -5534,38 +5951,18 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online #endif init_misc_aux_work(); -#if !HALFWORD_HEAP init_swtreq_alloc(); -#endif + init_screq_alloc(); + erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */ + debug_wait_completed_flags = 0; #ifdef ERTS_SMP - erts_atomic32_init_nob(&completed_dealloc_count, 0); /* debug only */ - aux_thread_aux_work_data = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, sizeof(ErtsAuxWorkData)); - erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); - erts_smp_cnd_init(&schdlr_sspnd.cnd); - - erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0); - schdlr_sspnd.online = no_schedulers_online; - schdlr_sspnd.curr_online = no_schedulers; - schdlr_sspnd.msb.ongoing = 0; - erts_smp_atomic32_init_nob(&schdlr_sspnd.active, no_schedulers); -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_cpu_changing, 0); - schdlr_sspnd.dirty_cpu_online = no_dirty_cpu_schedulers_online; - schdlr_sspnd.dirty_cpu_curr_online = no_dirty_cpu_schedulers; - erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_cpu_active, no_dirty_cpu_schedulers); - erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_io_changing, 0); - schdlr_sspnd.dirty_io_online = no_dirty_io_schedulers; - schdlr_sspnd.dirty_io_curr_online = no_dirty_io_schedulers; - erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_io_active, no_dirty_io_schedulers); -#endif - schdlr_sspnd.msb.procs = NULL; init_no_runqs(no_schedulers_online, no_schedulers_online); balance_info.last_active_runqs = no_schedulers; erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); @@ -5580,32 +5977,66 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online init_migration_paths(); - if (no_schedulers_online < no_schedulers) { + init_scheduler_suspend(); + + set_schdlr_sspnd_change_flags = 0; + + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL, + no_schedulers_online); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_NORMAL, + no_schedulers); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL, + no_schedulers); + + if (no_schedulers_online != no_schedulers) { + ASSERT(no_schedulers_online < no_schedulers); + set_schdlr_sspnd_change_flags |= ERTS_SCHDLR_SSPND_CHNG_ONLN; + schdlr_sspnd.changer = am_init; change_no_used_runqs(no_schedulers_online); for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) suspend_run_queue(ERTS_RUNQ_IX(ix)); } - schdlr_sspnd.wait_curr_online = no_schedulers_online; - schdlr_sspnd.curr_online *= 2; /* Boot strapping... */ - ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); #ifdef ERTS_DIRTY_SCHEDULERS - schdlr_sspnd.dirty_cpu_wait_curr_online = no_dirty_cpu_schedulers_online; - schdlr_sspnd.dirty_cpu_curr_online *= 2; - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - for (ix = no_dirty_cpu_schedulers_online; ix < no_dirty_cpu_schedulers; ix++) { - ErtsSchedulerData* esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); - erts_smp_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED); + + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU, + no_dirty_cpu_schedulers_online); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_DIRTY_CPU, + no_dirty_cpu_schedulers); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_CPU, + no_dirty_cpu_schedulers); + + if (no_dirty_cpu_schedulers_online != no_dirty_cpu_schedulers) { + ASSERT(no_dirty_cpu_schedulers_online < no_dirty_cpu_schedulers); + set_schdlr_sspnd_change_flags |= ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; + for (ix = no_dirty_cpu_schedulers_online; ix < no_dirty_cpu_schedulers; ix++) { + ErtsSchedulerData* esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); + erts_smp_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED); + } } - schdlr_sspnd.dirty_io_wait_curr_online = no_dirty_io_schedulers; - schdlr_sspnd.dirty_io_curr_online *= 2; - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_IO, + no_dirty_io_schedulers); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_DIRTY_IO, + no_dirty_io_schedulers); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_IO, + no_dirty_io_schedulers); + #endif + if (set_schdlr_sspnd_change_flags) + erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, + set_schdlr_sspnd_change_flags); + erts_smp_atomic32_init_nob(&doing_sys_schedule, 0); init_misc_aux_work(); @@ -5620,11 +6051,9 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online #endif } erts_no_schedulers = 1; -#ifdef ERTS_DIRTY_SCHEDULERS erts_no_dirty_cpu_schedulers = 0; erts_no_dirty_io_schedulers = 0; #endif -#endif erts_smp_atomic32_init_nob(&function_calls, 0); @@ -5716,19 +6145,96 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio) return proxy; } -static ERTS_INLINE void -free_proxy_proc(Process *proxy) -{ - ASSERT(erts_smp_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY); - erts_free(ERTS_ALC_T_PROC, proxy); -} - #define ERTS_ENQUEUE_NOT 0 #define ERTS_ENQUEUE_NORMAL_QUEUE 1 -#ifdef ERTS_DIRTY_SCHEDULERS #define ERTS_ENQUEUE_DIRTY_CPU_QUEUE 2 #define ERTS_ENQUEUE_DIRTY_IO_QUEUE 3 + +#ifdef ERTS_DIRTY_SCHEDULERS + +static int +check_dirty_enqueue_in_prio_queue(Process *c_p, + erts_aint32_t *newp, + erts_aint32_t actual, + erts_aint32_t aprio, + erts_aint32_t qbit) +{ + int queue; + erts_aint32_t dact, max_qbit; + + /* Termination should be done on an ordinary scheduler */ + if (actual & ERTS_PSFLG_EXITING) { + *newp &= ~ERTS_PSFLGS_DIRTY_WORK; + return ERTS_ENQUEUE_NORMAL_QUEUE; + } + + /* + * If we have system tasks, we enqueue on ordinary run-queue + * and take care of those system tasks first. + */ + if (actual & ERTS_PSFLG_ACTIVE_SYS) + return ERTS_ENQUEUE_NORMAL_QUEUE; + + dact = erts_smp_atomic32_read_mb(&c_p->dirty_state); + if (actual & (ERTS_PSFLG_DIRTY_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_CPU_PROC)) { + max_qbit = ((dact >> ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET) + & ERTS_PDSFLGS_QMASK); + queue = ERTS_ENQUEUE_DIRTY_CPU_QUEUE; + } + else { + ASSERT(actual & ERTS_PSFLG_DIRTY_IO_PROC); + max_qbit = ((dact >> ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET) + & ERTS_PDSFLGS_QMASK); + queue = ERTS_ENQUEUE_DIRTY_IO_QUEUE; + } + + max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS; + max_qbit &= -max_qbit; + + if (qbit >= max_qbit) + return ERTS_ENQUEUE_NOT; /* Already queued in higher or equal prio */ + if ((actual & (ERTS_PSFLG_IN_RUNQ|ERTS_PSFLGS_USR_PRIO_MASK)) + != (aprio << ERTS_PSFLGS_USR_PRIO_OFFSET)) { + /* + * Process struct already enqueued, or actual prio not + * equal to user prio, i.e., enqueue using proxy. + */ + return -1*queue; + } + + *newp |= ERTS_PSFLG_IN_RUNQ; + return queue; +} + +static ERTS_INLINE int +fin_dirty_enq_s_change(Process *p, + int pstruct_reserved, + erts_aint32_t enq_prio, + int qmask_offset) +{ + erts_aint32_t qbit = 1 << enq_prio; + qbit <<= qmask_offset; + + if (qbit & erts_smp_atomic32_read_bor_mb(&p->dirty_state, qbit)) { + /* Already enqueue by someone else... */ + if (pstruct_reserved) { + /* We reserved process struct for enqueue; clear it... */ +#ifdef DEBUG + erts_aint32_t old = +#else + (void) #endif + erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_IN_RUNQ); + ASSERT(old & ERTS_PSFLG_IN_RUNQ); + } + return 0; + } + + return !0; +} + +#endif /* ERTS_DIRTY_SCHEDULERS */ static ERTS_INLINE int check_enqueue_in_prio_queue(Process *c_p, @@ -5744,61 +6250,14 @@ check_enqueue_in_prio_queue(Process *c_p, *prq_prio_p = aprio; #ifdef ERTS_DIRTY_SCHEDULERS - if (actual & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) { - /* - * If we have system tasks of a priority higher - * or equal to the user priority, we enqueue - * on ordinary run-queue and take care of - * those system tasks first. - */ - if (actual & ERTS_PSFLG_ACTIVE_SYS) { - erts_aint32_t uprio, stprio, qmask; - uprio = (actual >> ERTS_PSFLGS_USR_PRIO_OFFSET) & ERTS_PSFLGS_PRIO_MASK; - if (aprio < uprio) - goto enqueue_normal_runq; /* system tasks with higher prio */ - erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_STATUS); - qmask = c_p->sys_task_qs->qmask; - erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); - switch (qmask & -qmask) { - case MAX_BIT: - stprio = PRIORITY_MAX; - break; - case HIGH_BIT: - stprio = PRIORITY_HIGH; - break; - case NORMAL_BIT: - stprio = PRIORITY_NORMAL; - break; - case LOW_BIT: - stprio = PRIORITY_LOW; - break; - default: - stprio = PRIORITY_LOW+1; - break; - } - if (stprio <= uprio) - goto enqueue_normal_runq; /* system tasks with higher prio */ - } - - /* Enqueue in dirty run queue if not already enqueued */ - if (actual & (ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q)) - return ERTS_ENQUEUE_NOT; /* already in queue */ - if (actual & ERTS_PSFLG_DIRTY_CPU_PROC) { - *newp |= ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q; - if (actual & ERTS_PSFLG_IN_RUNQ) - return -ERTS_ENQUEUE_DIRTY_CPU_QUEUE; /* use proxy */ - *newp |= ERTS_PSFLG_IN_RUNQ; - return ERTS_ENQUEUE_DIRTY_CPU_QUEUE; - } - *newp |= ERTS_PSFLG_DIRTY_IO_PROC_IN_Q; - if (actual & ERTS_PSFLG_IN_RUNQ) - return -ERTS_ENQUEUE_DIRTY_IO_QUEUE; /* use proxy */ - *newp |= ERTS_PSFLG_IN_RUNQ; - return ERTS_ENQUEUE_DIRTY_IO_QUEUE; + if (actual & ERTS_PSFLGS_DIRTY_WORK) { + int res = check_dirty_enqueue_in_prio_queue(c_p, newp, actual, + aprio, qbit); + if (res != ERTS_ENQUEUE_NORMAL_QUEUE) + return res; } - - enqueue_normal_runq: #endif + max_qbit = (actual >> ERTS_PSFLGS_IN_PRQ_MASK_OFFSET) & ERTS_PSFLGS_QMASK; max_qbit |= 1 << ERTS_PSFLGS_QMASK_BITS; max_qbit &= -max_qbit; @@ -5830,6 +6289,65 @@ check_enqueue_in_prio_queue(Process *c_p, return ERTS_ENQUEUE_NORMAL_QUEUE; } +static ERTS_INLINE ErtsRunQueue * +select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t state) +{ + + switch (enqueue) { + + case ERTS_ENQUEUE_NOT: + + return NULL; + +#ifdef ERTS_DIRTY_SCHEDULERS + + case ERTS_ENQUEUE_DIRTY_CPU_QUEUE: + case -ERTS_ENQUEUE_DIRTY_CPU_QUEUE: + + if (fin_dirty_enq_s_change(p, enqueue > 0, enq_prio, + ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET)) + return ERTS_DIRTY_CPU_RUNQ; + + return NULL; + + + case ERTS_ENQUEUE_DIRTY_IO_QUEUE: + case -ERTS_ENQUEUE_DIRTY_IO_QUEUE: + + if (fin_dirty_enq_s_change(p, enqueue > 0, enq_prio, + ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET)) + return ERTS_DIRTY_IO_RUNQ; + + return NULL; + +#endif + + default: { + ErtsRunQueue* runq; + + ASSERT(enqueue == ERTS_ENQUEUE_NORMAL_QUEUE + || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE); + + runq = erts_get_runq_proc(p); + +#ifdef ERTS_SMP + if (!(ERTS_PSFLG_BOUND & state)) { + ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio); + if (new_runq) { + RUNQ_SET_RQ(&p->run_queue, new_runq); + runq = new_runq; + } + } +#endif + + ASSERT(runq); + + return runq; + } + } +} + + /* * schedule_out_process() return with c_rq locked. */ @@ -5838,11 +6356,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces { erts_aint32_t a, e, n, enq_prio = -1; int enqueue; /* < 0 -> use proxy */ - Process* sched_p; ErtsRunQueue* runq; -#ifdef ERTS_SMP - int check_emigration_need; -#endif a = state; @@ -5854,7 +6368,7 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces enqueue = ERTS_ENQUEUE_NOT; n &= ~(ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS); - if (a & ERTS_PSFLG_ACTIVE_SYS + if (a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS) || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) { enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a); } @@ -5863,13 +6377,17 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces break; } - switch (enqueue) { - case ERTS_ENQUEUE_NOT: + runq = select_enqueue_run_queue(enqueue, enq_prio, p, n); + + if (!runq) { + if (erts_system_profile_flags.runnable_procs) { - if (!(a & ERTS_PSFLG_ACTIVE_SYS) - && (!(a & ERTS_PSFLG_ACTIVE) - || (a & ERTS_PSFLG_SUSPENDED))) { + /* Status lock prevents out of order "runnable proc" trace msgs */ + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + + if (!(a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS)) + && (!(a & ERTS_PSFLG_ACTIVE) || (a & ERTS_PSFLG_SUSPENDED))) { /* Process inactive */ profile_runnable_proc(p, am_inactive); } @@ -5879,100 +6397,79 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces free_proxy_proc(proxy); erts_smp_runq_lock(c_rq); + return 0; -#ifdef ERTS_DIRTY_SCHEDULERS -#ifdef ERTS_SMP - case ERTS_ENQUEUE_DIRTY_CPU_QUEUE: - case -ERTS_ENQUEUE_DIRTY_CPU_QUEUE: - runq = ERTS_DIRTY_CPU_RUNQ; - ASSERT(ERTS_SCHEDULER_IS_DIRTY_CPU(runq->scheduler)); -#ifdef ERTS_SMP - check_emigration_need = 0; -#endif - break; + } + else { + Process* sched_p; - case ERTS_ENQUEUE_DIRTY_IO_QUEUE: - case -ERTS_ENQUEUE_DIRTY_IO_QUEUE: - runq = ERTS_DIRTY_IO_RUNQ; - ASSERT(ERTS_SCHEDULER_IS_DIRTY_IO(runq->scheduler)); -#ifdef ERTS_SMP - check_emigration_need = 0; -#endif - break; -#endif -#endif + ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & (ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS))); - default: - ASSERT(enqueue == ERTS_ENQUEUE_NORMAL_QUEUE - || enqueue == -ERTS_ENQUEUE_NORMAL_QUEUE); + if (enqueue < 0) + sched_p = make_proxy_proc(proxy, p, enq_prio); + else { + sched_p = p; + if (proxy) + free_proxy_proc(proxy); + } - runq = erts_get_runq_proc(p); -#ifdef ERTS_SMP - check_emigration_need = !(ERTS_PSFLG_BOUND & n); -#endif - break; - } + ASSERT(runq); - ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & ERTS_PSFLG_ACTIVE_SYS)); + erts_smp_runq_lock(runq); - if (enqueue < 0) - sched_p = make_proxy_proc(proxy, p, enq_prio); - else { - sched_p = p; - if (proxy) - free_proxy_proc(proxy); - } + /* Enqueue the process */ + enqueue_process(runq, (int) enq_prio, sched_p); -#ifdef ERTS_SMP - if (check_emigration_need) { - ErtsRunQueue *new_runq = erts_check_emigration_need(runq, enq_prio); - if (new_runq) { - RUNQ_SET_RQ(&sched_p->run_queue, new_runq); - runq = new_runq; - } - } -#endif + if (runq == c_rq) + return 1; - ASSERT(runq); + erts_smp_runq_unlock(runq); - erts_smp_runq_lock(runq); + smp_notify_inc_runq(runq); - /* Enqueue the process */ - enqueue_process(runq, (int) enq_prio, sched_p); + erts_smp_runq_lock(c_rq); - if (runq == c_rq) return 1; - erts_smp_runq_unlock(runq); - smp_notify_inc_runq(runq); - erts_smp_runq_lock(c_rq); - return 1; + } + } static ERTS_INLINE void -add2runq(Process *p, erts_aint32_t state, erts_aint32_t prio) +add2runq(int enqueue, erts_aint32_t prio, + Process *proc, erts_aint32_t state, + Process **proxy) { - ErtsRunQueue *runq = erts_get_runq_proc(p); + ErtsRunQueue *runq; -#ifdef ERTS_SMP - if (!(ERTS_PSFLG_BOUND & state)) { - ErtsRunQueue *new_runq = erts_check_emigration_need(runq, (int) prio); - if (new_runq) { - RUNQ_SET_RQ(&p->run_queue, new_runq); - runq = new_runq; - } - } -#endif - ASSERT(runq); + runq = select_enqueue_run_queue(enqueue, prio, proc, state); - erts_smp_runq_lock(runq); + if (runq) { + Process *sched_p; - /* Enqueue the process */ - enqueue_process(runq, (int) prio, p); + if (enqueue > 0) + sched_p = proc; + else { + Process *pxy; - erts_smp_runq_unlock(runq); - smp_notify_inc_runq(runq); + if (!proxy) + pxy = NULL; + else { + pxy = *proxy; + *proxy = NULL; + } + sched_p = make_proxy_proc(pxy, proc, prio); + } + + erts_smp_runq_lock(runq); + /* Enqueue the process */ + enqueue_process(runq, (int) prio, sched_p); + + erts_smp_runq_unlock(runq); + smp_notify_inc_runq(runq); + } } static ERTS_INLINE int @@ -5980,7 +6477,8 @@ change_proc_schedule_state(Process *p, erts_aint32_t clear_state_flags, erts_aint32_t set_state_flags, erts_aint32_t *statep, - erts_aint32_t *enq_prio_p) + erts_aint32_t *enq_prio_p, + ErtsProcLocks locks) { /* * NOTE: ERTS_PSFLG_RUNNING, ERTS_PSFLG_RUNNING_SYS and @@ -5989,6 +6487,11 @@ change_proc_schedule_state(Process *p, */ erts_aint32_t a = *statep, n; int enqueue; /* < 0 -> use proxy */ + unsigned int prof_runnable_procs = erts_system_profile_flags.runnable_procs; + unsigned int lock_status = (prof_runnable_procs + && !(locks & ERTS_PROC_LOCK_STATUS)); + + ERTS_SMP_LC_ASSERT(locks == erts_proc_lc_my_proc_locks(p)); ASSERT(!(a & ERTS_PSFLG_PROXY)); ASSERT((clear_state_flags & (ERTS_PSFLG_RUNNING @@ -5998,6 +6501,9 @@ change_proc_schedule_state(Process *p, | ERTS_PSFLG_RUNNING_SYS | ERTS_PSFLG_ACTIVE_SYS)) == 0); + if (lock_status) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + while (1) { erts_aint32_t e; n = e = a; @@ -6033,7 +6539,9 @@ change_proc_schedule_state(Process *p, break; } - if (erts_system_profile_flags.runnable_procs) { + if (prof_runnable_procs) { + + /* Status lock prevents out of order "runnable proc" trace msgs */ if (((n & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_ACTIVE)) == ERTS_PSFLG_ACTIVE) @@ -6046,15 +6554,18 @@ change_proc_schedule_state(Process *p, profile_runnable_proc(p, am_active); } + if (lock_status) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } + *statep = a; return enqueue; } static ERTS_INLINE void -schedule_process(Process *p, erts_aint32_t in_state) +schedule_process(Process *p, erts_aint32_t in_state, ErtsProcLocks locks) { erts_aint32_t enq_prio = -1; erts_aint32_t state = in_state; @@ -6062,24 +6573,110 @@ schedule_process(Process *p, erts_aint32_t in_state) 0, ERTS_PSFLG_ACTIVE, &state, - &enq_prio); - if (enqueue != ERTS_ENQUEUE_NOT) - add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio), - state, - enq_prio); + &enq_prio, + locks); + add2runq(enqueue, enq_prio, p, state, NULL); } void -erts_schedule_process(Process *p, erts_aint32_t state) +erts_schedule_process(Process *p, erts_aint32_t state, ErtsProcLocks locks) { - schedule_process(p, state); + schedule_process(p, state, locks); } -static void -schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy) +static int +schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st) { - erts_aint32_t a = state, n, enq_prio = -1; + int res; + int locked; + ErtsProcSysTaskQs *stqs, *free_stqs; + erts_aint32_t state, a, n, enq_prio; int enqueue; /* < 0 -> use proxy */ + unsigned int prof_runnable_procs; + + res = 1; /* prepare for success */ + st->next = st->prev = st; /* Prep for empty prio queue */ + state = erts_smp_atomic32_read_nob(&p->state); + prof_runnable_procs = erts_system_profile_flags.runnable_procs; + locked = 0; + free_stqs = NULL; + if (state & ERTS_PSFLG_ACTIVE_SYS) + stqs = NULL; + else { + alloc_qs: + stqs = proc_sys_task_queues_alloc(); + stqs->qmask = 1 << prio; + stqs->ncount = 0; + stqs->q[PRIORITY_MAX] = NULL; + stqs->q[PRIORITY_HIGH] = NULL; + stqs->q[PRIORITY_NORMAL] = NULL; + stqs->q[PRIORITY_LOW] = NULL; + stqs->q[prio] = st; + } + + if (!locked) { + locked = 1; + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + + state = erts_smp_atomic32_read_nob(&p->state); + if (state & ERTS_PSFLG_EXITING) { + free_stqs = stqs; + res = 0; + goto cleanup; + } + } + + if (!p->sys_task_qs) { + if (stqs) + p->sys_task_qs = stqs; + else + goto alloc_qs; + } + else { + free_stqs = stqs; + stqs = p->sys_task_qs; + if (!stqs->q[prio]) { + stqs->q[prio] = st; + stqs->qmask |= 1 << prio; + } + else { + st->next = stqs->q[prio]; + st->prev = stqs->q[prio]->prev; + st->next->prev = st; + st->prev->next = st; + ASSERT(stqs->qmask & (1 << prio)); + } + } + + if (ERTS_PSFLGS_GET_ACT_PRIO(state) > prio) { + erts_aint32_t n, a, e; + /* Need to elevate actual prio */ + + a = state; + do { + if (ERTS_PSFLGS_GET_ACT_PRIO(a) <= prio) { + n = a; + break; + } + n = e = a; + n &= ~ERTS_PSFLGS_ACT_PRIO_MASK; + n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET); + a = erts_smp_atomic32_cmpxchg_nob(&p->state, n, e); + } while (a != e); + state = n; + } + + + a = state; + enq_prio = -1; + + /* Status lock prevents out of order "runnable proc" trace msgs */ + ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); + + if (!prof_runnable_procs) { + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + locked = 0; + } ASSERT(!(state & ERTS_PSFLG_PROXY)); @@ -6087,8 +6684,10 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy) erts_aint32_t e; n = e = a; - if (a & ERTS_PSFLG_FREE) - return; /* We don't want to schedule free processes... */ + if (a & ERTS_PSFLG_FREE) { + res = 0; + goto cleanup; /* We don't want to schedule free processes... */ + } enqueue = ERTS_ENQUEUE_NOT; n |= ERTS_PSFLG_ACTIVE_SYS; @@ -6101,7 +6700,7 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy) goto cleanup; } - if (erts_system_profile_flags.runnable_procs) { + if (prof_runnable_procs) { if (!(a & (ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_RUNNING @@ -6111,22 +6710,23 @@ schedule_process_sys_task(Process *p, erts_aint32_t state, Process *proxy) profile_runnable_proc(p, am_active); } + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + locked = 0; } - if (enqueue != ERTS_ENQUEUE_NOT) { - Process *sched_p; - if (enqueue > 0) - sched_p = p; - else { - sched_p = make_proxy_proc(proxy, p, enq_prio); - proxy = NULL; - } - add2runq(sched_p, n, enq_prio); - } + add2runq(enqueue, enq_prio, p, n, NULL); cleanup: - if (proxy) - free_proxy_proc(proxy); + + if (locked) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + + if (free_stqs) + proc_sys_task_queues_free(free_stqs); + + ERTS_SMP_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p))); + + return res; } static ERTS_INLINE int @@ -6144,7 +6744,7 @@ suspend_process(Process *c_p, Process *p) if (c_p == p) { state = erts_smp_atomic32_read_bor_relb(&p->state, ERTS_PSFLG_SUSPENDED); - ASSERT(state & ERTS_PSFLG_RUNNING); + ASSERT(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)); suspended = (state & ERTS_PSFLG_SUSPENDED) ? -1: 1; } else { @@ -6168,9 +6768,6 @@ suspend_process(Process *c_p, Process *p) if (suspended) { - ASSERT(!(ERTS_PSFLG_RUNNING & state) - || p == erts_get_current_process()); - if (suspended > 0 && erts_system_profile_flags.runnable_procs) { /* 'state' is before our change... */ @@ -6193,7 +6790,7 @@ suspend_process(Process *c_p, Process *p) } static ERTS_INLINE void -resume_process(Process *p) +resume_process(Process *p, ErtsProcLocks locks) { erts_aint32_t state, enq_prio = -1; int enqueue; @@ -6210,24 +6807,9 @@ resume_process(Process *p) ERTS_PSFLG_SUSPENDED, 0, &state, - &enq_prio); - if (enqueue) - add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio), - state, - enq_prio); -} - -int -erts_get_max_no_executing_schedulers(void) -{ -#ifdef ERTS_SMP - if (erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) - return (int) erts_no_schedulers; - ERTS_THR_MEMORY_BARRIER; - return (int) erts_smp_atomic32_read_nob(&schdlr_sspnd.active); -#else - return 1; -#endif + &enq_prio, + locks); + add2runq(enqueue, enq_prio, p, state, NULL); } #ifdef ERTS_SMP @@ -6332,42 +6914,82 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) } } -#ifdef ERTS_DIRTY_SCHEDULERS +static void +init_scheduler_suspend(void) +{ + erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); + schdlr_sspnd.online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); + schdlr_sspnd.curr_online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); + schdlr_sspnd.active = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); + erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0); + schdlr_sspnd.chngq = NULL; + schdlr_sspnd.changer = am_false; + schdlr_sspnd.nmsb.ongoing = 0; + schdlr_sspnd.nmsb.blckrs = NULL; + schdlr_sspnd.nmsb.chngq = NULL; + schdlr_sspnd.msb.ongoing = 0; + schdlr_sspnd.msb.blckrs = NULL; + schdlr_sspnd.msb.chngq = NULL; +} + +typedef struct { + struct { + Eterm chngr; + Eterm nxt; + } onln; + struct { + ErtsProcList *chngrs; + } msb; +} ErtsSchdlrSspndResume; + +static void +schdlr_sspnd_resume_proc(Eterm pid) +{ + Process *p = erts_pid2proc(NULL, 0, pid, ERTS_PROC_LOCK_STATUS); + if (p) { + resume_process(p, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } +} + +static ERTS_INLINE void +schdlr_sspnd_resume_procs(ErtsSchedType sched_type, + ErtsSchdlrSspndResume *resume) +{ + if (is_internal_pid(resume->onln.chngr)) { + schdlr_sspnd_resume_proc(resume->onln.chngr); + resume->onln.chngr = NIL; + } + if (is_internal_pid(resume->onln.nxt)) { + schdlr_sspnd_resume_proc(resume->onln.nxt); + resume->onln.nxt = NIL; + } + while (resume->msb.chngrs) { + ErtsProcList *plp = resume->msb.chngrs; + resume->msb.chngrs = plp->next; + schdlr_sspnd_resume_proc(plp->pid); + proclist_destroy(plp); + } +} static void suspend_scheduler(ErtsSchedulerData *esdp) { erts_aint32_t flgs; erts_aint32_t changing; -#ifdef ERTS_DIRTY_SCHEDULERS - long no = (long) (ERTS_SCHEDULER_IS_DIRTY(esdp) - ? ERTS_DIRTY_SCHEDULER_NO(esdp) - : esdp->no); -#else - long no = (long) esdp->no; -#endif + long no; ErtsSchedulerSleepInfo *ssi = esdp->ssi; - long active_schedulers; int curr_online = 1; - int wake = 0; + ErtsSchdlrSspndResume resume = {{NIL, NIL}, {NULL}}; erts_aint32_t aux_work; int thr_prgr_active = 1; ErtsStuckBoundProcesses sbp = {NULL, NULL}; - int* ss_onlinep; - int* ss_curr_onlinep; - int* ss_wait_curr_onlinep; - long* ss_wait_activep; - long ss_wait_active_target; - erts_smp_atomic32_t* ss_changingp; - erts_smp_atomic32_t* ss_activep; + ErtsSchedType sched_type; + erts_aint32_t online_flag; /* * Schedulers may be suspended in two different ways: * - A scheduler may be suspended since it is not online. - * All schedulers with scheduler ids greater than - * schdlr_sspnd.online are suspended; same for dirty - * schedulers and schdlr_sspnd.dirty_cpu_online and - * schdlr_sspnd.dirty_io_online. * - Multi scheduling is blocked. All schedulers except the * scheduler with scheduler id 1 are suspended, and all * dirty CPU and dirty I/O schedulers are suspended. @@ -6375,27 +6997,43 @@ suspend_scheduler(ErtsSchedulerData *esdp) * Regardless of why a scheduler is suspended, it ends up here. */ - ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp) || no != 1); - #ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { + no = ERTS_DIRTY_SCHEDULER_NO(esdp); + if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) { + online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; + sched_type = ERTS_SCHED_DIRTY_CPU; + } + else { + online_flag = 0; + sched_type = ERTS_SCHED_DIRTY_IO; + } + } + else +#endif + { + online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN; + no = esdp->no; + sched_type = ERTS_SCHED_NORMAL; + } + + ASSERT(sched_type != ERTS_SCHED_NORMAL || no != 1); + + if (sched_type != ERTS_SCHED_NORMAL) { if (erts_smp_mtx_trylock(&schdlr_sspnd.mtx) == EBUSY) { erts_smp_runq_unlock(esdp->run_queue); erts_smp_mtx_lock(&schdlr_sspnd.mtx); erts_smp_runq_lock(esdp->run_queue); } - if (ongoing_multi_scheduling_block()) + if (schdlr_sspnd.msb.ongoing) evacuate_run_queue(esdp->run_queue, &sbp); - } else -#endif + erts_smp_runq_unlock(esdp->run_queue); + } + else { evacuate_run_queue(esdp->run_queue, &sbp); - erts_smp_runq_unlock(esdp->run_queue); + erts_smp_runq_unlock(esdp->run_queue); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - { erts_sched_check_cpu_bind_prep_suspend(esdp); if (erts_system_profile_flags.scheduler) @@ -6409,364 +7047,245 @@ suspend_scheduler(ErtsSchedulerData *esdp) flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); if (flgs & ERTS_SSI_FLG_SUSPENDED) { -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) { - active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.dirty_cpu_active); - ASSERT(active_schedulers >= 0); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing); - ss_onlinep = &schdlr_sspnd.dirty_cpu_online; - ss_curr_onlinep = &schdlr_sspnd.dirty_cpu_curr_online; - ss_wait_curr_onlinep = &schdlr_sspnd.dirty_cpu_wait_curr_online; - ss_changingp = &schdlr_sspnd.dirty_cpu_changing; - ss_wait_activep = &schdlr_sspnd.msb.dirty_cpu_wait_active; - ss_activep = &schdlr_sspnd.dirty_cpu_active; - } else { - active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.dirty_io_active); - ASSERT(active_schedulers >= 0); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing); - ss_onlinep = &schdlr_sspnd.dirty_io_online; - ss_curr_onlinep = &schdlr_sspnd.dirty_io_curr_online; - ss_wait_curr_onlinep = &schdlr_sspnd.dirty_io_wait_curr_online; - ss_changingp = &schdlr_sspnd.dirty_io_changing; - ss_wait_activep = &schdlr_sspnd.msb.dirty_io_wait_active; - ss_activep = &schdlr_sspnd.dirty_io_active; - } - ss_wait_active_target = 0; - } - else -#endif - { - active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active); - ASSERT(active_schedulers >= 1); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - ss_onlinep = &schdlr_sspnd.online; - ss_curr_onlinep = &schdlr_sspnd.curr_online; - ss_wait_curr_onlinep = &schdlr_sspnd.wait_curr_online; - ss_changingp = &schdlr_sspnd.changing; - ss_wait_activep = &schdlr_sspnd.msb.wait_active; - ss_activep = &schdlr_sspnd.active; - ss_wait_active_target = 1; - } - if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { - if (active_schedulers == *ss_wait_activep) - wake = 1; - if (active_schedulers == ss_wait_active_target) { - changing = erts_smp_atomic32_read_band_nob(ss_changingp, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); - changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; - } - } + schdlr_sspnd_dec_nscheds(&schdlr_sspnd.active, sched_type); - while (1) { - if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { - int changed = 0; - if (no > *ss_onlinep && curr_online) { - (*ss_curr_onlinep)--; - curr_online = 0; - changed = 1; - } - else if (no <= *ss_onlinep && !curr_online) { - (*ss_curr_onlinep)++; - curr_online = 1; - changed = 1; - } - if (changed - && *ss_curr_onlinep == *ss_wait_curr_onlinep) - wake = 1; - if (*ss_onlinep == *ss_curr_onlinep) { - changing = erts_smp_atomic32_read_band_nob(ss_changingp, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; - } - } + ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) >= 1); - if (wake) { - erts_smp_cnd_signal(&schdlr_sspnd.cnd); - wake = 0; - } - - if (curr_online && !ongoing_multi_scheduling_block()) { - flgs = erts_smp_atomic32_read_acqb(&ssi->flags); - if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) - break; - } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - while (1) { - erts_aint32_t qmask; - erts_aint32_t flgs; + while (1) { - qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue) - & ERTS_RUNQ_FLGS_QMASK); - aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work|qmask) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); + if (changing & (ERTS_SCHDLR_SSPND_CHNG_NMSB + | ERTS_SCHDLR_SSPND_CHNG_MSB)) { + int i = 0; + ErtsMultiSchedulingBlock *msb[3] = {0}; + if (changing & ERTS_SCHDLR_SSPND_CHNG_NMSB) + msb[i++] = &schdlr_sspnd.nmsb; + if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) + msb[i++] = &schdlr_sspnd.msb; + + for (i = 0; msb[i]; i++) { + erts_aint32_t clr_flg = 0; + + if (msb[i] == &schdlr_sspnd.nmsb + && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) == 1) { + clr_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB; } - if (aux_work) - aux_work = handle_aux_work(&esdp->aux_work_data, - aux_work, - 1); - - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && - (aux_work && erts_thr_progress_update(esdp))) - erts_thr_progress_leader_update(esdp); - if (qmask) { -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - erts_smp_runq_lock(esdp->run_queue); - if (ongoing_multi_scheduling_block()) - evacuate_run_queue(esdp->run_queue, &sbp); - erts_smp_runq_unlock(esdp->run_queue); - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - } else -#endif - { - erts_smp_runq_lock(esdp->run_queue); - evacuate_run_queue(esdp->run_queue, &sbp); - erts_smp_runq_unlock(esdp->run_queue); - } - } - } - - if (!aux_work) { -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); - } - erts_thr_progress_prepare_wait(esdp); + else if (schdlr_sspnd.active + == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)) { + clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB; } - flgs = sched_spin_suspended(ssi, - ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - flgs = sched_set_suspended_sleeptype(ssi); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - int res; - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + if (clr_flg) { + ErtsProcList *plp, *end_plp; + changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~clr_flg); + changing &= ~clr_flg; + (void) erts_proclist_fetch(&msb[i]->chngq, &end_plp); + /* resume processes that initiated the multi scheduling block... */ + plp = msb[i]->chngq; + while (plp) { + erts_proclist_store_last(&msb[i]->blckrs, + proclist_copy(plp)); + plp = plp->next; } + if (end_plp) + end_plp->next = resume.msb.chngrs; + resume.msb.chngrs = msb[i]->chngq; + msb[i]->chngq = NULL; } -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - erts_thr_progress_finalize_wait(esdp); } - - flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)); - if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) - break; - changing = erts_smp_atomic32_read_nob(ss_changingp); - if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) - break; } - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read_nob(ss_changingp); - } - - active_schedulers = erts_smp_atomic32_inc_read_nob(ss_activep); - changing = erts_smp_atomic32_read_nob(ss_changingp); - if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) - && *ss_onlinep == active_schedulers) { - erts_smp_atomic32_read_band_nob(ss_changingp, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); - } - -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - ASSERT(no <= *ss_onlinep); - ASSERT(!ongoing_multi_scheduling_block()); - - } - - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - - ASSERT(curr_online); - -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - { - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(esdp->no), am_active); - - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - } - - erts_smp_runq_lock(esdp->run_queue); - non_empty_runq(esdp->run_queue); - -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - { - schedule_bound_processes(esdp->run_queue, &sbp); - - erts_sched_check_cpu_bind_post_suspend(esdp); - } -} - -#else /* !ERTS_DIRTY_SCHEDULERS */ - -static void -suspend_scheduler(ErtsSchedulerData *esdp) -{ - erts_aint32_t flgs; - erts_aint32_t changing; - long no = (long) esdp->no; - ErtsSchedulerSleepInfo *ssi = esdp->ssi; - long active_schedulers; - int curr_online = 1; - int wake = 0; - erts_aint32_t aux_work; - int thr_prgr_active = 1; - ErtsStuckBoundProcesses sbp = {NULL, NULL}; - - /* - * Schedulers may be suspended in two different ways: - * - A scheduler may be suspended since it is not online. - * All schedulers with scheduler ids greater than - * schdlr_sspnd.online are suspended. - * - Multi scheduling is blocked. All schedulers except the - * scheduler with scheduler id 1 are suspended. - * - * Regardless of why a scheduler is suspended, it ends up here. - */ - - ASSERT(no != 1); - - evacuate_run_queue(esdp->run_queue, &sbp); - - erts_smp_runq_unlock(esdp->run_queue); - - erts_sched_check_cpu_bind_prep_suspend(esdp); - - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(esdp->no), am_inactive); - - sched_wall_time_change(esdp, 0); - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); - if (flgs & ERTS_SSI_FLG_SUSPENDED) { - - active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active); - ASSERT(active_schedulers >= 1); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { - if (active_schedulers == schdlr_sspnd.msb.wait_active) - wake = 1; - if (active_schedulers == 1) { - changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); - changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; - } - } - - while (1) { - if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + if (changing & online_flag) { int changed = 0; - if (no > schdlr_sspnd.online && curr_online) { - schdlr_sspnd.curr_online--; + Uint32 st_online; + + st_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + sched_type); + if (no > st_online && curr_online) { + schdlr_sspnd_dec_nscheds(&schdlr_sspnd.curr_online, + sched_type); curr_online = 0; changed = 1; } - else if (no <= schdlr_sspnd.online && !curr_online) { - schdlr_sspnd.curr_online++; + else if (no <= st_online && !curr_online) { + schdlr_sspnd_inc_nscheds(&schdlr_sspnd.curr_online, + sched_type); curr_online = 1; changed = 1; } if (changed - && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) - wake = 1; - if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { + && (schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + sched_type) + == schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online, + sched_type))) { + ErtsProcList *plp; changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; + ~online_flag); + changing &= ~online_flag; + if (sched_type == ERTS_SCHED_NORMAL) { + ASSERT(is_internal_pid(schdlr_sspnd.changer) + || schdlr_sspnd.changer == am_init); + /* resume process that initiated this change... */ + resume.onln.chngr = schdlr_sspnd.changer; + plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + if (!plp) + schdlr_sspnd.changer = am_false; + else { + schdlr_sspnd.changer = am_true; /* change right in transit */ + /* resume process that is queued for next change... */ + resume.onln.nxt = plp->pid; + ASSERT(is_internal_pid(resume.onln.nxt)); + } + } } } - if (wake) { - erts_smp_cnd_signal(&schdlr_sspnd.cnd); - wake = 0; - } - - if (curr_online && !ongoing_multi_scheduling_block()) { + if (curr_online + && (sched_type == ERTS_SCHED_NORMAL + ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) + : !schdlr_sspnd.msb.ongoing)) { flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + schdlr_sspnd_resume_procs(sched_type, &resume); + while (1) { + ErtsMonotonicTime current_time; erts_aint32_t qmask; erts_aint32_t flgs; qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue) & ERTS_RUNQ_FLGS_QMASK); - aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work|qmask) { - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - if (aux_work) - aux_work = handle_aux_work(&esdp->aux_work_data, - aux_work, - 1); - if (aux_work && erts_thr_progress_update(esdp)) - erts_thr_progress_leader_update(esdp); + + if (sched_type != ERTS_SCHED_NORMAL) { if (qmask) { + erts_smp_mtx_lock(&schdlr_sspnd.mtx); erts_smp_runq_lock(esdp->run_queue); - evacuate_run_queue(esdp->run_queue, &sbp); + if (schdlr_sspnd.msb.ongoing) + evacuate_run_queue(esdp->run_queue, &sbp); erts_smp_runq_unlock(esdp->run_queue); + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); } + aux_work = 0; } + else { - if (!aux_work) { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + + if (aux_work|qmask) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + if (aux_work) + aux_work = handle_aux_work(&esdp->aux_work_data, + aux_work, + 1); + + if (aux_work && erts_thr_progress_update(esdp)) + erts_thr_progress_leader_update(esdp); + if (qmask) { + erts_smp_runq_lock(esdp->run_queue); + evacuate_run_queue(esdp->run_queue, &sbp); + erts_smp_runq_unlock(esdp->run_queue); + } } - erts_thr_progress_prepare_wait(esdp); - flgs = sched_spin_suspended(ssi, - ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - flgs = sched_set_suspended_sleeptype(ssi); + + } + + if (aux_work) { + ASSERT(sched_type == ERTS_SCHED_NORMAL); + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + erts_bump_timers(esdp->timer_wheel, current_time); + } + } + else { + ErtsMonotonicTime timeout_time; + int do_timeout; + + if (sched_type == ERTS_SCHED_NORMAL) { + timeout_time = erts_check_next_timeout_time(esdp); + current_time = erts_get_monotonic_time(esdp); + do_timeout = (current_time >= timeout_time); + } + else { + timeout_time = ERTS_MONOTONIC_TIME_MAX; + current_time = 0; + do_timeout = 0; + } + + if (do_timeout) { + ASSERT(sched_type == ERTS_SCHED_NORMAL); + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } + } + else { + if (sched_type == ERTS_SCHED_NORMAL) { + if (thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 0); + sched_wall_time_change(esdp, 0); + } + erts_thr_progress_prepare_wait(esdp); + } + flgs = sched_spin_suspended(ssi, + ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)) { - int res; - - do { - res = erts_tse_wait(ssi->event); - } while (res == EINTR); + flgs = sched_set_suspended_sleeptype(ssi); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + + if (sched_type == ERTS_SCHED_NORMAL) + current_time = erts_get_monotonic_time(esdp); + else + current_time = 0; + + do { + Sint64 timeout; + if (current_time >= timeout_time) + break; + if (sched_type != ERTS_SCHED_NORMAL) + timeout = -1; + else + timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time + - current_time + - 1) + 1; + res = erts_tse_twait(ssi->event, timeout); + + if (sched_type == ERTS_SCHED_NORMAL) + current_time = erts_get_monotonic_time(esdp); + else + current_time = 0; + + } while (res == EINTR); + } } + if (sched_type == ERTS_SCHED_NORMAL) + erts_thr_progress_finalize_wait(esdp); + } + + if (current_time >= timeout_time) { + ASSERT(sched_type == ERTS_SCHED_NORMAL); + erts_bump_timers(esdp->timer_wheel, current_time); } - erts_thr_progress_finalize_wait(esdp); } flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING @@ -6774,7 +7293,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) + if (changing) break; } @@ -6782,612 +7301,494 @@ suspend_scheduler(ErtsSchedulerData *esdp) changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); } - active_schedulers = erts_smp_atomic32_inc_read_nob(&schdlr_sspnd.active); + schdlr_sspnd_inc_nscheds(&schdlr_sspnd.active, sched_type); changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) - && schdlr_sspnd.online == active_schedulers) { + && schdlr_sspnd.online == schdlr_sspnd.active) { erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_MSB); } - ASSERT(no <= schdlr_sspnd.online); - ASSERT(!ongoing_multi_scheduling_block()); - + ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type)); + ASSERT((sched_type == ERTS_SCHED_NORMAL + ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) + : !schdlr_sspnd.msb.ongoing)); } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + ASSERT(!resume.msb.chngrs); + schdlr_sspnd_resume_procs(sched_type, &resume); + ASSERT(curr_online); - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(esdp->no), am_active); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (erts_system_profile_flags.scheduler) + profile_scheduler(make_small(esdp->no), am_active); - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } } + if (sched_type == ERTS_SCHED_NORMAL) + (void) erts_get_monotonic_time(esdp); erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); - schedule_bound_processes(esdp->run_queue, &sbp); + if (sched_type == ERTS_SCHED_NORMAL) { + schedule_bound_processes(esdp->run_queue, &sbp); - erts_sched_check_cpu_bind_post_suspend(esdp); + erts_sched_check_cpu_bind_post_suspend(esdp); + } } -#endif - -ErtsSchedSuspendResult +void erts_schedulers_state(Uint *total, Uint *online, Uint *active, Uint *dirty_cpu, Uint *dirty_cpu_online, + Uint *dirty_cpu_active, Uint *dirty_io, - int yield_allowed) + Uint *dirty_io_active) { - int res = ERTS_SCHDLR_SSPND_EINVAL; - erts_aint32_t changing; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); -#ifdef ERTS_DIRTY_SCHEDULERS - changing |= (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing) - | erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing)); -#endif - if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)) - res = ERTS_SCHDLR_SSPND_YIELD_RESTART; - else { + if (active || online || dirty_cpu_online + || dirty_cpu_active || dirty_io_active) { + erts_smp_mtx_lock(&schdlr_sspnd.mtx); if (active) - *active = schdlr_sspnd.online; + *active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL); if (online) - *online = schdlr_sspnd.online; - if (ongoing_multi_scheduling_block() && active) - *active = 1; -#ifdef ERTS_DIRTY_SCHEDULERS + *online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_NORMAL); + if (dirty_cpu_active) + *dirty_cpu_active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_CPU); if (dirty_cpu_online) - *dirty_cpu_online = schdlr_sspnd.dirty_cpu_online; -#endif - res = ERTS_SCHDLR_SSPND_DONE; + *dirty_cpu_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_DIRTY_CPU); + if (dirty_io_active) + *dirty_io_active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_IO); + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (total) *total = erts_no_schedulers; -#ifdef ERTS_DIRTY_SCHEDULERS if (dirty_cpu) *dirty_cpu = erts_no_dirty_cpu_schedulers; if (dirty_io) *dirty_io = erts_no_dirty_io_schedulers; -#endif - return res; } -#ifdef ERTS_DIRTY_SCHEDULERS +static void +abort_sched_onln_chng_waitq(Process *p) +{ + Eterm resume = NIL; + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + +#ifdef DEBUG + { + int found_it = 0; + ErtsProcList *plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + while (plp) { + if (erts_proclist_same(plp, p)) + found_it++; + plp = erts_proclist_peek_next(schdlr_sspnd.chngq, plp); + } + ASSERT(found_it == !!(p->flags & F_SCHDLR_ONLN_WAITQ)); + } +#endif + + if (p->flags & F_SCHDLR_ONLN_WAITQ) { + ErtsProcList *plp = NULL; + + plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + if (plp) { + if (erts_proclist_same(plp, p) + && schdlr_sspnd.changer == am_true) { + p->flags &= ~F_SCHDLR_ONLN_WAITQ; + /* + * Change right was in transit to us; + * transfer it to the next process by + * resuming it... + */ + erts_proclist_remove(&schdlr_sspnd.chngq, plp); + proclist_destroy(plp); + plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + if (plp) + resume = plp->pid; + else + schdlr_sspnd.changer = am_false; + } + else { + do { + if (erts_proclist_same(plp, p)) { + p->flags &= ~F_SCHDLR_ONLN_WAITQ; + erts_proclist_remove(&schdlr_sspnd.chngq, plp); + proclist_destroy(plp); + break; + } + plp = erts_proclist_peek_next(schdlr_sspnd.chngq, plp); + } while (plp); + } + } + } + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + + if (is_internal_pid(resume)) + schdlr_sspnd_resume_proc(resume); +} ErtsSchedSuspendResult erts_set_schedulers_online(Process *p, ErtsProcLocks plocks, Sint new_no, - Sint *old_no -#ifdef ERTS_DIRTY_SCHEDULERS - , int dirty_only -#endif - ) + Sint *old_no, + int dirty_only) { - ErtsSchedulerData *esdp; - int ix, res = -1, no, have_unlocked_plocks, end_wait; - erts_aint32_t changing = 0; + int resume_proc, ix, res = -1, no, have_unlocked_plocks; + erts_aint32_t changing = 0, change_flags; + int online, increase; + ErtsProcList *plp; #ifdef ERTS_DIRTY_SCHEDULERS - ErtsSchedulerSleepInfo* ssi; - int dirty_no, change_dirty; + int dirty_no, change_dirty, dirty_online; +#else + ASSERT(!dirty_only); #endif if (new_no < 1) return ERTS_SCHDLR_SSPND_EINVAL; -#ifdef ERTS_DIRTY_SCHEDULERS else if (dirty_only && erts_no_dirty_cpu_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; -#endif else if (erts_no_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; - esdp = ERTS_PROC_GET_SCHDATA(p); - end_wait = 0; +#ifdef ERTS_DIRTY_SCHEDULERS + if (dirty_only) + resume_proc = 0; + else +#endif + { + resume_proc = 1; + /* + * If we suspend current process we need to suspend before + * requesting the change; otherwise, we got a resume/suspend + * race... + */ + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + suspend_process(p, p); + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } erts_smp_mtx_lock(&schdlr_sspnd.mtx); + change_flags = 0; have_unlocked_plocks = 0; no = (int) new_no; #ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(schdlr_sspnd.dirty_cpu_online <= erts_no_dirty_cpu_schedulers); + if (!dirty_only) +#endif + { + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); + if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + enqueue_wait: + p->flags |= F_SCHDLR_ONLN_WAITQ; + plp = proclist_create(p); + erts_proclist_store_last(&schdlr_sspnd.chngq, plp); + resume_proc = 0; + res = ERTS_SCHDLR_SSPND_YIELD_RESTART; + goto done; + } + plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + if (!plp) { + ASSERT(schdlr_sspnd.changer == am_false); + } + else { + ASSERT(schdlr_sspnd.changer == am_true); + if (!erts_proclist_same(plp, p)) + goto enqueue_wait; + p->flags &= ~F_SCHDLR_ONLN_WAITQ; + erts_proclist_remove(&schdlr_sspnd.chngq, plp); + proclist_destroy(plp); + } + } + + *old_no = online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL); +#ifndef ERTS_DIRTY_SCHEDULERS + if (no == online) { + res = ERTS_SCHDLR_SSPND_DONE; + goto done; + } +#else /* ERTS_DIRTY_SCHEDULERS */ + dirty_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU); + if (dirty_only) + *old_no = dirty_online; + + ASSERT(dirty_online <= erts_no_dirty_cpu_schedulers); + if (dirty_only) { - if (no > schdlr_sspnd.online) { - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - return ERTS_SCHDLR_SSPND_EINVAL; + if (no > online) { + res = ERTS_SCHDLR_SSPND_EINVAL; + goto done; } dirty_no = no; + if (dirty_no == dirty_online) { + res = ERTS_SCHDLR_SSPND_DONE; + goto done; + } + change_dirty = 1; } else { /* * Adjust the number of dirty CPU schedulers online relative to the * adjustment made to the number of normal schedulers online. */ int total_pct = erts_no_dirty_cpu_schedulers*100/erts_no_schedulers; - int onln_pct = no*total_pct/schdlr_sspnd.online; - dirty_no = schdlr_sspnd.dirty_cpu_online*onln_pct/100; + int onln_pct = no*total_pct/online; + dirty_no = dirty_online*onln_pct/100; if (dirty_no == 0) dirty_no = 1; ASSERT(dirty_no <= erts_no_dirty_cpu_schedulers); - } -#endif - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); -#ifdef ERTS_DIRTY_SCHEDULERS - changing |= erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing); -#endif - if (changing) { - res = ERTS_SCHDLR_SSPND_YIELD_RESTART; - } - else { - int online = *old_no = schdlr_sspnd.online; -#ifdef ERTS_DIRTY_SCHEDULERS - int dirty_online = schdlr_sspnd.dirty_cpu_online; - if (dirty_only) { - *old_no = schdlr_sspnd.dirty_cpu_online; - if (dirty_no == schdlr_sspnd.dirty_cpu_online) { + if (no != online) + change_dirty = (dirty_no != dirty_online); + else { + dirty_only = 1; + if (dirty_no == dirty_online) { res = ERTS_SCHDLR_SSPND_DONE; + goto done; } change_dirty = 1; - } else { -#endif - if (no == schdlr_sspnd.online) { -#ifdef ERTS_DIRTY_SCHEDULERS - dirty_only = 1; - if (dirty_no == schdlr_sspnd.dirty_cpu_online) -#endif - res = ERTS_SCHDLR_SSPND_DONE; -#ifdef ERTS_DIRTY_SCHEDULERS - else - change_dirty = 1; -#endif - } -#ifdef ERTS_DIRTY_SCHEDULERS - else - change_dirty = (dirty_no != schdlr_sspnd.dirty_cpu_online); } -#endif - if (res == -1) - { - int increase = (no > online); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!dirty_only) { -#endif - ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - schdlr_sspnd.online = no; -#ifdef ERTS_DIRTY_SCHEDULERS - } else - increase = (dirty_no > dirty_online); - if (change_dirty) { - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - schdlr_sspnd.dirty_cpu_online = dirty_no; - } -#endif - if (increase) { - int ix; -#ifdef ERTS_DIRTY_SCHEDULERS - if (!dirty_only) { -#endif - schdlr_sspnd.wait_curr_online = no; - if (ongoing_multi_scheduling_block()) { - for (ix = online; ix < no; ix++) - erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); - } - else { - if (plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - change_no_used_runqs(no); + } + if (change_dirty) { + change_flags |= ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU, + dirty_no); + } - for (ix = online; ix < no; ix++) - resume_run_queue(ERTS_RUNQ_IX(ix)); + if (dirty_only) + increase = (dirty_no > dirty_online); + else +#endif /* ERTS_DIRTY_SCHEDULERS */ + { + change_flags |= ERTS_SCHDLR_SSPND_CHNG_ONLN; + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL, + no); + increase = (no > online); + } - for (ix = no; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); - } + erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, change_flags); + + res = ERTS_SCHDLR_SSPND_DONE; + if (increase) { + int ix; #ifdef ERTS_DIRTY_SCHEDULERS + if (change_dirty) { + ErtsSchedulerSleepInfo* ssi; + if (schdlr_sspnd.msb.ongoing) { + for (ix = dirty_online; ix < dirty_no; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_sched_poke(ssi); } - if (change_dirty) { - schdlr_sspnd.dirty_cpu_wait_curr_online = dirty_no; - ASSERT(schdlr_sspnd.dirty_cpu_curr_online != - schdlr_sspnd.dirty_cpu_wait_curr_online); - if (ongoing_multi_scheduling_block()) { - for (ix = dirty_online; ix < dirty_no; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_sched_poke(ssi); - } - } else { - for (ix = dirty_online; ix < dirty_no; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - erts_smp_atomic32_read_band_nob(&ssi->flags, - ~ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - } + } else { + for (ix = dirty_online; ix < dirty_no; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + scheduler_ssi_resume_wake(ssi); } -#endif - res = ERTS_SCHDLR_SSPND_DONE; } - else /* if (no < online) */ { -#ifdef ERTS_DIRTY_SCHEDULERS - if (change_dirty) { - schdlr_sspnd.dirty_cpu_wait_curr_online = dirty_no; - ASSERT(schdlr_sspnd.dirty_cpu_curr_online != - schdlr_sspnd.dirty_cpu_wait_curr_online); - if (ongoing_multi_scheduling_block()) { - for (ix = dirty_no; ix < dirty_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_sched_poke(ssi); - } - } else { - for (ix = dirty_no; ix < dirty_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - } - } - if (dirty_only) { - res = ERTS_SCHDLR_SSPND_DONE; - } - else + } + if (!dirty_only) #endif - { - if (p->scheduler_data->no <= no) { - res = ERTS_SCHDLR_SSPND_DONE; - schdlr_sspnd.wait_curr_online = no; - } - else { - /* - * Yield! Current process needs to migrate - * before bif returns. - */ - res = ERTS_SCHDLR_SSPND_YIELD_DONE; - schdlr_sspnd.wait_curr_online = no+1; - } - - if (ongoing_multi_scheduling_block()) { - for (ix = no; ix < online; ix++) - erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); - } - else { - if (plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - - change_no_used_runqs(no); - for (ix = no; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); - - for (ix = no; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq); - } - } - } + { + if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) { + for (ix = online; ix < no; ix++) + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); } - -#ifdef ERTS_DIRTY_SCHEDULERS - if (change_dirty) { - while (schdlr_sspnd.dirty_cpu_curr_online != schdlr_sspnd.dirty_cpu_wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_cpu_changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); - } - if (!dirty_only) -#endif - { - if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - if (plocks && !have_unlocked_plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - erts_thr_progress_active(esdp, 0); - erts_thr_progress_prepare_wait(esdp); - end_wait = 1; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + else { + if (plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); } + change_no_used_runqs(no); - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + for (ix = online; ix < no; ix++) + resume_run_queue(ERTS_RUNQ_IX(ix)); - ASSERT(res != ERTS_SCHDLR_SSPND_DONE - ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) - : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + for (ix = no; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); } } } - - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + else /* if decrease */ { #ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(schdlr_sspnd.dirty_cpu_online <= schdlr_sspnd.online); - if (!dirty_only) -#endif - { - if (end_wait) { - erts_thr_progress_finalize_wait(esdp); - erts_thr_progress_active(esdp, 1); - } - if (have_unlocked_plocks) - erts_smp_proc_lock(p, plocks); - } - - return res; -} - -#else /* !ERTS_DIRTY_SCHEDULERS */ - -ErtsSchedSuspendResult -erts_set_schedulers_online(Process *p, - ErtsProcLocks plocks, - Sint new_no, - Sint *old_no) -{ - ErtsSchedulerData *esdp; - int ix, res, no, have_unlocked_plocks, end_wait; - erts_aint32_t changing; - - if (new_no < 1 || erts_no_schedulers < new_no) - return ERTS_SCHDLR_SSPND_EINVAL; - - esdp = ERTS_PROC_GET_SCHDATA(p); - end_wait = 0; - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - have_unlocked_plocks = 0; - no = (int) new_no; - - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - if (changing) { - res = ERTS_SCHDLR_SSPND_YIELD_RESTART; - } - else { - int online = *old_no = schdlr_sspnd.online; - if (no == schdlr_sspnd.online) { - res = ERTS_SCHDLR_SSPND_DONE; - } - else { - ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - schdlr_sspnd.online = no; - if (no > online) { - int ix; - schdlr_sspnd.wait_curr_online = no; - if (ongoing_multi_scheduling_block()) { - for (ix = online; ix < no; ix++) - erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + if (change_dirty) { + ErtsSchedulerSleepInfo* ssi; + if (schdlr_sspnd.msb.ongoing) { + for (ix = dirty_no; ix < dirty_online; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_sched_poke(ssi); } - else { - if (plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - change_no_used_runqs(no); - - for (ix = online; ix < no; ix++) - resume_run_queue(ERTS_RUNQ_IX(ix)); - - for (ix = no; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); + } else { + for (ix = dirty_no; ix < dirty_online; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); } - res = ERTS_SCHDLR_SSPND_DONE; + wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); } - else /* if (no < online) */ { - if (p->scheduler_data->no <= no) { - res = ERTS_SCHDLR_SSPND_DONE; - schdlr_sspnd.wait_curr_online = no; - } - else { - /* - * Yield! Current process needs to migrate - * before bif returns. - */ - res = ERTS_SCHDLR_SSPND_YIELD_DONE; - schdlr_sspnd.wait_curr_online = no+1; - } - - if (ongoing_multi_scheduling_block()) { - for (ix = no; ix < online; ix++) - erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); - } - else { - if (plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - - change_no_used_runqs(no); - for (ix = no; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); - - for (ix = no; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq); - } - } + } + if (!dirty_only) +#endif + { + if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) { + for (ix = no; ix < online; ix++) + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); } - - if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - if (plocks && !have_unlocked_plocks) { + else { + if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - erts_thr_progress_active(esdp, 0); - erts_thr_progress_prepare_wait(esdp); - end_wait = 1; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - } - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - - ASSERT(res != ERTS_SCHDLR_SSPND_DONE - ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) - : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + change_no_used_runqs(no); + for (ix = no; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); + for (ix = no; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + wake_scheduler(rq); + } + } } } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - if (end_wait) { - erts_thr_progress_finalize_wait(esdp); - erts_thr_progress_active(esdp, 1); + if (change_flags & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + /* Suspend and wait for requested change to complete... */ + schdlr_sspnd.changer = p->common.id; + resume_proc = 0; + res = ERTS_SCHDLR_SSPND_YIELD_DONE; } + +done: + + ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU) + <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL)); + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (have_unlocked_plocks) erts_smp_proc_lock(p, plocks); + if (resume_proc) { + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + resume_process(p, plocks|ERTS_PROC_LOCK_STATUS); + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } + return res; } -#endif - ErtsSchedSuspendResult -erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) +erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal, int all) { - int ix, res, have_unlocked_plocks = 0, online; - erts_aint32_t changing; + int resume_proc, ix, res, have_unlocked_plocks = 0; ErtsProcList *plp; #ifdef ERTS_DIRTY_SCHEDULERS ErtsSchedulerSleepInfo* ssi; #endif + ErtsMultiSchedulingBlock *msbp; + erts_aint32_t chng_flg; + int have_blckd_flg; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); -#ifdef ERTS_DIRTY_SCHEDULERS - changing |= (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing) - | erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing)); -#endif - if (changing) { - res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ + if (normal) { + chng_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB; + have_blckd_flg = F_HAVE_BLCKD_NMSCHED; + msbp = &schdlr_sspnd.nmsb; } - else if (on) { /* ------ BLOCK ------ */ - if (schdlr_sspnd.msb.procs) { + else { + chng_flg = ERTS_SCHDLR_SSPND_CHNG_MSB; + have_blckd_flg = F_HAVE_BLCKD_MSCHED; + msbp = &schdlr_sspnd.msb; + } + + /* + * If we suspend current process we need to suspend before + * requesting the change; otherwise, we got a resume/suspend + * race... + */ + if (!on) { + /* We never suspend current process when unblocking... */ + resume_proc = 0; + } + else { + resume_proc = 1; + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + suspend_process(p, p); + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + if (on) { /* ------ BLOCK ------ */ + if (msbp->chngq) { + ASSERT(msbp->ongoing); + p->flags |= have_blckd_flg; + goto wait_until_msb; + } + else if (msbp->blckrs) { + ASSERT(msbp->ongoing); plp = proclist_create(p); - erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp); - p->flags |= F_HAVE_BLCKD_MSCHED; - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) == 0); - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) == 0); -#endif + erts_proclist_store_last(&msbp->blckrs, plp); + p->flags |= have_blckd_flg; + ASSERT(schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)); ASSERT(p->scheduler_data->no == 1); - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - } else { - int online = schdlr_sspnd.online; - p->flags |= F_HAVE_BLCKD_MSCHED; + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED; + } + else { + int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL); + ASSERT(!msbp->ongoing); + p->flags |= have_blckd_flg; if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - ASSERT(!ongoing_multi_scheduling_block()); - schdlr_sspnd.msb.ongoing = 1; - if (online == 1) { - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) == 1); - ASSERT(!(erts_smp_atomic32_read_nob(&ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(0)->flags) - & ERTS_SSI_FLG_SUSPENDED)); - schdlr_sspnd.msb.dirty_cpu_wait_active = 0; - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(0); - erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) - != schdlr_sspnd.msb.dirty_cpu_wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - - schdlr_sspnd.msb.dirty_io_wait_active = 0; - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) - != schdlr_sspnd.msb.dirty_io_wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); -#endif + ASSERT(!msbp->ongoing); + msbp->ongoing = 1; + if (schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0) + || (normal && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) == 1)) { ASSERT(p->scheduler_data->no == 1); + plp = proclist_create(p); + erts_proclist_store_last(&msbp->blckrs, plp); + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED; } else { - ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - if (p->scheduler_data->no == 1) { - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - schdlr_sspnd.msb.wait_active = 1; - } - else { - /* - * Yield! Current process needs to migrate - * before bif returns. - */ - res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; - schdlr_sspnd.msb.wait_active = 2; - } - -#ifdef ERTS_DIRTY_SCHEDULERS - schdlr_sspnd.msb.dirty_cpu_wait_active = 0; - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) - != schdlr_sspnd.msb.dirty_cpu_wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - ASSERT(schdlr_sspnd.dirty_cpu_curr_online == schdlr_sspnd.dirty_cpu_online); - - schdlr_sspnd.msb.dirty_io_wait_active = 0; - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) - != schdlr_sspnd.msb.dirty_io_wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - ASSERT(schdlr_sspnd.dirty_io_curr_online == schdlr_sspnd.dirty_io_online); -#endif + erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, + chng_flg); change_no_used_runqs(1); for (ix = 1; ix < erts_no_run_queues; ix++) suspend_run_queue(ERTS_RUNQ_IX(ix)); @@ -7397,84 +7798,84 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) wake_scheduler(rq); } - if (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) - != schdlr_sspnd.msb.wait_active) { - ErtsSchedulerData *esdp; - - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - - if (plocks && !have_unlocked_plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); +#ifdef ERTS_DIRTY_SCHEDULERS + if (!normal) { + for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); } + wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - esdp = ERTS_PROC_GET_SCHDATA(p); - - erts_thr_progress_active(esdp, 0); - erts_thr_progress_prepare_wait(esdp); - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) - != schdlr_sspnd.msb.wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, - &schdlr_sspnd.mtx); - - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - - erts_thr_progress_active(esdp, 1); - erts_thr_progress_finalize_wait(esdp); + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { + ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + } + wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); + } +#endif - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + wait_until_msb: - } + ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)); - ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED - ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) - : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + plp = proclist_create(p); + erts_proclist_store_last(&msbp->chngq, plp); + resume_proc = 0; + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED; } - plp = proclist_create(p); - erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp); ASSERT(p->scheduler_data); } } - else if (!ongoing_multi_scheduling_block()) { - /* unblock not ongoing */ - ASSERT(!schdlr_sspnd.msb.procs); - res = ERTS_SCHDLR_SSPND_DONE; + else if (!msbp->ongoing) { + ASSERT(!msbp->blckrs); + goto unblock_res; } else { /* ------ UNBLOCK ------ */ - if (p->flags & F_HAVE_BLCKD_MSCHED) { - ErtsProcList *plp = erts_proclist_peek_first(schdlr_sspnd.msb.procs); - - while (plp) { - ErtsProcList *tmp_plp = plp; - plp = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp); - if (erts_proclist_same(tmp_plp, p)) { - erts_proclist_remove(&schdlr_sspnd.msb.procs, tmp_plp); - proclist_destroy(tmp_plp); - if (!all) - break; + if (p->flags & have_blckd_flg) { + ErtsProcList *plps[2]; + ErtsProcList *plp; + int limit = 0; + + plps[limit++] = erts_proclist_peek_first(msbp->blckrs); + if (all) + plps[limit++] = erts_proclist_peek_first(msbp->chngq); + + for (ix = 0; ix < limit; ix++) { + plp = plps[ix]; + while (plp) { + ErtsProcList *tmp_plp = plp; + plp = erts_proclist_peek_next(msbp->blckrs, plp); + if (erts_proclist_same(tmp_plp, p)) { + erts_proclist_remove(&msbp->blckrs, tmp_plp); + proclist_destroy(tmp_plp); + if (!all) + break; + } } } } - if (schdlr_sspnd.msb.procs) - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - else { - ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); - p->flags &= ~F_HAVE_BLCKD_MSCHED; - schdlr_sspnd.msb.ongoing = 0; - if (schdlr_sspnd.online == 1) { + if (!msbp->blckrs && !msbp->chngq) { + int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL); + erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, + chng_flg); + p->flags &= ~have_blckd_flg; + msbp->ongoing = 0; + if (online == 1) { /* No normal schedulers to resume */ - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); - ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB); + ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) == 1); +#ifndef ERTS_DIRTY_SCHEDULERS + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~chng_flg); +#endif } - else { - online = schdlr_sspnd.online; + else if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) { if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); @@ -7490,83 +7891,91 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) suspend_run_queue(ERTS_RUNQ_IX(ix)); } #ifdef ERTS_DIRTY_SCHEDULERS - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); - schdlr_sspnd.msb.dirty_cpu_wait_active = schdlr_sspnd.dirty_cpu_online; - for (ix = 0; ix < schdlr_sspnd.dirty_cpu_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - erts_smp_atomic32_read_band_nob(&ssi->flags, - ~ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); - schdlr_sspnd.msb.dirty_io_wait_active = erts_no_dirty_io_schedulers; - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - erts_smp_atomic32_read_band_nob(&ssi->flags, - ~ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); + if (!normal) { + ASSERT(!schdlr_sspnd.msb.ongoing); + online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU); + for (ix = 0; ix < online; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + scheduler_ssi_resume_wake(ssi); + } + + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { + ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); + scheduler_ssi_resume_wake(ssi); + } + } #endif - res = ERTS_SCHDLR_SSPND_DONE; } + + unblock_res: + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; + else if (schdlr_sspnd.nmsb.ongoing) + res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_DONE; } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (have_unlocked_plocks) erts_smp_proc_lock(p, plocks); - return res; -} -#ifdef DEBUG -void -erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value) -{ - if (return_value == am_blocked) { - erts_aint32_t active = erts_smp_atomic32_read_nob(&schdlr_sspnd.active); - ASSERT(1 <= active && active <= 2); - ASSERT(ERTS_PROC_GET_SCHDATA(p)->no == 1); + if (resume_proc) { + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + resume_process(p, plocks|ERTS_PROC_LOCK_STATUS); + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } + + return res; } -#endif int erts_is_multi_scheduling_blocked(void) { int res; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - res = schdlr_sspnd.msb.procs != NULL; + if (schdlr_sspnd.msb.blckrs) + res = 1; + else if (schdlr_sspnd.nmsb.blckrs) + res = -1; + else + res = 0; erts_smp_mtx_unlock(&schdlr_sspnd.mtx); return res; } Eterm -erts_multi_scheduling_blockers(Process *p) +erts_multi_scheduling_blockers(Process *p, int normal) { Eterm res = NIL; + ErtsMultiSchedulingBlock *msbp; + + msbp = normal ? &schdlr_sspnd.nmsb : &schdlr_sspnd.msb; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - if (!erts_proclist_is_empty(schdlr_sspnd.msb.procs)) { + if (!erts_proclist_is_empty(msbp->blckrs)) { Eterm *hp, *hp_end; ErtsProcList *plp1, *plp2; Uint max_size = 0; - for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); + for (plp1 = erts_proclist_peek_first(msbp->blckrs); plp1; - plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) { + plp1 = erts_proclist_peek_next(msbp->blckrs, plp1)) { max_size += 2; } ASSERT(max_size); hp = HAlloc(p, max_size); hp_end = hp + max_size; - for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); + for (plp1 = erts_proclist_peek_first(msbp->blckrs); plp1; - plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) { - for (plp2 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); + plp1 = erts_proclist_peek_next(msbp->blckrs, plp1)) { + for (plp2 = erts_proclist_peek_first(msbp->blckrs); plp2->pid != plp1->pid; - plp2 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp2)); + plp2 = erts_proclist_peek_next(msbp->blckrs, plp2)); if (plp2 == plp1) { res = CONS(hp, plp1->pid, res); hp += 2; @@ -7586,13 +7995,26 @@ sched_thread_func(void *vesdp) ErtsSchedulerData *esdp = vesdp; Uint no = esdp->no; #ifdef ERTS_SMP - ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); + erts_tse_t *tse; +#endif + + erts_sched_init_time_sup(esdp); + + (void) ERTS_RUNQ_FLGS_SET_NOB(esdp->run_queue, + ERTS_RUNQ_FLG_EXEC); + +#ifdef ERTS_SMP + tse = erts_tse_fetch(); + erts_tse_prepare_timed(tse); + ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = tse; callbacks.arg = (void *) esdp->ssi; callbacks.wakeup = thr_prgr_wakeup; callbacks.prepare_wait = thr_prgr_prep_wait; callbacks.wait = thr_prgr_wait; callbacks.finalize_wait = thr_prgr_fin_wait; + erts_msacc_init_thread("scheduler", no, 1); + erts_thr_progress_register_managed_thread(esdp, &callbacks, 0); erts_alloc_register_scheduler(vesdp); #endif @@ -7622,39 +8044,6 @@ sched_thread_func(void *vesdp) #endif erts_thread_init_float(); - if (no == 1) { - erts_thr_progress_active(esdp, 0); - erts_thr_progress_prepare_wait(esdp); - } - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.changing) - & ERTS_SCHDLR_SSPND_CHNG_ONLN); - - if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) { - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - if (no != 1) -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); -#else - erts_smp_cnd_signal(&schdlr_sspnd.cnd); -#endif - } - - if (no == 1) { - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - - if (no == 1) { - erts_thr_progress_finalize_wait(esdp); - erts_thr_progress_active(esdp, 1); - } - #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC esdp->verify_unused_temp_alloc = erts_alloc_get_verify_unused_temp_alloc( @@ -7664,7 +8053,7 @@ sched_thread_func(void *vesdp) process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Scheduler thread number %beu terminated\n", no); return NULL; @@ -7687,6 +8076,8 @@ sched_dirty_cpu_thread_func(void *vesdp) callbacks.wait = NULL; callbacks.finalize_wait = NULL; + esdp->thr_id += erts_no_schedulers; + erts_thr_progress_register_unmanaged_thread(&callbacks); #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -7707,27 +8098,9 @@ sched_dirty_cpu_thread_func(void *vesdp) #endif erts_thread_init_float(); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing) - & ERTS_SCHDLR_SSPND_CHNG_ONLN); - - if (--schdlr_sspnd.dirty_cpu_curr_online == schdlr_sspnd.dirty_cpu_wait_curr_online) { - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_cpu_changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - if (no != 1) - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - } - - if (no == 1) { - while (schdlr_sspnd.dirty_cpu_curr_online != schdlr_sspnd.dirty_cpu_wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Dirty CPU scheduler thread number %beu terminated\n", no); return NULL; @@ -7748,6 +8121,8 @@ sched_dirty_io_thread_func(void *vesdp) callbacks.wait = NULL; callbacks.finalize_wait = NULL; + esdp->thr_id += erts_no_schedulers + erts_no_dirty_cpu_schedulers; + erts_thr_progress_register_unmanaged_thread(&callbacks); #ifdef ERTS_ENABLE_LOCK_CHECK { @@ -7768,27 +8143,9 @@ sched_dirty_io_thread_func(void *vesdp) #endif erts_thread_init_float(); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing) - & ERTS_SCHDLR_SSPND_CHNG_ONLN); - - if (--schdlr_sspnd.dirty_io_curr_online == schdlr_sspnd.dirty_io_wait_curr_online) { - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_io_changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - if (no != 1) - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - } - - if (no == 1) { - while (schdlr_sspnd.dirty_io_curr_online != schdlr_sspnd.dirty_io_wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Dirty I/O scheduler thread number %beu terminated\n", no); return NULL; @@ -7805,28 +8162,25 @@ erts_start_schedulers(void) Uint actual; Uint wanted = erts_no_schedulers; Uint wanted_no_schedulers = erts_no_schedulers; + char name[16]; ethr_thr_opts opts = ETHR_THR_OPTS_DEFAULT_INITER; opts.detached = 1; -#ifdef ETHR_HAVE_THREAD_NAMES - opts.name = malloc(80); -#endif + opts.name = name; #ifdef ERTS_SMP if (erts_runq_supervision_interval) { opts.suggested_stack_size = 16; -#ifdef ETHR_HAVE_THREAD_NAMES - sprintf(opts.name, "runq_supervisor"); -#endif + erts_snprintf(opts.name, 16, "runq_supervisor"); erts_atomic_init_nob(&runq_supervisor_sleeping, 0); if (0 != ethr_event_init(&runq_supervision_event)) - erl_exit(1, "Failed to create run-queue supervision event\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision event\n"); if (0 != ethr_thr_create(&runq_supervisor_tid, runq_supervisor, NULL, &opts)) - erl_exit(1, "Failed to create run-queue supervision thread\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread\n"); } #endif @@ -7845,14 +8199,7 @@ erts_start_schedulers(void) ASSERT(actual == esdp->no - 1); -#ifdef ETHR_HAVE_THREAD_NAMES - sprintf(opts.name, "scheduler_%d", actual + 1); -#endif - -#ifdef __OSE__ - /* This should be done in the bind strategy */ - opts.coreNo = (actual+1) % ose_num_cpus(); -#endif + erts_snprintf(opts.name, 16, "%lu_scheduler", actual + 1); res = ethr_thr_create(&esdp->tid, sched_thread_func, (void*)esdp, &opts); @@ -7860,7 +8207,6 @@ erts_start_schedulers(void) break; } } - erts_no_schedulers = actual; #ifdef ERTS_DIRTY_SCHEDULERS @@ -7869,21 +8215,17 @@ erts_start_schedulers(void) int ix; for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); -#ifdef ETHR_HAVE_THREAD_NAMES - sprintf(opts.name,"dirty_cpu_scheduler_%d", ix + 1); -#endif + erts_snprintf(opts.name, 16, "%d_dirty_cpu_scheduler", ix + 1); res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts); if (res != 0) - erl_exit(1, "Failed to create dirty cpu scheduler thread %d\n", ix); + erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d\n", ix); } for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); -#ifdef ETHR_HAVE_THREAD_NAMES - sprintf(opts.name,"dirty_io_scheduler_%d", ix + 1); -#endif + erts_snprintf(opts.name, 16, "%d_dirty_io_scheduler", ix + 1); res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts); if (res != 0) - erl_exit(1, "Failed to create dirty io scheduler thread %d\n", ix); + erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix); } } #endif @@ -7891,20 +8233,14 @@ erts_start_schedulers(void) ERTS_THR_MEMORY_BARRIER; -#ifdef ETHR_HAVE_THREAD_NAMES - sprintf(opts.name, "aux"); -#endif - -#ifdef __OSE__ - opts.coreNo = 0; -#endif /* __OSE__ */ + erts_snprintf(opts.name, 16, "aux"); res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts); if (res != 0) - erl_exit(1, "Failed to create aux thread\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread\n"); if (actual < 1) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Failed to create any scheduler-threads: %s (%d)\n", erl_errno_id(res), res); @@ -7918,10 +8254,6 @@ erts_start_schedulers(void) actual, actual == 1 ? " was" : "s were"); erts_send_error_to_logger_nogl(dsbufp); } - -#ifdef ETHR_HAVE_THREAD_NAMES - free(opts.name); -#endif } #endif /* ERTS_SMP */ @@ -7940,7 +8272,7 @@ add_pend_suspend(Process *suspendee, sizeof(ErtsPendingSuspend)); psp->next = NULL; #ifdef DEBUG -#if defined(ARCH_64) && !HALFWORD_HEAP +#if defined(ARCH_64) psp->end = (ErtsPendingSuspend *) 0xdeaddeaddeaddead; #else psp->end = (ErtsPendingSuspend *) 0xdeaddead; @@ -8023,7 +8355,8 @@ handle_pend_sync_suspend(Process *suspendee, } /* suspender is suspended waiting for suspendee to suspend; resume suspender */ - resume_process(suspender); + ASSERT(suspendee != suspender); + resume_process(suspender, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_STATUS); } } @@ -8058,12 +8391,11 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, ASSERT(c_p->flags & F_P2PNR_RESCHED); c_p->flags &= ~F_P2PNR_RESCHED; if (!suspend && rp) - resume_process(rp); + resume_process(rp, rp_locks); } else { - rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, - pid, pid_locks|ERTS_PROC_LOCK_STATUS); + pid, ERTS_PROC_LOCK_STATUS); if (!rp) { c_p->flags &= ~F_P2PNR_RESCHED; @@ -8072,40 +8404,84 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, ASSERT(!(c_p->flags & F_P2PNR_RESCHED)); - if (suspend) { - if (suspend_process(c_p, rp)) - goto done; - } - else { - if (!((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS) - & erts_smp_atomic32_read_acqb(&rp->state))) - goto done; + /* + * Suspend the other process in order to prevent + * it from being selected for normal execution. + * This will however not prevent it from being + * selected for execution of a system task. If + * it is selected for execution of a system task + * we might be blocked for quite a while if the + * try-lock below fails. That is, there is room + * for improvement here... + */ - } + if (!suspend_process(c_p, rp)) { + /* Other process running */ - /* Other process running */ + ASSERT(ERTS_PSFLG_RUNNING + & erts_smp_atomic32_read_nob(&rp->state)); - /* - * If we got pending suspenders and suspend ourselves waiting - * to suspend another process we might deadlock. - * In this case we have to yield, be suspended by - * someone else and then do it all over again. - */ - if (!c_p->pending_suspenders) { - /* Mark rp pending for suspend by c_p */ - add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend); - ASSERT(is_nil(c_p->suspendee)); + running: - /* Suspend c_p; when rp is suspended c_p will be resumed. */ - suspend_process(c_p, c_p); - c_p->flags |= F_P2PNR_RESCHED; + /* + * If we got pending suspenders and suspend ourselves waiting + * to suspend another process we might deadlock. + * In this case we have to yield, be suspended by + * someone else and then do it all over again. + */ + if (!c_p->pending_suspenders) { + /* Mark rp pending for suspend by c_p */ + add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend); + ASSERT(is_nil(c_p->suspendee)); + + /* Suspend c_p; when rp is suspended c_p will be resumed. */ + suspend_process(c_p, c_p); + c_p->flags |= F_P2PNR_RESCHED; + } + /* Yield (caller is assumed to yield immediately in bif). */ + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + rp = ERTS_PROC_LOCK_BUSY; + } + else { + ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS; + if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { + if (ERTS_PSFLG_RUNNING_SYS + & erts_smp_atomic32_read_nob(&rp->state)) { + /* Executing system task... */ + resume_process(rp, ERTS_PROC_LOCK_STATUS); + goto running; + } + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + /* + * If we are unlucky, the process just got selected for + * execution of a system task. In this case we may be + * blocked here for quite a while... Execution of system + * tasks are fortunately quite rare events. We try to + * avoid this by checking if it is in a state executing + * system tasks (above), but it will not prevent all + * scenarios for a long block here... + */ + rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, + pid, pid_locks|ERTS_PROC_LOCK_STATUS); + if (!rp) + goto done; + } + + /* + * The previous suspend has prevented the process + * from being selected for normal execution regardless + * of locks held or not held on it... + */ + ASSERT(!(ERTS_PSFLG_RUNNING + & erts_smp_atomic32_read_nob(&rp->state))); + + if (!suspend) + resume_process(rp, pid_locks|ERTS_PROC_LOCK_STATUS); } - /* Yield (caller is assumed to yield immediately in bif). */ - erts_smp_proc_unlock(rp, pid_locks|ERTS_PROC_LOCK_STATUS); - rp = ERTS_PROC_LOCK_BUSY; } done: + if (rp && rp != ERTS_PROC_LOCK_BUSY && !(pid_locks & ERTS_PROC_LOCK_STATUS)) erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); if (unlock_c_p_status) @@ -8216,7 +8592,8 @@ handle_pend_bif_sync_suspend(Process *suspendee, } /* suspender is suspended waiting for suspendee to suspend; resume suspender */ - resume_process(suspender); + ASSERT(suspender != suspendee); + resume_process(suspender, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(suspender, ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCK_STATUS); } @@ -8576,7 +8953,8 @@ resume_process_1(BIF_ALIST_1) ASSERT(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&suspendee->state)); - resume_process(suspendee); + ASSERT(BIF_P != suspendee); + resume_process(suspendee, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); } @@ -8598,24 +8976,39 @@ resume_process_1(BIF_ALIST_1) } Uint -erts_run_queues_len(Uint *qlen) +erts_run_queues_len(Uint *qlen, int atomic_queues_read, int incl_active_sched) { int i = 0; Uint len = 0; - ERTS_ATOMIC_FOREACH_RUNQ(rq, - { - Sint pqlen = 0; - int pix; - for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) - pqlen += RUNQ_READ_LEN(&rq->procs.prio_info[pix].len); + if (atomic_queues_read) + ERTS_ATOMIC_FOREACH_RUNQ(rq, + { + Sint rq_len = (Sint) erts_smp_atomic32_read_dirty(&rq->len); + ASSERT(rq_len >= 0); + if (incl_active_sched + && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) { + rq_len++; + } + if (qlen) + qlen[i++] = rq_len; + len += (Uint) rq_len; + } + ); + else { + for (i = 0; i < erts_no_run_queues; i++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(i); + Sint rq_len = (Sint) erts_smp_atomic32_read_nob(&rq->len); + ASSERT(rq_len >= 0); + if (incl_active_sched + && (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_EXEC)) { + rq_len++; + } + if (qlen) + qlen[i] = rq_len; + len += (Uint) rq_len; + } - if (pqlen < 0) - pqlen = 0; - if (qlen) - qlen[i++] = pqlen; - len += pqlen; } - ); return len; } @@ -8706,7 +9099,7 @@ erts_resume(Process* process, ErtsProcLocks process_locks) ERTS_SMP_LC_ASSERT(process_locks == erts_proc_lc_my_proc_locks(process)); if (!(process_locks & ERTS_PROC_LOCK_STATUS)) erts_smp_proc_lock(process, ERTS_PROC_LOCK_STATUS); - resume_process(process); + resume_process(process, process_locks|ERTS_PROC_LOCK_STATUS); if (!(process_locks & ERTS_PROC_LOCK_STATUS)) erts_smp_proc_unlock(process, ERTS_PROC_LOCK_STATUS); } @@ -8725,7 +9118,7 @@ erts_resume_processes(ErtsProcList *list) proc = erts_pid2proc(NULL, 0, plp->pid, ERTS_PROC_LOCK_STATUS); if (proc) { if (erts_proclist_same(plp, proc)) { - resume_process(proc); + resume_process(proc, ERTS_PROC_LOCK_STATUS); nresumed++; } erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_STATUS); @@ -8828,6 +9221,10 @@ erts_set_process_priority(Process *p, Eterm value) a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); } while (a != e); + + if (slocked) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } switch (oprio) { @@ -8861,7 +9258,6 @@ Process *schedule(Process *p, int calls) { Process *proxy_p = NULL; ErtsRunQueue *rq; - erts_aint_t dt; ErtsSchedulerData *esdp; int context_reds; int fcalls; @@ -8871,6 +9267,8 @@ Process *schedule(Process *p, int calls) Uint32 flags; erts_aint32_t state = 0; /* Supress warning... */ + ERTS_MSACC_DECLARE_CACHE(); + #ifdef USE_VM_PROBES if (p != NULL && DTRACE_ENABLED(process_unscheduled)) { DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE); @@ -8961,7 +9359,7 @@ Process *schedule(Process *p, int calls) schedule_out_process(rq, state, p, proxy_p); /* Returns with rq locked! */ proxy_p = NULL; - ERTS_PROC_REDUCTIONS_EXECUTED(rq, + ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq, (int) ERTS_PSFLGS_GET_USR_PRIO(state), reds, actual_reds); @@ -8973,17 +9371,16 @@ Process *schedule(Process *p, int calls) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); + if (state & ERTS_PSFLG_FREE) { #ifdef ERTS_SMP ASSERT(esdp->free_process == p); esdp->free_process = NULL; #else - state = erts_smp_atomic32_read_nob(&p->state); - if (!(state & ERTS_PSFLG_IN_RUNQ)) - erts_free_proc(p); + erts_proc_dec_refc(p); #endif } - #ifdef ERTS_SMP ASSERT(!esdp->free_process); #endif @@ -8991,11 +9388,15 @@ Process *schedule(Process *p, int calls) ERTS_SMP_CHK_NO_PROC_LOCKS; - dt = erts_do_time_read_and_reset(); - if (dt) { - erts_smp_runq_unlock(rq); - erts_bump_timer(dt); - erts_smp_runq_lock(rq); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS) + (void) erts_get_monotonic_time(esdp); + + if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + erts_smp_runq_unlock(rq); + erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time); + erts_smp_runq_lock(rq); + } } BM_STOP_TIMER(system); @@ -9008,15 +9409,16 @@ Process *schedule(Process *p, int calls) #ifdef ERTS_SMP ErtsMigrationPaths *mps; ErtsMigrationPath *mp; - ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; - if (erts_proclist_fetch(&pnd_xtrs, NULL)) { - rq->procs.pending_exiters = NULL; - erts_smp_runq_unlock(rq); - handle_pending_exiters(pnd_xtrs); - erts_smp_runq_lock(rq); - } if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; + if (erts_proclist_fetch(&pnd_xtrs, NULL)) { + rq->procs.pending_exiters = NULL; + erts_smp_runq_unlock(rq); + handle_pending_exiters(pnd_xtrs); + erts_smp_runq_lock(rq); + } + if (rq->check_balance_reds <= 0) check_balance(rq); @@ -9038,8 +9440,10 @@ Process *schedule(Process *p, int calls) if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) { if (flags & ERTS_RUNQ_FLG_SUSPENDED) { + (void) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_EXEC); suspend_scheduler(esdp); - flags = ERTS_RUNQ_FLGS_GET_NOB(rq); + flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); + flags |= ERTS_RUNQ_FLG_EXEC; } if (flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) { flags = ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_CHK_CPU_BIND); @@ -9054,24 +9458,20 @@ Process *schedule(Process *p, int calls) suspend_scheduler(esdp); #endif - { + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { erts_aint32_t aux_work; - int leader_update = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 - : erts_thr_progress_update(esdp); + int leader_update = erts_thr_progress_update(esdp); aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); - if (aux_work | leader_update | ERTS_SCHED_FAIR) { + if (aux_work | leader_update) { erts_smp_runq_unlock(rq); if (leader_update) erts_thr_progress_leader_update(esdp); - else if (ERTS_SCHED_FAIR) - ERTS_SCHED_FAIR_YIELD(); if (aux_work) handle_aux_work(&esdp->aux_work_data, aux_work, 0); erts_smp_runq_lock(rq); } - ERTS_SMP_LC_ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp) - || !erts_thr_progress_is_blocking()); + ERTS_SMP_LC_ASSERT(!erts_thr_progress_is_blocking()); } ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -9130,8 +9530,11 @@ Process *schedule(Process *p, int calls) } #endif + (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_EXEC); scheduler_wait(&fcalls, esdp, rq); - + flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC); + flags |= ERTS_RUNQ_FLG_EXEC; + ERTS_MSACC_UPDATE_CACHE(); #ifdef ERTS_SMP non_empty_runq(rq); #endif @@ -9140,23 +9543,28 @@ Process *schedule(Process *p, int calls) } else if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && (fcalls > input_reductions && - prepare_for_sys_schedule(esdp))) { + prepare_for_sys_schedule(!0))) { + ErtsMonotonicTime current_time; /* * Schedule system-level activities. */ + ERTS_MSACC_PUSH_STATE_CACHED_M(); + erts_smp_atomic32_set_relb(&function_calls, 0); fcalls = 0; - ASSERT(!erts_port_task_have_outstanding_io_tasks()); - #if 0 /* Not needed since we wont wait in sys schedule */ erts_sys_schedule_interrupt(0); #endif erts_smp_runq_unlock(rq); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_CHECK_IO); erl_sys_schedule(1); - dt = erts_do_time_read_and_reset(); - if (dt) erts_bump_timer(dt); + ERTS_MSACC_POP_STATE_M(); + + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) + erts_bump_timers(esdp->timer_wheel, current_time); #ifdef ERTS_SMP erts_smp_runq_lock(rq); @@ -9181,7 +9589,9 @@ Process *schedule(Process *p, int calls) if (RUNQ_READ_LEN(&rq->ports.info.len)) { int have_outstanding_io; have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port); - if ((have_outstanding_io && fcalls > 2*input_reductions) + if ((!erts_eager_check_io + && have_outstanding_io + && fcalls > 2*input_reductions) || rq->halt_in_progress) { /* * If we have performed more than 2*INPUT_REDUCTIONS since @@ -9208,11 +9618,12 @@ Process *schedule(Process *p, int calls) pick_next_process: { erts_aint32_t psflg_band_mask; int prio_q; - int qmask; + int qmask, qbit; flags = ERTS_RUNQ_FLGS_GET_NOB(rq); qmask = (int) (flags & ERTS_RUNQ_FLGS_PROCS_QMASK); - switch (qmask & -qmask) { + qbit = qmask & -qmask; + switch (qbit) { case MAX_BIT: prio_q = PRIORITY_MAX; break; @@ -9228,9 +9639,12 @@ Process *schedule(Process *p, int calls) case 0: /* No process at all */ default: ASSERT(qmask == 0); + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER); goto check_activities_to_run; } + ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR); + BM_START_TIMER(system); /* @@ -9240,20 +9654,11 @@ Process *schedule(Process *p, int calls) ASSERT(p); /* Wrong qmask in rq->flags? */ - psflg_band_mask = ~(((erts_aint32_t) 1) << (ERTS_PSFLGS_GET_PRQ_PRIO(state) - + ERTS_PSFLGS_IN_PRQ_MASK_OFFSET)); - -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT((state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) != - (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)); - if (state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) { - ASSERT((ERTS_SCHEDULER_IS_DIRTY_CPU(esdp) && (state & ERTS_PSFLG_DIRTY_CPU_PROC)) || - (ERTS_SCHEDULER_IS_DIRTY_IO(esdp) && (state & ERTS_PSFLG_DIRTY_IO_PROC))); - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !(state & ERTS_PSFLG_ACTIVE_SYS)) - goto pick_next_process; - state &= ~(ERTS_PSFLG_DIRTY_CPU_PROC_IN_Q|ERTS_PSFLG_DIRTY_IO_PROC_IN_Q); - } -#endif + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) + psflg_band_mask = ~((erts_aint32_t) 0); + else + psflg_band_mask = ~(((erts_aint32_t) 1) << (ERTS_PSFLGS_GET_PRQ_PRIO(state) + + ERTS_PSFLGS_IN_PRQ_MASK_OFFSET)); if (!(state & ERTS_PSFLG_PROXY)) psflg_band_mask &= ~ERTS_PSFLG_IN_RUNQ; @@ -9276,9 +9681,11 @@ Process *schedule(Process *p, int calls) | ERTS_PSFLG_RUNNING_SYS))) { tmp = state & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_PENDING_EXIT - | ERTS_PSFLG_ACTIVE_SYS); + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS); if (tmp != ERTS_PSFLG_SUSPENDED) { - if (state & ERTS_PSFLG_ACTIVE_SYS) + if (state & (ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) new |= ERTS_PSFLG_RUNNING_SYS; else new |= ERTS_PSFLG_RUNNING; @@ -9291,7 +9698,8 @@ Process *schedule(Process *p, int calls) | ERTS_PSFLG_FREE)) || ((state & (ERTS_PSFLG_SUSPENDED | ERTS_PSFLG_PENDING_EXIT - | ERTS_PSFLG_ACTIVE_SYS)) + | ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) == ERTS_PSFLG_SUSPENDED)) { if (proxy_p) { free_proxy_proc(proxy_p); @@ -9299,11 +9707,7 @@ Process *schedule(Process *p, int calls) } else if (state & ERTS_PSFLG_FREE) { /* free and not queued by proxy */ -#ifdef ERTS_SMP - erts_smp_proc_dec_refc(p); -#else - erts_free_proc(p); -#endif + erts_proc_dec_refc(p); } goto pick_next_process; } @@ -9316,10 +9720,42 @@ Process *schedule(Process *p, int calls) esdp->current_process = p; + + reds = context_reds; + +#ifdef ERTS_SMP + + erts_smp_runq_unlock(rq); + +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { +#ifdef DEBUG + int old_dqbit; +#endif + int dqbit = qbit; + + if (rq == ERTS_DIRTY_CPU_RUNQ) + dqbit <<= ERTS_PDSFLGS_IN_CPU_PRQ_MASK_OFFSET; + else { + ASSERT(rq == ERTS_DIRTY_IO_RUNQ); + dqbit <<= ERTS_PDSFLGS_IN_IO_PRQ_MASK_OFFSET; + } + +#ifdef DEBUG + old_dqbit = (int) +#else + (void) +#endif + erts_smp_atomic32_read_band_mb(&p->dirty_state, ~dqbit); + ASSERT(old_dqbit & dqbit); + } +#endif /* ERTS_DIRTY_SCHEDULERS */ + +#endif /* ERTS_SMP */ + } #ifdef ERTS_SMP - erts_smp_runq_unlock(rq); if (flags & ERTS_RUNQ_FLG_PROTECTED) (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED); @@ -9328,12 +9764,11 @@ Process *schedule(Process *p, int calls) erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); + state = erts_smp_atomic32_read_nob(&p->state); + if (erts_sched_stat.enabled) { int prio; - UWord old = ERTS_PROC_SCHED_ID(p, - (ERTS_PROC_LOCK_MAIN - | ERTS_PROC_LOCK_STATUS), - (UWord) esdp->no); + UWord old = ERTS_PROC_SCHED_ID(p, (UWord) esdp->no); int migrated = old && old != esdp->no; #ifdef ERTS_DIRTY_SCHEDULERS @@ -9352,15 +9787,56 @@ Process *schedule(Process *p, int calls) erts_smp_spin_unlock(&erts_sched_stat.lock); } - if (ERTS_PROC_PENDING_EXIT(p)) { + ASSERT(!p->scheduler_data); + p->scheduler_data = esdp; + + state = erts_smp_atomic32_read_nob(&p->state); + +#ifdef ERTS_DIRTY_SCHEDULERS + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (!!(state & ERTS_PSFLGS_DIRTY_WORK) + & !(state & ERTS_PSFLG_ACTIVE_SYS)) { + /* Migrate to dirty scheduler... */ + sunlock_sched_out_proc: + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + p->fcalls = reds; + goto sched_out_proc; + + } + } + else { + if (state & (ERTS_PSFLG_ACTIVE_SYS + | ERTS_PSFLG_PENDING_EXIT + | ERTS_PSFLG_EXITING)) { + /* Migrate to normal scheduler... */ + goto sunlock_sched_out_proc; + } + if ((state & ERTS_PSFLG_DIRTY_ACTIVE_SYS) + && rq == ERTS_DIRTY_IO_RUNQ) { + /* Migrate to dirty cpu scheduler... */ + goto sunlock_sched_out_proc; + } + + ASSERT((state & ERTS_PSFLG_DIRTY_ACTIVE_SYS) + || *p->i == (BeamInstr) em_call_nif); + + ASSERT(rq == ERTS_DIRTY_CPU_RUNQ + ? (state & (ERTS_PSFLG_DIRTY_CPU_PROC + | ERTS_PSFLG_DIRTY_ACTIVE_SYS)) + : (rq == ERTS_DIRTY_IO_RUNQ + && (state & ERTS_PSFLG_DIRTY_IO_PROC))); + } +#endif + + if (state & ERTS_PSFLG_PENDING_EXIT) { erts_handle_pending_exit(p, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS); state = erts_smp_atomic32_read_nob(&p->state); } - ASSERT(!p->scheduler_data); - p->scheduler_data = esdp; -#endif - reds = context_reds; + +#endif /* ERTS_SMP */ + + p->fcalls = reds; if (IS_TRACED(p)) { if (state & ERTS_PSFLG_EXITING) { @@ -9386,14 +9862,24 @@ Process *schedule(Process *p, int calls) #endif if (state & ERTS_PSFLG_RUNNING_SYS) { - reds -= execute_sys_tasks(p, &state, reds); - if (reds <= 0 + /* + * GC is normally never delayed when a process + * is scheduled out, but might be when executing + * hand written beam assembly in + * prim_eval:'receive'. If GC is delayed we are + * not allowed to execute system tasks. + */ + if (!(p->flags & F_DELAY_GC)) { + reds -= execute_sys_tasks(p, &state, reds); + if (reds <= 0 #ifdef ERTS_DIRTY_SCHEDULERS - || (state & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)) + || ERTS_SCHEDULER_IS_DIRTY(esdp) + || (state & ERTS_PSFLGS_DIRTY_WORK) #endif - ) { - p->fcalls = reds; - goto sched_out_proc; + ) { + p->fcalls = reds; + goto sched_out_proc; + } } ASSERT(state & ERTS_PSFLG_RUNNING_SYS); @@ -9422,13 +9908,13 @@ Process *schedule(Process *p, int calls) } } - if (!(state & ERTS_PSFLG_EXITING) - && ((FLAGS(p) & F_FORCE_GC) - || (MSO(p).overhead > BIN_VHEAP_SZ(p)))) { - reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity); - if (reds <= 0) { - p->fcalls = reds; - goto sched_out_proc; + if (ERTS_IS_GC_DESIRED(p)) { + if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) { + reds -= erts_garbage_collect_nobump(p, 0, p->arg_reg, p->arity); + if (reds <= 0) { + p->fcalls = reds; + goto sched_out_proc; + } } } @@ -9437,12 +9923,25 @@ Process *schedule(Process *p, int calls) proxy_p = NULL; } - p->fcalls = reds; ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* Never run a suspended process */ ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state))); + ASSERT(erts_proc_read_refc(p) > 0); + + if (!(state & ERTS_PSFLG_EXITING) && ERTS_PTMR_IS_TIMED_OUT(p)) { + BeamInstr** pi; +#ifdef ERTS_SMP + ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore); +#endif + pi = (BeamInstr **) p->def_arg_reg; + p->i = *pi; + p->flags &= ~F_INSLPQUEUE; + p->flags |= F_TIMO; + ERTS_PTMR_CLEAR(p); + } + return p; } } @@ -9454,7 +9953,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) if (rp) { ErtsProcLocks rp_locks; ErlOffHeap *ohp; - ErlHeapFragment* bp; + ErtsMessage *mp; Eterm *hp, msg, req_id, result; Uint st_result_sz, hsz; #ifdef DEBUG @@ -9466,11 +9965,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) st_result_sz = is_immed(st_result) ? 0 : size_object(st_result); hsz = st->req_id_sz + st_result_sz + 4 /* 3-tuple */; - hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - rp, - &rp_locks); + mp = erts_alloc_message_heap(rp, &rp_locks, hsz, &hp, &ohp); #ifdef DEBUG hp_start = hp; @@ -9495,15 +9990,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result) ASSERT(hp_start + hsz == hp); #endif - erts_queue_message(rp, - &rp_locks, - bp, - msg, - NIL -#ifdef USE_VM_PROBES - , NIL -#endif - ); + erts_queue_message(rp, &rp_locks, mp, msg, NIL); if (c_p == rp) rp_locks &= ~ERTS_PROC_LOCK_MAIN; @@ -9705,10 +10192,10 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) else { if (!garbage_collected) { FLAGS(c_p) |= F_NEED_FULLSWEEP; - reds += erts_garbage_collect(c_p, - 0, - c_p->arg_reg, - c_p->arity); + reds += erts_garbage_collect_nobump(c_p, + 0, + c_p->arg_reg, + c_p->arity); garbage_collected = 1; } st_res = am_true; @@ -9717,7 +10204,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) case ERTS_PSTT_CPC: st_res = erts_check_process_code(c_p, st->arg[0], - st->arg[1] == am_true, + unsigned_val(st->arg[1]), &reds); if (is_non_value(st_res)) { /* Needed gc, but gc was disabled */ @@ -9725,6 +10212,10 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) st = NULL; } break; + case ERTS_PSTT_COHMQ: + reds += erts_complete_off_heap_message_queue_change(c_p); + st_res = am_true; + break; default: ERTS_INTERNAL_ERROR("Invalid process sys task type"); st_res = am_false; @@ -9767,6 +10258,9 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) case ERTS_PSTT_CPC: st_res = am_false; break; + case ERTS_PSTT_COHMQ: + st_res = am_false; + break; default: ERTS_INTERNAL_ERROR("Invalid process sys task type"); st_res = am_false; @@ -9785,10 +10279,8 @@ BIF_RETTYPE erts_internal_request_system_task_3(BIF_ALIST_3) { Process *rp = erts_proc_lookup(BIF_ARG_1); - ErtsProcSysTaskQs *stqs, *free_stqs = NULL; ErtsProcSysTask *st = NULL; - erts_aint32_t prio, rp_state; - int rp_locked; + erts_aint32_t prio; Eterm noproc_res, req_type; if (!rp && !is_internal_pid(BIF_ARG_1)) { @@ -9845,7 +10337,6 @@ erts_internal_request_system_task_3(BIF_ALIST_3) } st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK, ERTS_PROC_SYS_TASK_SIZE(tot_sz)); - st->next = st->prev = st; /* Prep for empty prio queue */ ERTS_INIT_OFF_HEAP(&st->off_heap); hp = &st->heap[0]; @@ -9877,7 +10368,7 @@ erts_internal_request_system_task_3(BIF_ALIST_3) case am_check_process_code: if (is_not_atom(st->arg[0])) goto badarg; - if (st->arg[1] != am_true && st->arg[1] != am_false) + if (is_not_small(st->arg[1]) || (unsigned_val(st->arg[1]) & ~ERTS_CPC_ALL)) goto badarg; noproc_res = am_false; st->type = ERTS_PSTT_CPC; @@ -9889,93 +10380,11 @@ erts_internal_request_system_task_3(BIF_ALIST_3) goto badarg; } - rp_state = erts_smp_atomic32_read_nob(&rp->state); - - rp_locked = 0; - - free_stqs = NULL; - if (rp_state & ERTS_PSFLG_ACTIVE_SYS) - stqs = NULL; - else { - alloc_qs: - stqs = proc_sys_task_queues_alloc(); - stqs->qmask = 1 << prio; - stqs->ncount = 0; - stqs->q[PRIORITY_MAX] = NULL; - stqs->q[PRIORITY_HIGH] = NULL; - stqs->q[PRIORITY_NORMAL] = NULL; - stqs->q[PRIORITY_LOW] = NULL; - stqs->q[prio] = st; - } - - if (!rp_locked) { - rp_locked = 1; - erts_smp_proc_lock(rp, ERTS_PROC_LOCK_STATUS); - - rp_state = erts_smp_atomic32_read_nob(&rp->state); - if (rp_state & ERTS_PSFLG_EXITING) { - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - rp = NULL; - free_stqs = stqs; - goto noproc; - } - } - - if (!rp->sys_task_qs) { - if (stqs) - rp->sys_task_qs = stqs; - else - goto alloc_qs; - } - else { - if (stqs) - free_stqs = stqs; - stqs = rp->sys_task_qs; - if (!stqs->q[prio]) { - stqs->q[prio] = st; - stqs->qmask |= 1 << prio; - } - else { - st->next = stqs->q[prio]; - st->prev = stqs->q[prio]->prev; - st->next->prev = st; - st->prev->next = st; - ASSERT(stqs->qmask & (1 << prio)); - } - } - - if (ERTS_PSFLGS_GET_ACT_PRIO(rp_state) > prio) { - erts_aint32_t n, a, e; - /* Need to elevate actual prio */ - - a = rp_state; - do { - if (ERTS_PSFLGS_GET_ACT_PRIO(a) <= prio) { - n = a; - break; - } - n = e = a; - n &= ~ERTS_PSFLGS_ACT_PRIO_MASK; - n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET); - a = erts_smp_atomic32_cmpxchg_nob(&rp->state, n, e); - } while (a != e); - rp_state = n; + if (!schedule_process_sys_task(rp, prio, st)) { + noproc: + notify_sys_task_executed(BIF_P, st, noproc_res); } - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); - - schedule_process_sys_task(rp, rp_state, NULL); - - if (free_stqs) - proc_sys_task_queues_free(free_stqs); - - BIF_RET(am_ok); - -noproc: - - notify_sys_task_executed(BIF_P, st, noproc_res); - if (free_stqs) - proc_sys_task_queues_free(free_stqs); BIF_RET(am_ok); badarg: @@ -9984,11 +10393,35 @@ badarg: erts_cleanup_offheap(&st->off_heap); erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); } - if (free_stqs) - proc_sys_task_queues_free(free_stqs); BIF_ERROR(BIF_P, BADARG); } +void +erts_schedule_complete_off_heap_message_queue_change(Eterm pid) +{ + Process *rp = erts_proc_lookup(pid); + if (rp) { + ErtsProcSysTask *st; + erts_aint32_t state; + int i; + + st = erts_alloc(ERTS_ALC_T_PROC_SYS_TSK, + ERTS_PROC_SYS_TASK_SIZE(0)); + st->type = ERTS_PSTT_COHMQ; + st->requester = NIL; + st->reply_tag = NIL; + st->req_id = NIL; + st->req_id_sz = 0; + for (i = 0; i < ERTS_MAX_PROC_SYS_TASK_ARGS; i++) + st->arg[i] = NIL; + ERTS_INIT_OFF_HEAP(&st->off_heap); + state = erts_smp_atomic32_read_nob(&rp->state); + + if (!schedule_process_sys_task(rp, ERTS_PSFLGS_GET_USR_PRIO(state), st)) + erts_free(ERTS_ALC_T_PROC_SYS_TSK, st); + } +} + static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) { @@ -10008,7 +10441,7 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio) qs->q[PRIORITY_NORMAL] = NULL; qs->q[PRIORITY_LOW] = NULL; qs->q[prio] = st; - (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, ERTS_PROC_LOCK_MAIN, qs); + (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, qs); } else { if (!qs->q[prio]) { @@ -10155,7 +10588,7 @@ erts_set_gc_state(Process *c_p, int enable) erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_STATUS); - (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, ERTS_PROC_LOCK_MAIN, NULL); + (void) ERTS_PROC_SET_DELAYED_GC_TASK_QS(c_p, NULL); if (dgc_tsk_qs) proc_sys_task_queues_free(dgc_tsk_qs); @@ -10357,6 +10790,8 @@ erts_free_proc(Process *p) #ifdef ERTS_SMP erts_proc_lock_fin(p); #endif + ASSERT(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_FREE); + ASSERT(0 == erts_proc_read_refc(p)); erts_free(ERTS_ALC_T_PROC, (void *) p); } @@ -10372,6 +10807,9 @@ static void early_init_process_struct(void *varg, Eterm data) Process *proc = arg->proc; proc->common.id = make_internal_pid(data); +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_atomic32_init_nob(&proc->dirty_state, 0); +#endif erts_smp_atomic32_init_relb(&proc->state, arg->state); #ifdef ERTS_SMP @@ -10399,7 +10837,7 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state) init_arg.run_queue = rq; init_arg.state = state; - ASSERT(((char *) p) == ((char *) &p->common)); + ERTS_CT_ASSERT(offsetof(Process,common) == 0); if (!erts_ptab_new_element(&erts_proc, &p->common, @@ -10409,6 +10847,8 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state) return NULL; } + ASSERT(erts_proc_read_refc(p) > 0); + ASSERT(internal_pid_serial(p->common.id) <= ERTS_MAX_PID_SERIAL); p->approx_started = erts_get_approx_time(); @@ -10430,6 +10870,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm args, /* Arguments for function (must be well-formed list). */ ErlSpawnOpts* so) /* Options for spawn. */ { + Uint flags = 0; ErtsRunQueue *rq = NULL; Process *p; Sint arity; /* Number of arguments. */ @@ -10439,6 +10880,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm res = THE_NON_VALUE; erts_aint32_t state = 0; erts_aint32_t prio = (erts_aint32_t) PRIORITY_NORMAL; +#ifdef SHCOPY_SPAWN + erts_shcopy_t info; + INITIALIZE_SHCOPY(info); +#endif #ifdef ERTS_SMP erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); @@ -10458,6 +10903,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). int ix = so->scheduler-1; ASSERT(0 <= ix && ix < erts_no_run_queues); rq = ERTS_RUNQ_IX(ix); + /* Unsupported feature... */ state |= ERTS_PSFLG_BOUND; } prio = (erts_aint32_t) so->priority; @@ -10466,6 +10912,15 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). state |= (((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_ACT_PRIO_OFFSET) | ((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_USR_PRIO_OFFSET)); + if (so->flags & SPO_OFF_HEAP_MSGQ) { + state |= ERTS_PSFLG_OFF_HEAP_MSGQ; + flags |= F_OFF_HEAP_MSGQ; + } + else if (so->flags & SPO_ON_HEAP_MSGQ) { + state |= ERTS_PSFLG_ON_HEAP_MSGQ; + flags |= F_ON_HEAP_MSGQ; + } + if (!rq) rq = erts_get_runq_proc(parent); @@ -10484,12 +10939,19 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). BM_COUNT(processes_spawned); BM_SWAP_TIMER(system,size); +#ifdef SHCOPY_SPAWN + arg_size = copy_shared_calculate(args, &info); +#else arg_size = size_object(args); +#endif BM_SWAP_TIMER(size,system); heap_need = arg_size; - p->flags = erts_default_process_flags; + p->flags = flags; + p->static_flags = 0; + if (so->flags & SPO_SYSTEM_PROC) + p->static_flags |= ERTS_STC_FLG_SYSTEM_PROC; if (so->flags & SPO_USE_ARGS) { p->min_heap_size = so->min_heap_size; p->min_vheap_size = so->min_vheap_size; @@ -10502,9 +10964,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->schedule_count = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); - p->initial[INITIAL_MOD] = mod; - p->initial[INITIAL_FUN] = func; - p->initial[INITIAL_ARI] = (Uint) arity; + p->u.initial[INITIAL_MOD] = mod; + p->u.initial[INITIAL_FUN] = func; + p->u.initial[INITIAL_ARI] = (Uint) arity; /* * Must initialize binary lists here before copying binaries to process. @@ -10534,18 +10996,19 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->stop = p->hend = p->heap + sz; p->htop = p->heap; p->heap_sz = sz; + p->abandoned_heap = NULL; + p->live_hf_end = ERTS_INVALID_HFRAG_PTR; p->catches = 0; p->bin_vheap_sz = p->min_vheap_size; p->bin_old_vheap_sz = p->min_vheap_size; p->bin_old_vheap = 0; - p->bin_vheap_mature = 0; p->sys_task_qs = NULL; /* No need to initialize p->fcalls. */ - p->current = p->initial+INITIAL_MOD; + p->current = p->u.initial+INITIAL_MOD; p->i = (BeamInstr *) beam_apply; p->cp = (BeamInstr *) beam_apply+1; @@ -10558,7 +11021,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). BM_MESSAGE(args,p,parent); BM_START_TIMER(system); BM_SWAP_TIMER(system,copy); +#ifdef SHCOPY_SPAWN + p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap); + DESTROY_SHCOPY(info); +#else p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap); +#endif BM_MESSAGE_COPIED(arg_size); BM_SWAP_TIMER(copy,system); p->arity = 3; @@ -10568,11 +11036,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->ftrace = NIL; p->reds = 0; -#ifdef ERTS_SMP - p->common.u.alive.ptimer = NULL; -#else - sys_memset(&p->common.u.alive.tm, 0, sizeof(ErlTimer)); -#endif + ERTS_PTMR_INIT(p); p->common.u.alive.reg = NULL; ERTS_P_LINKS(p) = NULL; @@ -10603,10 +11067,14 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->msg_inq.last = &p->msg_inq.first; p->msg_inq.len = 0; #endif - p->u.bif_timers = NULL; + p->bif_timers = NULL; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + p->accessor_bif_timers = NULL; +#endif p->mbuf = NULL; + p->msg_frag = NULL; p->mbuf_sz = 0; - p->psd = NULL; + erts_smp_atomic_init_nob(&p->psd, (erts_aint_t) NULL); p->dictionary = NULL; p->seq_trace_lastcnt = 0; p->seq_trace_clock = 0; @@ -10708,7 +11176,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). * Schedule process for execution. */ - schedule_process(p, state); + schedule_process(p, state, 0); VERBOSE(DEBUG_PROCESSES, ("Created a new process: %T\n",p->common.id)); @@ -10740,6 +11208,8 @@ void erts_init_empty_process(Process *p) p->stop = NULL; p->hend = NULL; p->heap = NULL; + p->abandoned_heap = NULL; + p->live_hf_end = ERTS_INVALID_HFRAG_PTR; p->gen_gcs = 0; p->max_gen_gcs = 0; p->min_heap_size = 0; @@ -10760,12 +11230,7 @@ void erts_init_empty_process(Process *p) p->bin_old_vheap_sz = BIN_VH_MIN_SIZE; p->bin_old_vheap = 0; p->sys_task_qs = NULL; - p->bin_vheap_mature = 0; -#ifdef ERTS_SMP - p->common.u.alive.ptimer = NULL; -#else - memset(&(p->common.u.alive.tm), 0, sizeof(ErlTimer)); -#endif + ERTS_PTMR_INIT(p); p->next = NULL; p->off_heap.first = NULL; p->off_heap.overhead = 0; @@ -10776,8 +11241,9 @@ void erts_init_empty_process(Process *p) p->old_htop = NULL; p->old_heap = NULL; p->mbuf = NULL; + p->msg_frag = NULL; p->mbuf_sz = 0; - p->psd = NULL; + erts_smp_atomic_init_nob(&p->psd, (erts_aint_t) NULL); ERTS_P_MONITORS(p) = NULL; ERTS_P_LINKS(p) = NULL; /* List of links */ p->nodes_monitors = NULL; @@ -10786,14 +11252,17 @@ void erts_init_empty_process(Process *p) p->msg.last = &p->msg.first; p->msg.save = &p->msg.first; p->msg.len = 0; - p->u.bif_timers = NULL; + p->bif_timers = NULL; +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + p->accessor_bif_timers = NULL; +#endif p->dictionary = NULL; p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; p->seq_trace_token = NIL; - p->initial[0] = 0; - p->initial[1] = 0; - p->initial[2] = 0; + p->u.initial[0] = 0; + p->u.initial[1] = 0; + p->u.initial[2] = 0; p->catches = 0; p->cp = NULL; p->i = NULL; @@ -10814,6 +11283,8 @@ void erts_init_empty_process(Process *p) p->parent = NIL; p->approx_started = 0; + p->static_flags = 0; + p->common.u.alive.started_interval = 0; #ifdef HIPE @@ -10828,6 +11299,9 @@ void erts_init_empty_process(Process *p) p->last_old_htop = NULL; #endif +#ifdef ERTS_DIRTY_SCHEDULERS + erts_smp_atomic32_init_nob(&p->dirty_state, 0); +#endif erts_smp_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL); #ifdef ERTS_SMP @@ -10859,6 +11333,8 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->htop == NULL); ASSERT(p->stop == NULL); ASSERT(p->hend == NULL); + ASSERT(p->abandoned_heap == NULL); + ASSERT(p->live_hf_end == ERTS_INVALID_HFRAG_PTR); ASSERT(p->heap == NULL); ASSERT(p->common.id == ERTS_INVALID_PID); ASSERT(ERTS_TRACER_PROC(p) == NIL); @@ -10878,7 +11354,10 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->suspend_monitors == NULL); ASSERT(p->msg.first == NULL); ASSERT(p->msg.len == 0); - ASSERT(p->u.bif_timers == NULL); + ASSERT(p->bif_timers == NULL); +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + ASSERT(p->accessor_bif_timers == NULL); +#endif ASSERT(p->dictionary == NULL); ASSERT(p->catches == 0); ASSERT(p->cp == NULL); @@ -10933,14 +11412,18 @@ erts_cleanup_empty_process(Process* p) static void delete_process(Process* p) { - ErlMessage* mp; - + Eterm *heap; + ErtsPSD *psd; VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id)); + VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id, + HEAP_START(p), HEAP_END(p), OLD_HEAP(p), OLD_HEND(p))); /* Cleanup psd */ - if (p->psd) - erts_free(ERTS_ALC_T_PSD, p->psd); + psd = (ErtsPSD *) erts_smp_atomic_read_nob(&p->psd); + + if (psd) + erts_free(ERTS_ALC_T_PSD, psd); /* Clean binaries and funs */ erts_cleanup_offheap(&p->off_heap); @@ -10959,16 +11442,17 @@ delete_process(Process* p) * Release heaps. Clobber contents in DEBUG build. */ - -#ifdef DEBUG - sys_memset(p->heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm)); -#endif - #ifdef HIPE hipe_delete_process(&p->hipe); #endif - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) p->heap, p->heap_sz*sizeof(Eterm)); + heap = p->abandoned_heap ? p->abandoned_heap : p->heap; + +#ifdef DEBUG + sys_memset(heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm)); +#endif + + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) heap, p->heap_sz*sizeof(Eterm)); if (p->old_heap != NULL) { #ifdef DEBUG @@ -10987,27 +11471,14 @@ delete_process(Process* p) free_message_buffer(p->mbuf); } + if (p->msg_frag) + erts_cleanup_messages(p->msg_frag); + erts_erase_dicts(p); /* free all pending messages */ - mp = p->msg.first; - while(mp != NULL) { - ErlMessage* next_mp = mp->next; - if (mp->data.attached) { - if (is_value(mp->m[0])) - free_message_buffer(mp->data.heap_frag); - else { - if (is_not_nil(mp->m[1])) { - ErlHeapFragment *heap_frag; - heap_frag = (ErlHeapFragment *) mp->data.dist_ext->ext_endp; - erts_cleanup_offheap(&heap_frag->off_heap); - } - erts_free_dist_ext_copy(mp->data.dist_ext); - } - } - free_message(mp); - mp = next_mp; - } + erts_cleanup_messages(p->msg.first); + p->msg.first = NULL; ASSERT(!p->nodes_monitors); ASSERT(!p->suspend_monitors); @@ -11026,10 +11497,13 @@ set_proc_exiting(Process *p, ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(p) == ERTS_PROC_LOCKS_ALL); enqueue = change_proc_schedule_state(p, - ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT, + (ERTS_PSFLG_SUSPENDED + | ERTS_PSFLG_PENDING_EXIT + | ERTS_PSFLGS_DIRTY_WORK), ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE, &state, - &enq_prio); + &enq_prio, + ERTS_PROC_LOCKS_ALL); p->fvalue = reason; if (bp) @@ -11041,13 +11515,25 @@ set_proc_exiting(Process *p, */ p->freason = EXTAG_EXIT; KILL_CATCHES(p); - cancel_timer(p); p->i = (BeamInstr *) beam_exit; - if (enqueue) - add2runq(enqueue > 0 ? p : make_proxy_proc(NULL, p, enq_prio), - state, - enq_prio); +#ifndef ERTS_SMP + if (state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)) { + /* + * I non smp case: + * + * Currently executing process might be sent an exit + * signal if it is traced by a port that it also is + * linked to, and the port terminates during the + * trace. In this case we want schedule out the + * process as quickly as possible in order to detect + * the event as fast as possible. + */ + ERTS_VBUMP_ALL_REDS(p); + } +#endif + + add2runq(enqueue, enq_prio, p, state, NULL); } static ERTS_INLINE erts_aint32_t @@ -11070,7 +11556,8 @@ set_proc_self_exiting(Process *c_p) ERTS_PSFLG_SUSPENDED|ERTS_PSFLG_PENDING_EXIT, ERTS_PSFLG_EXITING|ERTS_PSFLG_ACTIVE, &state, - &enq_prio); + &enq_prio, + ERTS_PROC_LOCKS_ALL); ASSERT(!enqueue); return state; @@ -11140,10 +11627,19 @@ save_pending_exiter(Process *p) { ErtsProcList *plp; ErtsRunQueue *rq; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p)); - rq = erts_get_runq_current(NULL); + if (!esdp) + rq = RUNQ_READ_RQ(&p->run_queue); + else + rq = esdp->run_queue; + +#ifdef ERTS_DIRTY_SCHEDULERS + if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) + rq = ERTS_RUNQ_IX(0); /* Handle on ordinary scheduler */ +#endif plp = proclist_create(p); @@ -11154,12 +11650,8 @@ save_pending_exiter(Process *p) non_empty_runq(rq); erts_smp_runq_unlock(rq); -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) - wake_dirty_schedulers(rq, 0); - else -#endif - wake_scheduler(rq); + + wake_scheduler(rq); } #endif @@ -11173,43 +11665,46 @@ static ERTS_INLINE void send_exit_message(Process *to, ErtsProcLocks *to_locksp, Eterm exit_term, Uint term_size, Eterm token) { - if (token == NIL -#ifdef USE_VM_PROBES - || token == am_have_dt_utag -#endif - ) { - Eterm* hp; - Eterm mess; - ErlHeapFragment* bp; - ErlOffHeap *ohp; - - hp = erts_alloc_message_heap(term_size, &bp, &ohp, to, to_locksp); + ErtsMessage *mp; + ErlOffHeap *ohp; + Eterm* hp; + Eterm mess; +#ifdef SHCOPY_SEND + erts_shcopy_t info; +#endif + + if (!have_seqtrace(token)) { +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + term_size = copy_shared_calculate(exit_term, &info); + mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp); + mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else + mp = erts_alloc_message_heap(to, to_locksp, term_size, &hp, &ohp); mess = copy_struct(exit_term, term_size, &hp, ohp); - erts_queue_message(to, to_locksp, bp, mess, NIL -#ifdef USE_VM_PROBES - , NIL #endif - ); + erts_queue_message(to, to_locksp, mp, mess, NIL); } else { - ErlHeapFragment* bp; - Eterm* hp; - Eterm mess; Eterm temp_token; Uint sz_token; ASSERT(is_tuple(token)); sz_token = size_object(token); - bp = new_message_buffer(term_size+sz_token); - hp = bp->mem; - mess = copy_struct(exit_term, term_size, &hp, &bp->off_heap); +#ifdef SHCOPY_SEND + INITIALIZE_SHCOPY(info); + term_size = copy_shared_calculate(exit_term, &info); + mp = erts_alloc_message_heap(to, to_locksp, term_size+sz_token, &hp, &ohp); + mess = copy_shared_perform(exit_term, term_size, &info, &hp, ohp); + DESTROY_SHCOPY(info); +#else + mp = erts_alloc_message_heap(to, to_locksp, term_size+sz_token, &hp, &ohp); + mess = copy_struct(exit_term, term_size, &hp, ohp); +#endif /* the trace token must in this case be updated by the caller */ seq_trace_output(token, mess, SEQ_TRACE_SEND, to->common.id, NULL); - temp_token = copy_struct(token, sz_token, &hp, &bp->off_heap); - erts_queue_message(to, to_locksp, bp, mess, temp_token -#ifdef USE_VM_PROBES - , NIL -#endif - ); + temp_token = copy_struct(token, sz_token, &hp, ohp); + erts_queue_message(to, to_locksp, mp, mess, temp_token); } } @@ -11318,11 +11813,7 @@ send_exit_signal(Process *c_p, /* current process if and only if ((state & ERTS_PSFLG_TRAP_EXIT) && (reason != am_kill || (flags & ERTS_XSIG_FLG_IGN_KILL))) { - if (is_not_nil(token) -#ifdef USE_VM_PROBES - && token != am_have_dt_utag -#endif - && token_update) + if (have_seqtrace(token) && token_update) seq_trace_update_send(token_update); if (is_value(exit_tuple)) send_exit_message(rp, rp_locks, exit_tuple, exit_tuple_sz, token); @@ -11356,23 +11847,21 @@ send_exit_signal(Process *c_p, /* current process if and only if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { /* ... but we havn't got all locks on it ... */ - save_pending_exiter(rp); + save_pending_exiter(rp); /* * The pending exit will be discovered when next * process is scheduled in */ - goto set_pending_exit; - } - else { - /* ...and we have all locks on it... */ - *rp_locks = ERTS_PROC_LOCKS_ALL; - set_proc_exiting(rp, - state, - (is_immed(rsn) - ? rsn - : copy_object(rsn, rp)), - NULL); + goto set_pending_exit; } + /* ...and we have all locks on it... */ + *rp_locks = ERTS_PROC_LOCKS_ALL; + set_proc_exiting(rp, + state, + (is_immed(rsn) + ? rsn + : copy_object(rsn, rp)), + NULL); } else { /* Process running... */ @@ -11462,7 +11951,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) ErtsMonitor *rmon; Process *rp; - if (mon->type == MON_ORIGIN) { + switch (mon->type) { + case MON_ORIGIN: /* We are monitoring someone else, we need to demonitor that one.. */ if (is_atom(mon->pid)) { /* remote by name */ ASSERT(is_node_name_atom(mon->pid)); @@ -11525,7 +12015,8 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) } } } - } else { /* type == MON_TARGET */ + break; + case MON_TARGET: ASSERT(mon->type == MON_TARGET); ASSERT(is_pid(mon->pid) || is_internal_port(mon->pid)); if (is_internal_port(mon->pid)) { @@ -11584,6 +12075,12 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) } } } + break; + case MON_TIME_OFFSET: + erts_demonitor_time_offset(mon->ref); + break; + default: + ERTS_INTERNAL_ERROR("Invalid monitor type"); } done: /* As the monitors are previously removed from the process, @@ -11703,7 +12200,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) break; default: - erl_exit(1, "bad type in link list\n"); + erts_exit(ERTS_ERROR_EXIT, "bad type in link list\n"); break; } erts_destroy_link(lnk); @@ -11715,8 +12212,9 @@ resume_suspend_monitor(ErtsSuspendMonitor *smon, void *vc_p) Process *suspendee = erts_pid2proc((Process *) vc_p, ERTS_PROC_LOCK_MAIN, smon->pid, ERTS_PROC_LOCK_STATUS); if (suspendee) { + ASSERT(suspendee != vc_p); if (smon->active) - resume_process(suspendee); + resume_process(suspendee, ERTS_PROC_LOCK_STATUS); erts_smp_proc_unlock(suspendee, ERTS_PROC_LOCK_STATUS); } erts_destroy_suspend_monitor(smon); @@ -11729,6 +12227,7 @@ erts_do_exit_process(Process* p, Eterm reason) { p->arity = 0; /* No live registers */ p->fvalue = reason; + #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_exit)) { @@ -11741,6 +12240,10 @@ erts_do_exit_process(Process* p, Eterm reason) } #endif + if (p->static_flags & ERTS_STC_FLG_SYSTEM_PROC) + erts_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n", + p->common.id, reason); + #ifdef ERTS_SMP ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); /* By locking all locks (main lock is already locked) when going @@ -11780,20 +12283,20 @@ erts_do_exit_process(Process* p, Eterm reason) ASSERT((ERTS_TRACE_FLAGS(p) & F_INITIAL_TRACE_FLAGS) == F_INITIAL_TRACE_FLAGS); - cancel_timer(p); /* Always cancel timer just in case */ - - if (p->u.bif_timers) - erts_cancel_bif_timers(p, ERTS_PROC_LOCKS_ALL); + ASSERT(erts_proc_read_refc(p) > 0); + if (ERTS_PTMR_IS_SET(p)) { + erts_cancel_proc_timer(p); + ASSERT(erts_proc_read_refc(p) > 0); + } erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR); /* - * The p->u.bif_timers of this process can *not* be used anymore; + * p->u.initial of this process can *not* be used anymore; * will be overwritten by misc termination data. */ p->u.terminate = NULL; - erts_continue_exit_process(p); } @@ -11808,6 +12311,7 @@ erts_continue_exit_process(Process *p) struct saved_calls *scb; process_breakpoint_time_t *pbt; erts_aint32_t state; + void *nif_export; #ifdef DEBUG int yield_allowed = 1; @@ -11817,22 +12321,70 @@ erts_continue_exit_process(Process *p) ASSERT(ERTS_PROC_IS_EXITING(p)); + ASSERT(erts_proc_read_refc(p) > 0); + if (p->bif_timers) { + if (erts_cancel_bif_timers(p, p->bif_timers, &p->u.terminate)) { + ASSERT(erts_proc_read_refc(p) > 0); + goto yield; + } + ASSERT(erts_proc_read_refc(p) > 0); + p->bif_timers = NULL; + } + +#ifdef ERTS_BTM_ACCESSOR_SUPPORT + if (p->accessor_bif_timers) { + if (erts_detach_accessor_bif_timers(p, + p->accessor_bif_timers, + &p->u.terminate)) { + ASSERT(erts_proc_read_refc(p) > 0); + goto yield; + } + ASSERT(erts_proc_read_refc(p) > 0); + p->accessor_bif_timers = NULL; + } +#endif + #ifdef ERTS_SMP + if (p->flags & F_SCHDLR_ONLN_WAITQ) + abort_sched_onln_chng_waitq(p); + if (p->flags & F_HAVE_BLCKD_MSCHED) { ErtsSchedSuspendResult ssr; - ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 1); + ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 0, 1); + switch (ssr) { + case ERTS_SCHDLR_SSPND_YIELD_RESTART: + goto yield; + case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_DONE: + case ERTS_SCHDLR_SSPND_YIELD_DONE: + p->flags &= ~F_HAVE_BLCKD_MSCHED; + break; + case ERTS_SCHDLR_SSPND_EINVAL: + default: + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", + __FILE__, __LINE__, (int) ssr); + } + } + if (p->flags & F_HAVE_BLCKD_NMSCHED) { + ErtsSchedSuspendResult ssr; + ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 1, 1); switch (ssr) { case ERTS_SCHDLR_SSPND_YIELD_RESTART: goto yield; case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED: case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED: case ERTS_SCHDLR_SSPND_DONE: case ERTS_SCHDLR_SSPND_YIELD_DONE: p->flags &= ~F_HAVE_BLCKD_MSCHED; break; case ERTS_SCHDLR_SSPND_EINVAL: default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", __FILE__, __LINE__, (int) ssr); } } @@ -11910,6 +12462,8 @@ erts_continue_exit_process(Process *p) p->scheduler_data->current_process = NULL; p->scheduler_data->free_process = p; +#else + erts_proc_inc_refc(p); /* Decremented in schedule() */ #endif /* Time of death! */ @@ -11928,36 +12482,29 @@ erts_continue_exit_process(Process *p) { /* Inactivate and notify free */ erts_aint32_t n, e, a = erts_smp_atomic32_read_nob(&p->state); -#ifdef ERTS_SMP int refc_inced = 0; -#endif while (1) { n = e = a; ASSERT(a & ERTS_PSFLG_EXITING); n |= ERTS_PSFLG_FREE; n &= ~ERTS_PSFLG_ACTIVE; -#ifdef ERTS_SMP if ((n & ERTS_PSFLG_IN_RUNQ) && !refc_inced) { - erts_smp_proc_inc_refc(p); + erts_proc_inc_refc(p); refc_inced = 1; } -#endif a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e); if (a == e) break; } -#ifdef ERTS_SMP if (refc_inced && !(n & ERTS_PSFLG_IN_RUNQ)) - erts_smp_proc_dec_refc(p); -#endif + erts_proc_dec_refc(p); } - dep = ((p->flags & F_DISTRIBUTION) - ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL) - : NULL); - scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL); - pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL); + dep = (p->flags & F_DISTRIBUTION) ? erts_this_dist_entry : NULL; + scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, NULL); + pbt = ERTS_PROC_SET_CALL_TIME(p, NULL); + nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); #ifdef BM_COUNTERS @@ -11966,8 +12513,6 @@ erts_continue_exit_process(Process *p) if (dep) { erts_do_net_exits(dep, reason); - if(dep) - erts_deref_dist_entry(dep); } /* @@ -12005,6 +12550,9 @@ erts_continue_exit_process(Process *p) if (pbt) erts_free(ERTS_ALC_T_BPD, (void *) pbt); + if (nif_export) + erts_destroy_nif_export(nif_export); + delete_process(p); #ifdef ERTS_SMP @@ -12037,64 +12585,6 @@ erts_continue_exit_process(Process *p) } -/* Callback for process timeout */ -static void -timeout_proc(Process* p) -{ - erts_aint32_t state; - BeamInstr** pi = (BeamInstr **) p->def_arg_reg; - p->i = *pi; - p->flags |= F_TIMO; - p->flags &= ~F_INSLPQUEUE; - - state = erts_smp_atomic32_read_acqb(&p->state); - if (!(state & ERTS_PSFLG_ACTIVE)) - schedule_process(p, state); -} - - -void -cancel_timer(Process* p) -{ - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); - p->flags &= ~(F_INSLPQUEUE|F_TIMO); -#ifdef ERTS_SMP - erts_cancel_smp_ptimer(p->common.u.alive.ptimer); -#else - erts_cancel_timer(&p->common.u.alive.tm); -#endif -} - -/* - * Insert a process into the time queue, with a timeout 'timeout' in ms. - */ -void -set_timer(Process* p, Uint timeout) -{ - ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p)); - - /* check for special case timeout=0 DONT ADD TO time queue */ - if (timeout == 0) { - p->flags |= F_TIMO; - return; - } - p->flags |= F_INSLPQUEUE; - p->flags &= ~F_TIMO; - -#ifdef ERTS_SMP - erts_create_smp_ptimer(&p->common.u.alive.ptimer, - p->common.id, - (ErlTimeoutProc) timeout_proc, - timeout); -#else - erts_set_timer(&p->common.u.alive.tm, - (ErlTimeoutProc) timeout_proc, - NULL, - (void*) p, - timeout); -#endif -} - /* * Stack dump functions follow. */ @@ -12110,7 +12600,7 @@ erts_stack_dump(int to, void *to_arg, Process *p) } erts_program_counter_info(to, to_arg, p); for (sp = p->stop; sp < STACK_START(p); sp++) { - yreg = stack_element_dump(to, to_arg, p, sp, yreg); + yreg = stack_element_dump(to, to_arg, sp, yreg); } } @@ -12167,7 +12657,7 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x) } static int -stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) +stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg) { Eterm x = *sp; @@ -12181,7 +12671,7 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) } if (is_CP(x)) { - erts_print(to, to_arg, "Return addr %p (", (Eterm *) EXPAND_POINTER(x)); + erts_print(to, to_arg, "Return addr %p (", (Eterm *) x); print_function_from_pc(to, to_arg, cp_val(x)); erts_print(to, to_arg, ")\n"); yreg = 0; @@ -12196,6 +12686,188 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) } /* + * Print scheduler information + */ +void +erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) { + int i; + erts_aint32_t flg; + Process *p; + + erts_print(to, to_arg, "=scheduler:%u\n", esdp->no); + +#ifdef ERTS_SMP + flg = erts_smp_atomic32_read_dirty(&esdp->ssi->flags); + erts_print(to, to_arg, "Scheduler Sleep Info Flags: "); + for (i = 0; i < ERTS_SSI_FLGS_MAX && flg; i++) { + erts_aint32_t chk = (1 << i); + if (flg & chk) { + switch (chk) { + case ERTS_SSI_FLG_SLEEPING: + erts_print(to, to_arg, "SLEEPING"); break; + case ERTS_SSI_FLG_POLL_SLEEPING: + erts_print(to, to_arg, "POLL_SLEEPING"); break; + case ERTS_SSI_FLG_TSE_SLEEPING: + erts_print(to, to_arg, "TSE_SLEEPING"); break; + case ERTS_SSI_FLG_WAITING: + erts_print(to, to_arg, "WAITING"); break; + case ERTS_SSI_FLG_SUSPENDED: + erts_print(to, to_arg, "SUSPENDED"); break; + default: + erts_print(to, to_arg, "UNKNOWN(%d)", flg); break; + } + if (flg > chk) + erts_print(to, to_arg, " | "); + flg -= chk; + } + } + erts_print(to, to_arg, "\n"); +#endif + + flg = erts_atomic32_read_dirty(&esdp->ssi->aux_work); + erts_print(to, to_arg, "Scheduler Sleep Info Aux Work: "); + for (i = 0; i < ERTS_SSI_AUX_WORK_NO_FLAGS && flg; i++) { + erts_aint32_t chk = (1 << i); + if (flg & chk) { + if (erts_aux_work_flag_descr[i]) + erts_print(to, to_arg, "%s", erts_aux_work_flag_descr[i]); + else + erts_print(to, to_arg, "1<<%d", i); + if (flg > chk) + erts_print(to, to_arg, " | "); + flg -= chk; + } + } + erts_print(to, to_arg, "\n"); + + erts_print(to, to_arg, "Current Port: "); + if (esdp->current_port) + erts_print(to, to_arg, "%T", esdp->current_port->common.id); + erts_print(to, to_arg, "\n"); + + for (i = 0; i < ERTS_NO_PROC_PRIO_LEVELS; i++) { + erts_print(to, to_arg, "Run Queue "); + switch (i) { + case PRIORITY_MAX: + erts_print(to, to_arg, "Max "); + break; + case PRIORITY_HIGH: + erts_print(to, to_arg, "High "); + break; + case PRIORITY_NORMAL: + erts_print(to, to_arg, "Normal "); + break; + case PRIORITY_LOW: + erts_print(to, to_arg, "Low "); + break; + default: + erts_print(to, to_arg, "Unknown "); + break; + } + erts_print(to, to_arg, "Length: %d\n", + erts_smp_atomic32_read_dirty(&esdp->run_queue->procs.prio_info[i].len)); + } + erts_print(to, to_arg, "Run Queue Port Length: %d\n", + erts_smp_atomic32_read_dirty(&esdp->run_queue->ports.info.len)); + + flg = erts_smp_atomic32_read_dirty(&esdp->run_queue->flags); + erts_print(to, to_arg, "Run Queue Flags: "); + for (i = 0; i < ERTS_RUNQ_FLG_MAX && flg; i++) { + erts_aint32_t chk = (1 << i); + if (flg & chk) { + switch (chk) { + case (1 << PRIORITY_MAX): + erts_print(to, to_arg, "NONEMPTY_MAX"); break; + case (1 << PRIORITY_HIGH): + erts_print(to, to_arg, "NONEMPTY_HIGH"); break; + case (1 << PRIORITY_NORMAL): + erts_print(to, to_arg, "NONEMPTY_NORMAL"); break; + case (1 << PRIORITY_LOW): + erts_print(to, to_arg, "NONEMPTY_LOW"); break; + case (1 << (PRIORITY_MAX + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)): + erts_print(to, to_arg, "EMIGRATE_MAX"); break; + case (1 << (PRIORITY_HIGH + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)): + erts_print(to, to_arg, "EMIGRATE_HIGH"); break; + case (1 << (PRIORITY_NORMAL + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)): + erts_print(to, to_arg, "EMIGRATE_NORMAL"); break; + case (1 << (PRIORITY_LOW + ERTS_RUNQ_FLGS_EMIGRATE_SHFT)): + erts_print(to, to_arg, "EMIGRATE_LOW"); break; + case (1 << (PRIORITY_MAX + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)): + erts_print(to, to_arg, "IMMIGRATE_MAX"); break; + case (1 << (PRIORITY_HIGH + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)): + erts_print(to, to_arg, "IMMIGRATE_HIGH"); break; + case (1 << (PRIORITY_NORMAL + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)): + erts_print(to, to_arg, "IMMIGRATE_NORMAL"); break; + case (1 << (PRIORITY_LOW + ERTS_RUNQ_FLGS_IMMIGRATE_SHFT)): + erts_print(to, to_arg, "IMMIGRATE_LOW"); break; + case (1 << (PRIORITY_MAX + ERTS_RUNQ_FLGS_EVACUATE_SHFT)): + erts_print(to, to_arg, "EVACUATE_MAX"); break; + case (1 << (PRIORITY_HIGH + ERTS_RUNQ_FLGS_EVACUATE_SHFT)): + erts_print(to, to_arg, "EVACUATE_HIGH"); break; + case (1 << (PRIORITY_NORMAL + ERTS_RUNQ_FLGS_EVACUATE_SHFT)): + erts_print(to, to_arg, "EVACUATE_NORMAL"); break; + case (1 << (PRIORITY_LOW + ERTS_RUNQ_FLGS_EVACUATE_SHFT)): + erts_print(to, to_arg, "EVACUATE_LOW"); break; + case ERTS_RUNQ_FLG_OUT_OF_WORK: + erts_print(to, to_arg, "OUT_OF_WORK"); break; + case ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK: + erts_print(to, to_arg, "HALFTIME_OUT_OF_WORK"); break; + case ERTS_RUNQ_FLG_SUSPENDED: + erts_print(to, to_arg, "SUSPENDED"); break; + case ERTS_RUNQ_FLG_CHK_CPU_BIND: + erts_print(to, to_arg, "CHK_CPU_BIND"); break; + case ERTS_RUNQ_FLG_INACTIVE: + erts_print(to, to_arg, "INACTIVE"); break; + case ERTS_RUNQ_FLG_NONEMPTY: + erts_print(to, to_arg, "NONEMPTY"); break; + case ERTS_RUNQ_FLG_PROTECTED: + erts_print(to, to_arg, "PROTECTED"); break; + default: + erts_print(to, to_arg, "UNKNOWN(%d)", flg); break; + } + if (flg > chk) + erts_print(to, to_arg, " | "); + flg -= chk; + } + } + erts_print(to, to_arg, "\n"); + + /* This *MUST* to be the last information in scheduler block */ + p = esdp->current_process; + erts_print(to, to_arg, "Current Process: "); + if (esdp->current_process && !(ERTS_TRACE_FLAGS(p) & F_SENSITIVE)) { + flg = erts_smp_atomic32_read_dirty(&p->state); + erts_print(to, to_arg, "%T\n", p->common.id); + + erts_print(to, to_arg, "Current Process State: "); + erts_dump_process_state(to, to_arg, flg); + + erts_print(to, to_arg, "Current Process Internal State: "); + erts_dump_extended_process_state(to, to_arg, flg); + + erts_print(to, to_arg, "Current Process Program counter: %p (", p->i); + print_function_from_pc(to, to_arg, p->i); + erts_print(to, to_arg, ")\n"); + erts_print(to, to_arg, "Current Process CP: %p (", p->cp); + print_function_from_pc(to, to_arg, p->cp); + erts_print(to, to_arg, ")\n"); + + /* Getting this stacktrace can segfault if we are very very + unlucky if called while a process is being garbage collected. + Therefore we only call this on other schedulers if we either + have protection against segfaults, or we know that the process + is not garbage collecting. It *should* always be safe to call + on a process owned by us, even if it is currently being garbage + collected. + */ + erts_print(to, to_arg, "Current Process Limited Stack Trace:\n"); + erts_limited_stack_trace(to, to_arg, p); + } else + erts_print(to, to_arg, "\n"); + +} + +/* * A nice system halt closing all open port goes as follows: * 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS * on all schedulers, then schedules itself out. @@ -12209,10 +12881,10 @@ stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, int yreg) * The same global atomic is used as refcount. * * A BIF that calls this should make sure to schedule out to never come back: - * erl_halt((int)(- code)); + * erts_halt(code); * ERTS_BIF_YIELD1(bif_export[BIF_erlang_halt_1], BIF_P, NIL); */ -void erl_halt(int code) +void erts_halt(int code) { if (-1 == erts_smp_atomic32_cmpxchg_acqb(&erts_halt_progress, erts_no_schedulers, |