diff options
Diffstat (limited to 'erts')
| -rw-r--r-- | erts/emulator/beam/erl_process.c | 171 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_process.h | 33 |
2 files changed, 103 insertions, 101 deletions
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 57ef7bb8b8..b0f82e4c11 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -348,40 +348,25 @@ dbg_chk_aux_work_val(erts_aint32_t value) { erts_aint32_t valid = 0; -#ifdef ERTS_SSI_AUX_WORK_SET_TMO valid |= ERTS_SSI_AUX_WORK_SET_TMO; -#endif -#ifdef ERTS_SSI_AUX_WORK_CHECK_CHILDREN - valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; -#endif -#ifdef ERTS_SSI_AUX_WORK_MISC valid |= ERTS_SSI_AUX_WORK_MISC; -#endif -#ifdef ERTS_SSI_AUX_WORK_MISC_THR_PRGR - valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; -#endif -#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY - valid |= ERTS_SSI_AUX_WORK_ASYNC_READY; -#endif -#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN - valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; -#endif - -#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM; -#endif -#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; +#if ERTS_USE_ASYNC_READY_Q + valid |= ERTS_SSI_AUX_WORK_ASYNC_READY; + valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; #endif -#ifdef ERTS_SSI_AUX_WORK_DD +#ifdef ERTS_SMP + valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_DD; -#endif -#ifdef ERTS_SSI_AUX_WORK_DD valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR; #endif -#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK +#if HAVE_ERTS_MSEG valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; #endif +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; +#endif if (~valid & value) erl_exit(ERTS_ABORT_EXIT, @@ -802,7 +787,7 @@ misc_aux_work_clean(ErtsThrQ_t *q, return aux_work; } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_misc_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { @@ -822,13 +807,13 @@ handle_misc_aux_work(ErtsAuxWorkData *awdp, #ifdef ERTS_SMP -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { if (!erts_thr_progress_has_reached_this(thr_prgr_current(awdp), awdp->misc.thr_prgr)) - return aux_work; + return aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR; unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR); @@ -901,7 +886,7 @@ erts_notify_check_async_ready_queue(void *vno) ERTS_SSI_AUX_WORK_ASYNC_READY); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_async_ready(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { @@ -923,7 +908,7 @@ handle_async_ready(ErtsAuxWorkData *awdp, | ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_async_ready_clean(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { @@ -960,7 +945,7 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, #endif -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { ErtsSchedulerSleepInfo *ssi = awdp->ssi; @@ -988,7 +973,7 @@ erts_alloc_notify_delayed_dealloc(int ix) ERTS_SSI_AUX_WORK_DD); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { ErtsSchedulerSleepInfo *ssi = awdp->ssi; @@ -1026,7 +1011,7 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) return aux_work & ~ERTS_SSI_AUX_WORK_DD; } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { ErtsSchedulerSleepInfo *ssi; @@ -1154,7 +1139,7 @@ erts_smp_notify_check_children_needed(void) ERTS_SSI_AUX_WORK_CHECK_CHILDREN); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN); @@ -1164,9 +1149,9 @@ handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) #endif -#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK +#if HAVE_ERTS_MSEG -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK); @@ -1176,7 +1161,7 @@ handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) #endif -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO); @@ -1184,72 +1169,92 @@ handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO; } -static ERTS_INLINE erts_aint32_t -handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +static erts_aint32_t +handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work) { +#undef HANDLE_AUX_WORK +#define HANDLE_AUX_WORK(FLG, HNDLR) \ + ignore |= FLG; \ + if (aux_work & FLG) { \ + aux_work = HNDLR(awdp, aux_work); \ + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ + if (!(aux_work & ~ignore)) { \ + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ + return aux_work; \ + } \ + } + + erts_aint32_t aux_work = orig_aux_work; + erts_aint32_t ignore = 0; + #ifdef ERTS_SMP thr_prgr_current_reset(awdp); #endif + + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + ASSERT(aux_work); + /* * Handlers are *only* allowed to modify flags in return value * and ssi flags that are explicity handled by the handler. * Handlers are, e.g., not allowed to read the ssi flag field and * then unconditionally return that value. + * + * Flag field returned should only contain flags for work that + * can continue immediately. + */ + + /* + * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative + * eachother. Most frequent first. */ - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - if (aux_work & ERTS_SSI_AUX_WORK_SET_TMO) { - aux_work = handle_setup_aux_work_timer(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } #ifdef ERTS_SMP - if (aux_work & ERTS_SSI_AUX_WORK_MISC_THR_PRGR) { - aux_work = handle_misc_aux_work_thr_prgr(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD, + handle_delayed_dealloc); + /* DD must be before DD_THR_PRGR */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD_THR_PRGR, + handle_delayed_dealloc_thr_prgr); #endif - if (aux_work & ERTS_SSI_AUX_WORK_MISC) { - aux_work = handle_misc_aux_work(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + + HANDLE_AUX_WORK((ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + handle_fix_alloc); + #if ERTS_USE_ASYNC_READY_Q - if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY) { - aux_work = handle_async_ready(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } - if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN) { - aux_work = handle_async_ready_clean(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } -#endif -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { - aux_work = handle_check_children(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY, + handle_async_ready); + /* ASYNC_READY must be before ASYNC_READY_CLEAN */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN, + handle_async_ready_clean); #endif - if (aux_work & (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)) { - aux_work = handle_fix_alloc(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + #ifdef ERTS_SMP - if (aux_work & ERTS_SSI_AUX_WORK_DD) { - aux_work = handle_delayed_dealloc(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } - if (aux_work & ERTS_SSI_AUX_WORK_DD_THR_PRGR) { - aux_work = handle_delayed_dealloc_thr_prgr(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC_THR_PRGR, + handle_misc_aux_work_thr_prgr); #endif -#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK - if (aux_work & ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK) { - aux_work = handle_mseg_cache_check(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + /* MISC_THR_PRGR must be before MISC */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC, + handle_misc_aux_work); + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CHECK_CHILDREN, + handle_check_children); +#endif + + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO, + handle_setup_aux_work_timer); + +#if HAVE_ERTS_MSEG + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK, + handle_mseg_cache_check); #endif + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + return aux_work; + +#undef HANDLE_AUX_WORK + } typedef struct { diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index ad32dc11f9..c4f0ced294 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -248,25 +248,22 @@ typedef enum { | ERTS_SSI_FLG_WAITING \ | ERTS_SSI_FLG_SUSPENDED) -#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 0) -#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 1) -#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 2) -#ifdef ERTS_SMP -#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 3) -#endif -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 4) -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 5) -#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 6) -#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 7) -#ifdef ERTS_SMP -#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 8) -#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 9) -#endif -#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10) +/* + * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative + * eachother. Most frequent - lowest bit number. + */ -#if !HAVE_ERTS_MSEG -# undef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK -#endif +#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 0) +#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 1) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 2) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 3) +#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 4) +#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 5) +#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 6) +#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 7) +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 8) +#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 9) +#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10) typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; |