diff options
Diffstat (limited to 'erts/emulator/beam/erl_process.c')
-rw-r--r-- | erts/emulator/beam/erl_process.c | 144 |
1 files changed, 123 insertions, 21 deletions
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index bd4c56eaa4..0fa2def5af 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -359,6 +359,7 @@ dbg_chk_aux_work_val(erts_aint32_t value) valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; #endif #ifdef ERTS_SMP + valid |= ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP; valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_DD; valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR; @@ -899,13 +900,13 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) #ifdef ERTS_SMP static ERTS_INLINE void -thr_prgr_current_reset(ErtsAuxWorkData *awdp) +haw_thr_prgr_current_reset(ErtsAuxWorkData *awdp) { awdp->current_thr_prgr = ERTS_THR_PRGR_INVALID; } static ERTS_INLINE ErtsThrPrgrVal -thr_prgr_current(ErtsAuxWorkData *awdp) +haw_thr_prgr_current(ErtsAuxWorkData *awdp) { ErtsThrPrgrVal current = awdp->current_thr_prgr; if (current == ERTS_THR_PRGR_INVALID) { @@ -915,6 +916,60 @@ thr_prgr_current(ErtsAuxWorkData *awdp) return current; } +static ERTS_INLINE void +haw_thr_prgr_current_check_progress(ErtsAuxWorkData *awdp) +{ + ErtsThrPrgrVal current = awdp->current_thr_prgr; + if (current != ERTS_THR_PRGR_INVALID + && !erts_thr_progress_equal(current, erts_thr_progress_current())) { + /* + * We have used a previouly read current value that isn't the + * latest; need to poke ourselfs in order to guarantee no loss + * of wakeups. + */ + erts_sched_poke(awdp->ssi); + } +} + +static ERTS_INLINE erts_aint32_t +handle_delayed_aux_work_wakeup(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +{ + int jix, max_jix; + unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP); + + ERTS_THR_MEMORY_BARRIER; + + max_jix = awdp->delayed_wakeup.jix; + awdp->delayed_wakeup.jix = -1; + for (jix = 0; jix <= max_jix; jix++) { + int sched = awdp->delayed_wakeup.job[jix].sched; + erts_aint32_t aux_work = awdp->delayed_wakeup.job[jix].aux_work; + + ASSERT(awdp->delayed_wakeup.sched2jix[sched] == jix); + awdp->delayed_wakeup.sched2jix[sched] = -1; + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(sched-1), + aux_work); + } + return aux_work & ~ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP; +} + +static ERTS_INLINE void +schedule_aux_work_wakeup(ErtsAuxWorkData *awdp, int sched, erts_aint32_t aux_work) +{ + int jix = awdp->delayed_wakeup.sched2jix[sched]; + if (jix >= 0) { + ASSERT(awdp->delayed_wakeup.job[jix].sched == sched); + awdp->delayed_wakeup.job[jix].aux_work |= aux_work; + } + else { + jix = ++awdp->delayed_wakeup.jix; + awdp->delayed_wakeup.sched2jix[sched] = jix; + awdp->delayed_wakeup.job[jix].sched = sched; + awdp->delayed_wakeup.job[jix].aux_work = aux_work; + } + set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP); +} + #endif typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t; @@ -1013,7 +1068,7 @@ static ERTS_INLINE erts_aint32_t handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { - if (!erts_thr_progress_has_reached_this(thr_prgr_current(awdp), + if (!erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp), awdp->misc.thr_prgr)) return aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR; @@ -1118,7 +1173,7 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, #ifdef ERTS_SMP if (awdp->async_ready.need_thr_prgr - && !erts_thr_progress_has_reached_this(thr_prgr_current(awdp), + && !erts_thr_progress_has_reached_this(haw_thr_prgr_current(awdp), awdp->async_ready.thr_prgr)) { return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; } @@ -1171,8 +1226,14 @@ handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) void erts_alloc_notify_delayed_dealloc(int ix) { - set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix-1), - ERTS_SSI_AUX_WORK_DD); + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp) + schedule_aux_work_wakeup(&esdp->aux_work_data, + ix, + ERTS_SSI_AUX_WORK_DD); + else + set_aux_work_flags_wakeup_relb(ERTS_SCHED_SLEEP_INFO_IX(ix-1), + ERTS_SSI_AUX_WORK_DD); } static ERTS_INLINE erts_aint32_t @@ -1199,7 +1260,7 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) if (need_thr_progress) { if (wakeup == ERTS_THR_PRGR_INVALID) - wakeup = erts_thr_progress_later_than(thr_prgr_current(awdp)); + wakeup = erts_thr_progress_later(awdp->esdp); awdp->dd.thr_prgr = wakeup; set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); awdp->dd.thr_prgr = wakeup; @@ -1220,7 +1281,7 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) int need_thr_progress; int more_work; ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; - ErtsThrPrgrVal current = thr_prgr_current(awdp); + ErtsThrPrgrVal current = haw_thr_prgr_current(awdp); if (!erts_thr_progress_has_reached_this(current, awdp->dd.thr_prgr)) return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; @@ -1242,7 +1303,7 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) if (need_thr_progress) { if (wakeup == ERTS_THR_PRGR_INVALID) - wakeup = erts_thr_progress_later_than(current); + wakeup = erts_thr_progress_later(awdp->esdp); awdp->dd.thr_prgr = wakeup; erts_thr_progress_wakeup(awdp->esdp, wakeup); } @@ -1431,7 +1492,7 @@ handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) } static erts_aint32_t -handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work) +handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting) { #undef HANDLE_AUX_WORK #define HANDLE_AUX_WORK(FLG, HNDLR) \ @@ -1449,7 +1510,7 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work) erts_aint32_t ignore = 0; #ifdef ERTS_SMP - thr_prgr_current_reset(awdp); + haw_thr_prgr_current_reset(awdp); #endif ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); @@ -1470,6 +1531,8 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work) * eachother. Most frequent first. */ #ifdef ERTS_SMP + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP, + handle_delayed_aux_work_wakeup); HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD, handle_delayed_dealloc); /* DD must be before DD_THR_PRGR */ @@ -1515,6 +1578,11 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work) ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); +#ifdef ERTS_SMP + if (waiting && !aux_work) + haw_thr_prgr_current_check_progress(awdp); +#endif + return aux_work; #undef HANDLE_AUX_WORK @@ -1943,7 +2011,7 @@ thr_prgr_fin_wait(void *vssi) | ERTS_SSI_FLG_TSE_SLEEPING)); } -static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp); +static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp); static void * aux_thread(void *unused) @@ -1963,7 +2031,7 @@ aux_thread(void *unused) callbacks.finalize_wait = thr_prgr_fin_wait; erts_thr_progress_register_managed_thread(NULL, &callbacks, 1); - init_aux_work_data(awdp, NULL); + init_aux_work_data(awdp, NULL, NULL); awdp->ssi = ssi; sched_prep_spin_wait(ssi); @@ -1975,7 +2043,7 @@ aux_thread(void *unused) if (aux_work) { if (!thr_prgr_active) erts_thr_progress_active(NULL, thr_prgr_active = 1); - aux_work = handle_aux_work(awdp, aux_work); + aux_work = handle_aux_work(awdp, aux_work, 1); if (aux_work && erts_thr_progress_update(NULL)) erts_thr_progress_leader_update(NULL); } @@ -2054,7 +2122,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); } - aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); + aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); } @@ -2158,7 +2226,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) if (!thr_prgr_active) erts_thr_progress_active(esdp, thr_prgr_active = 1); #endif - aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); + aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); #ifdef ERTS_SMP if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); @@ -3830,7 +3898,7 @@ erts_sched_set_busy_wait_threshold(char *str) return 0; } static void -init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp) +init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp) { awdp->sched_id = esdp ? (int) esdp->no : 0; awdp->esdp = esdp; @@ -3848,12 +3916,32 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp) #endif awdp->async_ready.queue = NULL; #endif +#ifdef ERTS_SMP + if (!dawwp) { + awdp->delayed_wakeup.job = NULL; + awdp->delayed_wakeup.sched2jix = NULL; + awdp->delayed_wakeup.jix = -1; + } + else { + int i; + awdp->delayed_wakeup.job = (ErtsDelayedAuxWorkWakeupJob *) dawwp; + dawwp += sizeof(ErtsDelayedAuxWorkWakeupJob)*(erts_no_schedulers+1); + awdp->delayed_wakeup.sched2jix = (int *) dawwp; + awdp->delayed_wakeup.jix = -1; + for (i = 0; i <= erts_no_schedulers; i++) + awdp->delayed_wakeup.sched2jix[i] = -1; + } +#endif } void erts_init_scheduling(int no_schedulers, int no_schedulers_online) { int ix, n, no_ssi; + char *daww_ptr; +#ifdef ERTS_SMP + size_t daww_sz; +#endif init_misc_op_list_alloc(); @@ -3986,6 +4074,15 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) /* Create and initialize scheduler specific data */ +#ifdef ERTS_SMP + daww_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE((sizeof(ErtsDelayedAuxWorkWakeupJob) + + sizeof(int))*(n+1)); + daww_ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, + daww_sz*n); +#else + daww_ptr = NULL; +#endif + erts_aligned_scheduler_data = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, n*sizeof(ErtsAlignedSchedulerData)); @@ -4020,7 +4117,10 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) esdp->run_queue = ERTS_RUNQ_IX(ix); esdp->run_queue->scheduler = esdp; - init_aux_work_data(&esdp->aux_work_data, esdp); + init_aux_work_data(&esdp->aux_work_data, esdp, daww_ptr); +#ifdef ERTS_SMP + daww_ptr += daww_sz; +#endif init_sched_wall_time(&esdp->sched_wall_time); } @@ -4418,7 +4518,9 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); } - aux_work = handle_aux_work(&esdp->aux_work_data, aux_work); + aux_work = handle_aux_work(&esdp->aux_work_data, + aux_work, + 1); if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); } @@ -6660,7 +6762,7 @@ Process *schedule(Process *p, int calls) if (leader_update) erts_thr_progress_leader_update(esdp); if (aux_work) - handle_aux_work(&esdp->aux_work_data, aux_work); + handle_aux_work(&esdp->aux_work_data, aux_work, 0); erts_smp_runq_lock(rq); } } @@ -6673,7 +6775,7 @@ Process *schedule(Process *p, int calls) erts_aint32_t aux_work; aux_work = erts_atomic32_read_acqb(&esdp->ssi->aux_work); if (aux_work) - handle_aux_work(&esdp->aux_work_data, aux_work); + handle_aux_work(&esdp->aux_work_data, aux_work, 0); } #endif /* ERTS_SMP */ |