From b50650b4c56de5279395ea195e9e68b077a1d3de Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 8 Feb 2017 17:57:52 +0100 Subject: Fix aux-work timer implementation --- erts/emulator/beam/erl_init.c | 1 + erts/emulator/beam/erl_process.c | 32 ++++++++++++++++++++++++++------ erts/emulator/beam/erl_process.h | 1 + 3 files changed, 28 insertions(+), 6 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 541bfec532..ac0324d846 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2334,6 +2334,7 @@ erl_start(int argc, char **argv) set_main_stack_size(); erts_sched_init_time_sup(esdp); erts_ets_sched_spec_data_init(esdp); + erts_aux_work_timeout_late_init(esdp); process_main(esdp->x_reg_array, esdp->f_reg_array); } #endif diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 6b5b64993f..e2764cf86f 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -560,7 +560,6 @@ static int stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg); static void aux_work_timeout(void *unused); static void aux_work_timeout_early_init(int no_schedulers); -static void aux_work_timeout_late_init(void); static void setup_aux_work_timer(ErtsSchedulerData *esdp); static int execute_sys_tasks(Process *c_p, @@ -2790,6 +2789,9 @@ typedef struct { int initialized; erts_atomic32_t refc; +#ifdef DEBUG + erts_atomic32_t used; +#endif erts_atomic32_t type[1]; } ErtsAuxWorkTmo; @@ -2799,6 +2801,13 @@ static ERTS_INLINE void start_aux_work_timer(ErtsSchedulerData *esdp) { ErtsMonotonicTime tmo = erts_get_monotonic_time(esdp); +#ifdef DEBUG + Uint no = (Uint) erts_atomic32_xchg_mb(&aux_work_tmo->used, + (erts_aint32_t) esdp->no); + ASSERT(esdp->type == ERTS_SCHED_NORMAL); + ASSERT(!no); +#endif + tmo = ERTS_MONOTONIC_TO_CLKTCKS(tmo-1); tmo += ERTS_MSEC_TO_CLKTCKS(1000) + 1; erts_twheel_init_timer(&aux_work_tmo->timer.data); @@ -2835,16 +2844,19 @@ aux_work_timeout_early_init(int no_schedulers) aux_work_tmo = (ErtsAuxWorkTmo *) p; aux_work_tmo->initialized = 0; erts_atomic32_init_nob(&aux_work_tmo->refc, 0); +#ifdef DEBUG + erts_atomic32_init_nob(&aux_work_tmo->used, 0); +#endif for (i = 0; i <= no_schedulers; i++) erts_atomic32_init_nob(&aux_work_tmo->type[i], 0); } void -aux_work_timeout_late_init(void) +erts_aux_work_timeout_late_init(ErtsSchedulerData *esdp) { aux_work_tmo->initialized = 1; - if (erts_atomic32_read_nob(&aux_work_tmo->refc)) - start_aux_work_timer(erts_get_scheduler_data()); + if (erts_atomic32_read_acqb(&aux_work_tmo->refc)) + start_aux_work_timer(esdp); } static void @@ -2852,6 +2864,13 @@ aux_work_timeout(void *vesdp) { erts_aint32_t refc; int i; +#ifdef DEBUG + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + Uint no = (Uint) erts_atomic32_xchg_mb(&aux_work_tmo->used, 0); + ASSERT(no == esdp->no); + ASSERT(esdp == (ErtsSchedulerData *) vesdp); +#endif + #ifdef ERTS_SMP i = 0; #else @@ -6465,8 +6484,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online /* init port tasks */ erts_port_task_init(); - aux_work_timeout_late_init(); - #ifndef ERTS_SMP #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC erts_scheduler_data->verify_unused_temp_alloc @@ -8752,6 +8769,9 @@ sched_thread_func(void *vesdp) erts_sched_init_time_sup(esdp); + if (no == 1) + erts_aux_work_timeout_late_init(esdp); + (void) ERTS_RUNQ_FLGS_SET_NOB(esdp->run_queue, ERTS_RUNQ_FLG_EXEC); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 883d9f2a4c..f6683b6e7b 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1830,6 +1830,7 @@ void erts_schedule_multi_misc_aux_work(int ignore_self, void (*func)(void *), void *arg); erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int); +void erts_aux_work_timeout_late_init(ErtsSchedulerData *esdp); void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); void erts_init_process(int, int, int); -- cgit v1.2.3 From cb7514ab298c715af435d9a8e5a019e91105cb98 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 30 Jan 2017 15:25:57 +0100 Subject: Remove accessor BIF timer implementation --- erts/emulator/beam/erl_alloc.c | 10 -- erts/emulator/beam/erl_alloc.types | 1 - erts/emulator/beam/erl_hl_timer.c | 206 +------------------------------------ erts/emulator/beam/erl_process.c | 22 ---- erts/emulator/beam/erl_process.h | 3 - 5 files changed, 3 insertions(+), 239 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index a374593c5d..71957b2259 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -678,10 +678,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) = erts_timer_type_size(ERTS_ALC_T_HL_PTIMER); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)] = erts_timer_type_size(ERTS_ALC_T_BIF_TIMER); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_ABIF_TIMER)] - = erts_timer_type_size(ERTS_ALC_T_ABIF_TIMER); -#endif fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_EXP_TRACE)] = sizeof(NifExportTrace); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MREF_NSCHED_ENT)] @@ -2440,12 +2436,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) &size.processes_used, fi, ERTS_ALC_T_BIF_TIMER); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - add_fix_values(&size.processes, - &size.processes_used, - fi, - ERTS_ALC_T_ABIF_TIMER); -#endif add_fix_values(&size.processes, &size.processes_used, fi, diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 43f43f9034..f296a98125 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -168,7 +168,6 @@ type TIMER_SERVICE LONG_LIVED SYSTEM timer_service type LL_PTIMER FIXED_SIZE PROCESSES ll_ptimer type HL_PTIMER FIXED_SIZE PROCESSES hl_ptimer type BIF_TIMER FIXED_SIZE PROCESSES bif_timer -# type ABIF_TIMER FIXED_SIZE PROCESSES accessor_bif_timer type TIMER_REQUEST SHORT_LIVED PROCESSES timer_request type BTM_YIELD_STATE SHORT_LIVED PROCESSES btm_yield_state type REG_TABLE STANDARD SYSTEM reg_tab diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 26be8c7edf..cfa1709518 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -106,9 +106,6 @@ typedef enum { #define ERTS_TMR_ROFLG_PROC (((Uint32) 1) << 14) #define ERTS_TMR_ROFLG_PORT (((Uint32) 1) << 15) #define ERTS_TMR_ROFLG_CALLBACK (((Uint32) 1) << 16) -#ifdef ERTS_BTM_ACCESSOR_SUPPORT -#define ERTS_TMR_ROFLG_ABIF_TMR (((Uint32) 1) << 17) -#endif #define ERTS_TMR_ROFLG_SID_MASK \ (ERTS_TMR_ROFLG_HLT - (Uint32) 1) @@ -185,21 +182,10 @@ struct ErtsHLTimer_ { Eterm message; ErlHeapFragment *bp; } btm; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - struct { - Eterm accessor; - ErtsHLTimerTree tree; - } abtm; -#endif }; #define ERTS_HL_PTIMER_SIZE offsetof(ErtsHLTimer, btm) -#ifdef ERTS_BTM_ACCESSOR_SUPPORT -#define ERTS_BIF_TIMER_SIZE offsetof(ErtsHLTimer, abtm) -#define ERTS_ABIF_TIMER_SIZE sizeof(ErtsHLTimer) -#else #define ERTS_BIF_TIMER_SIZE sizeof(ErtsHLTimer) -#endif typedef struct { ErtsTmrHead head; /* NEED to be first! */ @@ -606,60 +592,6 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #include "erl_rbtree.h" -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - -#define ERTS_RBT_PREFIX abtm -#define ERTS_RBT_T ErtsHLTimer -#define ERTS_RBT_KEY_T Uint32 * -#define ERTS_RBT_FLAGS_T UWord -#define ERTS_RBT_INIT_EMPTY_TNODE(T) \ - do { \ - (T)->abtm.tree.parent = (UWord) NULL; \ - (T)->abtm.tree.right = NULL; \ - (T)->abtm.tree.left = NULL; \ - } while (0) -#define ERTS_RBT_IS_RED(T) \ - ((int) ((T)->abtm.tree.parent & ERTS_HLT_PFLG_RED)) -#define ERTS_RBT_SET_RED(T) \ - ((T)->abtm.tree.parent |= ERTS_HLT_PFLG_RED) -#define ERTS_RBT_IS_BLACK(T) \ - (!ERTS_RBT_IS_RED((T))) -#define ERTS_RBT_SET_BLACK(T) \ - ((T)->abtm.tree.parent &= ~ERTS_HLT_PFLG_RED) -#define ERTS_RBT_GET_FLAGS(T) \ - ((T)->abtm.tree.parent & ERTS_HLT_PFLGS_MASK) -#define ERTS_RBT_SET_FLAGS(T, F) \ - do { \ - ERTS_HLT_ASSERT((((UWord) (F)) & ~ERTS_HLT_PFLGS_MASK) == 0); \ - (T)->abtm.tree.parent &= ~ERTS_HLT_PFLGS_MASK; \ - (T)->abtm.tree.parent |= (F); \ - } while (0) -#define ERTS_RBT_GET_PARENT(T) \ - ((ErtsHLTimer *) ((T)->abtm.tree.parent & ~ERTS_HLT_PFLGS_MASK)) -#define ERTS_RBT_SET_PARENT(T, P) \ - do { \ - ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ - (T)->abtm.tree.parent &= ERTS_HLT_PFLGS_MASK; \ - (T)->abtm.tree.parent |= (UWord) (P); \ - } while (0) -#define ERTS_RBT_GET_RIGHT(T) ((T)->abtm.tree.right) -#define ERTS_RBT_SET_RIGHT(T, R) ((T)->abtm.tree.right = (R)) -#define ERTS_RBT_GET_LEFT(T) ((T)->abtm.tree.left) -#define ERTS_RBT_SET_LEFT(T, L) ((T)->abtm.tree.left = (L)) -#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) -#define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) -#define ERTS_RBT_IS_EQ(KX, KY) \ - (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) -#define ERTS_RBT_WANT_DELETE -#define ERTS_RBT_WANT_INSERT -#define ERTS_RBT_WANT_LOOKUP -#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING -#define ERTS_RBT_UNDEF - -#include "erl_rbtree.h" - -#endif /* ERTS_BTM_ACCESSOR_SUPPORT */ - #ifdef ERTS_SMP static void init_canceled_queue(ErtsHLTCncldTmrQ *cq); #endif @@ -699,9 +631,6 @@ erts_timer_type_size(ErtsAlcType_t type) case ERTS_ALC_T_LL_PTIMER: return sizeof(ErtsTWTimer); case ERTS_ALC_T_HL_PTIMER: return ERTS_HL_PTIMER_SIZE; case ERTS_ALC_T_BIF_TIMER: return ERTS_BIF_TIMER_SIZE; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - case ERTS_ALC_T_ABIF_TIMER: return ERTS_ABIF_TIMER_SIZE; -#endif default: ERTS_INTERNAL_ERROR("Unknown type"); } return 0; @@ -920,10 +849,6 @@ hl_timer_destroy(ErtsHLTimer *tmr) else { if (roflgs & ERTS_TMR_ROFLG_PRE_ALC) bif_timer_pre_free(tmr); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - else if (roflgs & ERTS_TMR_ROFLG_ABIF_TMR) - erts_free(ERTS_ALC_T_ABIF_TIMER, tmr); -#endif else erts_free(ERTS_ALC_T_BIF_TIMER, tmr); } @@ -1018,33 +943,6 @@ check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv) #endif } -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - -static void -hlt_delete_abtm(ErtsHLTimer *tmr) -{ - Process *proc; - - ERTS_HLT_ASSERT(tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR); - - proc = erts_proc_lookup(tmr->abtm.accessor); - - if (proc) { - int deref = 0; - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); - if (tmr->abtm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { - abtm_rbt_delete(&proc->accessor_bif_timers, tmr); - deref = 1; - tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - } - erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); - if (deref) - hl_timer_pre_dec_refc(tmr); - } -} - -#endif - static ErtsHLTimer * create_hl_timer(ErtsSchedulerData *esdp, ErtsMonotonicTime timeout_pos, @@ -1106,9 +1004,6 @@ create_hl_timer(ErtsSchedulerData *esdp, } else { /* ERTS_TMR_BIF */ Uint hsz; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - int is_abif_tmr = is_value(acsr) && acsr != rcvr; -#endif if (short_time) { tmr = bif_timer_pre_alloc(); @@ -1118,14 +1013,8 @@ create_hl_timer(ErtsSchedulerData *esdp, } else { alloc_bif_timer: -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (is_abif_tmr) - tmr = erts_alloc(ERTS_ALC_T_ABIF_TIMER, - ERTS_ABIF_TIMER_SIZE); - else -#endif - tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER, - ERTS_BIF_TIMER_SIZE); + tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER, + ERTS_BIF_TIMER_SIZE); } tmr->timeout = timeout_pos; @@ -1160,23 +1049,6 @@ create_hl_timer(ErtsSchedulerData *esdp, tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (is_abif_tmr) { - Process *aproc; - roflgs |= ERTS_TMR_ROFLG_ABIF_TMR; - tmr->abtm.accessor = acsr; - aproc = erts_proc_lookup(acsr); - if (!aproc) - tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - else { - refc++; - erts_smp_proc_lock(aproc, ERTS_PROC_LOCK_BTM); - abtm_rbt_insert(&aproc->accessor_bif_timers, tmr); - erts_smp_proc_unlock(aproc, ERTS_PROC_LOCK_BTM); - } - } -#endif - btm_rbt_insert(&srv->btm_tree, tmr); } @@ -1222,11 +1094,6 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME); ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR) - hlt_delete_abtm(tmr); -#endif - if (is_reg_name) { Eterm pid; ERTS_HLT_ASSERT(is_atom(tmr->receiver.name)); @@ -1409,10 +1276,6 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; } -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR) - hlt_delete_abtm(tmr); -#endif } if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) { @@ -2032,15 +1895,10 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, /* * Check if the timer is aimed at current - * process of if this process is an accessor - * of the timer... + * process... */ erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_BTM); tmr = proc_btm_rbt_lookup(c_p->bif_timers, trefn); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (!tmr) - tmr = abtm_rbt_lookup(c_p->accessor_bif_timers, trefn); -#endif erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_BTM); if (!tmr) return 0; @@ -2290,13 +2148,6 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info, if (!abs || !bool_arg(tp[2], abs)) return 0; break; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - case am_accessor: - if (!accessor || is_not_internal_pid(tp[2])) - return 0; - *accessor = tp[2]; - break; -#endif default: return 0; } @@ -2358,9 +2209,6 @@ typedef struct { ErtsBifTimers *bif_timers; union { proc_btm_rbt_yield_state_t proc_btm_yield_state; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - abtm_rbt_yield_state_t abtm_yield_state; -#endif } u; } ErtsBifTimerYieldState; @@ -2401,54 +2249,6 @@ int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) return res; } -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - -static void -detach_bif_timer(ErtsHLTimer *tmr, void *vesdp) -{ - tmr->abtm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - hl_timer_dec_refc(tmr, tmr->head.roflgs); -} - -int erts_detach_accessor_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) -{ - ErtsSchedulerData *esdp = erts_proc_sched_data(p); - ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}}; - ErtsBifTimerYieldState *ysp; - int res; - - ysp = (ErtsBifTimerYieldState *) *vyspp; - if (!ysp) - ysp = &ys; - - res = abtm_rbt_foreach_destroy_yielding(&ysp->bif_timers, - detach_bif_timer, - (void *) esdp, - &ysp->u.abtm_yield_state, - ERTS_BTM_MAX_DESTROY_LIMIT); - - if (res == 0) { - if (ysp != &ys) - erts_free(ERTS_ALC_T_BTM_YIELD_STATE, ysp); - *vyspp = NULL; - } - else { - - if (ysp == &ys) { - ysp = erts_alloc(ERTS_ALC_T_BTM_YIELD_STATE, - sizeof(ErtsBifTimerYieldState)); - sys_memcpy((void *) ysp, (void *) &ys, - sizeof(ErtsBifTimerYieldState)); - } - - *vyspp = (void *) ysp; - } - - return res; -} - -#endif /* ERTS_BTM_ACCESSOR_SUPPORT */ - static ERTS_INLINE int parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, ErtsMonotonicTime *conv_arg, int abs, diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index e2764cf86f..cb95eab67e 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -12447,9 +12447,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->msg_inq.len = 0; #endif p->bif_timers = NULL; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - p->accessor_bif_timers = NULL; -#endif p->mbuf = NULL; p->msg_frag = NULL; p->mbuf_sz = 0; @@ -12648,9 +12645,6 @@ void erts_init_empty_process(Process *p) p->msg.save = &p->msg.first; p->msg.len = 0; p->bif_timers = NULL; -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - p->accessor_bif_timers = NULL; -#endif p->dictionary = NULL; p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; @@ -12751,9 +12745,6 @@ erts_debug_verify_clean_empty_process(Process* p) ASSERT(p->msg.first == NULL); ASSERT(p->msg.len == 0); ASSERT(p->bif_timers == NULL); -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - ASSERT(p->accessor_bif_timers == NULL); -#endif ASSERT(p->dictionary == NULL); ASSERT(p->catches == 0); ASSERT(p->cp == NULL); @@ -13827,19 +13818,6 @@ erts_continue_exit_process(Process *p) p->bif_timers = NULL; } -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - if (p->accessor_bif_timers) { - if (erts_detach_accessor_bif_timers(p, - p->accessor_bif_timers, - &p->u.terminate)) { - ASSERT(erts_proc_read_refc(p) > 0); - goto yield; - } - ASSERT(erts_proc_read_refc(p) > 0); - p->accessor_bif_timers = NULL; - } -#endif - #ifdef ERTS_SMP if (p->flags & F_SCHDLR_ONLN_WAITQ) abort_sched_onln_chng_waitq(p); diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index f6683b6e7b..b21597d63b 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1026,9 +1026,6 @@ struct process { ErlMessageQueue msg; /* Message queue */ ErtsBifTimers *bif_timers; /* Bif timers aiming at this process */ -#ifdef ERTS_BTM_ACCESSOR_SUPPORT - ErtsBifTimers *accessor_bif_timers; /* Accessor bif timers */ -#endif ProcDict *dictionary; /* Process dictionary, may be NULL */ -- cgit v1.2.3 From e46671ce6ea9429fe3d6723b4f93c08c7fa85cd0 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 31 Jan 2017 13:30:49 +0100 Subject: Use magic refs for BIF timers --- erts/emulator/beam/erl_hl_timer.c | 757 +++++++++++++++++++++++++++----------- erts/emulator/beam/erl_hl_timer.h | 2 +- erts/emulator/beam/erl_process.c | 2 +- 3 files changed, 553 insertions(+), 208 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index cfa1709518..6d570d28ec 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -29,6 +29,8 @@ # include "config.h" #endif +#define ERTS_MAGIC_REF_BIF_TIMERS + #include "sys.h" #include "global.h" #include "bif.h" @@ -36,6 +38,9 @@ #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" #include "erl_hl_timer.h" +#ifdef ERTS_MAGIC_REF_BIF_TIMERS +#include "erl_binary.h" +#endif #define ERTS_TMR_CHECK_CANCEL_ON_CREATE 0 @@ -124,6 +129,11 @@ typedef struct ErtsHLTimer_ ErtsHLTimer; #define ERTS_HLT_PFIELD_NOT_IN_TABLE (~((UWord) 0)) +typedef struct { + ErtsHLTimer *next; + ErtsHLTimer *prev; +} ErtsHLTimerList; + typedef struct { UWord parent; /* parent pointer and flags... */ union { @@ -140,11 +150,6 @@ typedef struct { } ErtsHLTimerTimeTree; typedef struct { - UWord parent; /* parent pointer and flags... */ - ErtsHLTimer *right; - ErtsHLTimer *left; -} ErtsHLTimerTree; - typedef struct { Uint32 roflgs; erts_smp_atomic32_t refc; @@ -176,9 +181,14 @@ struct ErtsHLTimer_ { /* BIF timer only fields follow... */ struct { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin; + ErtsHLTimerList proc_list; +#else Uint32 refn[ERTS_REF_NUMBERS]; ErtsHLTimerTree proc_tree; ErtsHLTimerTree tree; +#endif Eterm message; ErlHeapFragment *bp; } btm; @@ -282,12 +292,16 @@ struct ErtsHLTimerService_ { ErtsHLTCncldTmrQ canceled_queue; #endif ErtsHLTimer *time_tree; +#ifndef ERTS_MAGIC_REF_BIF_TIMERS ErtsHLTimer *btm_tree; +#endif ErtsHLTimer *next_timeout; ErtsYieldingTimeoutState yield; ErtsTWheelTimer service_timer; }; +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + static ERTS_INLINE int refn_is_lt(Uint32 *x, Uint32 *y) { @@ -303,6 +317,14 @@ refn_is_lt(Uint32 *x, Uint32 *y) return x[0] < y[0]; } +static ERTS_INLINE int +refn_is_eq(Uint32 *x, Uint32 *y) +{ + return (x[0] == y[0]) & (x[1] == y[1]) & (x[2] == y[2]); +} + +#endif + #define ERTS_RBT_PREFIX time #define ERTS_RBT_T ErtsHLTimer #define ERTS_RBT_KEY_T ErtsMonotonicTime @@ -492,6 +514,14 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #endif /* ERTS_HLT_HARD_DEBUG */ +#ifdef ERTS_MAGIC_REF_BIF_TIMERS +#define ERTS_BTM_HLT2REFN(T) ((T)->btm.mbin->refn) +#else +#define ERTS_BTM_HLT2REFN(T) ((T)->btm.refn) +#endif + +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + #define ERTS_RBT_PREFIX btm #define ERTS_RBT_T ErtsHLTimer #define ERTS_RBT_KEY_T Uint32 * @@ -530,18 +560,92 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #define ERTS_RBT_SET_RIGHT(T, R) ((T)->btm.tree.right = (R)) #define ERTS_RBT_GET_LEFT(T) ((T)->btm.tree.left) #define ERTS_RBT_SET_LEFT(T, L) ((T)->btm.tree.left = (L)) -#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) +#define ERTS_RBT_GET_KEY(T) ERTS_BTM_HLT2REFN((T)) #define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) -#define ERTS_RBT_IS_EQ(KX, KY) \ - (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) +#define ERTS_RBT_IS_EQ(KX, KY) refn_is_eq((KX), (KY)) #define ERTS_RBT_WANT_DELETE #define ERTS_RBT_WANT_INSERT +#ifndef ERTS_MAGIC_REF_BIF_TIMERS #define ERTS_RBT_WANT_LOOKUP +#endif #define ERTS_RBT_WANT_FOREACH #define ERTS_RBT_UNDEF #include "erl_rbtree.h" +#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */ + +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static ERTS_INLINE void +proc_btm_list_insert(ErtsHLTimer **list, ErtsHLTimer *x) +{ + ErtsHLTimer *y = *list; + if (!y) { + x->btm.proc_list.next = x; + x->btm.proc_list.prev = x; + *list = x; + } + else { + ERTS_HLT_ASSERT(y->btm.proc_list.prev->btm.proc_list.next == y); + x->btm.proc_list.next = y; + x->btm.proc_list.prev = y->btm.proc_list.prev; + y->btm.proc_list.prev->btm.proc_list.next = x; + y->btm.proc_list.prev = x; + } +} + +static ERTS_INLINE void +proc_btm_list_delete(ErtsHLTimer **list, ErtsHLTimer *x) +{ + ErtsHLTimer *y = *list; + if (y == x && x->btm.proc_list.next == x) { + ERTS_HLT_ASSERT(x->btm.proc_list.prev == x); + *list = NULL; + } + else { + if (y == x) + *list = x->btm.proc_list.next; + ERTS_HLT_ASSERT(x->btm.proc_list.prev->btm.proc_list.next == x); + ERTS_HLT_ASSERT(x->btm.proc_list.next->btm.proc_list.prev == x); + x->btm.proc_list.prev->btm.proc_list.next = x->btm.proc_list.next; + x->btm.proc_list.next->btm.proc_list.prev = x->btm.proc_list.prev; + } + x->btm.proc_list.next = NULL; +} + +static ERTS_INLINE int +proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list, + void (*destroy)(ErtsHLTimer *, void *), + void *arg, + int limit) +{ + int i; + ErtsHLTimer *first, *last; + + first = *list; + if (!first) + return 0; + + last = first->btm.proc_list.prev; + for (i = 0; i < limit; i++) { + ErtsHLTimer *x = last; + last = last->btm.proc_list.prev; + (*destroy)(x, arg); + x->btm.proc_list.next = NULL; + if (x == first) { + *list = NULL; + return 0; + } + } + + last->btm.proc_list.next = first; + first->btm.proc_list.prev = last; + return 1; +} + +#else /* !ERTS_MAGIC_REF_BIF_TIMERS */ + #define ERTS_RBT_PREFIX proc_btm #define ERTS_RBT_T ErtsHLTimer #define ERTS_RBT_KEY_T Uint32 * @@ -580,18 +684,21 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #define ERTS_RBT_SET_RIGHT(T, R) ((T)->btm.proc_tree.right = (R)) #define ERTS_RBT_GET_LEFT(T) ((T)->btm.proc_tree.left) #define ERTS_RBT_SET_LEFT(T, L) ((T)->btm.proc_tree.left = (L)) -#define ERTS_RBT_GET_KEY(T) ((T)->btm.refn) +#define ERTS_RBT_GET_KEY(T) ERTS_BTM_HLT2REFN((T)) #define ERTS_RBT_IS_LT(KX, KY) refn_is_lt((KX), (KY)) -#define ERTS_RBT_IS_EQ(KX, KY) \ - (((KX)[0] == (KY)[0]) & ((KX)[1] == (KY)[1]) & ((KX)[2] == (KY)[2])) +#define ERTS_RBT_IS_EQ(KX, KY) refn_is_eq((KX), (KY)) #define ERTS_RBT_WANT_DELETE #define ERTS_RBT_WANT_INSERT +#ifndef ERTS_MAGIC_REF_BIF_TIMERS #define ERTS_RBT_WANT_LOOKUP +#endif #define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING #define ERTS_RBT_UNDEF #include "erl_rbtree.h" +#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */ + #ifdef ERTS_SMP static void init_canceled_queue(ErtsHLTCncldTmrQ *cq); #endif @@ -612,7 +719,9 @@ erts_create_timer_service(void) srv = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_SERVICE, sizeof(ErtsHLTimerService)); srv->time_tree = NULL; +#ifndef ERTS_MAGIC_REF_BIF_TIMERS srv->btm_tree = NULL; +#endif srv->next_timeout = NULL; srv->yield = init_yield; erts_twheel_init_timer(&srv->service_timer); @@ -847,6 +956,11 @@ hl_timer_destroy(ErtsHLTimer *tmr) if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR)) erts_free(ERTS_ALC_T_HL_PTIMER, tmr); else { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + Binary *bp = (Binary *) tmr->btm.mbin; + if (erts_refc_dectest(&bp->refc, 0) == 0) + erts_bin_free(bp); +#endif if (roflgs & ERTS_TMR_ROFLG_PRE_ALC) bif_timer_pre_free(tmr); else @@ -892,6 +1006,7 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs) * once... */ size = sizeof(ErtsHLTimer); + } erts_schedule_thr_prgr_later_cleanup_op( @@ -943,12 +1058,64 @@ check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv) #endif } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static erts_smp_atomic_t * +mbin_to_btmref__(ErtsMagicBinary *mbin) +{ + return erts_smp_binary_to_magic_indirection((Binary *) mbin); +} + +static ERTS_INLINE void +magic_binary_init(ErtsMagicBinary *mbin, ErtsHLTimer *tmr) +{ + erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); + erts_smp_atomic_init_nob(aptr, (erts_aint_t) tmr); +} + +static ERTS_INLINE ErtsHLTimer * +magic_binary_to_btm(ErtsMagicBinary *mbin) +{ + erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); + ErtsHLTimer *tmr = (ErtsHLTimer *) erts_smp_atomic_read_nob(aptr); + ERTS_HLT_ASSERT(!tmr || tmr->btm.mbin == mbin); + return tmr; +} + +static ERTS_INLINE void +btm_clear_magic_binary(ErtsHLTimer *tmr) +{ + erts_smp_atomic_t *aptr = mbin_to_btmref__(tmr->btm.mbin); +#ifdef ERTS_HLT_DEBUG + erts_aint_t tval = erts_smp_atomic_xchg_nob(aptr, + (erts_aint_t) NULL); + ERTS_HLT_ASSERT(tval == (erts_aint_t) tmr); +#else + erts_smp_atomic_set_nob(aptr, (erts_aint_t) NULL); +#endif + hl_timer_dec_refc(tmr, tmr->head.roflgs); +} + +static int +bif_timer_ref_destructor(Binary *unused) +{ + return 1; +} + +#endif + + static ErtsHLTimer * create_hl_timer(ErtsSchedulerData *esdp, ErtsMonotonicTime timeout_pos, int short_time, ErtsTmrType type, void *rcvrp, Eterm rcvr, Eterm acsr, - Eterm msg, Uint32 *refn, + Eterm msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin, +#else + Uint32 *refn, +#endif void (*callback)(void *), void *arg) { ErtsHLTimerService *srv = esdp->timer_service; @@ -1043,6 +1210,13 @@ create_hl_timer(ErtsSchedulerData *esdp, tmr->btm.message = copy_struct(msg, hsz, &hp, &bp->off_heap); tmr->btm.bp = bp; } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + tmr->btm.mbin = mbin; + erts_refc_inc(&mbin->refc, 1); + magic_binary_init(mbin, tmr); + refc += 1; /* from magic binary... */ + tmr->btm.proc_list.next = NULL; +#else tmr->btm.refn[0] = refn[0]; tmr->btm.refn[1] = refn[1]; tmr->btm.refn[2] = refn[2]; @@ -1050,6 +1224,7 @@ create_hl_timer(ErtsSchedulerData *esdp, tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; btm_rbt_insert(&srv->btm_tree, tmr); +#endif } tmr->head.roflgs = roflgs; @@ -1094,6 +1269,10 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME); ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); +#endif + if (is_reg_name) { Eterm pid; ERTS_HLT_ASSERT(is_atom(tmr->receiver.name)); @@ -1123,11 +1302,18 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) queued_message = 1; proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND; tmr->btm.bp = NULL; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + if (tmr->btm.proc_list.next) { + proc_btm_list_delete(&proc->bif_timers, tmr); + dec_refc = 1; + } +#else if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { proc_btm_rbt_delete(&proc->bif_timers, tmr); tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; dec_refc = 1; } +#endif } if (proc_locks) erts_smp_proc_unlock(proc, proc_locks); @@ -1154,7 +1340,9 @@ hlt_port_timeout(ErtsHLTimer *tmr) static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv) { +#if !defined(ERTS_MAGIC_REF_BIF_TIMERS) || defined(ERTS_HLT_HARD_DEBUG) ErtsHLTimerService *srv = (ErtsHLTimerService *) vsrv; +#endif Uint32 roflgs; erts_aint32_t state; @@ -1186,11 +1374,14 @@ static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv) } tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + +#ifndef ERTS_MAGIC_REF_BIF_TIMERS if ((roflgs & ERTS_TMR_ROFLG_BIF_TMR) && tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { btm_rbt_delete(&srv->btm_tree, tmr); tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; } +#endif ERTS_HLT_HDBG_CHK_SRV(srv); @@ -1269,6 +1460,7 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) ERTS_HLT_HDBG_CHK_SRV(srv); +#ifndef ERTS_MAGIC_REF_BIF_TIMERS if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) { if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { @@ -1277,6 +1469,7 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) } } +#endif if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) { /* Already removed... */ @@ -1613,6 +1806,7 @@ continue_cancel_ptimer(ErtsSchedulerData *esdp, ErtsTimer *tmr) * BIF timer specific */ + Uint erts_bif_timer_memory_size(void) { return (Uint) 0; @@ -1627,41 +1821,59 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, Eterm ref, tmo_msg, *hp; ErtsHLTimer *tmr; ErtsSchedulerData *esdp; - DeclareTmpHeap(tmp_hp, 4, c_p); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + Binary *mbin; +#endif + Eterm tmp_hp[4]; if (is_not_internal_pid(rcvr) && is_not_atom(rcvr)) goto badarg; esdp = erts_proc_sched_data(c_p); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + mbin = erts_create_magic_indirection(bif_timer_ref_destructor); + hp = HAlloc(c_p, ERTS_MAGIC_REF_THING_SIZE); + ref = erts_mk_magic_ref(&hp, &c_p->off_heap, mbin); + ASSERT(erts_get_ref_numbers_thr_id(((ErtsMagicBinary *)mbin)->refn) + == (Uint32) esdp->no); +#else hp = HAlloc(c_p, ERTS_REF_THING_SIZE); ref = erts_sched_make_ref_in_buffer(esdp, hp); - - ASSERT(erts_get_ref_numbers_thr_id( - internal_ref_numbers(ref)) == (Uint32) esdp->no); - - UseTmpHeap(4, c_p); + ASSERT(erts_get_ref_numbers_thr_id(internal_ordinary_ref_numbers(ref)) + == (Uint32) esdp->no); +#endif tmo_msg = wrap ? TUPLE3(tmp_hp, am_timeout, ref, msg) : msg; tmr = create_hl_timer(esdp, timeout_pos, short_time, ERTS_TMR_BIF, NULL, rcvr, acsr, tmo_msg, - internal_ref_numbers(ref), NULL, NULL); - - UnUseTmpHeap(4, c_p); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + (ErtsMagicBinary *) mbin, +#else + internal_ordinary_ref_numbers(ref), +#endif + NULL, NULL); if (is_internal_pid(rcvr)) { Process *proc = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, rcvr, ERTS_PROC_LOCK_BTM, ERTS_P2P_FLG_INC_REFC); if (!proc) { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); +#endif if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); hlt_delete_timer(esdp, tmr); hl_timer_destroy(tmr); } else { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + proc_btm_list_insert(&proc->bif_timers, tmr); +#else proc_btm_rbt_insert(&proc->bif_timers, tmr); +#endif erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); tmr->receiver.proc = proc; } @@ -1689,6 +1901,10 @@ cancel_bif_timer(ErtsHLTimer *tmr) if (state != ERTS_TMR_STATE_ACTIVE) return 0; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); +#endif + if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); @@ -1705,25 +1921,215 @@ cancel_bif_timer(ErtsHLTimer *tmr) * the btm tree by itself (it may be in * the middle of tree destruction). */ +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + if (!ERTS_PROC_IS_EXITING(proc) && tmr->btm.proc_list.next) { + proc_btm_list_delete(&proc->bif_timers, tmr); + res = 1; + } +#else if (!ERTS_PROC_IS_EXITING(proc) && tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { proc_btm_rbt_delete(&proc->bif_timers, tmr); tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; res = 1; } +#endif erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); } return res; } +static ERTS_INLINE Sint64 +access_btm(ErtsHLTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel) +{ + int cncl_res; + Sint64 time_left; + + if (!tmr) + return -1; + + if (!cancel) { + erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state); + if (state == ERTS_TMR_STATE_ACTIVE) + return get_time_left(esdp, tmr->timeout); + return -1; + } + + cncl_res = cancel_bif_timer(tmr); + if (!cncl_res) + return -1; + + time_left = get_time_left(esdp, tmr->timeout); + + if (sid != (Uint32) esdp->no) { + if (cncl_res > 0) + queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); + } + else { + if (cncl_res > 0) + hl_timer_dec_refc(tmr, tmr->head.roflgs); + + hlt_delete_timer(esdp, tmr); + } + + return time_left; +} + +static ERTS_INLINE Eterm +return_info(Process *c_p, Sint64 time_left) +{ + Uint hsz; + Eterm *hp; + + if (time_left < 0) + return am_false; + + if (time_left <= (Sint64) MAX_SMALL) + return make_small((Sint) time_left); + + hsz = ERTS_SINT64_HEAP_SIZE(time_left); + hp = HAlloc(c_p, hsz); + return erts_sint64_to_big(time_left, &hp); +} + +static ERTS_INLINE Eterm +send_async_info(Process *proc, ErtsProcLocks initial_locks, + Eterm tref, int cancel, Sint64 time_left) +{ + ErtsProcLocks locks = initial_locks; + ErtsMessage *mp; + Eterm tag, res, msg, ref; + Uint hsz; + Eterm *hp; + ErlOffHeap *ohp; + + hsz = 4; + hsz += NC_HEAP_SIZE(tref); + + if (time_left > (Sint64) MAX_SMALL) + hsz += ERTS_SINT64_HEAP_SIZE(time_left); + + mp = erts_alloc_message_heap(proc, &locks, hsz, &hp, &ohp); + + if (cancel) + tag = am_cancel_timer; + else + tag = am_read_timer; + + ref = STORE_NC(&hp, ohp, tref); + + if (time_left < 0) + res = am_false; + else if (time_left <= (Sint64) MAX_SMALL) + res = make_small((Sint) time_left); + else + res = erts_sint64_to_big(time_left, &hp); + + msg = TUPLE3(hp, tag, ref, res); + + erts_queue_message(proc, locks, mp, msg, am_clock_service); + + locks &= ~initial_locks; + if (locks) + erts_smp_proc_unlock(proc, locks); + + return am_ok; +} + +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static BIF_RETTYPE +access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) +{ + BIF_RETTYPE ret; + Eterm res; + Sint64 time_left; + + if (!is_internal_magic_ref(tref)) { + if (is_not_ref(tref)) { + ERTS_BIF_PREP_ERROR(ret, c_p, BADARG); + return ret; + } + time_left = -1; + } + else { + ErtsMagicBinary *mbin; + mbin = (ErtsMagicBinary *) erts_magic_ref2bin(tref); + if (mbin->destructor != bif_timer_ref_destructor) + time_left = -1; + else { + ErtsHLTimer *tmr; + Uint32 sid; + tmr = magic_binary_to_btm(mbin); + sid = erts_get_ref_numbers_thr_id(internal_magic_ref_numbers(tref)); + ASSERT(1 <= sid && sid <= erts_no_schedulers); + time_left = access_btm(tmr, sid, erts_proc_sched_data(c_p), cancel); + } + } + + if (!info) + res = am_ok; + else if (!async) + res = return_info(c_p, time_left); + else + res = send_async_info(c_p, ERTS_PROC_LOCK_MAIN, + tref, cancel, time_left); + + ERTS_BIF_PREP_RET(ret, res); + + return ret; +} + +#else /* !ERTS_MAGIC_REF_BIF_TIMERS */ + +static ERTS_INLINE Eterm +send_sync_info(Process *proc, ErtsProcLocks initial_locks, + Uint32 *refn, int cancel, Sint64 time_left) +{ + ErtsProcLocks locks = initial_locks; + ErtsMessage *mp; + Eterm res, msg, ref; + Uint hsz; + Eterm *hp; + ErlOffHeap *ohp; + + hsz = 3 + ERTS_REF_THING_SIZE; + + if (time_left > (Sint64) MAX_SMALL) + hsz += ERTS_SINT64_HEAP_SIZE(time_left); + + mp = erts_alloc_message_heap(proc, &locks, hsz, &hp, &ohp); + + write_ref_thing(hp, refn[0], refn[1], refn[2]); + ref = make_internal_ref(hp); + hp += ERTS_REF_THING_SIZE; + + if (time_left < 0) + res = am_false; + else if (time_left <= (Sint64) MAX_SMALL) + res = make_small((Sint) time_left); + else + res = erts_sint64_to_big(time_left, &hp); + + msg = TUPLE2(hp, ref, res); + + erts_queue_message(proc, locks, mp, msg, am_clock_service); + + locks &= ~initial_locks; + if (locks) + erts_smp_proc_unlock(proc, locks); + + return am_ok; +} + static ERTS_INLINE Eterm access_sched_local_btm(Process *c_p, Eterm pid, - Eterm tref, Uint32 *trefn, - Uint32 *rrefn, - int async, int cancel, - int return_res, - int info) + Eterm tref, Uint32 *trefn, + Uint32 *rrefn, + int async, int cancel, + int return_res, + int info) { ErtsSchedulerData *esdp; ErtsHLTimerService *srv; @@ -1747,111 +2153,40 @@ access_sched_local_btm(Process *c_p, Eterm pid, srv = esdp->timer_service; tmr = btm_rbt_lookup(srv->btm_tree, trefn); - if (tmr) { - if (!cancel) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state); - if (state == ERTS_TMR_STATE_ACTIVE) - time_left = get_time_left(esdp, tmr->timeout); - } - else { - int cncl_res = cancel_bif_timer(tmr); - if (cncl_res) { - - time_left = get_time_left(esdp, tmr->timeout); - if (cncl_res > 0) - hl_timer_dec_refc(tmr, tmr->head.roflgs); - - hlt_delete_timer(esdp, tmr); - } - } - } + time_left = access_btm(tmr, (Uint32) esdp->no, esdp, cancel); if (!info) - return am_ok; - - if (return_res) { - ERTS_HLT_ASSERT(c_p); - if (time_left < 0) - return am_false; - else if (time_left <= (Sint64) MAX_SMALL) - return make_small((Sint) time_left); - else { - Uint hsz = ERTS_SINT64_HEAP_SIZE(time_left); - Eterm *hp = HAlloc(c_p, hsz); - return erts_sint64_to_big(time_left, &hp); - } - } + return am_ok; if (c_p) { - proc = c_p; - proc_locks = ERTS_PROC_LOCK_MAIN; + proc = c_p; + proc_locks = ERTS_PROC_LOCK_MAIN; } else { - proc = erts_proc_lookup(pid); - proc_locks = 0; + proc = erts_proc_lookup(pid); + proc_locks = 0; } - if (proc) { - Uint hsz; - ErtsMessage *mp; - Eterm *hp, msg, ref, result; - ErlOffHeap *ohp; - Uint32 *refn; -#ifdef ERTS_HLT_DEBUG - Eterm *hp_end; -#endif - - hsz = ERTS_REF_THING_SIZE; - if (async) { - refn = trefn; /* timer ref */ - hsz += 4; /* 3-tuple */ - } - else { - refn = rrefn; /* request ref */ - hsz += 3; /* 2-tuple */ - } - - ERTS_HLT_ASSERT(refn); - - if (time_left > (Sint64) MAX_SMALL) - hsz += ERTS_SINT64_HEAP_SIZE(time_left); - - mp = erts_alloc_message_heap(proc, &proc_locks, - hsz, &hp, &ohp); - -#ifdef ERTS_HLT_DEBUG - hp_end = hp + hsz; -#endif - - if (time_left < 0) - result = am_false; - else if (time_left <= (Sint64) MAX_SMALL) - result = make_small((Sint) time_left); - else - result = erts_sint64_to_big(time_left, &hp); - - write_ref_thing(hp, - refn[0], - refn[1], - refn[2]); - ref = make_internal_ref(hp); - hp += ERTS_REF_THING_SIZE; - - msg = (async - ? TUPLE3(hp, (cancel - ? am_cancel_timer - : am_read_timer), ref, result) - : TUPLE2(hp, ref, result)); - - ERTS_HLT_ASSERT(hp + (async ? 4 : 3) == hp_end); - - erts_queue_message(proc, proc_locks, mp, msg, am_clock_service); - - if (c_p) - proc_locks &= ~ERTS_PROC_LOCK_MAIN; - if (proc_locks) - erts_smp_proc_unlock(proc, proc_locks); + if (!async) { + if (c_p) + return return_info(c_p, time_left); + + if (proc) + return send_sync_info(proc, proc_locks, + rrefn, cancel, time_left); + } + else if (proc) { + Eterm ref; + Eterm heap[ERTS_REF_THING_SIZE]; + if (is_value(tref)) + ref = tref; + else { + write_ref_thing(&heap[0], trefn[0], trefn[1], trefn[2]); + ref = make_internal_ref(&heap[0]); + } + return send_async_info(proc, proc_locks, + ref, cancel, time_left); } return am_ok; @@ -1884,7 +2219,7 @@ bif_timer_access_request(void *vreq) static int try_access_sched_remote_btm(ErtsSchedulerData *esdp, Process *c_p, Uint32 sid, - Uint32 *trefn, + Eterm tref, Uint32 *trefn, int async, int cancel, int info, Eterm *resp) { @@ -1903,84 +2238,45 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, if (!tmr) return 0; - if (!cancel) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state); - if (state == ERTS_TMR_STATE_ACTIVE) - time_left = get_time_left(esdp, tmr->timeout); - else - time_left = -1; - } - else { - int cncl_res = cancel_bif_timer(tmr); - if (!cncl_res) - time_left = -1; - else { - time_left = get_time_left(esdp, tmr->timeout); - if (cncl_res > 0) - queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); - } - } + time_left = access_btm(tmr, sid, esdp, cancel); - if (!info) { + if (!info) *resp = am_ok; - return 1; - } - - if (!async) { - if (time_left < 0) - *resp = am_false; - else if (time_left <= (Sint64) MAX_SMALL) - *resp = make_small((Sint) time_left); - else { - Uint hsz = ERTS_SINT64_HEAP_SIZE(time_left); - Eterm *hp = HAlloc(c_p, hsz); - *resp = erts_sint64_to_big(time_left, &hp); - } - } - else { - ErtsMessage *mp; - Eterm tag, res, msg, tref; - Uint hsz; - Eterm *hp; - ErtsProcLocks proc_locks = ERTS_PROC_LOCK_MAIN; - ErlOffHeap *ohp; - - hsz = 4 + ERTS_REF_THING_SIZE; - if (time_left > (Sint64) MAX_SMALL) - hsz += ERTS_SINT64_HEAP_SIZE(time_left); - - mp = erts_alloc_message_heap(c_p, &proc_locks, - hsz, &hp, &ohp); - if (cancel) - tag = am_cancel_timer; - else - tag = am_read_timer; - - write_ref_thing(hp, - trefn[0], - trefn[1], - trefn[2]); - tref = make_internal_ref(hp); - hp += ERTS_REF_THING_SIZE; - - if (time_left < 0) - res = am_false; - else if (time_left <= (Sint64) MAX_SMALL) - res = make_small((Sint) time_left); - else - res = erts_sint64_to_big(time_left, &hp); - - msg = TUPLE3(hp, tag, tref, res); + else if (!async) + *resp = return_info(c_p, time_left); + else + *resp = send_async_info(c_p, ERTS_PROC_LOCK_MAIN, + tref, cancel, time_left); - erts_queue_message(c_p, proc_locks, mp, msg, am_clock_service); + return 1; +} - proc_locks &= ~ERTS_PROC_LOCK_MAIN; - if (proc_locks) - erts_smp_proc_unlock(c_p, proc_locks); +static Eterm +no_timer_result(Process *c_p, Eterm tref, int cancel, int async, int info) +{ + ErtsMessage *mp; + Uint hsz; + Eterm *hp, msg, ref, tag; + ErlOffHeap *ohp; + ErtsProcLocks locks; - *resp = am_ok; - } - return 1; + if (!async) + return am_false; + if (!info) + return am_ok; + + hsz = 4; + hsz += NC_HEAP_SIZE(tref); + locks = ERTS_PROC_LOCK_MAIN; + mp = erts_alloc_message_heap(c_p, &locks, hsz, &hp, &ohp); + ref = STORE_NC(&hp, ohp, tref); + tag = cancel ? am_cancel_timer : am_read_timer; + msg = TUPLE3(hp, tag, ref, am_false); + erts_queue_message(c_p, locks, mp, msg, am_clock_service); + locks &= ~ERTS_PROC_LOCK_MAIN; + if (locks) + erts_smp_proc_unlock(c_p, locks); + return am_ok; } static BIF_RETTYPE @@ -2014,7 +2310,7 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) ERTS_BIF_PREP_RET(ret, res); } else if (try_access_sched_remote_btm(esdp, c_p, - sid, trefn, + sid, tref, trefn, async, cancel, info, &res)) { ERTS_BIF_PREP_RET(ret, res); @@ -2093,11 +2389,11 @@ badarg: return ret; no_timer: - ERTS_BIF_PREP_RET(ret, am_false); - return ret; - + return no_timer_result(c_p, tref, cancel, async, info); } +#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */ + static ERTS_INLINE int bool_arg(Eterm val, int *argp) { @@ -2174,14 +2470,20 @@ exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp) roflgs = tmr->head.roflgs; sid = roflgs & ERTS_TMR_ROFLG_SID_MASK; - ERTS_HLT_ASSERT(sid == erts_get_ref_numbers_thr_id(tmr->btm.refn)); + ERTS_HLT_ASSERT(sid == erts_get_ref_numbers_thr_id(ERTS_BTM_HLT2REFN(tmr))); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ERTS_HLT_ASSERT(tmr->btm.proc_list.next); +#else ERTS_HLT_ASSERT(tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE); - tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; +#endif if (sid == (Uint32) esdp->no) { if (state == ERTS_TMR_STATE_ACTIVE) { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); +#endif if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); hlt_delete_timer(esdp, tmr); @@ -2190,6 +2492,9 @@ exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp) } else { if (state == ERTS_TMR_STATE_ACTIVE) { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); +#endif if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); @@ -2205,17 +2510,29 @@ exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp) # define ERTS_BTM_MAX_DESTROY_LIMIT 50 #endif +#ifndef ERTS_MAGIC_REF_BIF_TIMERS typedef struct { ErtsBifTimers *bif_timers; union { proc_btm_rbt_yield_state_t proc_btm_yield_state; } u; } ErtsBifTimerYieldState; +#endif -int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) +int erts_cancel_bif_timers(Process *p, ErtsBifTimers **btm, void **vyspp) { ErtsSchedulerData *esdp = erts_proc_sched_data(p); - ErtsBifTimerYieldState ys = {btm, {ERTS_RBT_YIELD_STAT_INITER}}; + +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + + return proc_btm_list_foreach_destroy_yielding(btm, + exit_cancel_bif_timer, + (void *) esdp, + ERTS_BTM_MAX_DESTROY_LIMIT); + +#else /* !ERTS_MAGIC_REF_BIF_TIMERS */ + + ErtsBifTimerYieldState ys = {*btm, {ERTS_RBT_YIELD_STAT_INITER}}; ErtsBifTimerYieldState *ysp; int res; @@ -2247,6 +2564,8 @@ int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp) } return res; + +#endif /* !ERTS_MAGIC_REF_BIF_TIMERS */ } static ERTS_INLINE int @@ -2663,6 +2982,11 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp) ErtsMonotonicTime left; Eterm receiver; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + return; +#endif + if (tmr->timeout <= btmp->now) left = 0; else @@ -2698,7 +3022,11 @@ erts_print_bif_timer_info(fmtfn_t to, void *to_arg) for (six = 0; six < erts_no_schedulers; six++) { ErtsHLTimerService *srv = erts_aligned_scheduler_data[six].esd.timer_service; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + time_rbt_foreach(srv->time_tree, btm_print, (void *) &btmp); +#else btm_rbt_foreach(srv->btm_tree, btm_print, (void *) &btmp); +#endif } } @@ -2713,6 +3041,10 @@ typedef struct { static void debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd) { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + return; +#endif if (erts_smp_atomic32_read_nob(&tmr->state) == ERTS_TMR_STATE_ACTIVE) { ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd; (*btmfd->func)(((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) @@ -2743,9 +3075,16 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm, for (six = 0; six < erts_no_schedulers; six++) { ErtsHLTimerService *srv = erts_aligned_scheduler_data[six].esd.timer_service; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + time_rbt_foreach(srv->time_tree, + debug_btm_foreach, + (void *) &btmfd); + +#else btm_rbt_foreach(srv->btm_tree, debug_btm_foreach, (void *) &btmfd); +#endif } } @@ -2868,7 +3207,9 @@ st_hdbg_func(ErtsHLTimer *tmr, void *vhdbg) } ERTS_HLT_ASSERT(tmr->time.tree.u.l.next->time.tree.u.l.prev == tmr); ERTS_HLT_ASSERT(tmr->time.tree.u.l.prev->time.tree.u.l.next == tmr); - ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr); +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, ERTS_BTM_HLT2REFN(tmr)) == tmr); +#endif } static void @@ -2897,8 +3238,10 @@ tt_hdbg_func(ErtsHLTimer *tmr, void *vhdbg) & ~ERTS_HLT_PFLGS_MASK); ERTS_HLT_ASSERT(tmr == prnt); } +#ifndef ERTS_MAGIC_REF_BIF_TIMERS if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) - ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, tmr->btm.refn) == tmr); + ERTS_HLT_ASSERT(btm_rbt_lookup(hdbg->srv->btm_tree, ERTS_BTM_HLT2REFN(tmr)) == tmr); +#endif if (tmr->time.tree.same_time) { ErtsHdbgHLT st_hdbg; st_hdbg.srv = hdbg->srv; @@ -2964,6 +3307,7 @@ hdbg_chk_srv(ErtsHLTimerService *srv) time_rbt_foreach(srv->time_tree, tt_hdbg_func, (void *) &hdbg); ERTS_HLT_ASSERT(hdbg.found_root); } +#ifndef ERTS_MAGIC_REF_BIF_TIMERS if (srv->btm_tree) { ErtsHdbgHLT hdbg; hdbg.srv = srv; @@ -2972,6 +3316,7 @@ hdbg_chk_srv(ErtsHLTimerService *srv) btm_rbt_foreach(srv->btm_tree, bt_hdbg_func, (void *) &hdbg); ERTS_HLT_ASSERT(hdbg.found_root); } +#endif } #endif /* ERTS_HLT_HARD_DEBUG */ diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h index 9cdcd581a0..33cdf217d7 100644 --- a/erts/emulator/beam/erl_hl_timer.h +++ b/erts/emulator/beam/erl_hl_timer.h @@ -56,7 +56,7 @@ void erts_cancel_proc_timer(Process *); void erts_set_port_timer(Port *, Sint64); void erts_cancel_port_timer(Port *); Sint64 erts_read_port_timer(Port *); -int erts_cancel_bif_timers(Process *, ErtsBifTimers *, void **); +int erts_cancel_bif_timers(Process *, ErtsBifTimers **, void **); int erts_detach_accessor_bif_timers(Process *, ErtsBifTimers *, void **); ErtsHLTimerService *erts_create_timer_service(void); void erts_hl_timer_init(void); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index cb95eab67e..5baf533acd 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -13810,7 +13810,7 @@ erts_continue_exit_process(Process *p) ASSERT(erts_proc_read_refc(p) > 0); if (p->bif_timers) { - if (erts_cancel_bif_timers(p, p->bif_timers, &p->u.terminate)) { + if (erts_cancel_bif_timers(p, &p->bif_timers, &p->u.terminate)) { ASSERT(erts_proc_read_refc(p) > 0); goto yield; } -- cgit v1.2.3 From a09f867979d8039434271a2177c90aabced3a255 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 3 Feb 2017 03:00:37 +0100 Subject: Use timer wheel for short BIF timers --- erts/emulator/beam/erl_hl_timer.c | 1055 +++++++++++++++++++++++-------------- erts/emulator/beam/erl_hl_timer.h | 2 +- 2 files changed, 662 insertions(+), 395 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 6d570d28ec..102a739b13 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -29,7 +29,7 @@ # include "config.h" #endif -#define ERTS_MAGIC_REF_BIF_TIMERS +/* #define ERTS_MAGIC_REF_BIF_TIMERS */ #include "sys.h" #include "global.h" @@ -129,10 +129,12 @@ typedef struct ErtsHLTimer_ ErtsHLTimer; #define ERTS_HLT_PFIELD_NOT_IN_TABLE (~((UWord) 0)) +typedef struct ErtsBifTimer_ ErtsBifTimer; + typedef struct { - ErtsHLTimer *next; - ErtsHLTimer *prev; -} ErtsHLTimerList; + ErtsBifTimer *next; + ErtsBifTimer *prev; +} ErtsBifTimerList; typedef struct { UWord parent; /* parent pointer and flags... */ @@ -150,6 +152,11 @@ typedef struct { } ErtsHLTimerTimeTree; typedef struct { + UWord parent; /* parent pointer and flags... */ + ErtsBifTimer *right; + ErtsBifTimer *left; +} ErtsBifTimerTree; + typedef struct { Uint32 roflgs; erts_smp_atomic32_t refc; @@ -176,42 +183,59 @@ struct ErtsHLTimer_ { #ifdef ERTS_HLT_HARD_DEBUG int pending_timeout; #endif +}; - erts_smp_atomic32_t state; +typedef struct { + ErtsTmrHead head; /* NEED to be first! */ + union { + Eterm name; + Process *proc; + Port *port; + void (*callback)(void *); + } receiver; + ErtsTWheelTimer tw_tmr; +} ErtsTWTimer; - /* BIF timer only fields follow... */ +struct ErtsBifTimer_ { + union { + ErtsTmrHead head; + ErtsHLTimer hlt; + ErtsTWTimer twt; + } type; struct { + erts_smp_atomic32_t state; #ifdef ERTS_MAGIC_REF_BIF_TIMERS ErtsMagicBinary *mbin; ErtsHLTimerList proc_list; #else Uint32 refn[ERTS_REF_NUMBERS]; - ErtsHLTimerTree proc_tree; - ErtsHLTimerTree tree; + ErtsBifTimerTree proc_tree; + ErtsBifTimerTree tree; #endif - Eterm message; - ErlHeapFragment *bp; + Eterm message; + ErlHeapFragment *bp; } btm; }; -#define ERTS_HL_PTIMER_SIZE offsetof(ErtsHLTimer, btm) -#define ERTS_BIF_TIMER_SIZE sizeof(ErtsHLTimer) - -typedef struct { - ErtsTmrHead head; /* NEED to be first! */ - union { - void *p; - void (*callback)(void *); - } u; - ErtsTWheelTimer tw_tmr; -} ErtsTWTimer; - typedef union { ErtsTmrHead head; ErtsHLTimer hlt; ErtsTWTimer twt; + ErtsBifTimer btm; } ErtsTimer; +typedef ErtsTimer *(*ErtsCreateTimerFunc)(ErtsSchedulerData *esdp, + ErtsMonotonicTime timeout_pos, + int short_time, ErtsTmrType type, + void *rcvrp, Eterm rcvr, + Eterm msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin, +#else + Uint32 *refn, +#endif + void (*callback)(void *), void *arg); + #ifdef SMALL_MEMORY #define BIF_TIMER_PREALC_SZ 10 #define PTIMER_PREALC_SZ 10 @@ -221,7 +245,7 @@ typedef union { #endif ERTS_SCHED_PREF_PALLOC_IMPL(bif_timer_pre, - ErtsHLTimer, + ErtsBifTimer, BIF_TIMER_PREALC_SZ) ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(tw_timer, @@ -293,7 +317,7 @@ struct ErtsHLTimerService_ { #endif ErtsHLTimer *time_tree; #ifndef ERTS_MAGIC_REF_BIF_TIMERS - ErtsHLTimer *btm_tree; + ErtsBifTimer *btm_tree; #endif ErtsHLTimer *next_timeout; ErtsYieldingTimeoutState yield; @@ -523,7 +547,7 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #ifndef ERTS_MAGIC_REF_BIF_TIMERS #define ERTS_RBT_PREFIX btm -#define ERTS_RBT_T ErtsHLTimer +#define ERTS_RBT_T ErtsBifTimer #define ERTS_RBT_KEY_T Uint32 * #define ERTS_RBT_FLAGS_T UWord #define ERTS_RBT_INIT_EMPTY_TNODE(T) \ @@ -549,7 +573,7 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) (T)->btm.tree.parent |= (F); \ } while (0) #define ERTS_RBT_GET_PARENT(T) \ - ((ErtsHLTimer *) ((T)->btm.tree.parent & ~ERTS_HLT_PFLGS_MASK)) + ((ErtsBifTimer *) ((T)->btm.tree.parent & ~ERTS_HLT_PFLGS_MASK)) #define ERTS_RBT_SET_PARENT(T, P) \ do { \ ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ @@ -578,9 +602,9 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x) #ifdef ERTS_MAGIC_REF_BIF_TIMERS static ERTS_INLINE void -proc_btm_list_insert(ErtsHLTimer **list, ErtsHLTimer *x) +proc_btm_list_insert(ErtsBifTimer **list, ErtsBifTimer *x) { - ErtsHLTimer *y = *list; + ErtsBifTimer *y = *list; if (!y) { x->btm.proc_list.next = x; x->btm.proc_list.prev = x; @@ -596,9 +620,9 @@ proc_btm_list_insert(ErtsHLTimer **list, ErtsHLTimer *x) } static ERTS_INLINE void -proc_btm_list_delete(ErtsHLTimer **list, ErtsHLTimer *x) +proc_btm_list_delete(ErtsBifTimer **list, ErtsBifTimer *x) { - ErtsHLTimer *y = *list; + ErtsBifTimer *y = *list; if (y == x && x->btm.proc_list.next == x) { ERTS_HLT_ASSERT(x->btm.proc_list.prev == x); *list = NULL; @@ -615,13 +639,13 @@ proc_btm_list_delete(ErtsHLTimer **list, ErtsHLTimer *x) } static ERTS_INLINE int -proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list, - void (*destroy)(ErtsHLTimer *, void *), +proc_btm_list_foreach_destroy_yielding(ErtsBifTimer **list, + void (*destroy)(ErtsBifTimer *, void *), void *arg, int limit) { int i; - ErtsHLTimer *first, *last; + ErtsBifTimer *first, *last; first = *list; if (!first) @@ -629,7 +653,7 @@ proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list, last = first->btm.proc_list.prev; for (i = 0; i < limit; i++) { - ErtsHLTimer *x = last; + ErtsBifTimer *x = last; last = last->btm.proc_list.prev; (*destroy)(x, arg); x->btm.proc_list.next = NULL; @@ -647,7 +671,7 @@ proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list, #else /* !ERTS_MAGIC_REF_BIF_TIMERS */ #define ERTS_RBT_PREFIX proc_btm -#define ERTS_RBT_T ErtsHLTimer +#define ERTS_RBT_T ErtsBifTimer #define ERTS_RBT_KEY_T Uint32 * #define ERTS_RBT_FLAGS_T UWord #define ERTS_RBT_INIT_EMPTY_TNODE(T) \ @@ -673,7 +697,7 @@ proc_btm_list_foreach_destroy_yielding(ErtsHLTimer **list, (T)->btm.proc_tree.parent |= (F); \ } while (0) #define ERTS_RBT_GET_PARENT(T) \ - ((ErtsHLTimer *) ((T)->btm.proc_tree.parent & ~ERTS_HLT_PFLGS_MASK)) + ((ErtsBifTimer *) ((T)->btm.proc_tree.parent & ~ERTS_HLT_PFLGS_MASK)) #define ERTS_RBT_SET_PARENT(T, P) \ do { \ ERTS_HLT_ASSERT((((UWord) (P)) & ERTS_HLT_PFLGS_MASK) == 0); \ @@ -738,8 +762,8 @@ erts_timer_type_size(ErtsAlcType_t type) { switch (type) { case ERTS_ALC_T_LL_PTIMER: return sizeof(ErtsTWTimer); - case ERTS_ALC_T_HL_PTIMER: return ERTS_HL_PTIMER_SIZE; - case ERTS_ALC_T_BIF_TIMER: return ERTS_BIF_TIMER_SIZE; + case ERTS_ALC_T_HL_PTIMER: return sizeof(ErtsHLTimer); + case ERTS_ALC_T_BIF_TIMER: return sizeof(ErtsBifTimer); default: ERTS_INTERNAL_ERROR("Unknown type"); } return 0; @@ -798,6 +822,111 @@ port_timeout_common(Port *port, void *tmr) return 0; } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static erts_smp_atomic_t * +mbin_to_btmref__(ErtsMagicBinary *mbin) +{ + return erts_smp_binary_to_magic_indirection((Binary *) mbin); +} + +static ERTS_INLINE void +magic_binary_init(ErtsMagicBinary *mbin, ErtsBifTimer *tmr) +{ + erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); + erts_smp_atomic_init_nob(aptr, (erts_aint_t) tmr); +} + +static ERTS_INLINE ErtsBifTimer * +magic_binary_to_btm(ErtsMagicBinary *mbin) +{ + erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); + ErtsBifTimer *tmr = (ErtsBifTimer *) erts_smp_atomic_read_nob(aptr); + ERTS_HLT_ASSERT(!tmr || tmr->btm.mbin == mbin); + return tmr; +} + +#endif /* ERTS_MAGIC_REF_BIF_TIMERS */ + +static ERTS_INLINE erts_aint_t +init_btm_specifics(ErtsSchedulerData *esdp, + ErtsBifTimer *tmr, Eterm msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin +#else + Uint32 *refn +#endif + ) +{ + Uint hsz = is_immed(msg) ? ((Uint) 0) : size_object(msg); + int refc; + if (!hsz) { + tmr->btm.message = msg; + tmr->btm.bp = NULL; + } + else { + ErlHeapFragment *bp = new_message_buffer(hsz); + Eterm *hp = bp->mem; + tmr->btm.message = copy_struct(msg, hsz, &hp, &bp->off_heap); + tmr->btm.bp = bp; + } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + refc = 1; + tmr->btm.mbin = mbin; + erts_refc_inc(&mbin->refc, 1); + magic_binary_init(mbin, tmr); + tmr->btm.proc_list.next = NULL; +#else + refc = 0; + tmr->btm.refn[0] = refn[0]; + tmr->btm.refn[1] = refn[1]; + tmr->btm.refn[2] = refn[2]; + + tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + + btm_rbt_insert(&esdp->timer_service->btm_tree, tmr); +#endif + + erts_smp_atomic32_init_nob(&tmr->btm.state, ERTS_TMR_STATE_ACTIVE); + return refc; /* refc from magic binary... */ +} + +static void tw_bif_timer_timeout(void *vbtmp); + +static ERTS_INLINE void +timer_destroy(ErtsTimer *tmr, int twt, int btm) +{ + if (!btm) { + if (twt) + tw_timer_free(&tmr->twt); + else + erts_free(ERTS_ALC_T_HL_PTIMER, tmr); + } + else { +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + Binary *bp = (Binary *) tmr->btm.btm.mbin; + if (erts_refc_dectest(&bp->refc, 0) == 0) + erts_bin_free(bp); +#endif + if (tmr->head.roflgs & ERTS_TMR_ROFLG_PRE_ALC) + bif_timer_pre_free(&tmr->btm); + else + erts_free(ERTS_ALC_T_BIF_TIMER, &tmr->btm); + } +} + +static ERTS_INLINE void +timer_pre_dec_refc(ErtsTimer *tmr) +{ +#ifdef ERTS_HLT_DEBUG + erts_aint_t refc; + refc = erts_smp_atomic32_dec_read_nob(&tmr->head.refc); + ERTS_HLT_ASSERT(refc > 0); +#else + erts_smp_atomic32_dec_nob(&tmr->head.refc); +#endif +} + /* * Basic timer wheel timer stuff */ @@ -805,26 +934,39 @@ port_timeout_common(Port *port, void *tmr) static void scheduled_tw_timer_destroy(void *vtmr) { - tw_timer_free((ErtsTWTimer *) vtmr); + ErtsTimer * tmr = (ErtsTimer *) vtmr; + int btm = !!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR); + timer_destroy((ErtsTimer *) vtmr, 1, btm); } static void schedule_tw_timer_destroy(ErtsTWTimer *tmr) { + Uint size; /* * Reference to process/port can be * dropped at once... */ if (tmr->head.roflgs & ERTS_TMR_ROFLG_PROC) - erts_proc_dec_refc((Process *) tmr->u.p); + erts_proc_dec_refc(tmr->receiver.proc); else if (tmr->head.roflgs & ERTS_TMR_ROFLG_PORT) - erts_port_dec_refc((Port *) tmr->u.p); + erts_port_dec_refc(tmr->receiver.port); + + if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + size = sizeof(ErtsHLTimer); + else { + /* Message buffer already dropped... */ + size = sizeof(ErtsBifTimer); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + size += sizeof(ErtsMagicIndirectionWord); +#endif + } erts_schedule_thr_prgr_later_cleanup_op( scheduled_tw_timer_destroy, (void *) tmr, &tmr->tw_tmr.u.cleanup, - sizeof(ErtsTWTimer)); + size); } static ERTS_INLINE void @@ -840,7 +982,7 @@ static void tw_proc_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - Process *proc = (Process *) twtp->u.p; + Process *proc = twtp->receiver.proc; if (proc_timeout_common(proc, vtwtp)) tw_timer_dec_refc(twtp); tw_timer_dec_refc(twtp); @@ -850,7 +992,7 @@ static void tw_port_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - Port *port = (Port *) twtp->u.p; + Port *port = twtp->receiver.port; if (port_timeout_common(port, vtwtp)) tw_timer_dec_refc(twtp); tw_timer_dec_refc(twtp); @@ -874,53 +1016,78 @@ static void tw_callback_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - void (*callback)(void *) = twtp->u.callback; + void (*callback)(void *) = twtp->receiver.callback; void *arg = twtp->head.u.arg; tw_timer_dec_refc(twtp); (*callback)(arg); } -static ErtsTWTimer * -create_tw_timer(ErtsSchedulerData *esdp, - ErtsTmrType type, void *p, - void (*callback)(void *), void *arg, - ErtsMonotonicTime timeout_pos) +static ErtsTimer * +create_tw_timer(ErtsSchedulerData *esdp, + ErtsMonotonicTime timeout_pos, + int short_time, ErtsTmrType type, + void *rcvrp, Eterm rcvr, + Eterm msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsMagicBinary *mbin, +#else + Uint32 *refn, +#endif + void (*callback)(void *), void *arg) { ErtsTWTimer *tmr; void (*timeout_func)(void *); void (*cancel_func)(void *); erts_aint32_t refc; - tmr = tw_timer_alloc(); - erts_twheel_init_timer(&tmr->tw_tmr); + if (type != ERTS_TMR_BIF) { + tmr = tw_timer_alloc(); + tmr->head.roflgs = 0; + } + else { + if (short_time) { + tmr = (ErtsTWTimer *) bif_timer_pre_alloc(); + if (!tmr) + goto alloc_bif_timer; + tmr->head.roflgs = (ERTS_TMR_ROFLG_BIF_TMR + | ERTS_TMR_ROFLG_PRE_ALC); + } + else { + alloc_bif_timer: + tmr = (ErtsTWTimer *) erts_alloc(ERTS_ALC_T_BIF_TIMER, + sizeof(ErtsBifTimer)); + tmr->head.roflgs = ERTS_TMR_ROFLG_BIF_TMR; + } + } - tmr->head.roflgs = (Uint32) esdp->no; - ERTS_HLT_ASSERT((tmr->head.roflgs - & ~ERTS_TMR_ROFLG_SID_MASK) == 0); + erts_twheel_init_timer(&tmr->tw_tmr); + tmr->head.roflgs |= (Uint32) esdp->no; + ERTS_HLT_ASSERT((((Uint32) esdp->no) + & ~ERTS_TMR_ROFLG_SID_MASK) == 0); switch (type) { case ERTS_TMR_PROC: - tmr->u.p = p; + tmr->receiver.proc = (Process *) rcvrp; tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; timeout_func = tw_proc_timeout; cancel_func = tw_ptimer_cancel; - erts_proc_inc_refc((Process *) p); + erts_proc_inc_refc((Process *) rcvrp); refc = 2; break; case ERTS_TMR_PORT: - tmr->u.p = p; + tmr->receiver.port = (Port *) rcvrp; tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT; timeout_func = tw_port_timeout; cancel_func = tw_ptimer_cancel; - erts_port_inc_refc((Port *) p); + erts_port_inc_refc((Port *) rcvrp); refc = 2; break; case ERTS_TMR_CALLBACK: tmr->head.u.arg = arg; - tmr->u.callback = callback; + tmr->receiver.callback = callback; tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK; timeout_func = tw_callback_timeout; @@ -928,6 +1095,33 @@ create_tw_timer(ErtsSchedulerData *esdp, refc = 1; break; + case ERTS_TMR_BIF: + + timeout_func = tw_bif_timer_timeout; + cancel_func = NULL; + if (is_internal_pid(rcvr)) { + tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; + tmr->receiver.proc = (Process *) rcvrp; + refc = 2; + } + else { + ERTS_HLT_ASSERT(is_atom(rcvr)); + tmr->head.roflgs |= ERTS_TMR_ROFLG_REG_NAME; + tmr->receiver.name = (Eterm) rcvr; + refc = 1; + } + + refc += init_btm_specifics(esdp, + (ErtsBifTimer *) tmr, + msg, +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + mbin +#else + refn +#endif + ); + break; + default: ERTS_INTERNAL_ERROR("Unsupported timer type"); return NULL; @@ -942,36 +1136,19 @@ create_tw_timer(ErtsSchedulerData *esdp, tmr, timeout_pos); - return tmr; + return (ErtsTimer *) tmr; } /* * Basic high level timer stuff */ -static ERTS_INLINE void -hl_timer_destroy(ErtsHLTimer *tmr) -{ - Uint32 roflgs = tmr->head.roflgs; - if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR)) - erts_free(ERTS_ALC_T_HL_PTIMER, tmr); - else { -#ifdef ERTS_MAGIC_REF_BIF_TIMERS - Binary *bp = (Binary *) tmr->btm.mbin; - if (erts_refc_dectest(&bp->refc, 0) == 0) - erts_bin_free(bp); -#endif - if (roflgs & ERTS_TMR_ROFLG_PRE_ALC) - bif_timer_pre_free(tmr); - else - erts_free(ERTS_ALC_T_BIF_TIMER, tmr); - } -} - static void scheduled_hl_timer_destroy(void *vtmr) { - hl_timer_destroy((ErtsHLTimer *) vtmr); + ErtsTimer * tmr = (ErtsTimer *) vtmr; + int btm = !!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR); + timer_destroy((ErtsTimer *) vtmr, 0, btm); } static void @@ -999,14 +1176,13 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs) } if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR)) - size = ERTS_HL_PTIMER_SIZE; - else { - /* - * Message buffer can be dropped at - * once... - */ size = sizeof(ErtsHLTimer); - + else { + /* Message buffer already dropped... */ + size = sizeof(ErtsBifTimer); +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + size += sizeof(ErtsMagicIndirectionWord); +#endif } erts_schedule_thr_prgr_later_cleanup_op( @@ -1014,18 +1190,6 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs) &tmr->time.cleanup, size); } -static ERTS_INLINE void -hl_timer_pre_dec_refc(ErtsHLTimer *tmr) -{ -#ifdef ERTS_HLT_DEBUG - erts_aint_t refc; - refc = erts_smp_atomic32_dec_read_nob(&tmr->head.refc); - ERTS_HLT_ASSERT(refc > 0); -#else - erts_smp_atomic32_dec_nob(&tmr->head.refc); -#endif -} - static ERTS_INLINE void hl_timer_dec_refc(ErtsHLTimer *tmr, Uint32 roflgs) { @@ -1060,32 +1224,17 @@ check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv) #ifdef ERTS_MAGIC_REF_BIF_TIMERS -static erts_smp_atomic_t * -mbin_to_btmref__(ErtsMagicBinary *mbin) -{ - return erts_smp_binary_to_magic_indirection((Binary *) mbin); -} - -static ERTS_INLINE void -magic_binary_init(ErtsMagicBinary *mbin, ErtsHLTimer *tmr) -{ - erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); - erts_smp_atomic_init_nob(aptr, (erts_aint_t) tmr); -} - -static ERTS_INLINE ErtsHLTimer * -magic_binary_to_btm(ErtsMagicBinary *mbin) +static int +bif_timer_ref_destructor(Binary *unused) { - erts_smp_atomic_t *aptr = mbin_to_btmref__(mbin); - ErtsHLTimer *tmr = (ErtsHLTimer *) erts_smp_atomic_read_nob(aptr); - ERTS_HLT_ASSERT(!tmr || tmr->btm.mbin == mbin); - return tmr; + return 1; } static ERTS_INLINE void -btm_clear_magic_binary(ErtsHLTimer *tmr) +btm_clear_magic_binary(ErtsBifTimer *tmr) { erts_smp_atomic_t *aptr = mbin_to_btmref__(tmr->btm.mbin); + Uint32 roflgs = tmr->type.head.roflgs; #ifdef ERTS_HLT_DEBUG erts_aint_t tval = erts_smp_atomic_xchg_nob(aptr, (erts_aint_t) NULL); @@ -1093,23 +1242,117 @@ btm_clear_magic_binary(ErtsHLTimer *tmr) #else erts_smp_atomic_set_nob(aptr, (erts_aint_t) NULL); #endif - hl_timer_dec_refc(tmr, tmr->head.roflgs); + if (roflgs & ERTS_TMR_ROFLG_HLT) + hl_timer_dec_refc(&tmr->type.hlt, roflgs); + else + tw_timer_dec_refc(&tmr->type.twt); } -static int -bif_timer_ref_destructor(Binary *unused) +#endif /* ERTS_MAGIC_REF_BIF_TIMERS */ + +static ERTS_INLINE void +bif_timer_timeout(ErtsHLTimerService *srv, ErtsBifTimer *tmr, + Uint32 roflgs, int is_hlt) { - return 1; -} + erts_aint32_t state; + + ERTS_HLT_ASSERT(tmr->type.head.roflgs == roflgs); + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); + ERTS_HLT_ASSERT(!!is_hlt == !!(roflgs & ERTS_TMR_ROFLG_HLT)); + + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state, + ERTS_TMR_STATE_TIMED_OUT, + ERTS_TMR_STATE_ACTIVE); + ERTS_HLT_ASSERT(state == ERTS_TMR_STATE_CANCELED + || state == ERTS_TMR_STATE_ACTIVE); + + if (state == ERTS_TMR_STATE_ACTIVE) { + Process *proc; + int dec_refc = 0; + Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME); + +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + btm_clear_magic_binary(tmr); #endif + if (is_reg_name) { + Eterm term; + if (is_hlt) + term = tmr->type.hlt.receiver.name; + else + term = tmr->type.twt.receiver.name; + ERTS_HLT_ASSERT(is_atom(term)); + term = erts_whereis_name_to_id(NULL, term); + proc = erts_proc_lookup(term); + } + else { + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PROC); + if (is_hlt) + proc = tmr->type.hlt.receiver.proc; + else + proc = tmr->type.twt.receiver.proc; + ERTS_HLT_ASSERT(proc); + } + if (proc) { + if (!ERTS_PROC_IS_EXITING(proc)) { + ErtsMessage *mp; + erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); + mp = erts_alloc_message(0, NULL); + mp->data.heap_frag = tmr->btm.bp; + erts_queue_message(proc, ERTS_PROC_LOCK_BTM, mp, + tmr->btm.message, am_clock_service); + tmr->btm.bp = NULL; +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + if (tmr->btm.proc_list.next) { + proc_btm_list_delete(&proc->bif_timers, tmr); + dec_refc = 1; + } +#else + if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + proc_btm_rbt_delete(&proc->bif_timers, tmr); + tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + dec_refc = 1; + } +#endif + erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + } + if (dec_refc) + timer_pre_dec_refc((ErtsTimer *) tmr); + } + if (tmr->btm.bp) + free_message_buffer(tmr->btm.bp); + } + +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&srv->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } +#endif + + +} + +static void +tw_bif_timer_timeout(void *vbtmp) +{ +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsHLTimerService *srv = NULL; +#else + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsHLTimerService *srv = esdp->timer_service; +#endif + ErtsBifTimer *btmp = (ErtsBifTimer *) vbtmp; + bif_timer_timeout(srv, btmp, btmp->type.head.roflgs, 0); + tw_timer_dec_refc(&btmp->type.twt); +} -static ErtsHLTimer * +static ErtsTimer * create_hl_timer(ErtsSchedulerData *esdp, ErtsMonotonicTime timeout_pos, int short_time, ErtsTmrType type, - void *rcvrp, Eterm rcvr, Eterm acsr, + void *rcvrp, Eterm rcvr, Eterm msg, #ifdef ERTS_MAGIC_REF_BIF_TIMERS ErtsMagicBinary *mbin, @@ -1134,7 +1377,7 @@ create_hl_timer(ErtsSchedulerData *esdp, if (type != ERTS_TMR_BIF) { tmr = erts_alloc(ERTS_ALC_T_HL_PTIMER, - ERTS_HL_PTIMER_SIZE); + sizeof(ErtsHLTimer)); tmr->timeout = timeout_pos; switch (type) { @@ -1170,19 +1413,18 @@ create_hl_timer(ErtsSchedulerData *esdp, } else { /* ERTS_TMR_BIF */ - Uint hsz; if (short_time) { - tmr = bif_timer_pre_alloc(); + tmr = (ErtsHLTimer *) bif_timer_pre_alloc(); if (!tmr) goto alloc_bif_timer; roflgs |= ERTS_TMR_ROFLG_PRE_ALC; } else { alloc_bif_timer: - tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER, - ERTS_BIF_TIMER_SIZE); - } + tmr = (ErtsHLTimer *) erts_alloc(ERTS_ALC_T_BIF_TIMER, + sizeof(ErtsBifTimer)); + } tmr->timeout = timeout_pos; @@ -1199,37 +1441,19 @@ create_hl_timer(ErtsSchedulerData *esdp, refc = 1; } - hsz = is_immed(msg) ? ((Uint) 0) : size_object(msg); - if (!hsz) { - tmr->btm.message = msg; - tmr->btm.bp = NULL; - } - else { - ErlHeapFragment *bp = new_message_buffer(hsz); - Eterm *hp = bp->mem; - tmr->btm.message = copy_struct(msg, hsz, &hp, &bp->off_heap); - tmr->btm.bp = bp; - } + refc += init_btm_specifics(esdp, + (ErtsBifTimer *) tmr, + msg, #ifdef ERTS_MAGIC_REF_BIF_TIMERS - tmr->btm.mbin = mbin; - erts_refc_inc(&mbin->refc, 1); - magic_binary_init(mbin, tmr); - refc += 1; /* from magic binary... */ - tmr->btm.proc_list.next = NULL; + mbin #else - tmr->btm.refn[0] = refn[0]; - tmr->btm.refn[1] = refn[1]; - tmr->btm.refn[2] = refn[2]; - - tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - - btm_rbt_insert(&srv->btm_tree, tmr); + refn #endif + ); } tmr->head.roflgs = roflgs; erts_smp_atomic32_init_nob(&tmr->head.refc, refc); - erts_smp_atomic32_init_nob(&tmr->state, ERTS_TMR_STATE_ACTIVE); if (!srv->next_timeout || tmr->timeout < srv->next_timeout->timeout) { @@ -1256,72 +1480,7 @@ create_hl_timer(ErtsSchedulerData *esdp, ERTS_HLT_HDBG_CHK_SRV(srv); - return tmr; -} - -static ERTS_INLINE void -hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs) -{ - ErtsProcLocks proc_locks = ERTS_PROC_LOCKS_MSG_SEND; - Process *proc; - int queued_message = 0; - int dec_refc = 0; - Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME); - ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); - -#ifdef ERTS_MAGIC_REF_BIF_TIMERS - btm_clear_magic_binary(tmr); -#endif - - if (is_reg_name) { - Eterm pid; - ERTS_HLT_ASSERT(is_atom(tmr->receiver.name)); - pid = erts_whereis_name_to_id(NULL, tmr->receiver.name); - proc = erts_proc_lookup(pid); - } - else { - ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PROC); - ERTS_HLT_ASSERT(tmr->receiver.proc); - - proc = tmr->receiver.proc; - proc_locks |= ERTS_PROC_LOCK_BTM; - } - if (proc) { - erts_smp_proc_lock(proc, proc_locks); - /* - * If process is exiting, let it clean up - * the btm tree by itself (it may be in - * the middle of tree destruction). - */ - if (!ERTS_PROC_IS_EXITING(proc)) { - ErtsMessage *mp = erts_alloc_message(0, NULL); - mp->data.heap_frag = tmr->btm.bp; - erts_queue_message(proc, proc_locks, mp, - tmr->btm.message, am_clock_service); - erts_smp_proc_unlock(proc, ERTS_PROC_LOCKS_MSG_SEND); - queued_message = 1; - proc_locks &= ~ERTS_PROC_LOCKS_MSG_SEND; - tmr->btm.bp = NULL; -#ifdef ERTS_MAGIC_REF_BIF_TIMERS - if (tmr->btm.proc_list.next) { - proc_btm_list_delete(&proc->bif_timers, tmr); - dec_refc = 1; - } -#else - if (tmr->btm.proc_tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { - proc_btm_rbt_delete(&proc->bif_timers, tmr); - tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - dec_refc = 1; - } -#endif - } - if (proc_locks) - erts_smp_proc_unlock(proc, proc_locks); - if (dec_refc) - hl_timer_pre_dec_refc(tmr); - } - if (!queued_message && tmr->btm.bp) - free_message_buffer(tmr->btm.bp); + return (ErtsTimer *) tmr; } static ERTS_INLINE void @@ -1340,49 +1499,27 @@ hlt_port_timeout(ErtsHLTimer *tmr) static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv) { -#if !defined(ERTS_MAGIC_REF_BIF_TIMERS) || defined(ERTS_HLT_HARD_DEBUG) ErtsHLTimerService *srv = (ErtsHLTimerService *) vsrv; -#endif Uint32 roflgs; - erts_aint32_t state; ERTS_HLT_HDBG_CHK_SRV(srv); roflgs = tmr->head.roflgs; ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_HLT); - state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, - ERTS_TMR_STATE_TIMED_OUT, - ERTS_TMR_STATE_ACTIVE); - - ERTS_HLT_ASSERT(state == ERTS_TMR_STATE_CANCELED - || state == ERTS_TMR_STATE_ACTIVE); - - if (state == ERTS_TMR_STATE_ACTIVE) { - - if (roflgs & ERTS_TMR_ROFLG_BIF_TMR) - hlt_bif_timer_timeout(tmr, roflgs); - else if (roflgs & ERTS_TMR_ROFLG_PROC) - hlt_proc_timeout(tmr); - else if (roflgs & ERTS_TMR_ROFLG_PORT) - hlt_port_timeout(tmr); - else { - ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK); - (*tmr->receiver.callback)(tmr->head.u.arg); - } - + if (roflgs & ERTS_TMR_ROFLG_BIF_TMR) + bif_timer_timeout(srv, (ErtsBifTimer *) tmr, roflgs, 1); + else if (roflgs & ERTS_TMR_ROFLG_PROC) + hlt_proc_timeout(tmr); + else if (roflgs & ERTS_TMR_ROFLG_PORT) + hlt_port_timeout(tmr); + else { + ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK); + (*tmr->receiver.callback)(tmr->head.u.arg); } tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; -#ifndef ERTS_MAGIC_REF_BIF_TIMERS - if ((roflgs & ERTS_TMR_ROFLG_BIF_TMR) - && tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { - btm_rbt_delete(&srv->btm_tree, tmr); - tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - } -#endif - ERTS_HLT_HDBG_CHK_SRV(srv); hl_timer_dec_refc(tmr, roflgs); @@ -1460,17 +1597,6 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) ERTS_HLT_HDBG_CHK_SRV(srv); -#ifndef ERTS_MAGIC_REF_BIF_TIMERS - if (tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR) { - - if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { - btm_rbt_delete(&srv->btm_tree, tmr); - tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; - } - - } -#endif - if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) { /* Already removed... */ ERTS_HLT_HDBG_CHK_SRV(srv); @@ -1541,6 +1667,17 @@ cleanup_sched_local_canceled_timer(ErtsSchedulerData *esdp, ERTS_HLT_ASSERT(esdp == erts_get_scheduler_data()); ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK) == (Uint32) esdp->no); + +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + if (roflgs & ERTS_TMR_ROFLG_BIF_TMR) { + ErtsBifTimer *btm = (ErtsBifTimer *) tmr; + if (btm->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&esdp->timer_service->btm_tree, btm); + btm->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } + } +#endif + if (roflgs & ERTS_TMR_ROFLG_HLT) { hlt_delete_timer(esdp, &tmr->hlt); hl_timer_dec_refc(&tmr->hlt, roflgs); @@ -1813,18 +1950,18 @@ Uint erts_bif_timer_memory_size(void) } static BIF_RETTYPE -setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, - int short_time, Eterm rcvr, Eterm acsr, - Eterm msg, int wrap) +setup_bif_timer(Process *c_p, int twheel, ErtsMonotonicTime timeout_pos, + int short_time, Eterm rcvr, Eterm msg, int wrap) { BIF_RETTYPE ret; Eterm ref, tmo_msg, *hp; - ErtsHLTimer *tmr; + ErtsBifTimer *tmr; ErtsSchedulerData *esdp; #ifdef ERTS_MAGIC_REF_BIF_TIMERS Binary *mbin; #endif Eterm tmp_hp[4]; + ErtsCreateTimerFunc create_timer; if (is_not_internal_pid(rcvr) && is_not_atom(rcvr)) goto badarg; @@ -1846,14 +1983,16 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, tmo_msg = wrap ? TUPLE3(tmp_hp, am_timeout, ref, msg) : msg; - tmr = create_hl_timer(esdp, timeout_pos, short_time, - ERTS_TMR_BIF, NULL, rcvr, acsr, tmo_msg, + create_timer = twheel ? create_tw_timer : create_hl_timer; + tmr = (ErtsBifTimer *) create_timer(esdp, timeout_pos, + short_time, ERTS_TMR_BIF, + NULL, rcvr, tmo_msg, #ifdef ERTS_MAGIC_REF_BIF_TIMERS - (ErtsMagicBinary *) mbin, + (ErtsMagicBinary *) mbin, #else - internal_ordinary_ref_numbers(ref), + internal_ordinary_ref_numbers(ref), #endif - NULL, NULL); + NULL, NULL); if (is_internal_pid(rcvr)) { Process *proc = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, @@ -1862,11 +2001,19 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, if (!proc) { #ifdef ERTS_MAGIC_REF_BIF_TIMERS btm_clear_magic_binary(tmr); +#else + if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&esdp->timer_service->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } #endif if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); - hlt_delete_timer(esdp, tmr); - hl_timer_destroy(tmr); + if (twheel) + cancel_tw_timer(esdp, &tmr->type.twt); + else + hlt_delete_timer(esdp, &tmr->type.hlt); + timer_destroy((ErtsTimer *) tmr, twheel, 1); } else { #ifdef ERTS_MAGIC_REF_BIF_TIMERS @@ -1875,7 +2022,10 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos, proc_btm_rbt_insert(&proc->bif_timers, tmr); #endif erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); - tmr->receiver.proc = proc; + if (twheel) + tmr->type.twt.receiver.proc = proc; + else + tmr->type.hlt.receiver.proc = proc; } } @@ -1889,13 +2039,13 @@ badarg: } static int -cancel_bif_timer(ErtsHLTimer *tmr) +cancel_bif_timer(ErtsBifTimer *tmr) { erts_aint_t state; Uint32 roflgs; int res; - state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state, ERTS_TMR_STATE_CANCELED, ERTS_TMR_STATE_ACTIVE); if (state != ERTS_TMR_STATE_ACTIVE) @@ -1910,10 +2060,14 @@ cancel_bif_timer(ErtsHLTimer *tmr) res = -1; - roflgs = tmr->head.roflgs; + roflgs = tmr->type.head.roflgs; if (roflgs & ERTS_TMR_ROFLG_PROC) { - Process *proc = tmr->receiver.proc; - ERTS_HLT_ASSERT(!(tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME)); + Process *proc; + + proc = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT) + ? tmr->type.hlt.receiver.proc + : tmr->type.twt.receiver.proc); + ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)); erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); /* @@ -1941,18 +2095,25 @@ cancel_bif_timer(ErtsHLTimer *tmr) } static ERTS_INLINE Sint64 -access_btm(ErtsHLTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel) +access_btm(ErtsBifTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel) { int cncl_res; Sint64 time_left; + ErtsMonotonicTime timeout; + int is_hlt; if (!tmr) return -1; + is_hlt = !!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT); + timeout = (is_hlt + ? tmr->type.hlt.timeout + : tmr->type.twt.tw_tmr.timeout_pos); + if (!cancel) { - erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->state); + erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->btm.state); if (state == ERTS_TMR_STATE_ACTIVE) - return get_time_left(esdp, tmr->timeout); + return get_time_left(esdp, timeout); return -1; } @@ -1960,17 +2121,29 @@ access_btm(ErtsHLTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel) if (!cncl_res) return -1; - time_left = get_time_left(esdp, tmr->timeout); + time_left = get_time_left(esdp, timeout); if (sid != (Uint32) esdp->no) { if (cncl_res > 0) queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); } else { - if (cncl_res > 0) - hl_timer_dec_refc(tmr, tmr->head.roflgs); - - hlt_delete_timer(esdp, tmr); +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&esdp->timer_service->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; + } +#endif + if (is_hlt) { + if (cncl_res > 0) + hl_timer_dec_refc(&tmr->type.hlt, tmr->type.hlt.head.roflgs); + hlt_delete_timer(esdp, &tmr->type.hlt); + } + else { + if (cncl_res > 0) + tw_timer_dec_refc(&tmr->type.twt); + cancel_tw_timer(esdp, &tmr->type.twt); + } } return time_left; @@ -2059,7 +2232,7 @@ access_bif_timer(Process *c_p, Eterm tref, int cancel, int async, int info) if (mbin->destructor != bif_timer_ref_destructor) time_left = -1; else { - ErtsHLTimer *tmr; + ErtsBifTimer *tmr; Uint32 sid; tmr = magic_binary_to_btm(mbin); sid = erts_get_ref_numbers_thr_id(internal_magic_ref_numbers(tref)); @@ -2133,7 +2306,7 @@ access_sched_local_btm(Process *c_p, Eterm pid, { ErtsSchedulerData *esdp; ErtsHLTimerService *srv; - ErtsHLTimer *tmr; + ErtsBifTimer *tmr; Sint64 time_left; Process *proc; ErtsProcLocks proc_locks; @@ -2223,7 +2396,7 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp, int async, int cancel, int info, Eterm *resp) { - ErtsHLTimer *tmr; + ErtsBifTimer *tmr; Sint64 time_left; ERTS_HLT_ASSERT(c_p); @@ -2405,8 +2578,8 @@ bool_arg(Eterm val, int *argp) } static ERTS_INLINE int -parse_bif_timer_options(Eterm option_list, int *async, int *info, - int *abs, Eterm *accessor) +parse_bif_timer_options(Eterm option_list, int *async, + int *info, int *abs) { Eterm list = option_list; @@ -2416,8 +2589,6 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info, *info = 1; if (abs) *abs = 0; - if (accessor) - *accessor = THE_NON_VALUE; while (is_list(list)) { Eterm *consp, *tp, opt; @@ -2457,18 +2628,20 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info, } static void -exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp) +exit_cancel_bif_timer(ErtsBifTimer *tmr, void *vesdp) { ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; Uint32 sid, roflgs; erts_aint_t state; + int is_hlt; - state = erts_smp_atomic32_cmpxchg_acqb(&tmr->state, + state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state, ERTS_TMR_STATE_CANCELED, ERTS_TMR_STATE_ACTIVE); - roflgs = tmr->head.roflgs; + roflgs = tmr->type.head.roflgs; sid = roflgs & ERTS_TMR_ROFLG_SID_MASK; + is_hlt = !!(roflgs & ERTS_TMR_ROFLG_HLT); ERTS_HLT_ASSERT(sid == erts_get_ref_numbers_thr_id(ERTS_BTM_HLT2REFN(tmr))); #ifdef ERTS_MAGIC_REF_BIF_TIMERS @@ -2479,29 +2652,33 @@ exit_cancel_bif_timer(ErtsHLTimer *tmr, void *vesdp) tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; #endif - if (sid == (Uint32) esdp->no) { - if (state == ERTS_TMR_STATE_ACTIVE) { + if (state == ERTS_TMR_STATE_ACTIVE) { #ifdef ERTS_MAGIC_REF_BIF_TIMERS - btm_clear_magic_binary(tmr); + btm_clear_magic_binary(tmr); #endif - if (tmr->btm.bp) - free_message_buffer(tmr->btm.bp); - hlt_delete_timer(esdp, tmr); + if (tmr->btm.bp) + free_message_buffer(tmr->btm.bp); + + if (sid != (Uint32) esdp->no) { + queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); + return; + } + +#ifndef ERTS_MAGIC_REF_BIF_TIMERS + if (tmr->btm.tree.parent != ERTS_HLT_PFIELD_NOT_IN_TABLE) { + btm_rbt_delete(&esdp->timer_service->btm_tree, tmr); + tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; } - hl_timer_dec_refc(tmr, roflgs); - } - else { - if (state == ERTS_TMR_STATE_ACTIVE) { -#ifdef ERTS_MAGIC_REF_BIF_TIMERS - btm_clear_magic_binary(tmr); #endif - if (tmr->btm.bp) - free_message_buffer(tmr->btm.bp); - queue_canceled_timer(esdp, sid, (ErtsTimer *) tmr); - } - else - hl_timer_dec_refc(tmr, roflgs); + if (is_hlt) + hlt_delete_timer(esdp, &tmr->type.hlt); + else + cancel_tw_timer(esdp, &tmr->type.twt); } + if (is_hlt) + hl_timer_dec_refc(&tmr->type.hlt, roflgs); + else + tw_timer_dec_refc(&tmr->type.twt); } #ifdef ERTS_HLT_DEBUG @@ -2571,10 +2748,11 @@ int erts_cancel_bif_timers(Process *p, ErtsBifTimers **btm, void **vyspp) static ERTS_INLINE int parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, ErtsMonotonicTime *conv_arg, int abs, - ErtsMonotonicTime *tposp, int *stimep) + ErtsMonotonicTime *tposp, int *stimep, + ErtsMonotonicTime *msp) { - ErtsMonotonicTime t; - + ErtsMonotonicTime t, now; + if (!term_to_Sint64(arg, &t)) { ERTS_HLT_ASSERT(!is_small(arg)); if (!is_big(arg)) @@ -2589,22 +2767,30 @@ parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, if (conv_arg) *conv_arg = t; + now = erts_get_monotonic_time(esdp); + if (abs) { t += -1*ERTS_MONOTONIC_OFFSET_MSEC; /* external to internal */ if (t < ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_BEGIN)) return 1; if (t > ERTS_MONOTONIC_TO_MSEC(ERTS_MONOTONIC_END)) return 1; + if (msp) + *msp = t - ERTS_MONOTONIC_TO_MSEC(now); + *stimep = (t - ERTS_MONOTONIC_TO_MSEC(esdp->last_monotonic_time) < ERTS_BIF_TIMER_SHORT_TIME); *tposp = ERTS_MSEC_TO_CLKTCKS(t); } else { - ErtsMonotonicTime now, ticks; + ErtsMonotonicTime ticks; if (t < 0) return -1; + if (msp) + *msp = t; + ticks = ERTS_MSEC_TO_CLKTCKS(t); if (ERTS_CLKTCK_RESOLUTION > 1000 && ticks < 0) @@ -2612,7 +2798,6 @@ parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, ERTS_HLT_ASSERT(ticks >= 0); - now = erts_get_monotonic_time(esdp); ticks += ERTS_MONOTONIC_TO_CLKTCKS(now-1); ticks += 1; @@ -2635,66 +2820,68 @@ parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg, BIF_RETTYPE send_after_3(BIF_ALIST_3) { - ErtsMonotonicTime timeout_pos; + ErtsMonotonicTime timeout_pos, tmo; int short_time, tres; - tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, - 0, &timeout_pos, &short_time); + tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, + NULL, 0, &timeout_pos, &short_time, &tmo); if (tres != 0) BIF_ERROR(BIF_P, BADARG); - return setup_bif_timer(BIF_P, timeout_pos, short_time, - BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, 0); + return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC, + timeout_pos, short_time, BIF_ARG_2, + BIF_ARG_3, 0); } BIF_RETTYPE send_after_4(BIF_ALIST_4) { - ErtsMonotonicTime timeout_pos; - Eterm accessor; + ErtsMonotonicTime timeout_pos, tmo; int short_time, abs, tres; - if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor)) + if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs)) BIF_ERROR(BIF_P, BADARG); tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, - abs, &timeout_pos, &short_time); + abs, &timeout_pos, &short_time, &tmo); if (tres != 0) BIF_ERROR(BIF_P, BADARG); - return setup_bif_timer(BIF_P, timeout_pos, short_time, - BIF_ARG_2, accessor, BIF_ARG_3, 0); + return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC, + timeout_pos, short_time, BIF_ARG_2, + BIF_ARG_3, 0); } BIF_RETTYPE start_timer_3(BIF_ALIST_3) { - ErtsMonotonicTime timeout_pos; + ErtsMonotonicTime timeout_pos, tmo; int short_time, tres; tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, - 0, &timeout_pos, &short_time); + 0, &timeout_pos, &short_time, &tmo); if (tres != 0) BIF_ERROR(BIF_P, BADARG); - return setup_bif_timer(BIF_P, timeout_pos, short_time, - BIF_ARG_2, BIF_ARG_2, BIF_ARG_3, !0); + return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC, + timeout_pos, short_time, BIF_ARG_2, + BIF_ARG_3, !0); } BIF_RETTYPE start_timer_4(BIF_ALIST_4) { - ErtsMonotonicTime timeout_pos; - Eterm accessor; + ErtsMonotonicTime timeout_pos, tmo; int short_time, abs, tres; - if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs, &accessor)) + if (!parse_bif_timer_options(BIF_ARG_4, NULL, NULL, &abs)) BIF_ERROR(BIF_P, BADARG); tres = parse_timeout_pos(erts_proc_sched_data(BIF_P), BIF_ARG_1, NULL, - abs, &timeout_pos, &short_time); + abs, &timeout_pos, &short_time, &tmo); if (tres != 0) BIF_ERROR(BIF_P, BADARG); - return setup_bif_timer(BIF_P, timeout_pos, short_time, - BIF_ARG_2, accessor, BIF_ARG_3, !0); + return setup_bif_timer(BIF_P, tmo < ERTS_TIMER_WHEEL_MSEC, + timeout_pos, short_time, BIF_ARG_2, + BIF_ARG_3, !0); } BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) @@ -2707,7 +2894,7 @@ BIF_RETTYPE cancel_timer_2(BIF_ALIST_2) BIF_RETTYPE ret; int async, info; - if (parse_bif_timer_options(BIF_ARG_2, &async, &info, NULL, NULL)) + if (parse_bif_timer_options(BIF_ARG_2, &async, &info, NULL)) return access_bif_timer(BIF_P, BIF_ARG_1, 1, async, info); ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); @@ -2724,7 +2911,7 @@ BIF_RETTYPE read_timer_2(BIF_ALIST_2) BIF_RETTYPE ret; int async; - if (parse_bif_timer_options(BIF_ARG_2, &async, NULL, NULL, NULL)) + if (parse_bif_timer_options(BIF_ARG_2, &async, NULL, NULL)) return access_bif_timer(BIF_P, BIF_ARG_1, 0, async, 1); ERTS_BIF_PREP_ERROR(ret, BIF_P, BADARG); @@ -2739,14 +2926,13 @@ start_callback_timer(ErtsSchedulerData *esdp, void *arg) { - if (twt) - create_tw_timer(esdp, ERTS_TMR_CALLBACK, NULL, - callback, arg, timeout_pos); - else - create_hl_timer(esdp, timeout_pos, 0, - ERTS_TMR_CALLBACK, NULL, - NIL, THE_NON_VALUE, NIL, - NULL, callback, arg); + ErtsCreateTimerFunc create_timer = (twt + ? create_tw_timer + : create_hl_timer); + (void) create_timer(esdp, timeout_pos, 0, + ERTS_TMR_CALLBACK, NULL, + NIL, THE_NON_VALUE, NULL, + callback, arg); } typedef struct { @@ -2823,18 +3009,18 @@ set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo, if (tmo == 0) c_p->flags |= F_TIMO; else { + ErtsCreateTimerFunc create_timer; c_p->flags |= F_INSLPQUEUE; c_p->flags &= ~F_TIMO; - if (tmo < ERTS_TIMER_WHEEL_MSEC) - tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PROC, (void *) c_p, - NULL, NULL, timeout_pos); - else - tmr = (void *) create_hl_timer(esdp, timeout_pos, short_time, - ERTS_TMR_PROC, (void *) c_p, - c_p->common.id, THE_NON_VALUE, - NIL, NULL, NULL, NULL); + create_timer = (tmo < ERTS_TIMER_WHEEL_MSEC + ? create_tw_timer + : create_hl_timer); + tmr = (void *) create_timer(esdp, timeout_pos, short_time, + ERTS_TMR_PROC, (void *) c_p, + c_p->common.id, THE_NON_VALUE, + NULL, NULL, NULL); erts_smp_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr); } } @@ -2850,7 +3036,7 @@ erts_set_proc_timer_term(Process *c_p, Eterm etmo) == ERTS_PTMR_NONE); tres = parse_timeout_pos(esdp, etmo, &tmo, 0, - &timeout_pos, &short_time); + &timeout_pos, &short_time, NULL); if (tres != 0) return tres; @@ -2908,6 +3094,7 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo) void *tmr; ErtsSchedulerData *esdp = erts_get_scheduler_data(); ErtsMonotonicTime timeout_pos; + ErtsCreateTimerFunc create_timer; if (erts_smp_atomic_read_nob(&c_prt->common.timer) != ERTS_PTMR_NONE) erts_cancel_port_timer(c_prt); @@ -2916,13 +3103,12 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo) timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), tmo); - if (tmo < ERTS_TIMER_WHEEL_MSEC) - tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PORT, (void *) c_prt, - NULL, NULL, timeout_pos); - else - tmr = (void *) create_hl_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT, - (void *) c_prt, c_prt->common.id, - THE_NON_VALUE, NIL, NULL, NULL, NULL); + create_timer = (tmo < ERTS_TIMER_WHEEL_MSEC + ? create_tw_timer + : create_hl_timer); + tmr = (void *) create_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT, + (void *) c_prt, c_prt->common.id, + THE_NON_VALUE, NULL, NULL, NULL); erts_smp_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr); } @@ -2976,25 +3162,39 @@ typedef struct { } ErtsBTMPrint; static void -btm_print(ErtsHLTimer *tmr, void *vbtmp) +btm_print(ErtsBifTimer *tmr, void *vbtmp, ErtsMonotonicTime tpos, int is_hlt) { ErtsBTMPrint *btmp = (ErtsBTMPrint *) vbtmp; ErtsMonotonicTime left; Eterm receiver; #ifdef ERTS_MAGIC_REF_BIF_TIMERS - if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + if (!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) return; #endif - if (tmr->timeout <= btmp->now) - left = 0; - else - left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now); + if (is_hlt) { + ERTS_HLT_ASSERT(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT); + if (tmr->type.hlt.timeout <= btmp->now) + left = 0; + else + left = ERTS_CLKTCKS_TO_MSEC(tmr->type.hlt.timeout - btmp->now); - receiver = ((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) - ? tmr->receiver.name - : tmr->receiver.proc->common.id); + receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->type.hlt.receiver.name + : tmr->type.hlt.receiver.proc->common.id); + } + else { + ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT)); + if (tpos <= btmp->now) + left = 0; + else + left = ERTS_CLKTCKS_TO_MSEC(tpos - btmp->now); + + receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->type.twt.receiver.name + : tmr->type.twt.receiver.proc->common.id); + } erts_print(btmp->to, btmp->to_arg, "=timer:%T\n" @@ -3005,6 +3205,36 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp) (Sint64) left); } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static void +hlt_btm_print(ErtsHLTimer *tmr, void *vbtmp) +{ + btm_print((ErtsBifTimer *) tmr, vbtmp, 0, 1); +} + +static void +twt_btm_print(void *vbtmp, ErtsMonotonicTime tpos, void *vtwtp) +{ + btm_print((ErtsBifTimer *) vtwtp, vbtmp, tpos, 0); +} + +#else + +static void +btm_tree_print(ErtsBifTimer *tmr, void *vbtmp) +{ + int is_hlt = !!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT); + ErtsMonotonicTime tpos; + if (is_hlt) + tpos = 0; + else + tpos = tmr->type.twt.tw_tmr.timeout_pos; + btm_print(tmr, vbtmp, tpos, is_hlt); +} + +#endif + void erts_print_bif_timer_info(fmtfn_t to, void *to_arg) { @@ -3023,9 +3253,13 @@ erts_print_bif_timer_info(fmtfn_t to, void *to_arg) ErtsHLTimerService *srv = erts_aligned_scheduler_data[six].esd.timer_service; #ifdef ERTS_MAGIC_REF_BIF_TIMERS - time_rbt_foreach(srv->time_tree, btm_print, (void *) &btmp); + ErtsTimerWheel *twheel = + erts_aligned_scheduler_data[six].esd.timer_wheel; + erts_twheel_debug_foreach(twheel, tw_bif_timer_timeout, + twt_btm_print, (void *) &btmp); + time_rbt_foreach(srv->time_tree, hlt_btm_print, (void *) &btmp); #else - btm_rbt_foreach(srv->btm_tree, btm_print, (void *) &btmp); + btm_rbt_foreach(srv->btm_tree, btm_tree_print, (void *) &btmp); #endif } } @@ -3039,23 +3273,52 @@ typedef struct { } ErtsBTMForeachDebug; static void -debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd) +debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd, int is_hlt) { #ifdef ERTS_MAGIC_REF_BIF_TIMERS - if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) + if (!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) return; #endif - if (erts_smp_atomic32_read_nob(&tmr->state) == ERTS_TMR_STATE_ACTIVE) { + if (erts_smp_atomic32_read_nob(&tmr->btm.state) == ERTS_TMR_STATE_ACTIVE) { ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd; - (*btmfd->func)(((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME) - ? tmr->receiver.name - : tmr->receiver.proc->common.id), - tmr->btm.message, - tmr->btm.bp, - btmfd->arg); + Eterm id; + if (is_hlt) + id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->type.hlt.receiver.name + : tmr->type.hlt.receiver.proc->common.id); + else + id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->type.twt.receiver.name + : tmr->type.twt.receiver.proc->common.id); + (*btmfd->func)(id, tmr->btm.message, tmr->btm.bp, btmfd->arg); } } +#ifdef ERTS_MAGIC_REF_BIF_TIMERS + +static void +hlt_debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd) +{ + debug_btm_foreach((ErtsBifTimer *) tmr, vbtmfd, 1); +} + +static void +twt_debug_btm_foreach(void *vbtmfd, ErtsMonotonicTime tpos, void *vtwtp) +{ + debug_btm_foreach((ErtsBifTimer *) vtwtp, vbtmfd, 0); +} + +#else + +static void +debug_btm_tree_foreach(ErtsBifTimer *tmr, void *vbtmfd) +{ + debug_btm_foreach(tmr, vbtmfd, !!(tmr->type.head.roflgs + & ERTS_TMR_ROFLG_HLT)); +} + +#endif + void erts_debug_bif_timer_foreach(void (*func)(Eterm, Eterm, @@ -3076,13 +3339,17 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm, ErtsHLTimerService *srv = erts_aligned_scheduler_data[six].esd.timer_service; #ifdef ERTS_MAGIC_REF_BIF_TIMERS + ErtsTimerWheel *twheel = + erts_aligned_scheduler_data[six].esd.timer_wheel; + erts_twheel_debug_foreach(twheel, tw_bif_timer_timeout, + twt_debug_btm_foreach, + (void *) &btmfd); time_rbt_foreach(srv->time_tree, - debug_btm_foreach, + hlt_debug_btm_foreach, (void *) &btmfd); - #else btm_rbt_foreach(srv->btm_tree, - debug_btm_foreach, + debug_btm_tree_foreach, (void *) &btmfd); #endif } @@ -3136,7 +3403,7 @@ debug_tw_callback_timer(void *vdfct, ErtsDebugForeachCallbackTimer *dfct = (ErtsDebugForeachCallbackTimer *) vdfct; - if (twtp->u.callback == dfct->tclbk) + if (twtp->receiver.callback == dfct->tclbk) (*dfct->func)(dfct->arg, timeout_pos, twtp->head.u.arg); diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h index 33cdf217d7..f70fcdd1c0 100644 --- a/erts/emulator/beam/erl_hl_timer.h +++ b/erts/emulator/beam/erl_hl_timer.h @@ -21,7 +21,7 @@ #ifndef ERL_HL_TIMER_H__ #define ERL_HL_TIMER_H__ -typedef struct ErtsHLTimer_ ErtsBifTimers; +typedef struct ErtsBifTimer_ ErtsBifTimers; typedef struct ErtsHLTimerService_ ErtsHLTimerService; #include "sys.h" -- cgit v1.2.3 From 79ac974710ecab6dfe37b23b4fd3b52e760b3504 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 3 Feb 2017 15:50:51 +0100 Subject: Rearrange timer struct fields in order to simplify --- erts/emulator/beam/erl_hl_timer.c | 177 +++++++++++++++----------------------- erts/emulator/beam/erl_time.h | 20 +++-- erts/emulator/beam/time.c | 32 +++---- 3 files changed, 98 insertions(+), 131 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 102a739b13..8e47fdd674 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -164,21 +164,21 @@ typedef struct { void *arg; erts_atomic_t next; } u; + union { + Process *proc; + Port *port; + Eterm name; + void (*callback)(void *); + } receiver; } ErtsTmrHead; struct ErtsHLTimer_ { ErtsTmrHead head; /* NEED to be first! */ + ErtsMonotonicTime timeout; union { ErtsThrPrgrLaterOp cleanup; ErtsHLTimerTimeTree tree; } time; - ErtsMonotonicTime timeout; - union { - Process *proc; - Port *port; - Eterm name; - void (*callback)(void *); - } receiver; #ifdef ERTS_HLT_HARD_DEBUG int pending_timeout; @@ -188,12 +188,9 @@ struct ErtsHLTimer_ { typedef struct { ErtsTmrHead head; /* NEED to be first! */ union { - Eterm name; - Process *proc; - Port *port; - void (*callback)(void *); - } receiver; - ErtsTWheelTimer tw_tmr; + ErtsTWheelTimer tw_tmr; + ErtsThrPrgrLaterOp cleanup; + } u; } ErtsTWTimer; struct ErtsBifTimer_ { @@ -948,9 +945,9 @@ schedule_tw_timer_destroy(ErtsTWTimer *tmr) * dropped at once... */ if (tmr->head.roflgs & ERTS_TMR_ROFLG_PROC) - erts_proc_dec_refc(tmr->receiver.proc); + erts_proc_dec_refc(tmr->head.receiver.proc); else if (tmr->head.roflgs & ERTS_TMR_ROFLG_PORT) - erts_port_dec_refc(tmr->receiver.port); + erts_port_dec_refc(tmr->head.receiver.port); if (!(tmr->head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) size = sizeof(ErtsHLTimer); @@ -965,7 +962,7 @@ schedule_tw_timer_destroy(ErtsTWTimer *tmr) erts_schedule_thr_prgr_later_cleanup_op( scheduled_tw_timer_destroy, (void *) tmr, - &tmr->tw_tmr.u.cleanup, + &tmr->u.cleanup, size); } @@ -982,7 +979,7 @@ static void tw_proc_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - Process *proc = twtp->receiver.proc; + Process *proc = twtp->head.receiver.proc; if (proc_timeout_common(proc, vtwtp)) tw_timer_dec_refc(twtp); tw_timer_dec_refc(twtp); @@ -992,7 +989,7 @@ static void tw_port_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - Port *port = twtp->receiver.port; + Port *port = twtp->head.receiver.port; if (port_timeout_common(port, vtwtp)) tw_timer_dec_refc(twtp); tw_timer_dec_refc(twtp); @@ -1009,14 +1006,14 @@ cancel_tw_timer(ErtsSchedulerData *esdp, ErtsTWTimer *tmr) { ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK) == (Uint32) esdp->no); - erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->tw_tmr); + erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->u.tw_tmr); } static void tw_callback_timeout(void *vtwtp) { ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp; - void (*callback)(void *) = twtp->receiver.callback; + void (*callback)(void *) = twtp->head.receiver.callback; void *arg = twtp->head.u.arg; tw_timer_dec_refc(twtp); (*callback)(arg); @@ -1060,7 +1057,7 @@ create_tw_timer(ErtsSchedulerData *esdp, } } - erts_twheel_init_timer(&tmr->tw_tmr); + erts_twheel_init_timer(&tmr->u.tw_tmr); tmr->head.roflgs |= (Uint32) esdp->no; ERTS_HLT_ASSERT((((Uint32) esdp->no) & ~ERTS_TMR_ROFLG_SID_MASK) == 0); @@ -1068,7 +1065,7 @@ create_tw_timer(ErtsSchedulerData *esdp, switch (type) { case ERTS_TMR_PROC: - tmr->receiver.proc = (Process *) rcvrp; + tmr->head.receiver.proc = (Process *) rcvrp; tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; timeout_func = tw_proc_timeout; cancel_func = tw_ptimer_cancel; @@ -1077,7 +1074,7 @@ create_tw_timer(ErtsSchedulerData *esdp, break; case ERTS_TMR_PORT: - tmr->receiver.port = (Port *) rcvrp; + tmr->head.receiver.port = (Port *) rcvrp; tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT; timeout_func = tw_port_timeout; cancel_func = tw_ptimer_cancel; @@ -1087,7 +1084,7 @@ create_tw_timer(ErtsSchedulerData *esdp, case ERTS_TMR_CALLBACK: tmr->head.u.arg = arg; - tmr->receiver.callback = callback; + tmr->head.receiver.callback = callback; tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK; timeout_func = tw_callback_timeout; @@ -1101,13 +1098,13 @@ create_tw_timer(ErtsSchedulerData *esdp, cancel_func = NULL; if (is_internal_pid(rcvr)) { tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; - tmr->receiver.proc = (Process *) rcvrp; + tmr->head.receiver.proc = (Process *) rcvrp; refc = 2; } else { ERTS_HLT_ASSERT(is_atom(rcvr)); tmr->head.roflgs |= ERTS_TMR_ROFLG_REG_NAME; - tmr->receiver.name = (Eterm) rcvr; + tmr->head.receiver.name = (Eterm) rcvr; refc = 1; } @@ -1130,7 +1127,7 @@ create_tw_timer(ErtsSchedulerData *esdp, erts_smp_atomic32_init_nob(&tmr->head.refc, refc); erts_twheel_set_timer(esdp->timer_wheel, - &tmr->tw_tmr, + &tmr->u.tw_tmr, timeout_func, cancel_func, tmr, @@ -1164,15 +1161,15 @@ schedule_hl_timer_destroy(ErtsHLTimer *tmr, Uint32 roflgs) ERTS_HLT_ASSERT(erts_smp_atomic32_read_nob(&tmr->head.refc) == 0); if (roflgs & ERTS_TMR_ROFLG_REG_NAME) { - ERTS_HLT_ASSERT(is_atom(tmr->receiver.name)); + ERTS_HLT_ASSERT(is_atom(tmr->head.receiver.name)); } else if (roflgs & ERTS_TMR_ROFLG_PROC) { - ERTS_HLT_ASSERT(tmr->receiver.proc); - erts_proc_dec_refc(tmr->receiver.proc); + ERTS_HLT_ASSERT(tmr->head.receiver.proc); + erts_proc_dec_refc(tmr->head.receiver.proc); } else if (roflgs & ERTS_TMR_ROFLG_PORT) { - ERTS_HLT_ASSERT(tmr->receiver.port); - erts_port_dec_refc(tmr->receiver.port); + ERTS_HLT_ASSERT(tmr->head.receiver.port); + erts_port_dec_refc(tmr->head.receiver.port); } if (!(roflgs & ERTS_TMR_ROFLG_BIF_TMR)) @@ -1251,14 +1248,14 @@ btm_clear_magic_binary(ErtsBifTimer *tmr) #endif /* ERTS_MAGIC_REF_BIF_TIMERS */ static ERTS_INLINE void -bif_timer_timeout(ErtsHLTimerService *srv, ErtsBifTimer *tmr, - Uint32 roflgs, int is_hlt) +bif_timer_timeout(ErtsHLTimerService *srv, + ErtsBifTimer *tmr, + Uint32 roflgs) { erts_aint32_t state; ERTS_HLT_ASSERT(tmr->type.head.roflgs == roflgs); ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR); - ERTS_HLT_ASSERT(!!is_hlt == !!(roflgs & ERTS_TMR_ROFLG_HLT)); state = erts_smp_atomic32_cmpxchg_acqb(&tmr->btm.state, ERTS_TMR_STATE_TIMED_OUT, @@ -1269,40 +1266,32 @@ bif_timer_timeout(ErtsHLTimerService *srv, ErtsBifTimer *tmr, if (state == ERTS_TMR_STATE_ACTIVE) { Process *proc; - int dec_refc = 0; - Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME); #ifdef ERTS_MAGIC_REF_BIF_TIMERS btm_clear_magic_binary(tmr); #endif - if (is_reg_name) { + if (roflgs & ERTS_TMR_ROFLG_REG_NAME) { Eterm term; - if (is_hlt) - term = tmr->type.hlt.receiver.name; - else - term = tmr->type.twt.receiver.name; + term = tmr->type.head.receiver.name; ERTS_HLT_ASSERT(is_atom(term)); term = erts_whereis_name_to_id(NULL, term); proc = erts_proc_lookup(term); } else { ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PROC); - if (is_hlt) - proc = tmr->type.hlt.receiver.proc; - else - proc = tmr->type.twt.receiver.proc; + proc = tmr->type.head.receiver.proc; ERTS_HLT_ASSERT(proc); } if (proc) { if (!ERTS_PROC_IS_EXITING(proc)) { - ErtsMessage *mp; - erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); - mp = erts_alloc_message(0, NULL); + int dec_refc = 0; + ErtsMessage *mp = erts_alloc_message(0, NULL); mp->data.heap_frag = tmr->btm.bp; - erts_queue_message(proc, ERTS_PROC_LOCK_BTM, mp, - tmr->btm.message, am_clock_service); tmr->btm.bp = NULL; + erts_queue_message(proc, 0, mp, tmr->btm.message, + am_clock_service); + erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); #ifdef ERTS_MAGIC_REF_BIF_TIMERS if (tmr->btm.proc_list.next) { proc_btm_list_delete(&proc->bif_timers, tmr); @@ -1316,9 +1305,9 @@ bif_timer_timeout(ErtsHLTimerService *srv, ErtsBifTimer *tmr, } #endif erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); + if (dec_refc) + timer_pre_dec_refc((ErtsTimer *) tmr); } - if (dec_refc) - timer_pre_dec_refc((ErtsTimer *) tmr); } if (tmr->btm.bp) free_message_buffer(tmr->btm.bp); @@ -1344,7 +1333,7 @@ tw_bif_timer_timeout(void *vbtmp) ErtsHLTimerService *srv = esdp->timer_service; #endif ErtsBifTimer *btmp = (ErtsBifTimer *) vbtmp; - bif_timer_timeout(srv, btmp, btmp->type.head.roflgs, 0); + bif_timer_timeout(srv, btmp, btmp->type.head.roflgs); tw_timer_dec_refc(&btmp->type.twt); } @@ -1386,7 +1375,7 @@ create_hl_timer(ErtsSchedulerData *esdp, ERTS_HLT_ASSERT(is_internal_pid(rcvr)); erts_proc_inc_refc((Process *) rcvrp); - tmr->receiver.proc = (Process *) rcvrp; + tmr->head.receiver.proc = (Process *) rcvrp; roflgs |= ERTS_TMR_ROFLG_PROC; refc = 2; break; @@ -1394,14 +1383,14 @@ create_hl_timer(ErtsSchedulerData *esdp, case ERTS_TMR_PORT: ERTS_HLT_ASSERT(is_internal_port(rcvr)); erts_port_inc_refc((Port *) rcvrp); - tmr->receiver.port = (Port *) rcvrp; + tmr->head.receiver.port = (Port *) rcvrp; roflgs |= ERTS_TMR_ROFLG_PORT; refc = 2; break; case ERTS_TMR_CALLBACK: roflgs |= ERTS_TMR_ROFLG_CALLBACK; - tmr->receiver.callback = callback; + tmr->head.receiver.callback = callback; tmr->head.u.arg = arg; refc = 1; break; @@ -1431,13 +1420,13 @@ create_hl_timer(ErtsSchedulerData *esdp, roflgs |= ERTS_TMR_ROFLG_BIF_TMR; if (is_internal_pid(rcvr)) { roflgs |= ERTS_TMR_ROFLG_PROC; - tmr->receiver.proc = (Process *) rcvrp; + tmr->head.receiver.proc = (Process *) rcvrp; refc = 2; } else { ERTS_HLT_ASSERT(is_atom(rcvr)); roflgs |= ERTS_TMR_ROFLG_REG_NAME; - tmr->receiver.name = rcvr; + tmr->head.receiver.name = rcvr; refc = 1; } @@ -1486,14 +1475,14 @@ create_hl_timer(ErtsSchedulerData *esdp, static ERTS_INLINE void hlt_proc_timeout(ErtsHLTimer *tmr) { - if (proc_timeout_common(tmr->receiver.proc, (void *) tmr)) + if (proc_timeout_common(tmr->head.receiver.proc, (void *) tmr)) hl_timer_dec_refc(tmr, tmr->head.roflgs); } static ERTS_INLINE void hlt_port_timeout(ErtsHLTimer *tmr) { - if (port_timeout_common(tmr->receiver.port, (void *) tmr)) + if (port_timeout_common(tmr->head.receiver.port, (void *) tmr)) hl_timer_dec_refc(tmr, tmr->head.roflgs); } @@ -1508,14 +1497,14 @@ static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv) ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_HLT); if (roflgs & ERTS_TMR_ROFLG_BIF_TMR) - bif_timer_timeout(srv, (ErtsBifTimer *) tmr, roflgs, 1); + bif_timer_timeout(srv, (ErtsBifTimer *) tmr, roflgs); else if (roflgs & ERTS_TMR_ROFLG_PROC) hlt_proc_timeout(tmr); else if (roflgs & ERTS_TMR_ROFLG_PORT) hlt_port_timeout(tmr); else { ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK); - (*tmr->receiver.callback)(tmr->head.u.arg); + (*tmr->head.receiver.callback)(tmr->head.u.arg); } tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE; @@ -2022,10 +2011,7 @@ setup_bif_timer(Process *c_p, int twheel, ErtsMonotonicTime timeout_pos, proc_btm_rbt_insert(&proc->bif_timers, tmr); #endif erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_BTM); - if (twheel) - tmr->type.twt.receiver.proc = proc; - else - tmr->type.hlt.receiver.proc = proc; + tmr->type.head.receiver.proc = proc; } } @@ -2064,9 +2050,7 @@ cancel_bif_timer(ErtsBifTimer *tmr) if (roflgs & ERTS_TMR_ROFLG_PROC) { Process *proc; - proc = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT) - ? tmr->type.hlt.receiver.proc - : tmr->type.twt.receiver.proc); + proc = tmr->type.head.receiver.proc; ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME)); erts_smp_proc_lock(proc, ERTS_PROC_LOCK_BTM); @@ -2108,7 +2092,7 @@ access_btm(ErtsBifTimer *tmr, Uint32 sid, ErtsSchedulerData *esdp, int cancel) is_hlt = !!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT); timeout = (is_hlt ? tmr->type.hlt.timeout - : tmr->type.twt.tw_tmr.timeout_pos); + : erts_tweel_read_timeout(&tmr->type.twt.u.tw_tmr)); if (!cancel) { erts_aint32_t state = erts_smp_atomic32_read_acqb(&tmr->btm.state); @@ -3147,7 +3131,7 @@ erts_read_port_timer(Port *c_prt) if (tmr->head.roflgs & ERTS_TMR_ROFLG_HLT) timeout_pos = tmr->hlt.timeout; else - timeout_pos = tmr->twt.tw_tmr.timeout_pos; + timeout_pos = erts_tweel_read_timeout(&tmr->twt.u.tw_tmr); return get_time_left(NULL, timeout_pos); } @@ -3179,10 +3163,6 @@ btm_print(ErtsBifTimer *tmr, void *vbtmp, ErtsMonotonicTime tpos, int is_hlt) left = 0; else left = ERTS_CLKTCKS_TO_MSEC(tmr->type.hlt.timeout - btmp->now); - - receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) - ? tmr->type.hlt.receiver.name - : tmr->type.hlt.receiver.proc->common.id); } else { ERTS_HLT_ASSERT(!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_HLT)); @@ -3190,12 +3170,12 @@ btm_print(ErtsBifTimer *tmr, void *vbtmp, ErtsMonotonicTime tpos, int is_hlt) left = 0; else left = ERTS_CLKTCKS_TO_MSEC(tpos - btmp->now); - - receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) - ? tmr->type.twt.receiver.name - : tmr->type.twt.receiver.proc->common.id); } + receiver = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->type.head.receiver.name + : tmr->type.head.receiver.proc->common.id); + erts_print(btmp->to, btmp->to_arg, "=timer:%T\n" "Message: %T\n" @@ -3229,7 +3209,7 @@ btm_tree_print(ErtsBifTimer *tmr, void *vbtmp) if (is_hlt) tpos = 0; else - tpos = tmr->type.twt.tw_tmr.timeout_pos; + tpos = erts_tweel_read_timeout(&tmr->type.twt.u.tw_tmr); btm_print(tmr, vbtmp, tpos, is_hlt); } @@ -3273,7 +3253,7 @@ typedef struct { } ErtsBTMForeachDebug; static void -debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd, int is_hlt) +debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd) { #ifdef ERTS_MAGIC_REF_BIF_TIMERS if (!(tmr->type.head.roflgs & ERTS_TMR_ROFLG_BIF_TMR)) @@ -3281,15 +3261,9 @@ debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd, int is_hlt) #endif if (erts_smp_atomic32_read_nob(&tmr->btm.state) == ERTS_TMR_STATE_ACTIVE) { ErtsBTMForeachDebug *btmfd = (ErtsBTMForeachDebug *) vbtmfd; - Eterm id; - if (is_hlt) - id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) - ? tmr->type.hlt.receiver.name - : tmr->type.hlt.receiver.proc->common.id); - else - id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) - ? tmr->type.twt.receiver.name - : tmr->type.twt.receiver.proc->common.id); + Eterm id = ((tmr->type.head.roflgs & ERTS_TMR_ROFLG_REG_NAME) + ? tmr->type.head.receiver.name + : tmr->type.head.receiver.proc->common.id); (*btmfd->func)(id, tmr->btm.message, tmr->btm.bp, btmfd->arg); } } @@ -3299,22 +3273,13 @@ debug_btm_foreach(ErtsBifTimer *tmr, void *vbtmfd, int is_hlt) static void hlt_debug_btm_foreach(ErtsHLTimer *tmr, void *vbtmfd) { - debug_btm_foreach((ErtsBifTimer *) tmr, vbtmfd, 1); + debug_btm_foreach((ErtsBifTimer *) tmr, vbtmfd); } static void twt_debug_btm_foreach(void *vbtmfd, ErtsMonotonicTime tpos, void *vtwtp) { - debug_btm_foreach((ErtsBifTimer *) vtwtp, vbtmfd, 0); -} - -#else - -static void -debug_btm_tree_foreach(ErtsBifTimer *tmr, void *vbtmfd) -{ - debug_btm_foreach(tmr, vbtmfd, !!(tmr->type.head.roflgs - & ERTS_TMR_ROFLG_HLT)); + debug_btm_foreach((ErtsBifTimer *) vtwtp, vbtmfd); } #endif @@ -3349,7 +3314,7 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm, (void *) &btmfd); #else btm_rbt_foreach(srv->btm_tree, - debug_btm_tree_foreach, + debug_btm_foreach, (void *) &btmfd); #endif } @@ -3370,7 +3335,7 @@ debug_callback_timer_foreach_list(ErtsHLTimer *tmr, void *vdfct) = (ErtsDebugForeachCallbackTimer *) vdfct; if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK) - && (tmr->receiver.callback == dfct->tclbk)) + && (tmr->head.receiver.callback == dfct->tclbk)) (*dfct->func)(dfct->arg, tmr->timeout, tmr->head.u.arg); @@ -3388,7 +3353,7 @@ debug_callback_timer_foreach(ErtsHLTimer *tmr, void *vdfct) vdfct); if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK) - && (tmr->receiver.callback == dfct->tclbk)) + && (tmr->head.receiver.callback == dfct->tclbk)) (*dfct->func)(dfct->arg, tmr->timeout, tmr->head.u.arg); @@ -3403,7 +3368,7 @@ debug_tw_callback_timer(void *vdfct, ErtsDebugForeachCallbackTimer *dfct = (ErtsDebugForeachCallbackTimer *) vdfct; - if (twtp->receiver.callback == dfct->tclbk) + if (twtp->head.receiver.callback == dfct->tclbk) (*dfct->func)(dfct->arg, timeout_pos, twtp->head.u.arg); diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index a1c4220633..2fb4b754d7 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -419,17 +419,12 @@ void erts_sched_init_time_sup(ErtsSchedulerData *esdp); ** Timer entry: */ typedef struct erl_timer { + ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */ struct erl_timer* next; /* next entry tiw slot or chain */ struct erl_timer* prev; /* prev entry tiw slot or chain */ - union { - struct { - void (*timeout)(void*); /* called when timeout */ - void (*cancel)(void*); /* called when cancel (may be NULL) */ - void* arg; /* argument to timeout/cancel procs */ - } func; - ErtsThrPrgrLaterOp cleanup; - } u; - ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */ + void (*timeout)(void*); /* called when timeout */ + void (*cancel)(void*); /* called when cancel (may be NULL) */ + void* arg; /* argument to timeout/cancel procs */ int slot; } ErtsTWheelTimer; @@ -447,6 +442,7 @@ ErtsMonotonicTime erts_check_next_timeout_time(ErtsSchedulerData *); ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p); ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef); +ERTS_GLB_INLINE ErtsMonotonicTime erts_tweel_read_timeout(ErtsTWheelTimer *twt); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -460,6 +456,12 @@ ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_ return *((ErtsMonotonicTime *) nxt_tmo_ref); } +ERTS_GLB_INLINE ErtsMonotonicTime +erts_tweel_read_timeout(ErtsTWheelTimer *twt) +{ + return twt->timeout_pos; +} + #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ void diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 6f15082130..fdf74498c9 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -330,8 +330,8 @@ timeout_timer(ErtsTWheelTimer *p) ErlTimeoutProc timeout; void *arg; p->slot = ERTS_TWHEEL_SLOT_INACTIVE; - timeout = p->u.func.timeout; - arg = p->u.func.arg; + timeout = p->timeout; + arg = p->arg; (*timeout)(arg); ASSERT_NO_LOCKED_LOCKS; } @@ -542,9 +542,9 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY; tiw->sentinel.next = &tiw->sentinel; tiw->sentinel.prev = &tiw->sentinel; - tiw->sentinel.u.func.timeout = NULL; - tiw->sentinel.u.func.cancel = NULL; - tiw->sentinel.u.func.arg = NULL; + tiw->sentinel.timeout = NULL; + tiw->sentinel.cancel = NULL; + tiw->sentinel.arg = NULL; return tiw; } @@ -583,9 +583,9 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, ErtsMonotonicTime timeout_time; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); - p->u.func.timeout = timeout; - p->u.func.cancel = cancel; - p->u.func.arg = arg; + p->timeout = timeout; + p->cancel = cancel; + p->arg = arg; ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE); @@ -636,8 +636,8 @@ erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) void *arg; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); remove_timer(tiw, p); - cancel = p->u.func.cancel; - arg = p->u.func.arg; + cancel = p->cancel; + arg = p->arg; if (cancel) (*cancel)(arg); ERTS_MSACC_POP_STATE_M_X(); @@ -657,22 +657,22 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw, tmr = tiw->sentinel.next; while (tmr != &tiw->sentinel) { - if (tmr->u.func.timeout == tclbk) - (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + if (tmr->timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->arg); tmr = tmr->next; } for (tmr = tiw->at_once.head; tmr; tmr = tmr->next) { - if (tmr->u.func.timeout == tclbk) - (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + if (tmr->timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->arg); } for (ix = 0; ix < ERTS_TIW_SIZE; ix++) { tmr = tiw->w[ix]; if (tmr) { do { - if (tmr->u.func.timeout == tclbk) - (*func)(arg, tmr->timeout_pos, tmr->u.func.arg); + if (tmr->timeout == tclbk) + (*func)(arg, tmr->timeout_pos, tmr->arg); tmr = tmr->next; } while (tmr != tiw->w[ix]); } -- cgit v1.2.3 From d37df80764c9c2e517d7c9a90291d75975a5e9bb Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 3 Feb 2017 16:08:12 +0100 Subject: Remove unnecessary cancel callback from timer-wheel timers --- erts/emulator/beam/erl_hl_timer.c | 16 +--------------- erts/emulator/beam/erl_process.c | 1 - erts/emulator/beam/erl_time.h | 7 ++----- erts/emulator/beam/erl_time_sup.c | 3 --- erts/emulator/beam/time.c | 11 +---------- 5 files changed, 4 insertions(+), 34 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c index 8e47fdd674..13d6136672 100644 --- a/erts/emulator/beam/erl_hl_timer.c +++ b/erts/emulator/beam/erl_hl_timer.c @@ -995,18 +995,13 @@ tw_port_timeout(void *vtwtp) tw_timer_dec_refc(twtp); } -static void -tw_ptimer_cancel(void *vtwtp) -{ - tw_timer_dec_refc((ErtsTWTimer *) vtwtp); -} - static void cancel_tw_timer(ErtsSchedulerData *esdp, ErtsTWTimer *tmr) { ERTS_HLT_ASSERT((tmr->head.roflgs & ERTS_TMR_ROFLG_SID_MASK) == (Uint32) esdp->no); erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->u.tw_tmr); + tw_timer_dec_refc(tmr); } static void @@ -1034,7 +1029,6 @@ create_tw_timer(ErtsSchedulerData *esdp, { ErtsTWTimer *tmr; void (*timeout_func)(void *); - void (*cancel_func)(void *); erts_aint32_t refc; if (type != ERTS_TMR_BIF) { @@ -1068,7 +1062,6 @@ create_tw_timer(ErtsSchedulerData *esdp, tmr->head.receiver.proc = (Process *) rcvrp; tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; timeout_func = tw_proc_timeout; - cancel_func = tw_ptimer_cancel; erts_proc_inc_refc((Process *) rcvrp); refc = 2; break; @@ -1077,7 +1070,6 @@ create_tw_timer(ErtsSchedulerData *esdp, tmr->head.receiver.port = (Port *) rcvrp; tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT; timeout_func = tw_port_timeout; - cancel_func = tw_ptimer_cancel; erts_port_inc_refc((Port *) rcvrp); refc = 2; break; @@ -1088,14 +1080,12 @@ create_tw_timer(ErtsSchedulerData *esdp, tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK; timeout_func = tw_callback_timeout; - cancel_func = NULL; refc = 1; break; case ERTS_TMR_BIF: timeout_func = tw_bif_timer_timeout; - cancel_func = NULL; if (is_internal_pid(rcvr)) { tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC; tmr->head.receiver.proc = (Process *) rcvrp; @@ -1129,7 +1119,6 @@ create_tw_timer(ErtsSchedulerData *esdp, erts_twheel_set_timer(esdp->timer_wheel, &tmr->u.tw_tmr, timeout_func, - cancel_func, tmr, timeout_pos); @@ -1452,7 +1441,6 @@ create_hl_timer(ErtsSchedulerData *esdp, erts_twheel_set_timer(esdp->timer_wheel, &srv->service_timer, hlt_service_timeout, - NULL, (void *) esdp, tmr->timeout); srv->next_timeout = tmr; @@ -1574,7 +1562,6 @@ hlt_service_timeout(void *vesdp) erts_twheel_set_timer(esdp->timer_wheel, &srv->service_timer, hlt_service_timeout, - NULL, vesdp, tmr->timeout); } @@ -1631,7 +1618,6 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr) erts_twheel_set_timer(esdp->timer_wheel, &srv->service_timer, hlt_service_timeout, - NULL, (void *) esdp, smlst->timeout); } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 5baf533acd..267ca7e8a6 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -2815,7 +2815,6 @@ start_aux_work_timer(ErtsSchedulerData *esdp) erts_twheel_set_timer(esdp->timer_wheel, &aux_work_tmo->timer.data, aux_work_timeout, - NULL, (void *) esdp, tmo); } diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 2fb4b754d7..00b4a84b4a 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -420,21 +420,18 @@ void erts_sched_init_time_sup(ErtsSchedulerData *esdp); */ typedef struct erl_timer { ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */ - struct erl_timer* next; /* next entry tiw slot or chain */ + struct erl_timer* next; /* next entry tiw slot or chain */ struct erl_timer* prev; /* prev entry tiw slot or chain */ void (*timeout)(void*); /* called when timeout */ - void (*cancel)(void*); /* called when cancel (may be NULL) */ void* arg; /* argument to timeout/cancel procs */ int slot; } ErtsTWheelTimer; typedef void (*ErlTimeoutProc)(void*); -typedef void (*ErlCancelProc)(void*); void erts_twheel_set_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p, ErlTimeoutProc timeout, - ErlCancelProc cancel, void *arg, - ErtsMonotonicTime timeout_pos); + void *arg, ErtsMonotonicTime timeout_pos); void erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p); ErtsTimerWheel *erts_create_timer_wheel(ErtsSchedulerData *esdp); diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index cf9d3adc86..c69fec3c80 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -678,7 +678,6 @@ check_time_correction(void *vesdp) erts_twheel_set_timer(esdp->timer_wheel, &time_sup.inf.c.parmon.timer, check_time_correction, - NULL, (void *) esdp, timeout_pos); } @@ -729,7 +728,6 @@ check_time_offset(void *vesdp) erts_twheel_set_timer(esdp->timer_wheel, &time_sup.inf.c.parmon.timer, check_time_offset, - NULL, vesdp, timeout_pos); } @@ -836,7 +834,6 @@ late_init_time_correction(ErtsSchedulerData *esdp) erts_twheel_set_timer(esdp->timer_wheel, &time_sup.inf.c.parmon.timer, check_func, - NULL, (quick_init_drift_adj ? NULL : esdp), diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index fdf74498c9..53896be19e 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -543,7 +543,6 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) tiw->sentinel.next = &tiw->sentinel; tiw->sentinel.prev = &tiw->sentinel; tiw->sentinel.timeout = NULL; - tiw->sentinel.cancel = NULL; tiw->sentinel.arg = NULL; return tiw; } @@ -577,14 +576,12 @@ erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode) void erts_twheel_set_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p, ErlTimeoutProc timeout, - ErlCancelProc cancel, void *arg, - ErtsMonotonicTime timeout_pos) + void *arg, ErtsMonotonicTime timeout_pos) { ErtsMonotonicTime timeout_time; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); p->timeout = timeout; - p->cancel = cancel; p->arg = arg; ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE); @@ -632,14 +629,8 @@ void erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) { if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) { - ErlCancelProc cancel; - void *arg; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); remove_timer(tiw, p); - cancel = p->cancel; - arg = p->arg; - if (cancel) - (*cancel)(arg); ERTS_MSACC_POP_STATE_M_X(); } } -- cgit v1.2.3 From 0981bfd7afee64c3c1f48db134d81e57c30e31c7 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 6 Feb 2017 20:39:02 +0100 Subject: Timer wheel divided into a "soon wheel" and a "later wheel" The old single wheel implementation handled about 65 seconds. The new dual wheel implementation handles more than 37 hours. The dual wheels have also been shrunk compared to the single wheel, so the total memory consumption for timer wheels have been cut in half. --- erts/emulator/beam/break.c | 2 +- erts/emulator/beam/erl_time.h | 50 ++- erts/emulator/beam/time.c | 810 ++++++++++++++++++++++++++++++++---------- 3 files changed, 671 insertions(+), 191 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 0b40d70cb7..538ab0d947 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -590,7 +590,7 @@ do_break(void) #endif #ifdef DEBUG case 't': - erts_p_slpq(); + /* erts_p_slpq(); */ return; case 'b': bin_check(); diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 00b4a84b4a..699e69e991 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -21,19 +21,52 @@ #ifndef ERL_TIME_H__ #define ERL_TIME_H__ -/* timer wheel size NEED to be a power of 2 */ -#ifdef SMALL_MEMORY -#define ERTS_TIW_SIZE (1 << 13) -#else -#define ERTS_TIW_SIZE (1 << 16) +#if 0 +# define ERTS_TW_DEBUG +#endif +#if defined(DEBUG) && !defined(ERTS_TW_DEBUG) +# define ERTS_TW_DEBUG #endif -#if defined(DEBUG) || 0 +#if defined(ERTS_TW_DEBUG) #define ERTS_TIME_ASSERT(B) ERTS_ASSERT(B) #else #define ERTS_TIME_ASSERT(B) ((void) 1) #endif +#ifdef ERTS_TW_DEBUG +/* + * Soon wheel will handle about 1 seconds + * Later wheel will handle about 8 minutes + */ +# define ERTS_TW_SOON_WHEEL_BITS 10 +# define ERTS_TW_LATER_WHEEL_BITS 10 +#else +# ifdef SMALL_MEMORY +/* + * Soon wheel will handle about 4 seconds + * Later wheel will handle about 2 hours and 19 minutes + */ +# define ERTS_TW_SOON_WHEEL_BITS 12 +# define ERTS_TW_LATER_WHEEL_BITS 12 +# else +/* + * Soon wheel will handle about 16 seconds + * Later wheel will handle about 37 hours and 16 minutes + */ +# define ERTS_TW_SOON_WHEEL_BITS 14 +# define ERTS_TW_LATER_WHEEL_BITS 14 +# endif +#endif + +/* + * Number of slots in each timer wheel... + * + * These *need* to be a power of 2 + */ +#define ERTS_TW_SOON_WHEEL_SIZE (1 << ERTS_TW_SOON_WHEEL_BITS) +#define ERTS_TW_LATER_WHEEL_SIZE (1 << ERTS_TW_LATER_WHEEL_BITS) + typedef enum { ERTS_NO_TIME_WARP_MODE, ERTS_SINGLE_TIME_WARP_MODE, @@ -103,7 +136,10 @@ Eterm erts_system_time_source(struct process*c_p); #define ERTS_CLKTCK_RESOLUTION (erts_time_sup__.r.o.clktck_resolution) #endif -#define ERTS_TIMER_WHEEL_MSEC (ERTS_TIW_SIZE/(ERTS_CLKTCK_RESOLUTION/1000)) +#define ERTS_TW_SOON_WHEEL_MSEC (ERTS_TW_SOON_WHEEL_SIZE/(ERTS_CLKTCK_RESOLUTION/1000)) +#define ERTS_TW_LATER_WHEEL_MSEC (ERTS_TW_LATER_WHEEL_SIZE*ERTS_TW_SOON_WHEEL_MSEC/2) + +#define ERTS_TIMER_WHEEL_MSEC ERTS_TW_LATER_WHEEL_MSEC struct erts_time_sup_read_only__ { ErtsMonotonicTime monotonic_time_unit; diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 53896be19e..7b5ff7b992 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -17,57 +17,124 @@ * * %CopyrightEnd% */ + /* - * TIMING WHEEL + * TIMER WHEEL + * + * + * The time scale used for timers is Erlang monotonic time. The + * time unit used is ERTS specific clock ticks. A clock tick is + * currently defined to 1 millisecond. That is, the resolution of + * timers triggered by the runtime system is 1 millisecond. * - * Timeouts kept in an wheel. A timeout is measured relative to the - * current slot (tiw_pos) in the wheel, and inserted at slot - * (tiw_pos + timeout) % TIW_SIZE. Each timeout also has a count - * equal to timeout/TIW_SIZE, which is needed since the time axis - * is wrapped arount the wheel. + * When a timer is set, it is determined at what Erlang monotonic + * time, in clock ticks, it should be triggered. * - * Several slots may be processed in one operation. If the number of - * slots is greater that the wheel size, the wheel is only traversed - * once, + * The 'pos' field of the wheel corresponds to current time of + * the wheel. That is, it corresponds to Erlang monotonic time in + * clock tick time unit. The 'pos' field of the wheel is + * monotonically increased when erts_bump_timers() is called. All + * timers in the wheel that have a time less than or equal to + * 'pos' are triggered by the bump operation. The bump operation + * may however be spread over multiple calls to erts_bump_timers() + * if there are a lots of timers to trigger. + * + * Each scheduler thread maintains its own timer wheel. The timer + * wheel of a scheduler, however, actually consists of two wheels. + * A soon wheel and a later wheel. + * + * + * -- The Soon Wheel -- + * + * The soon wheel contain timers that should be triggered soon. + * That is, they are soon to be triggered. Each slot in the soon + * wheel is 1 clock tick wide. The number of slots in the soon + * wheel is currently 2¹⁴. That is, it contains timers in the + * range ('pos', 'pos' + 2¹⁴] which corresponds to a bit more + * than 16 seconds. + * + * When the bump operation is started, 'pos' is moved forward to a + * position that corresponds to current Erlang monotonic time. Then + * all timers that are in the range (old 'pos', new 'pos'] are + * triggered. During a bump operation, the soon wheel may contain + * timers in the two, possibly overlapping, ranges (old 'pos', + * old 'pos' + 2¹⁴], and (new 'pos', new 'pos' + 2¹⁴]. This may + * occur even if the bump operation doesn't yield, due to timeout + * callbacks inserting new timers. + * + * + * -- The Later Wheel -- + * + * The later wheel contain timers that are further away from 'pos' + * than the width of the soon timer wheel. That is, currently + * timers further away from 'pos' than 2¹⁴ clock ticks. The width + * of each slot in the later wheel is half the width of the soon + * wheel. That is, each slot is currently 2¹³ clock ticks wide + * which corresponds to about 8 seconds. If three timers of the + * times 'pos' + 17000, 'pos' + 18000, and 'pos' + 19000 are + * inserted, they will all end up in the same slot in the later + * wheel. + * + * The number of slots in the later wheel is currently the same as + * in the soon wheel, i.e. 2¹⁴. That is, one revolution of the later + * wheel currently corresponds to 2¹⁴×2¹³ clock ticks which is + * almost 37 ½ hour. Timers even further away than that are put in + * the later slot identified by their time modulo the size of the later + * wheel. Such timers are however very uncommon. Most timers used + * by the runtime system will utilize the high level timer API. + * The high level timer implementation will not insert timers + * further away then one revolution into the later wheel. It will + * instead keep such timers in a tree of very long timers. The + * high level timer implementation utilize one timer wheel timer + * for the management of this tree of timers. This timer is set to + * the closest timeout in the tree. This timer may however be + * further away than one revolution in the later wheel. + * + * The 'later.pos' field identifies next position in the later wheel. + * 'later.pos' is always increased by the width of a later wheel slot. + * That is, currently 2¹³ clock ticks. When 'pos' is moved (during + * a bump operation) closer to 'later.pos' than the width of a later + * wheel slot, i.e. currently when 'pos' + 2¹³ ≥ 'later.pos', we + * inspect the slot identified by 'later.pos' and then move 'later.pos' + * forward. When inspecting the later slot we move all timers in the + * slot, that are in the soon wheel range, from the later wheel to + * the soon wheel. Timers one or more revolutions of the later wheel + * away are kept in the slot. + * + * During normal operation, timers originally located in the later + * wheel will currently be moved into the soon wheel about 8 to + * 16 seconds before they should be triggered. During extremely + * heavy load, the scheduler might however be heavily delayed, so + * the code must be prepared for situations where time for + * triggering the timer has passed when we inspect the later wheel + * slot, and then trigger the timer immediately. We must also be + * prepared to inspect multiple later wheel slots at once due to the + * delay. + * + * + * -- Slot Management -- + * + * All timers of a slot are placed in a circular double linked + * list. This makes insertion and removal of a timer O(1). + * + * While bumping timers in a slot, we move the circular list + * away from the slot, and refer to it from the 'sentinel' + * field. The list will stay there until we are done with it + * even if the bump operation should yield. The cancel operation + * can remove the timer from this position as well as from the + * slot position by just removing it from the circular double + * linked list that it is in. + * + * -- At Once List -- + * + * If a timer is set that has a time earlier or equal to 'pos', + * it is not inserted into the wheel. It is instead inserted, + * into a list referred to by the 'at_once' field. When the + * bump operation is performed thise timers will be triggered + * at once. * - * The following example shows a time axis where there is one timeout - * at each "tick", and where 1, 2, 3 ... wheel slots are released in - * one operation. The notation "pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE >= (TIW)->later.pos) + +static int bump_later_wheel(ErtsTimerWheel *tiw, int *yield_count_p); + +static ERTS_INLINE int +soon_slot(ErtsMonotonicTime soon_pos) +{ + ErtsMonotonicTime slot = soon_pos; + slot &= ERTS_TW_SOON_WHEEL_MASK; + + ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= slot); + ERTS_TW_ASSERT(slot < ERTS_TW_SOON_WHEEL_END_SLOT); + + return (int) slot; +} + +static ERTS_INLINE int +later_slot(ErtsMonotonicTime later_pos) +{ + ErtsMonotonicTime slot = later_pos; + slot >>= ERTS_TW_LATER_WHEEL_SHIFT; + slot &= ERTS_TW_LATER_WHEEL_MASK; + slot += ERTS_TW_LATER_WHEEL_FIRST_SLOT; + + ERTS_TW_ASSERT(ERTS_TW_LATER_WHEEL_FIRST_SLOT <= slot); + ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT); + + return (int) slot; +} + +#ifdef ERTS_TW_HARD_DEBUG +#define ERTS_HARD_DBG_CHK_WHEELS(TIW, CHK_MIN_TPOS) \ + hrd_dbg_check_wheels((TIW), (CHK_MIN_TPOS)) +static void hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos); +#else +#define ERTS_HARD_DBG_CHK_WHEELS(TIW, CHK_MIN_TPOS) +#endif + static ERTS_INLINE ErtsMonotonicTime find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, @@ -143,11 +289,14 @@ find_next_timeout(ErtsSchedulerData *esdp, ErtsMonotonicTime curr_time, /* When !search_all */ ErtsMonotonicTime max_search_time) /* When !search_all */ { - int start_ix, tiw_pos_ix; - ErtsTWheelTimer *p; + int start_ix, tiw_pos_ix, pos_inc, wheel_start_ix, wheel_end_ix; int true_min_timeout = 0; ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos; + ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); + + ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TWHEEL_SLOT_INACTIVE); + if (tiw->nto == 0) { /* no timeouts in wheel */ if (!search_all) min_timeout_pos = tiw->pos; @@ -156,58 +305,115 @@ find_next_timeout(ErtsSchedulerData *esdp, tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); } min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); - goto found_next; + goto done; } - slot_timeout_pos = min_timeout_pos = tiw->pos; if (search_all) - min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); + min_timeout_pos = tiw->pos + ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); else - min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time); + min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time); + + if (!tiw->soon.nto) { + /* Select later wheel... */ + slot_timeout_pos = tiw->later.pos; + start_ix = tiw_pos_ix = later_slot(slot_timeout_pos); + pos_inc = ERTS_TW_LATER_WHEEL_SLOT_SIZE; + wheel_start_ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + wheel_end_ix = ERTS_TW_LATER_WHEEL_END_SLOT; + /* Pre-timeout for move from later to soon wheel... */ + slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + } + else { + /* Select soon wheel... */ + + /* + * Besides inspecting the soon wheel we + * may also have to inspect two slots in the + * later wheel which potentially can trigger + * timeouts before timeouts in soon wheel... + */ + slot_timeout_pos = tiw->later.pos; + slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (slot_timeout_pos < min_timeout_pos) { + int fslot = later_slot(tiw->later.pos); + if (tiw->w[fslot]) { + min_timeout_pos = slot_timeout_pos; + true_min_timeout = 1; + } + else { + slot_timeout_pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (slot_timeout_pos < min_timeout_pos) { + fslot++; + if (fslot == ERTS_TW_LATER_WHEEL_END_SLOT) + fslot = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + if (tiw->w[fslot]) { + min_timeout_pos = slot_timeout_pos; + true_min_timeout = 1; + } + } + } + } - start_ix = tiw_pos_ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1)); + slot_timeout_pos = tiw->pos; + start_ix = tiw_pos_ix = soon_slot(tiw->pos); + pos_inc = 1; + wheel_start_ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + wheel_end_ix = ERTS_TW_SOON_WHEEL_END_SLOT; + } + /* + * Scan selected wheel... + */ do { - if (++slot_timeout_pos >= min_timeout_pos) - break; - - p = tiw->w[tiw_pos_ix]; - - if (p) { - ErtsTWheelTimer *end = p; - - do { - ErtsMonotonicTime timeout_pos; - timeout_pos = p->timeout_pos; - if (min_timeout_pos > timeout_pos) { - true_min_timeout = 1; - min_timeout_pos = timeout_pos; - if (min_timeout_pos <= slot_timeout_pos) - goto found_next; - } - p = p->next; - } while (p != end); - } - - tiw_pos_ix++; - if (tiw_pos_ix == ERTS_TIW_SIZE) - tiw_pos_ix = 0; + if (tiw->w[tiw_pos_ix]) { + min_timeout_pos = slot_timeout_pos; + true_min_timeout = 1; + break; + } + + slot_timeout_pos += pos_inc; + if (slot_timeout_pos >= min_timeout_pos) + break; + + tiw_pos_ix++; + if (tiw_pos_ix == wheel_end_ix) + tiw_pos_ix = wheel_start_ix; } while (start_ix != tiw_pos_ix); -found_next: +done: min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos); tiw->next_timeout_time = min_timeout; tiw->true_next_timeout_time = true_min_timeout; + ERTS_HARD_DBG_CHK_WHEELS(tiw, 1); + return min_timeout; } +static ERTS_INLINE void +insert_timer_into_at_once_list(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) +{ + tiw->at_once.nto++; + p->next = NULL; + p->prev = tiw->at_once.tail; + if (tiw->at_once.tail) { + ERTS_TW_ASSERT(tiw->at_once.head); + tiw->at_once.tail->next = p; + } + else { + ERTS_TW_ASSERT(!tiw->at_once.head); + tiw->at_once.head = p; + } + tiw->at_once.tail = p; + p->slot = ERTS_TWHEEL_SLOT_AT_ONCE; +} + static ERTS_INLINE void insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) { - ERTS_TW_ASSERT(slot >= 0); - ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE); + ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= slot); + ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT); p->slot = slot; if (!tiw->w[slot]) { tiw->w[slot] = p; @@ -223,6 +429,10 @@ insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) prev->next = p; next->prev = p; } + if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) { + ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + tiw->soon.nto++; + } } static ERTS_INLINE void @@ -231,13 +441,13 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) int slot = p->slot; ERTS_TW_ASSERT(slot != ERTS_TWHEEL_SLOT_INACTIVE); - if (slot >= 0) { + if (slot >= ERTS_TW_SOON_WHEEL_FIRST_SLOT) { /* * Timer in wheel or in circular * list of timers currently beeing * triggered (referred by sentinel). */ - ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE); + ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT); if (p->next == p) { ERTS_TW_ASSERT(tiw->w[slot] == p); @@ -249,6 +459,8 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) p->prev->next = p->next; p->next->prev = p->prev; } + if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) + tiw->soon.nto--; } else { /* Timer in "at once" queue... */ @@ -270,8 +482,6 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) } p->slot = ERTS_TWHEEL_SLOT_INACTIVE; - - tiw->nto--; } ErtsMonotonicTime @@ -299,13 +509,13 @@ debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos) ErtsTWheelTimer *tmr; ErtsMonotonicTime tmp; - ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1)); + ix = soon_slot(tiw->pos); tmp = skip_to_pos - tiw->pos; ERTS_TW_ASSERT(tmp >= 0); - if (tmp < (ErtsMonotonicTime) ERTS_TIW_SIZE) + if (tmp < (ErtsMonotonicTime) ERTS_TW_SOON_WHEEL_SIZE) slots = (int) tmp; else - slots = ERTS_TIW_SIZE; + slots = ERTS_TW_SOON_WHEEL_SIZE; while (slots > 0) { tmr = tiw->w[ix]; @@ -317,10 +527,41 @@ debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos) } while (tmr != end); } ix++; - if (ix == ERTS_TIW_SIZE) - ix = 0; + if (ix == ERTS_TW_SOON_WHEEL_END_SLOT) + ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; slots--; } + + ix = later_slot(tiw->later.pos); + tmp = skip_to_pos; + tmp &= ERTS_TW_LATER_WHEEL_POS_MASK; + if (tmp > tiw->later.pos) { + tmp -= tiw->later.pos; + tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + ERTS_TW_ASSERT(tmp >= 0); + if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE) + slots = (int) tmp; + else + slots = ERTS_TW_LATER_WHEEL_SIZE; + + while (slots > 0) { + tmr = tiw->w[ix]; + if (tmr) { + ErtsMonotonicTime tpos = tmr->timeout_pos; + ErtsTWheelTimer *end = tmr; + do { + tpos &= ERTS_TW_LATER_WHEEL_POS_MASK; + tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + ERTS_TW_ASSERT(tpos > skip_to_pos); + tmr = tmr->next; + } while (tmr != end); + } + ix++; + if (ix == ERTS_TW_LATER_WHEEL_END_SLOT) + ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + slots--; + } + } } #endif @@ -339,8 +580,8 @@ timeout_timer(ErtsTWheelTimer *p) void erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) { - int tiw_pos_ix, slots, yielded_slot_restarted, yield_count; - ErtsMonotonicTime bump_to, tmp_slots, old_pos; + int tiw_pos_ix, yielded_slot_restarted, yield_count, slots; + ErtsMonotonicTime bump_to; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); yield_count = ERTS_TWHEEL_BUMP_YIELD_LIMIT; @@ -350,13 +591,16 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) * where we left off when restarting after a yield. */ - if (tiw->yield_slot >= 0) { + if (tiw->yield_slot >= ERTS_TW_SOON_WHEEL_FIRST_SLOT) { yielded_slot_restarted = 1; + bump_to = tiw->pos; + if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) + goto restart_yielded_later_slot; tiw_pos_ix = tiw->yield_slot; slots = tiw->yield_slots_left; - bump_to = tiw->pos; - old_pos = tiw->yield_start_pos; - goto restart_yielded_slot; + ASSERT(0 <= slots && slots <= ERTS_TW_SOON_WHEEL_SIZE); + tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + goto restart_yielded_soon_slot; } do { @@ -368,8 +612,6 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) while (1) { ErtsTWheelTimer *p; - old_pos = tiw->pos; - if (tiw->nto == 0) { empty_wheel: ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to); @@ -383,12 +625,12 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) p = tiw->at_once.head; while (p) { - if (--yield_count <= 0) { + if (yield_count <= 0) { ERTS_TW_ASSERT(tiw->nto > 0); ERTS_TW_ASSERT(tiw->at_once.nto > 0); tiw->yield_slot = ERTS_TWHEEL_SLOT_AT_ONCE; tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos); + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); ERTS_MSACC_POP_STATE_M_X(); return; } @@ -405,6 +647,8 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) timeout_timer(p); + yield_count -= ERTS_TW_COST_TIMEOUT; + p = tiw->at_once.head; } @@ -433,22 +677,34 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos); tiw->pos = skip_until_pos; + + skip_until_pos++; + skip_until_pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + if (tiw->later.pos < skip_until_pos) + tiw->later.pos = skip_until_pos; } } - tiw_pos_ix = (int) ((tiw->pos+1) & (ERTS_TIW_SIZE-1)); - tmp_slots = (bump_to - tiw->pos); - if (tmp_slots < (ErtsMonotonicTime) ERTS_TIW_SIZE) - slots = (int) tmp_slots; - else - slots = ERTS_TIW_SIZE; + { + ErtsMonotonicTime tmp_slots = bump_to - tiw->pos; + tmp_slots = (bump_to - tiw->pos); + if (tmp_slots < ERTS_TW_SOON_WHEEL_SIZE) + slots = (int) tmp_slots; + else + slots = ERTS_TW_SOON_WHEEL_SIZE; + } + tiw_pos_ix = soon_slot(tiw->pos+1); tiw->pos = bump_to; - while (slots > 0) { + /* Timeout timers in soon wheel */ + while (slots) { + + yield_count -= ERTS_TW_COST_SLOT; p = tiw->w[tiw_pos_ix]; if (p) { + /* timeout callback need tiw->pos to be up to date */ if (p->next == p) { ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel); ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); @@ -463,18 +719,21 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) while (1) { - if (p->timeout_pos > bump_to) { - /* Very unusual case... */ - ++yield_count; - insert_timer_into_slot(tiw, tiw_pos_ix, p); - } - else { - /* Normal case... */ - timeout_timer(p); - tiw->nto--; - } - - restart_yielded_slot: + ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= p->slot + && p->slot < ERTS_TW_SOON_WHEEL_END_SLOT); + tiw->soon.nto--; + if (p->timeout_pos <= bump_to) { + timeout_timer(p); + tiw->nto--; + yield_count -= ERTS_TW_COST_TIMEOUT; + } + else { + /* uncommon case */ + insert_timer_into_slot(tiw, tiw_pos_ix, p); + yield_count -= ERTS_TW_COST_SLOT_MOVE; + } + + restart_yielded_soon_slot: p = tiw->sentinel.next; if (p == &tiw->sentinel) { @@ -482,12 +741,11 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) break; } - if (--yield_count <= 0) { + if (yield_count <= 0) { tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos); + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); tiw->yield_slot = tiw_pos_ix; tiw->yield_slots_left = slots; - tiw->yield_start_pos = old_pos; ERTS_MSACC_POP_STATE_M_X(); return; /* Yield! */ } @@ -496,16 +754,22 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) p->next->prev = &tiw->sentinel; } } - tiw_pos_ix++; - if (tiw_pos_ix == ERTS_TIW_SIZE) - tiw_pos_ix = 0; - slots--; + + tiw_pos_ix++; + if (tiw_pos_ix == ERTS_TW_SOON_WHEEL_END_SLOT) + tiw_pos_ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + slots--; } + + if (ERTS_TW_BUMP_LATER_WHEEL(tiw)) { + restart_yielded_later_slot: + if (bump_later_wheel(tiw, &yield_count)) + return; /* Yield! */ + } } } while (yielded_slot_restarted); - tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; tiw->true_next_timeout_time = 0; tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; @@ -514,6 +778,119 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) ERTS_MSACC_POP_STATE_M_X(); } +static int +bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) +{ + ErtsMonotonicTime cpos = tiw->pos; + ErtsMonotonicTime later_pos = tiw->later.pos; + int ycount = *ycount_p; + int slots, fslot; + + ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); + + if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) { + fslot = tiw->yield_slot; + slots = tiw->yield_slots_left; + ASSERT(0 <= slots && slots <= ERTS_TW_LATER_WHEEL_SIZE); + tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + goto restart_yielded_slot; + } + else { + ErtsMonotonicTime tmp_slots = cpos; + tmp_slots += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + tmp_slots -= later_pos; + tmp_slots /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (tmp_slots < ERTS_TW_LATER_WHEEL_SIZE) + slots = (int) tmp_slots; + else + slots = ERTS_TW_LATER_WHEEL_SIZE; + } + + while (slots >= 0) { + ErtsTWheelTimer *p; + + fslot = later_slot(later_pos); + + ycount -= ERTS_TW_COST_SLOT; + + p = tiw->w[fslot]; + + if (p) { + + if (p->next == p) { + ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel); + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + } + else { + tiw->sentinel.next = p->next; + tiw->sentinel.prev = p->prev; + tiw->sentinel.next->prev = &tiw->sentinel; + tiw->sentinel.prev->next = &tiw->sentinel; + } + tiw->w[fslot] = NULL; + + while (1) { + ErtsMonotonicTime tpos = p->timeout_pos; + + ERTS_TW_ASSERT(tpos >= later_pos); + ERTS_TW_ASSERT(p->slot == fslot); + + if (tpos >= later_pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) { + /* keep in later slot; very uncommon... */ + insert_timer_into_slot(tiw, fslot, p); + ycount -= ERTS_TW_COST_SLOT_MOVE; + } + else { + ERTS_TW_ASSERT(tpos < cpos + ERTS_TW_SOON_WHEEL_SIZE); + if (tpos > cpos) { + /* move into soon wheel */ + insert_timer_into_slot(tiw, soon_slot(tpos), p); + ycount -= ERTS_TW_COST_SLOT_MOVE; + } + else { + /* trigger at once */ + timeout_timer(p); + tiw->nto--; + ycount -= ERTS_TW_COST_TIMEOUT; + } + } + + restart_yielded_slot: + + p = tiw->sentinel.next; + if (p == &tiw->sentinel) { + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + break; + } + + if (ycount < 0) { + tiw->later.pos = later_pos; + tiw->true_next_timeout_time = 1; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(cpos); + tiw->yield_slot = fslot; + tiw->yield_slots_left = slots; + *ycount_p = 0; + ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); + return 1; /* Yield! */ + } + + tiw->sentinel.next = p->next; + p->next->prev = &tiw->sentinel; + } + } + slots--; + later_pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + } + + ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); + + tiw->later.pos = later_pos; + + *ycount_p = ycount; + + return 0; +} + Uint erts_timer_wheel_memory_size(void) { @@ -524,11 +901,11 @@ ErtsTimerWheel * erts_create_timer_wheel(ErtsSchedulerData *esdp) { ErtsMonotonicTime mtime; - int i; + int i = ERTS_TW_SOON_WHEEL_FIRST_SLOT; ErtsTimerWheel *tiw; tiw = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_WHEEL, sizeof(ErtsTimerWheel)); - for(i = 0; i < ERTS_TIW_SIZE; i++) + for(; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++) tiw->w[i] = NULL; mtime = erts_get_monotonic_time(esdp); @@ -537,6 +914,10 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) tiw->at_once.head = NULL; tiw->at_once.tail = NULL; tiw->at_once.nto = 0; + tiw->soon.nto = 0; + tiw->later.pos = tiw->pos; + tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + tiw->later.pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; tiw->true_next_timeout_time = 0; tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY; @@ -586,36 +967,40 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE); + tiw->nto++; if (timeout_pos <= tiw->pos) { - tiw->nto++; - tiw->at_once.nto++; - p->next = NULL; - p->prev = tiw->at_once.tail; - if (tiw->at_once.tail) { - ERTS_TW_ASSERT(tiw->at_once.head); - tiw->at_once.tail->next = p; - } - else { - ERTS_TW_ASSERT(!tiw->at_once.head); - tiw->at_once.head = p; - } - tiw->at_once.tail = p; - p->timeout_pos = tiw->pos; - p->slot = ERTS_TWHEEL_SLOT_AT_ONCE; + p->timeout_pos = tiw->pos; + insert_timer_into_at_once_list(tiw, p); timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->pos); } else { int slot; + p->timeout_pos = timeout_pos; + /* calculate slot */ - slot = (int) (timeout_pos & (ERTS_TIW_SIZE-1)); + if (timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) { + /* soon wheel */ + slot = soon_slot(timeout_pos); + } + else { + /* later wheel */ + slot = later_slot(timeout_pos); + + /* + * Next timeout due to this timeout + * should be in good time before the + * actual timeout (one later wheel slot + * size). This, in order to move it + * from the later wheel to the soon + * wheel. + */ + timeout_pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + } insert_timer_into_slot(tiw, slot, p); - - tiw->nto++; - - timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); - p->timeout_pos = timeout_pos; + timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); } if (timeout_time < tiw->next_timeout_time) { @@ -631,6 +1016,7 @@ erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) { ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); remove_timer(tiw, p); + tiw->nto--; ERTS_MSACC_POP_STATE_M_X(); } } @@ -658,7 +1044,9 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw, (*func)(arg, tmr->timeout_pos, tmr->arg); } - for (ix = 0; ix < ERTS_TIW_SIZE; ix++) { + for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + ix < ERTS_TW_LATER_WHEEL_END_SLOT; + ix++) { tmr = tiw->w[ix]; if (tmr) { do { @@ -670,36 +1058,92 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw, } } -#ifdef ERTS_TW_DEBUG -void erts_p_slpq(void) +#ifdef ERTS_TW_HARD_DEBUG + +static void +hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) { - erts_printf("Not yet implemented...\n"); -#if 0 - ErtsMonotonicTime current_time = erts_get_monotonic_time(NULL); - int i; - ErtsTWheelTimer* p; - - /* print the whole wheel, starting at the current position */ - erts_printf("\ncurrent time = %bps tiw_pos = %d tiw_nto %d\n", - current_time, tiw->pos, tiw->nto); - i = tiw->pos; - if (tiw->w[i] != NULL) { - erts_printf("%d:\n", i); - for(p = tiw->w[i]; p != NULL; p = p->next) { - erts_printf(" (timeout time %bps, slot %d)\n", - ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), - p->slot); - } + int ix, soon_tmo, later_tmo, at_once_tmo; + ErtsMonotonicTime min_tpos; + + min_tpos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); + + soon_tmo = 0; + for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + ix < ERTS_TW_SOON_WHEEL_END_SLOT; + ix++) { + ErtsTWheelTimer *p; + + p = tiw->w[ix]; + if (p) { + ErtsTWheelTimer *first = p; + do { + ErtsMonotonicTime tpos = p->timeout_pos; + ERTS_TW_ASSERT(p->slot == ix); + soon_tmo++; + ERTS_TW_ASSERT(ix == soon_slot(tpos)); + ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + ERTS_TW_ASSERT(!check_min_tpos || tpos >= min_tpos); + ERTS_TW_ASSERT(p->next->prev == p); + p = p->next; + } while (p != first); + } } - for(i = ((i+1) & (ERTS_TIW_SIZE-1)); i != (tiw->pos & (ERTS_TIW_SIZE-1)); i = ((i+1) & (ERTS_TIW_SIZE-1))) { - if (tiw->w[i] != NULL) { - erts_printf("%d:\n", i); - for(p = tiw->w[i]; p != NULL; p = p->next) { - erts_printf(" (timeout time %bps, slot %d)\n", - ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), p->slot); - } - } + + later_tmo = 0; + for (ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + ix < ERTS_TW_LATER_WHEEL_END_SLOT; + ix++) { + ErtsTWheelTimer *p; + + p = tiw->w[ix]; + if (p) { + ErtsTWheelTimer *first = p; + do { + ErtsMonotonicTime tpos = p->timeout_pos; + ERTS_TW_ASSERT(p->slot == ix); + later_tmo++; + ERTS_TW_ASSERT(later_slot(tpos) == ix); + tpos &= ERTS_TW_LATER_WHEEL_POS_MASK; + tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + ERTS_TW_ASSERT(!check_min_tpos || tpos >= min_tpos); + ERTS_TW_ASSERT(p->next->prev == p); + p = p->next; + } while (p != first); + } } -#endif + + if (tiw->yield_slot >= 0) { + ErtsTWheelTimer *p = tiw->sentinel.next; + while (p != &tiw->sentinel) { + ErtsMonotonicTime tpos = p->timeout_pos; + if (tiw->yield_slot >= ERTS_TW_SOON_WHEEL_SIZE) { + later_tmo++; + ERTS_TW_ASSERT(p->slot == later_slot(tpos)); + } + else { + soon_tmo++; + ERTS_TW_ASSERT(p->slot == (tpos & ERTS_TW_SOON_WHEEL_MASK)); + ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + } + p = p->next; + } + } + + at_once_tmo = 0; + if (tiw->at_once.head) { + ErtsTWheelTimer *p = tiw->at_once.head; + while (p) { + ErtsMonotonicTime tpos = p->timeout_pos; + at_once_tmo++; + ERTS_TW_ASSERT(tpos <= tiw->pos); + p = p->next; + } + } + + ERTS_TW_ASSERT(tiw->at_once.nto == at_once_tmo); + ERTS_TW_ASSERT(tiw->soon.nto == soon_tmo); + ERTS_TW_ASSERT(tiw->nto == soon_tmo + later_tmo + at_once_tmo); } -#endif /* ERTS_TW_DEBUG */ + +#endif -- cgit v1.2.3 From 2ed143ac11d858db728bfa69f14fb34d7f449305 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 10 Feb 2017 17:16:51 +0100 Subject: Introduce timer slot range counters Timer counters for ranges of slots in order to be able to avoid inspecting large ranges of slots without timers. --- erts/emulator/beam/time.c | 374 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 301 insertions(+), 73 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 7b5ff7b992..1b6ca2b269 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -18,7 +18,6 @@ * %CopyrightEnd% */ - /* * TIMER WHEEL * @@ -134,6 +133,32 @@ * bump operation is performed thise timers will be triggered * at once. * + * -- Searching for Next Timeout -- + * + * In order to limit the amount of work needed in order to find + * next timeout, we keep track of total amount of timers in the + * wheels, total amount of timers in soon wheel, and the total + * amount of timers in each range of slots. Each slot range + * currently contain 512 slots. + * + * When next timeout is less than half the soon wheel width away + * we determine the exact timeout. Due to the timer counts of + * slot ranges, we currently at most need to search 1024 slots + * in the soon wheel. This besides inspecting slot range counts + * and two slots in the later wheel which potentially might trigger + * timeouts for moving timers from the later wheel to the soon wheel + * earlier than timeouts in the soon wheel. + * + * When next timeout is further away than half the soon wheel + * width we settle for the earliest possible timeout in the first + * non-empty slot range. The further away the next timeout is, the + * more likely it is that the next timeout change before we + * actually get there. That is, a change due to another timer is + * set to an earlier time and/or the timer is cancelled. It is + * therefore in this case no point determining next timeout + * exactly. If the state should not change, we will wake up a bit + * early and do a recalculation of next timeout and eventually + * we will be so close to it that we determine it exactly. * */ @@ -147,8 +172,8 @@ #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" -#define ERTS_MONOTONIC_DAY ERTS_SEC_TO_MONOTONIC(60*60*24) -#define ERTS_CLKTCKS_DAY ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY) +#define ERTS_MONOTONIC_WEEK ERTS_SEC_TO_MONOTONIC(7*60*60*24) +#define ERTS_CLKTCKS_WEEK ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK) #ifdef ERTS_ENABLE_LOCK_CHECK #define ASSERT_NO_LOCKED_LOCKS erts_lc_check_exact(NULL, 0) @@ -211,6 +236,21 @@ #define ERTS_TW_LATER_WHEEL_MASK (ERTS_TW_LATER_WHEEL_SIZE-1) +#define ERTS_TW_SCNT_BITS 9 +#define ERTS_TW_SCNT_SHIFT +#define ERTS_TW_SCNT_SIZE \ + ((ERTS_TW_SOON_WHEEL_SIZE + ERTS_TW_LATER_WHEEL_SIZE) \ + >> ERTS_TW_SCNT_BITS) + +#ifdef __GNUC__ +#if ERTS_TW_SOON_WHEEL_BITS < ERTS_TW_SCNT_BITS +# warning Consider larger soon timer wheel +#endif +#if ERTS_TW_SOON_WHEEL_BITS < ERTS_TW_SCNT_BITS +# warning Consider larger later timer wheel +#endif +#endif + /* Actual interval time chosen by sys_init_time() */ #if SYS_CLOCK_RESOLUTION == 1 @@ -223,6 +263,8 @@ static int tiw_itime; /* Constant after init */ struct ErtsTimerWheel_ { ErtsTWheelTimer *w[ERTS_TW_SOON_WHEEL_SIZE + ERTS_TW_LATER_WHEEL_SIZE]; + Sint scnt[ERTS_TW_SCNT_SIZE]; + Sint bump_scnt[ERTS_TW_SCNT_SIZE]; ErtsMonotonicTime pos; Uint nto; struct { @@ -248,6 +290,108 @@ struct ErtsTimerWheel_ { static int bump_later_wheel(ErtsTimerWheel *tiw, int *yield_count_p); +static ERTS_INLINE int +scnt_get_ix(int slot) +{ + return slot >> ERTS_TW_SCNT_BITS; +} + +static ERTS_INLINE void +scnt_inc(Sint *scnt, int slot) +{ + scnt[slot >> ERTS_TW_SCNT_BITS]++; +} + +#ifdef ERTS_TW_HARD_DEBUG + +static ERTS_INLINE void +scnt_ix_inc(Sint *scnt, int six) +{ + scnt[six]++; +} + +#endif + +static ERTS_INLINE void +scnt_dec(Sint *scnt, int slot) +{ + scnt[slot >> ERTS_TW_SCNT_BITS]--; + ERTS_TW_ASSERT(scnt[slot >> ERTS_TW_SCNT_BITS] >= 0); +} + +static ERTS_INLINE void +scnt_ix_dec(Sint *scnt, int six) +{ + scnt[six]--; + ERTS_TW_ASSERT(scnt[six] >= 0); +} + +static ERTS_INLINE void +scnt_wheel_next(int *slotp, int *leftp, ErtsMonotonicTime *posp, + int *sixp, Sint *scnt, int first_slot, + int end_slot, ErtsMonotonicTime slot_sz) +{ + int slot = *slotp; + int left = *leftp; + int ix; + + ERTS_TW_ASSERT(*leftp >= 0); + + left--; + slot++; + if (slot == end_slot) + slot = first_slot; + ix = slot >> ERTS_TW_SCNT_BITS; + + while (!scnt[ix] && left > 0) { + int diff, old_slot = slot; + ix++; + slot = (ix << ERTS_TW_SCNT_BITS); + diff = slot - old_slot; + if (left < diff) { + slot = old_slot + left; + diff = left; + } + if (slot < end_slot) + left -= diff; + else { + left -= end_slot - old_slot; + slot = first_slot; + ix = slot >> ERTS_TW_SCNT_BITS; + } + } + + ERTS_TW_ASSERT(left >= -1); + + if (posp) + *posp += slot_sz * ((ErtsMonotonicTime) (*leftp - left)); + if (sixp) + *sixp = slot >> ERTS_TW_SCNT_BITS; + *leftp = left; + *slotp = slot; +} + + +static ERTS_INLINE void +scnt_soon_wheel_next(int *slotp, int *leftp, ErtsMonotonicTime *posp, + int *sixp, Sint *scnt) +{ + scnt_wheel_next(slotp, leftp, posp, sixp, scnt, + ERTS_TW_SOON_WHEEL_FIRST_SLOT, + ERTS_TW_SOON_WHEEL_END_SLOT, 1); +} + +static ERTS_INLINE void +scnt_later_wheel_next(int *slotp, int *leftp, ErtsMonotonicTime *posp, + int *sixp, Sint *scnt) +{ + scnt_wheel_next(slotp, leftp, posp, sixp, scnt, + ERTS_TW_LATER_WHEEL_FIRST_SLOT, + ERTS_TW_LATER_WHEEL_END_SLOT, + ERTS_TW_LATER_WHEEL_SLOT_SIZE); +} + + static ERTS_INLINE int soon_slot(ErtsMonotonicTime soon_pos) { @@ -282,14 +426,10 @@ static void hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos); #define ERTS_HARD_DBG_CHK_WHEELS(TIW, CHK_MIN_TPOS) #endif -static ERTS_INLINE ErtsMonotonicTime -find_next_timeout(ErtsSchedulerData *esdp, - ErtsTimerWheel *tiw, - int search_all, - ErtsMonotonicTime curr_time, /* When !search_all */ - ErtsMonotonicTime max_search_time) /* When !search_all */ +static ErtsMonotonicTime +find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough) { - int start_ix, tiw_pos_ix, pos_inc, wheel_start_ix, wheel_end_ix; + int slot, slots; int true_min_timeout = 0; ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos; @@ -298,32 +438,51 @@ find_next_timeout(ErtsSchedulerData *esdp, ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TWHEEL_SLOT_INACTIVE); if (tiw->nto == 0) { /* no timeouts in wheel */ - if (!search_all) + if (!thorough) min_timeout_pos = tiw->pos; else { - curr_time = erts_get_monotonic_time(esdp); + ErtsMonotonicTime curr_time = erts_get_monotonic_time(esdp); tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); } - min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); + min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK); goto done; } - if (search_all) - min_timeout_pos = tiw->pos + ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY); + min_timeout_pos = tiw->pos; + if (thorough) + min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK); else - min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time); + min_timeout_pos += ERTS_TW_SOON_WHEEL_SIZE/2; if (!tiw->soon.nto) { + ErtsMonotonicTime tmp; /* Select later wheel... */ slot_timeout_pos = tiw->later.pos; - start_ix = tiw_pos_ix = later_slot(slot_timeout_pos); - pos_inc = ERTS_TW_LATER_WHEEL_SLOT_SIZE; - wheel_start_ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; - wheel_end_ix = ERTS_TW_LATER_WHEEL_END_SLOT; + slot = later_slot(slot_timeout_pos); /* Pre-timeout for move from later to soon wheel... */ slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + tmp = min_timeout_pos - slot_timeout_pos; + tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + tmp++; + if (tmp > ERTS_TW_LATER_WHEEL_SIZE) + slots = ERTS_TW_LATER_WHEEL_SIZE; + else + slots = (int) tmp; + + /* + * We never search for an exact timeout in the + * later wheel, but instead settle for the first + * scnt range used. + */ + if (tiw->w[slot]) + true_min_timeout = 1; + else + scnt_later_wheel_next(&slot, &slots, &slot_timeout_pos, + NULL, tiw->scnt); + min_timeout_pos = slot_timeout_pos; } else { + ErtsMonotonicTime tmp; /* Select soon wheel... */ /* @@ -355,30 +514,31 @@ find_next_timeout(ErtsSchedulerData *esdp, } slot_timeout_pos = tiw->pos; - start_ix = tiw_pos_ix = soon_slot(tiw->pos); - pos_inc = 1; - wheel_start_ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; - wheel_end_ix = ERTS_TW_SOON_WHEEL_END_SLOT; - } + slot = soon_slot(tiw->pos); + tmp = min_timeout_pos - slot_timeout_pos; + if (tmp > ERTS_TW_SOON_WHEEL_SIZE) + slots = ERTS_TW_SOON_WHEEL_SIZE; + else + slots = (int) tmp; - /* - * Scan selected wheel... - */ - do { - if (tiw->w[tiw_pos_ix]) { - min_timeout_pos = slot_timeout_pos; - true_min_timeout = 1; - break; + while (slot_timeout_pos < min_timeout_pos) { + if (tiw->w[slot]) { + ERTS_TW_ASSERT(tiw->w[slot]->timeout_pos == slot_timeout_pos); + min_timeout_pos = slot_timeout_pos; + true_min_timeout = 1; + break; + } + scnt_soon_wheel_next(&slot, &slots, &slot_timeout_pos, + NULL, tiw->scnt); } - slot_timeout_pos += pos_inc; - if (slot_timeout_pos >= min_timeout_pos) - break; - - tiw_pos_ix++; - if (tiw_pos_ix == wheel_end_ix) - tiw_pos_ix = wheel_start_ix; - } while (start_ix != tiw_pos_ix); + if (slots > 0 && !true_min_timeout) { + /* Find first non-empty range... */ + scnt_soon_wheel_next(&slot, &slots, &slot_timeout_pos, + NULL, tiw->scnt); + min_timeout_pos = slot_timeout_pos; + } + } done: @@ -433,6 +593,7 @@ insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); tiw->soon.nto++; } + scnt_inc(tiw->scnt, slot); } static ERTS_INLINE void @@ -461,6 +622,7 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) } if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) tiw->soon.nto--; + scnt_dec(tiw->scnt, slot); } else { /* Timer in "at once" queue... */ @@ -480,7 +642,6 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) ERTS_TW_ASSERT(tiw->at_once.nto > 0); tiw->at_once.nto--; } - p->slot = ERTS_TWHEEL_SLOT_INACTIVE; } @@ -493,7 +654,7 @@ erts_check_next_timeout_time(ErtsSchedulerData *esdp) if (tiw->true_next_timeout_time) return tiw->next_timeout_time; ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(ERTS_MSACC_STATE_TIMERS); - time = find_next_timeout(esdp, tiw, 1, 0, 0); + time = find_next_timeout(esdp, tiw, 1); ERTS_MSACC_POP_STATE_M_X(); return time; } @@ -535,9 +696,10 @@ debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos) ix = later_slot(tiw->later.pos); tmp = skip_to_pos; tmp &= ERTS_TW_LATER_WHEEL_POS_MASK; - if (tmp > tiw->later.pos) { + if (tmp >= tiw->later.pos) { tmp -= tiw->later.pos; tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + tmp++; ERTS_TW_ASSERT(tmp >= 0); if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE) slots = (int) tmp; @@ -580,12 +742,16 @@ timeout_timer(ErtsTWheelTimer *p) void erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) { - int tiw_pos_ix, yielded_slot_restarted, yield_count, slots; + int slot, yielded_slot_restarted, yield_count, slots, scnt_ix; ErtsMonotonicTime bump_to; + Sint *scnt, *bump_scnt; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); yield_count = ERTS_TWHEEL_BUMP_YIELD_LIMIT; + scnt = &tiw->scnt[0]; + bump_scnt = &tiw->bump_scnt[0]; + /* * In order to be fair we always continue with work * where we left off when restarting after a yield. @@ -596,7 +762,8 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) bump_to = tiw->pos; if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) goto restart_yielded_later_slot; - tiw_pos_ix = tiw->yield_slot; + slot = tiw->yield_slot; + scnt_ix = scnt_get_ix(slot); slots = tiw->yield_slots_left; ASSERT(0 <= slots && slots <= ERTS_TW_SOON_WHEEL_SIZE); tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; @@ -616,7 +783,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) empty_wheel: ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to); tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; + tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK; tiw->pos = bump_to; tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; ERTS_MSACC_POP_STATE_M_X(); @@ -660,6 +827,22 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) if (tiw->nto == 0) goto empty_wheel; + /* + * Save slot counts in bump operation local + * array. + * + * The amount of timers to trigger (or move) + * will only decrease from now until we have + * completed this bump operation (even if we + * yield in the middle of it). + * + * The amount of timers in the wheels may + * however increase due to timers being set + * by timeout callbacks. + */ + sys_memcpy((void *) bump_scnt, (void *) scnt, + sizeof(Sint) * ERTS_TW_SCNT_SIZE); + if (tiw->true_next_timeout_time) { ErtsMonotonicTime skip_until_pos; /* @@ -694,15 +877,17 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) slots = ERTS_TW_SOON_WHEEL_SIZE; } - tiw_pos_ix = soon_slot(tiw->pos+1); + slot = soon_slot(tiw->pos+1); tiw->pos = bump_to; + scnt_ix = scnt_get_ix(slot); + /* Timeout timers in soon wheel */ - while (slots) { + while (slots > 0) { yield_count -= ERTS_TW_COST_SLOT; - p = tiw->w[tiw_pos_ix]; + p = tiw->w[slot]; if (p) { /* timeout callback need tiw->pos to be up to date */ if (p->next == p) { @@ -715,21 +900,23 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) tiw->sentinel.next->prev = &tiw->sentinel; tiw->sentinel.prev->next = &tiw->sentinel; } - tiw->w[tiw_pos_ix] = NULL; + tiw->w[slot] = NULL; while (1) { ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= p->slot && p->slot < ERTS_TW_SOON_WHEEL_END_SLOT); tiw->soon.nto--; + scnt_ix_dec(scnt, scnt_ix); if (p->timeout_pos <= bump_to) { timeout_timer(p); tiw->nto--; + scnt_ix_dec(bump_scnt, scnt_ix); yield_count -= ERTS_TW_COST_TIMEOUT; } else { /* uncommon case */ - insert_timer_into_slot(tiw, tiw_pos_ix, p); + insert_timer_into_slot(tiw, slot, p); yield_count -= ERTS_TW_COST_SLOT_MOVE; } @@ -744,7 +931,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) if (yield_count <= 0) { tiw->true_next_timeout_time = 1; tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); - tiw->yield_slot = tiw_pos_ix; + tiw->yield_slot = slot; tiw->yield_slots_left = slots; ERTS_MSACC_POP_STATE_M_X(); return; /* Yield! */ @@ -755,10 +942,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) } } - tiw_pos_ix++; - if (tiw_pos_ix == ERTS_TW_SOON_WHEEL_END_SLOT) - tiw_pos_ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; - slots--; + scnt_soon_wheel_next(&slot, &slots, NULL, &scnt_ix, bump_scnt); } if (ERTS_TW_BUMP_LATER_WHEEL(tiw)) { @@ -771,10 +955,9 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) } while (yielded_slot_restarted); tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; + tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK; - /* Search at most two seconds ahead... */ - (void) find_next_timeout(NULL, tiw, 0, curr_time, ERTS_SEC_TO_MONOTONIC(2)); + (void) find_next_timeout(NULL, tiw, 0); ERTS_MSACC_POP_STATE_M_X(); } @@ -784,12 +967,17 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) ErtsMonotonicTime cpos = tiw->pos; ErtsMonotonicTime later_pos = tiw->later.pos; int ycount = *ycount_p; - int slots, fslot; + int slots, fslot, scnt_ix; + Sint *scnt, *bump_scnt; + + scnt = &tiw->scnt[0]; + bump_scnt = &tiw->bump_scnt[0]; ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) { fslot = tiw->yield_slot; + scnt_ix = scnt_get_ix(fslot); slots = tiw->yield_slots_left; ASSERT(0 <= slots && slots <= ERTS_TW_LATER_WHEEL_SIZE); tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; @@ -800,17 +988,18 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) tmp_slots += ERTS_TW_LATER_WHEEL_SLOT_SIZE; tmp_slots -= later_pos; tmp_slots /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + tmp_slots++; if (tmp_slots < ERTS_TW_LATER_WHEEL_SIZE) slots = (int) tmp_slots; else slots = ERTS_TW_LATER_WHEEL_SIZE; + fslot = later_slot(later_pos); + scnt_ix = scnt_get_ix(fslot); } - while (slots >= 0) { + while (slots > 0) { ErtsTWheelTimer *p; - fslot = later_slot(later_pos); - ycount -= ERTS_TW_COST_SLOT; p = tiw->w[fslot]; @@ -835,12 +1024,15 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) ERTS_TW_ASSERT(tpos >= later_pos); ERTS_TW_ASSERT(p->slot == fslot); + scnt_ix_dec(scnt, scnt_ix); + if (tpos >= later_pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) { /* keep in later slot; very uncommon... */ insert_timer_into_slot(tiw, fslot, p); ycount -= ERTS_TW_COST_SLOT_MOVE; } else { + scnt_ix_dec(bump_scnt, scnt_ix); ERTS_TW_ASSERT(tpos < cpos + ERTS_TW_SOON_WHEEL_SIZE); if (tpos > cpos) { /* move into soon wheel */ @@ -878,8 +1070,8 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) p->next->prev = &tiw->sentinel; } } - slots--; - later_pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + + scnt_later_wheel_next(&fslot, &slots, &later_pos, &scnt_ix, bump_scnt); } ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); @@ -908,6 +1100,9 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) for(; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++) tiw->w[i] = NULL; + for (i = 0; i < ERTS_TW_SCNT_SIZE; i++) + tiw->scnt[i] = 0; + mtime = erts_get_monotonic_time(esdp); tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime); tiw->nto = 0; @@ -920,7 +1115,7 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) tiw->later.pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY; + tiw->next_timeout_time = mtime + ERTS_MONOTONIC_WEEK; tiw->sentinel.next = &tiw->sentinel; tiw->sentinel.prev = &tiw->sentinel; tiw->sentinel.timeout = NULL; @@ -1063,24 +1258,37 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw, static void hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) { - int ix, soon_tmo, later_tmo, at_once_tmo; + int ix, six, soon_tmo, later_tmo, at_once_tmo, + scnt_slot, scnt_slots, scnt_six; ErtsMonotonicTime min_tpos; + Sint scnt[ERTS_TW_SCNT_SIZE]; + + for (six = 0; six < ERTS_TW_SCNT_SIZE; six++) + scnt[six] = 0; min_tpos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); soon_tmo = 0; + scnt_slot = ERTS_TW_SOON_WHEEL_END_SLOT-1; + scnt_slots = ERTS_TW_SOON_WHEEL_SIZE; + scnt_six = 0; + scnt_soon_wheel_next(&scnt_slot, &scnt_slots, + NULL, &scnt_six, tiw->scnt); for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; ix < ERTS_TW_SOON_WHEEL_END_SLOT; ix++) { ErtsTWheelTimer *p; p = tiw->w[ix]; + six = scnt_get_ix(ix); + ERTS_TW_ASSERT(!p || six == scnt_six); if (p) { ErtsTWheelTimer *first = p; do { ErtsMonotonicTime tpos = p->timeout_pos; - ERTS_TW_ASSERT(p->slot == ix); soon_tmo++; + scnt_ix_inc(scnt, six); + ERTS_TW_ASSERT(p->slot == ix); ERTS_TW_ASSERT(ix == soon_slot(tpos)); ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); ERTS_TW_ASSERT(!check_min_tpos || tpos >= min_tpos); @@ -1088,21 +1296,33 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) p = p->next; } while (p != first); } + if (ix == scnt_slot) + scnt_soon_wheel_next(&scnt_slot, &scnt_slots, + NULL, &scnt_six, tiw->scnt); } later_tmo = 0; + scnt_slot = ERTS_TW_SOON_WHEEL_END_SLOT-1; + scnt_slots = ERTS_TW_SOON_WHEEL_SIZE; + scnt_six = 0; + scnt_later_wheel_next(&scnt_slot, &scnt_slots, + NULL, &scnt_six, tiw->scnt); for (ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; ix < ERTS_TW_LATER_WHEEL_END_SLOT; ix++) { ErtsTWheelTimer *p; p = tiw->w[ix]; + six = scnt_get_ix(ix); + ERTS_TW_ASSERT(!p || six == scnt_six); if (p) { ErtsTWheelTimer *first = p; + six = scnt_get_ix(ix); do { ErtsMonotonicTime tpos = p->timeout_pos; - ERTS_TW_ASSERT(p->slot == ix); later_tmo++; + scnt_ix_inc(scnt, six); + ERTS_TW_ASSERT(p->slot == ix); ERTS_TW_ASSERT(later_slot(tpos) == ix); tpos &= ERTS_TW_LATER_WHEEL_POS_MASK; tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; @@ -1111,19 +1331,24 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) p = p->next; } while (p != first); } + if (ix == scnt_slot) + scnt_later_wheel_next(&scnt_slot, &scnt_slots, + NULL, &scnt_six, tiw->scnt); } if (tiw->yield_slot >= 0) { ErtsTWheelTimer *p = tiw->sentinel.next; + ix = tiw->yield_slot; while (p != &tiw->sentinel) { ErtsMonotonicTime tpos = p->timeout_pos; - if (tiw->yield_slot >= ERTS_TW_SOON_WHEEL_SIZE) { + scnt_inc(scnt, ix); + if (ix >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) { later_tmo++; - ERTS_TW_ASSERT(p->slot == later_slot(tpos)); + ERTS_TW_ASSERT(ix == later_slot(tpos)); } else { soon_tmo++; - ERTS_TW_ASSERT(p->slot == (tpos & ERTS_TW_SOON_WHEEL_MASK)); + ERTS_TW_ASSERT(ix == (tpos & ERTS_TW_SOON_WHEEL_MASK)); ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); } p = p->next; @@ -1144,6 +1369,9 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) ERTS_TW_ASSERT(tiw->at_once.nto == at_once_tmo); ERTS_TW_ASSERT(tiw->soon.nto == soon_tmo); ERTS_TW_ASSERT(tiw->nto == soon_tmo + later_tmo + at_once_tmo); + + for (six = 0; six < ERTS_TW_SCNT_SIZE; six++) + ERTS_TW_ASSERT(scnt[six] == tiw->scnt[six]); } #endif -- cgit v1.2.3 From 25e9f3abd395c26eccb7962acc47dffe90beec81 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Sun, 12 Feb 2017 04:03:55 +0100 Subject: Manage timers to trigger at once in a slot similar to other timers --- erts/emulator/beam/erl_time.h | 5 +- erts/emulator/beam/time.c | 365 ++++++++++++++++++++++-------------------- 2 files changed, 193 insertions(+), 177 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 699e69e991..46d6da6448 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -448,8 +448,7 @@ erts_time_unit_conversion(Uint64 value, void erts_sched_init_time_sup(ErtsSchedulerData *esdp); -#define ERTS_TWHEEL_SLOT_AT_ONCE -1 -#define ERTS_TWHEEL_SLOT_INACTIVE -2 +#define ERTS_TW_SLOT_INACTIVE (-2) /* ** Timer entry: @@ -481,7 +480,7 @@ ERTS_GLB_INLINE ErtsMonotonicTime erts_tweel_read_timeout(ErtsTWheelTimer *twt); ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p) { - p->slot = ERTS_TWHEEL_SLOT_INACTIVE; + p->slot = ERTS_TW_SLOT_INACTIVE; } ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_tmo_ref) diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 1b6ca2b269..16e4328a05 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -125,13 +125,17 @@ * slot position by just removing it from the circular double * linked list that it is in. * - * -- At Once List -- + * -- At Once Slot -- * * If a timer is set that has a time earlier or equal to 'pos', * it is not inserted into the wheel. It is instead inserted, - * into a list referred to by the 'at_once' field. When the - * bump operation is performed thise timers will be triggered - * at once. + * into a circular double linked list referred to by the "at + * once" slot. When the bump operation is performed these timers + * will be triggered at once. The circular list of the slot will + * be moved to the 'sentinel' field while bumping these timers + * as when bumping an ordinary wheel slot. A yielding bump + * operation and cancelation of timers is handled the same way + * as if the timer was in a wheel slot. * * -- Searching for Next Timeout -- * @@ -262,14 +266,15 @@ static int tiw_itime; /* Constant after init */ #endif struct ErtsTimerWheel_ { - ErtsTWheelTimer *w[ERTS_TW_SOON_WHEEL_SIZE + ERTS_TW_LATER_WHEEL_SIZE]; + ErtsTWheelTimer *slots[1 /* At Once Slot */ + + ERTS_TW_SOON_WHEEL_SIZE /* Soon Wheel Slots */ + + ERTS_TW_LATER_WHEEL_SIZE]; /* Later Wheel Slots */ + ErtsTWheelTimer **w; Sint scnt[ERTS_TW_SCNT_SIZE]; Sint bump_scnt[ERTS_TW_SCNT_SIZE]; ErtsMonotonicTime pos; Uint nto; struct { - ErtsTWheelTimer *head; - ErtsTWheelTimer *tail; Uint nto; } at_once; struct { @@ -285,6 +290,8 @@ struct ErtsTimerWheel_ { ErtsMonotonicTime next_timeout_time; }; +#define ERTS_TW_SLOT_AT_ONCE (-1) + #define ERTS_TW_BUMP_LATER_WHEEL(TIW) \ ((tiw)->pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE >= (TIW)->later.pos) @@ -435,7 +442,7 @@ find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough) ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); - ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TWHEEL_SLOT_INACTIVE); + ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TW_SLOT_INACTIVE); if (tiw->nto == 0) { /* no timeouts in wheel */ if (!thorough) @@ -551,29 +558,11 @@ done: return min_timeout; } -static ERTS_INLINE void -insert_timer_into_at_once_list(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) -{ - tiw->at_once.nto++; - p->next = NULL; - p->prev = tiw->at_once.tail; - if (tiw->at_once.tail) { - ERTS_TW_ASSERT(tiw->at_once.head); - tiw->at_once.tail->next = p; - } - else { - ERTS_TW_ASSERT(!tiw->at_once.head); - tiw->at_once.head = p; - } - tiw->at_once.tail = p; - p->slot = ERTS_TWHEEL_SLOT_AT_ONCE; -} - static ERTS_INLINE void insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) { - ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= slot); - ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT); + ERTS_TW_ASSERT(ERTS_TW_SLOT_AT_ONCE <= slot + && slot < ERTS_TW_LATER_WHEEL_END_SLOT); p->slot = slot; if (!tiw->w[slot]) { tiw->w[slot] = p; @@ -589,60 +578,53 @@ insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) prev->next = p; next->prev = p; } - if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) { - ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); - tiw->soon.nto++; + if (slot == ERTS_TW_SLOT_AT_ONCE) + tiw->at_once.nto++; + else { + if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) { + ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + tiw->soon.nto++; + } + scnt_inc(tiw->scnt, slot); } - scnt_inc(tiw->scnt, slot); } static ERTS_INLINE void remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) { int slot = p->slot; - ERTS_TW_ASSERT(slot != ERTS_TWHEEL_SLOT_INACTIVE); - - if (slot >= ERTS_TW_SOON_WHEEL_FIRST_SLOT) { - /* - * Timer in wheel or in circular - * list of timers currently beeing - * triggered (referred by sentinel). - */ - ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT); - - if (p->next == p) { - ERTS_TW_ASSERT(tiw->w[slot] == p); - tiw->w[slot] = NULL; - } - else { - if (tiw->w[slot] == p) - tiw->w[slot] = p->next; - p->prev->next = p->next; - p->next->prev = p->prev; - } - if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) - tiw->soon.nto--; - scnt_dec(tiw->scnt, slot); + ERTS_TW_ASSERT(slot != ERTS_TW_SLOT_INACTIVE); + + /* + * Timer is in circular list either referred to + * by at once slot, slot in soon wheel, slot + * in later wheel, or by sentinel (timers currently + * being triggered). + */ + ERTS_TW_ASSERT(ERTS_TW_SLOT_AT_ONCE <= slot + && slot < ERTS_TW_LATER_WHEEL_END_SLOT); + + if (p->next == p) { + /* Cannot be referred by sentinel, i.e. must be referred by slot... */ + ERTS_TW_ASSERT(tiw->w[slot] == p); + tiw->w[slot] = NULL; } else { - /* Timer in "at once" queue... */ - ERTS_TW_ASSERT(slot == ERTS_TWHEEL_SLOT_AT_ONCE); - if (p->prev) - p->prev->next = p->next; - else { - ERTS_TW_ASSERT(tiw->at_once.head == p); - tiw->at_once.head = p->next; - } - if (p->next) - p->next->prev = p->prev; - else { - ERTS_TW_ASSERT(tiw->at_once.tail == p); - tiw->at_once.tail = p->prev; - } + if (tiw->w[slot] == p) + tiw->w[slot] = p->next; + p->prev->next = p->next; + p->next->prev = p->prev; + } + if (slot == ERTS_TW_SLOT_AT_ONCE) { ERTS_TW_ASSERT(tiw->at_once.nto > 0); tiw->at_once.nto--; } - p->slot = ERTS_TWHEEL_SLOT_INACTIVE; + else { + if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) + tiw->soon.nto--; + scnt_dec(tiw->scnt, slot); + } + p->slot = ERTS_TW_SLOT_INACTIVE; } ErtsMonotonicTime @@ -732,7 +714,7 @@ timeout_timer(ErtsTWheelTimer *p) { ErlTimeoutProc timeout; void *arg; - p->slot = ERTS_TWHEEL_SLOT_INACTIVE; + p->slot = ERTS_TW_SLOT_INACTIVE; timeout = p->timeout; arg = p->arg; (*timeout)(arg); @@ -742,7 +724,7 @@ timeout_timer(ErtsTWheelTimer *p) void erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) { - int slot, yielded_slot_restarted, yield_count, slots, scnt_ix; + int slot, restarted, yield_count, slots, scnt_ix; ErtsMonotonicTime bump_to; Sint *scnt, *bump_scnt; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); @@ -757,23 +739,24 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) * where we left off when restarting after a yield. */ - if (tiw->yield_slot >= ERTS_TW_SOON_WHEEL_FIRST_SLOT) { - yielded_slot_restarted = 1; + slot = tiw->yield_slot; + restarted = slot != ERTS_TW_SLOT_INACTIVE; + if (restarted) { bump_to = tiw->pos; - if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) + if (slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) goto restart_yielded_later_slot; - slot = tiw->yield_slot; + tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; + if (slot == ERTS_TW_SLOT_AT_ONCE) + goto restart_yielded_at_once_slot; scnt_ix = scnt_get_ix(slot); slots = tiw->yield_slots_left; ASSERT(0 <= slots && slots <= ERTS_TW_SOON_WHEEL_SIZE); - tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; goto restart_yielded_soon_slot; } do { - yielded_slot_restarted = 0; - + restarted = 0; bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); while (1) { @@ -785,38 +768,59 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) tiw->true_next_timeout_time = 0; tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK; tiw->pos = bump_to; - tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; ERTS_MSACC_POP_STATE_M_X(); return; } - p = tiw->at_once.head; - while (p) { - if (yield_count <= 0) { - ERTS_TW_ASSERT(tiw->nto > 0); - ERTS_TW_ASSERT(tiw->at_once.nto > 0); - tiw->yield_slot = ERTS_TWHEEL_SLOT_AT_ONCE; - tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); - ERTS_MSACC_POP_STATE_M_X(); - return; - } + p = tiw->w[ERTS_TW_SLOT_AT_ONCE]; - ERTS_TW_ASSERT(tiw->nto > 0); - ERTS_TW_ASSERT(tiw->at_once.nto > 0); - tiw->nto--; - tiw->at_once.nto--; - tiw->at_once.head = p->next; - if (p->next) - p->next->prev = NULL; - else - tiw->at_once.tail = NULL; + if (p) { - timeout_timer(p); + if (p->next == p) { + ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel); + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + } + else { + tiw->sentinel.next = p->next; + tiw->sentinel.prev = p->prev; + tiw->sentinel.next->prev = &tiw->sentinel; + tiw->sentinel.prev->next = &tiw->sentinel; + } + tiw->w[ERTS_TW_SLOT_AT_ONCE] = NULL; + + while (1) { + ERTS_TW_ASSERT(tiw->nto > 0); + ERTS_TW_ASSERT(tiw->at_once.nto > 0); + tiw->nto--; + tiw->at_once.nto--; + + timeout_timer(p); + + yield_count -= ERTS_TW_COST_TIMEOUT; + + restart_yielded_at_once_slot: - yield_count -= ERTS_TW_COST_TIMEOUT; + p = tiw->sentinel.next; + if (p == &tiw->sentinel) { + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + break; + } + + if (yield_count <= 0) { + ERTS_TW_ASSERT(tiw->nto > 0); + ERTS_TW_ASSERT(tiw->at_once.nto > 0); + tiw->yield_slot = ERTS_TW_SLOT_AT_ONCE; + tiw->true_next_timeout_time = 1; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); + ERTS_MSACC_POP_STATE_M_X(); + return; /* Yield! */ + } + + tiw->sentinel.next = p->next; + p->next->prev = &tiw->sentinel; + } - p = tiw->at_once.head; } if (tiw->pos >= bump_to) { @@ -952,7 +956,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) } } - } while (yielded_slot_restarted); + } while (restarted); tiw->true_next_timeout_time = 0; tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK; @@ -980,7 +984,7 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) scnt_ix = scnt_get_ix(fslot); slots = tiw->yield_slots_left; ASSERT(0 <= slots && slots <= ERTS_TW_LATER_WHEEL_SIZE); - tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; goto restart_yielded_slot; } else { @@ -1093,11 +1097,29 @@ ErtsTimerWheel * erts_create_timer_wheel(ErtsSchedulerData *esdp) { ErtsMonotonicTime mtime; - int i = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + int i; ErtsTimerWheel *tiw; + + /* Some compile time sanity checks... */ + + /* Slots... */ + ERTS_CT_ASSERT(ERTS_TW_SLOT_AT_ONCE == -1); + ERTS_CT_ASSERT(ERTS_TW_SLOT_INACTIVE < ERTS_TW_SLOT_AT_ONCE); + ERTS_CT_ASSERT(ERTS_TW_SLOT_AT_ONCE + 1 == ERTS_TW_SOON_WHEEL_FIRST_SLOT); + ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT < ERTS_TW_SOON_WHEEL_END_SLOT); + ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_END_SLOT == ERTS_TW_LATER_WHEEL_FIRST_SLOT); + ERTS_CT_ASSERT(ERTS_TW_LATER_WHEEL_FIRST_SLOT < ERTS_TW_LATER_WHEEL_END_SLOT); + + /* Both wheel sizes should be a powers of 2 */ + ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_SIZE + && !(ERTS_TW_SOON_WHEEL_SIZE & (ERTS_TW_SOON_WHEEL_SIZE-1))); + ERTS_CT_ASSERT(ERTS_TW_LATER_WHEEL_SIZE + && !(ERTS_TW_LATER_WHEEL_SIZE & (ERTS_TW_LATER_WHEEL_SIZE-1))); + tiw = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_WHEEL, sizeof(ErtsTimerWheel)); - for(; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++) + tiw->w = &tiw->slots[1]; + for(i = ERTS_TW_SLOT_AT_ONCE; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++) tiw->w[i] = NULL; for (i = 0; i < ERTS_TW_SCNT_SIZE; i++) @@ -1106,14 +1128,12 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) mtime = erts_get_monotonic_time(esdp); tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime); tiw->nto = 0; - tiw->at_once.head = NULL; - tiw->at_once.tail = NULL; tiw->at_once.nto = 0; tiw->soon.nto = 0; tiw->later.pos = tiw->pos; tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK; tiw->later.pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; - tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; tiw->true_next_timeout_time = 0; tiw->next_timeout_time = mtime + ERTS_MONOTONIC_WEEK; tiw->sentinel.next = &tiw->sentinel; @@ -1154,50 +1174,48 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p, ErlTimeoutProc timeout, void *arg, ErtsMonotonicTime timeout_pos) { + int slot; ErtsMonotonicTime timeout_time; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); p->timeout = timeout; p->arg = arg; - ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE); + ERTS_TW_ASSERT(p->slot == ERTS_TW_SLOT_INACTIVE); tiw->nto++; + + /* calculate slot */ if (timeout_pos <= tiw->pos) { - p->timeout_pos = tiw->pos; - insert_timer_into_at_once_list(tiw, p); - timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->pos); + /* at once */ + p->timeout_pos = timeout_pos = tiw->pos; + slot = ERTS_TW_SLOT_AT_ONCE; + } + else if (timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) { + /* soon wheel */ + p->timeout_pos = timeout_pos; + slot = soon_slot(timeout_pos); } else { - int slot; - - p->timeout_pos = timeout_pos; - - /* calculate slot */ - if (timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) { - /* soon wheel */ - slot = soon_slot(timeout_pos); - } - else { - /* later wheel */ - slot = later_slot(timeout_pos); - - /* - * Next timeout due to this timeout - * should be in good time before the - * actual timeout (one later wheel slot - * size). This, in order to move it - * from the later wheel to the soon - * wheel. - */ - timeout_pos &= ERTS_TW_LATER_WHEEL_POS_MASK; - timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - } + /* later wheel */ + p->timeout_pos = timeout_pos; + slot = later_slot(timeout_pos); - insert_timer_into_slot(tiw, slot, p); - timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); + /* + * Next timeout due to this timeout + * should be in good time before the + * actual timeout (one later wheel slot + * size). This, in order to move it + * from the later wheel to the soon + * wheel. + */ + timeout_pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; } + insert_timer_into_slot(tiw, slot, p); + timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); + if (timeout_time < tiw->next_timeout_time) { tiw->true_next_timeout_time = 1; tiw->next_timeout_time = timeout_time; @@ -1208,7 +1226,7 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, void erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) { - if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) { + if (p->slot != ERTS_TW_SLOT_INACTIVE) { ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); remove_timer(tiw, p); tiw->nto--; @@ -1234,14 +1252,7 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw, tmr = tmr->next; } - for (tmr = tiw->at_once.head; tmr; tmr = tmr->next) { - if (tmr->timeout == tclbk) - (*func)(arg, tmr->timeout_pos, tmr->arg); - } - - for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; - ix < ERTS_TW_LATER_WHEEL_END_SLOT; - ix++) { + for (ix = ERTS_TW_SLOT_AT_ONCE; ix < ERTS_TW_LATER_WHEEL_END_SLOT; ix++) { tmr = tiw->w[ix]; if (tmr) { do { @@ -1262,12 +1273,27 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) scnt_slot, scnt_slots, scnt_six; ErtsMonotonicTime min_tpos; Sint scnt[ERTS_TW_SCNT_SIZE]; + ErtsTWheelTimer *p; for (six = 0; six < ERTS_TW_SCNT_SIZE; six++) scnt[six] = 0; min_tpos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); + at_once_tmo = 0; + p = tiw->w[ERTS_TW_SLOT_AT_ONCE]; + if (p) { + ErtsTWheelTimer *first = p; + do { + at_once_tmo++; + ERTS_TW_ASSERT(p->slot == ERTS_TW_SLOT_AT_ONCE); + ERTS_TW_ASSERT(p->timeout_pos <= tiw->pos); + ERTS_TW_ASSERT(!check_min_tpos || tiw->pos >= min_tpos); + ERTS_TW_ASSERT(p->next->prev == p); + p = p->next; + } while (p != first); + } + soon_tmo = 0; scnt_slot = ERTS_TW_SOON_WHEEL_END_SLOT-1; scnt_slots = ERTS_TW_SOON_WHEEL_SIZE; @@ -1277,8 +1303,6 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; ix < ERTS_TW_SOON_WHEEL_END_SLOT; ix++) { - ErtsTWheelTimer *p; - p = tiw->w[ix]; six = scnt_get_ix(ix); ERTS_TW_ASSERT(!p || six == scnt_six); @@ -1310,8 +1334,6 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) for (ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; ix < ERTS_TW_LATER_WHEEL_END_SLOT; ix++) { - ErtsTWheelTimer *p; - p = tiw->w[ix]; six = scnt_get_ix(ix); ERTS_TW_ASSERT(!p || six == scnt_six); @@ -1336,35 +1358,30 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) NULL, &scnt_six, tiw->scnt); } - if (tiw->yield_slot >= 0) { - ErtsTWheelTimer *p = tiw->sentinel.next; + if (tiw->yield_slot != ERTS_TW_SLOT_INACTIVE) { + p = tiw->sentinel.next; ix = tiw->yield_slot; while (p != &tiw->sentinel) { ErtsMonotonicTime tpos = p->timeout_pos; - scnt_inc(scnt, ix); - if (ix >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) { - later_tmo++; - ERTS_TW_ASSERT(ix == later_slot(tpos)); - } + ERTS_TW_ASSERT(ix == p->slot); + if (ix == ERTS_TW_SLOT_AT_ONCE) + at_once_tmo++; else { - soon_tmo++; - ERTS_TW_ASSERT(ix == (tpos & ERTS_TW_SOON_WHEEL_MASK)); - ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + scnt_inc(scnt, ix); + if (ix >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) { + later_tmo++; + ERTS_TW_ASSERT(ix == later_slot(tpos)); + } + else { + soon_tmo++; + ERTS_TW_ASSERT(ix == (tpos & ERTS_TW_SOON_WHEEL_MASK)); + ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + } + p = p->next; } - p = p->next; } } - at_once_tmo = 0; - if (tiw->at_once.head) { - ErtsTWheelTimer *p = tiw->at_once.head; - while (p) { - ErtsMonotonicTime tpos = p->timeout_pos; - at_once_tmo++; - ERTS_TW_ASSERT(tpos <= tiw->pos); - p = p->next; - } - } ERTS_TW_ASSERT(tiw->at_once.nto == at_once_tmo); ERTS_TW_ASSERT(tiw->soon.nto == soon_tmo); -- cgit v1.2.3 From cfd009221d53d2f19dfa2852962f48b85b34d485 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Sun, 12 Feb 2017 23:25:39 +0100 Subject: Minimum timeout position in each timer wheel Minimum known timeout position is saved in bot far and near wheel. This information is used to avoid scanning from current position in the cases were we know the minimum timeout position. --- erts/emulator/beam/time.c | 499 ++++++++++++++++++++++++++---------------- erts/etc/unix/etp-commands.in | 75 +++++++ 2 files changed, 390 insertions(+), 184 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 16e4328a05..f530ee5de0 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -141,20 +141,24 @@ * * In order to limit the amount of work needed in order to find * next timeout, we keep track of total amount of timers in the - * wheels, total amount of timers in soon wheel, and the total - * amount of timers in each range of slots. Each slot range - * currently contain 512 slots. + * wheels, total amount of timers in the later wheel, total amount + * of timers in soon wheel, and the total amount of timers in + * each range of slots. Each slot range currently contain 512 + * slots. * - * When next timeout is less than half the soon wheel width away - * we determine the exact timeout. Due to the timer counts of + * When next timeout is less than the soon wheel width away we + * determine the exact timeout. Due to the timer counts of * slot ranges, we currently at most need to search 1024 slots * in the soon wheel. This besides inspecting slot range counts * and two slots in the later wheel which potentially might trigger * timeouts for moving timers from the later wheel to the soon wheel - * earlier than timeouts in the soon wheel. + * earlier than timeouts in the soon wheel. We also keep track + * of latest known minimum timeout position in each wheel which + * makes it possible to avoid scanning from current position + * each time. * - * When next timeout is further away than half the soon wheel - * width we settle for the earliest possible timeout in the first + * When next timeout is further away than the soon wheel width + * we settle for the earliest possible timeout in the first * non-empty slot range. The further away the next timeout is, the * more likely it is that the next timeout change before we * actually get there. That is, a change due to another timer is @@ -176,8 +180,11 @@ #define ERTS_WANT_TIMER_WHEEL_API #include "erl_time.h" -#define ERTS_MONOTONIC_WEEK ERTS_SEC_TO_MONOTONIC(7*60*60*24) -#define ERTS_CLKTCKS_WEEK ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK) +#define ERTS_MAX_CLKTCKS \ + ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_TIME_MAX) + +#define ERTS_CLKTCKS_WEEK \ + ERTS_MONOTONIC_TO_CLKTCKS(ERTS_SEC_TO_MONOTONIC(7*60*60*24)) #ifdef ERTS_ENABLE_LOCK_CHECK #define ASSERT_NO_LOCKED_LOCKS erts_lc_check_exact(NULL, 0) @@ -265,10 +272,21 @@ static int tiw_itime; /* Constant after init */ # define TIW_ITIME tiw_itime #endif +const int etp_tw_soon_wheel_size = ERTS_TW_SOON_WHEEL_SIZE; +const ErtsMonotonicTime etp_tw_soon_wheel_mask = ERTS_TW_SOON_WHEEL_MASK; +const int etp_tw_soon_wheel_first_slot = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + +const int etp_tw_later_wheel_size = ERTS_TW_LATER_WHEEL_SIZE; +const ErtsMonotonicTime etp_tw_later_wheel_slot_size = ERTS_TW_LATER_WHEEL_SLOT_SIZE; +const int etp_tw_later_wheel_shift = ERTS_TW_LATER_WHEEL_SHIFT; +const ErtsMonotonicTime etp_tw_later_wheel_mask = ERTS_TW_LATER_WHEEL_MASK; +const ErtsMonotonicTime etp_tw_later_wheel_pos_mask = ERTS_TW_LATER_WHEEL_POS_MASK; +const int etp_tw_later_wheel_first_slot = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + struct ErtsTimerWheel_ { ErtsTWheelTimer *slots[1 /* At Once Slot */ + ERTS_TW_SOON_WHEEL_SIZE /* Soon Wheel Slots */ - + ERTS_TW_LATER_WHEEL_SIZE]; /* Later Wheel Slots */ + + ERTS_TW_LATER_WHEEL_SIZE]; /* Later Wheel Slots */ ErtsTWheelTimer **w; Sint scnt[ERTS_TW_SCNT_SIZE]; Sint bump_scnt[ERTS_TW_SCNT_SIZE]; @@ -278,15 +296,20 @@ struct ErtsTimerWheel_ { Uint nto; } at_once; struct { + ErtsMonotonicTime min_tpos; Uint nto; } soon; struct { + ErtsMonotonicTime min_tpos; + int min_tpos_slot; ErtsMonotonicTime pos; + Uint nto; } later; int yield_slot; int yield_slots_left; ErtsTWheelTimer sentinel; int true_next_timeout_time; + ErtsMonotonicTime next_timeout_pos; ErtsMonotonicTime next_timeout_time; }; @@ -297,6 +320,18 @@ struct ErtsTimerWheel_ { static int bump_later_wheel(ErtsTimerWheel *tiw, int *yield_count_p); +#ifdef ERTS_TW_DEBUG +#define ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(TIW, TO_POS) \ + dbg_verify_empty_soon_slots((TIW), (TO_POS)) +#define ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(TIW, TO_POS) \ + dbg_verify_empty_later_slots((TIW), (TO_POS)) +void dbg_verify_empty_soon_slots(ErtsTimerWheel *, ErtsMonotonicTime); +void dbg_verify_empty_later_slots(ErtsTimerWheel *, ErtsMonotonicTime); +#else +#define ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(TIW, TO_POS) +#define ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(TIW, TO_POS) +#endif + static ERTS_INLINE int scnt_get_ix(int slot) { @@ -434,47 +469,59 @@ static void hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos); #endif static ErtsMonotonicTime -find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough) +find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw) { int slot, slots; int true_min_timeout = 0; - ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos; + ErtsMonotonicTime min_timeout_pos; + + ERTS_TW_ASSERT(tiw->pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE < tiw->later.pos + && tiw->later.pos <= tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TW_SLOT_INACTIVE); if (tiw->nto == 0) { /* no timeouts in wheel */ - if (!thorough) - min_timeout_pos = tiw->pos; - else { - ErtsMonotonicTime curr_time = erts_get_monotonic_time(esdp); - tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); - } - min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK); + ErtsMonotonicTime curr_time = erts_get_monotonic_time(esdp); + tiw->pos = min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); + tiw->later.pos = min_timeout_pos + ERTS_TW_SOON_WHEEL_SIZE; + tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + min_timeout_pos += ERTS_CLKTCKS_WEEK; goto done; } - min_timeout_pos = tiw->pos; - if (thorough) - min_timeout_pos += ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_WEEK); - else - min_timeout_pos += ERTS_TW_SOON_WHEEL_SIZE/2; + ERTS_TW_ASSERT(tiw->soon.nto || tiw->later.nto); if (!tiw->soon.nto) { - ErtsMonotonicTime tmp; - /* Select later wheel... */ - slot_timeout_pos = tiw->later.pos; - slot = later_slot(slot_timeout_pos); - /* Pre-timeout for move from later to soon wheel... */ - slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - tmp = min_timeout_pos - slot_timeout_pos; - tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - tmp++; - if (tmp > ERTS_TW_LATER_WHEEL_SIZE) + ErtsMonotonicTime tpos, min_tpos; + + /* Search later wheel... */ + + min_tpos = tiw->later.min_tpos & ERTS_TW_LATER_WHEEL_POS_MASK; + + if (min_tpos <= tiw->later.pos) { + tpos = tiw->later.pos; slots = ERTS_TW_LATER_WHEEL_SIZE; - else - slots = (int) tmp; + } + else { + ErtsMonotonicTime tmp; + /* Don't inspect slots we know are empty... */ + tmp = min_tpos - tiw->later.pos; + tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (tmp >= ERTS_TW_LATER_WHEEL_SIZE) { + /* Timeout more than one revolution ahead... */ + + /* Pre-timeout for move from later to soon wheel... */ + min_timeout_pos = min_tpos - ERTS_TW_LATER_WHEEL_SLOT_SIZE; + goto done; + } + tpos = min_tpos; + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, min_tpos); + slots = ERTS_TW_LATER_WHEEL_SIZE - ((int) tmp); + } + + slot = later_slot(tpos); /* * We never search for an exact timeout in the @@ -484,13 +531,21 @@ find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough) if (tiw->w[slot]) true_min_timeout = 1; else - scnt_later_wheel_next(&slot, &slots, &slot_timeout_pos, - NULL, tiw->scnt); - min_timeout_pos = slot_timeout_pos; + scnt_later_wheel_next(&slot, &slots, &tpos, NULL, tiw->scnt); + + tiw->later.min_tpos = tpos; + tiw->later.min_tpos_slot = slot; + ERTS_TW_ASSERT(slot == later_slot(tpos)); + + /* Pre-timeout for move from later to soon wheel... */ + tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + min_timeout_pos = tpos; } else { - ErtsMonotonicTime tmp; - /* Select soon wheel... */ + ErtsMonotonicTime tpos; + /* Search soon wheel... */ + + min_timeout_pos = tiw->pos + ERTS_TW_SOON_WHEEL_SIZE; /* * Besides inspecting the soon wheel we @@ -498,64 +553,72 @@ find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough) * later wheel which potentially can trigger * timeouts before timeouts in soon wheel... */ - slot_timeout_pos = tiw->later.pos; - slot_timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - if (slot_timeout_pos < min_timeout_pos) { - int fslot = later_slot(tiw->later.pos); - if (tiw->w[fslot]) { - min_timeout_pos = slot_timeout_pos; - true_min_timeout = 1; - } + if (tiw->later.min_tpos > (tiw->later.pos + + 2*ERTS_TW_LATER_WHEEL_SLOT_SIZE)) { + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS( + tiw, 2*ERTS_TW_LATER_WHEEL_SLOT_SIZE); + } + else { + int fslot; + tpos = tiw->later.pos; + tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + fslot = later_slot(tiw->later.pos); + if (tiw->w[fslot]) + min_timeout_pos = tpos; else { - slot_timeout_pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; - if (slot_timeout_pos < min_timeout_pos) { + tpos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (tpos < min_timeout_pos) { fslot++; if (fslot == ERTS_TW_LATER_WHEEL_END_SLOT) fslot = ERTS_TW_LATER_WHEEL_FIRST_SLOT; - if (tiw->w[fslot]) { - min_timeout_pos = slot_timeout_pos; - true_min_timeout = 1; - } + if (tiw->w[fslot]) + min_timeout_pos = tpos; } } } - slot_timeout_pos = tiw->pos; - slot = soon_slot(tiw->pos); - tmp = min_timeout_pos - slot_timeout_pos; - if (tmp > ERTS_TW_SOON_WHEEL_SIZE) + if (tiw->soon.min_tpos <= tiw->pos) { + tpos = tiw->pos; slots = ERTS_TW_SOON_WHEEL_SIZE; - else - slots = (int) tmp; + } + else { + ErtsMonotonicTime tmp; + /* Don't inspect slots we know are empty... */ + tmp = tiw->soon.min_tpos - tiw->pos; + ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_SIZE > tmp); + ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(tiw, tiw->soon.min_tpos); + slots = ERTS_TW_SOON_WHEEL_SIZE - ((int) tmp); + tpos = tiw->soon.min_tpos; + } - while (slot_timeout_pos < min_timeout_pos) { + slot = soon_slot(tpos); + + /* find next non-empty slot */ + while (tpos < min_timeout_pos) { if (tiw->w[slot]) { - ERTS_TW_ASSERT(tiw->w[slot]->timeout_pos == slot_timeout_pos); - min_timeout_pos = slot_timeout_pos; - true_min_timeout = 1; + ERTS_TW_ASSERT(tiw->w[slot]->timeout_pos == tpos); + min_timeout_pos = tpos; break; } - scnt_soon_wheel_next(&slot, &slots, &slot_timeout_pos, - NULL, tiw->scnt); + scnt_soon_wheel_next(&slot, &slots, &tpos, NULL, tiw->scnt); } - if (slots > 0 && !true_min_timeout) { - /* Find first non-empty range... */ - scnt_soon_wheel_next(&slot, &slots, &slot_timeout_pos, - NULL, tiw->scnt); - min_timeout_pos = slot_timeout_pos; - } + tiw->soon.min_tpos = min_timeout_pos; + true_min_timeout = 1; } -done: +done: { + ErtsMonotonicTime min_timeout; - min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos); - tiw->next_timeout_time = min_timeout; - tiw->true_next_timeout_time = true_min_timeout; + min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos); + tiw->next_timeout_pos = min_timeout_pos; + tiw->next_timeout_time = min_timeout; + tiw->true_next_timeout_time = true_min_timeout; - ERTS_HARD_DBG_CHK_WHEELS(tiw, 1); + ERTS_HARD_DBG_CHK_WHEELS(tiw, 1); - return min_timeout; + return min_timeout; + } } static ERTS_INLINE void @@ -581,9 +644,20 @@ insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) if (slot == ERTS_TW_SLOT_AT_ONCE) tiw->at_once.nto++; else { + ErtsMonotonicTime tpos = p->timeout_pos; if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) { ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); tiw->soon.nto++; + if (tiw->soon.min_tpos > tpos) + tiw->soon.min_tpos = tpos; + } + else { + ERTS_TW_ASSERT(p->timeout_pos >= tiw->pos + ERTS_TW_SOON_WHEEL_SIZE); + tiw->later.nto++; + if (tiw->later.min_tpos > tpos) { + tiw->later.min_tpos = tpos; + tiw->later.min_tpos_slot = slot; + } } scnt_inc(tiw->scnt, slot); } @@ -593,6 +667,7 @@ static ERTS_INLINE void remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) { int slot = p->slot; + int empty_slot; ERTS_TW_ASSERT(slot != ERTS_TW_SLOT_INACTIVE); /* @@ -608,21 +683,45 @@ remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) /* Cannot be referred by sentinel, i.e. must be referred by slot... */ ERTS_TW_ASSERT(tiw->w[slot] == p); tiw->w[slot] = NULL; + empty_slot = 1; } else { if (tiw->w[slot] == p) tiw->w[slot] = p->next; p->prev->next = p->next; p->next->prev = p->prev; + empty_slot = 0; } if (slot == ERTS_TW_SLOT_AT_ONCE) { ERTS_TW_ASSERT(tiw->at_once.nto > 0); tiw->at_once.nto--; } else { - if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) - tiw->soon.nto--; scnt_dec(tiw->scnt, slot); + if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) { + if (empty_slot + && tiw->true_next_timeout_time + && p->timeout_pos == tiw->next_timeout_pos) { + tiw->true_next_timeout_time = 0; + } + if (--tiw->soon.nto == 0) + tiw->soon.min_tpos = ERTS_MAX_CLKTCKS; + } + else { + if (empty_slot + && tiw->true_next_timeout_time + && tiw->later.min_tpos_slot == slot) { + ErtsMonotonicTime tpos = tiw->later.min_tpos; + tpos &= ERTS_TW_LATER_WHEEL_POS_MASK; + tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + if (tpos == tiw->next_timeout_pos) + tiw->true_next_timeout_time = 0; + } + if (--tiw->later.nto == 0) { + tiw->later.min_tpos = ERTS_MAX_CLKTCKS; + tiw->later.min_tpos_slot = ERTS_TW_LATER_WHEEL_END_SLOT; + } + } } p->slot = ERTS_TW_SLOT_INACTIVE; } @@ -633,82 +732,18 @@ erts_check_next_timeout_time(ErtsSchedulerData *esdp) ErtsTimerWheel *tiw = esdp->timer_wheel; ErtsMonotonicTime time; ERTS_MSACC_DECLARE_CACHE_X(); + ERTS_TW_ASSERT(tiw->next_timeout_time + == ERTS_CLKTCKS_TO_MONOTONIC(tiw->next_timeout_pos)); if (tiw->true_next_timeout_time) - return tiw->next_timeout_time; + return tiw->next_timeout_time; /* known timeout... */ + if (tiw->next_timeout_pos > tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) + return tiw->next_timeout_time; /* sufficiently later away... */ ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_X(ERTS_MSACC_STATE_TIMERS); - time = find_next_timeout(esdp, tiw, 1); + time = find_next_timeout(esdp, tiw); ERTS_MSACC_POP_STATE_M_X(); return time; } -#ifndef ERTS_TW_DEBUG -#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) ((void) 0) -#else -#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) debug_check_safe_to_skip_to((TIW), (TO)) -static void -debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos) -{ - int slots, ix; - ErtsTWheelTimer *tmr; - ErtsMonotonicTime tmp; - - ix = soon_slot(tiw->pos); - tmp = skip_to_pos - tiw->pos; - ERTS_TW_ASSERT(tmp >= 0); - if (tmp < (ErtsMonotonicTime) ERTS_TW_SOON_WHEEL_SIZE) - slots = (int) tmp; - else - slots = ERTS_TW_SOON_WHEEL_SIZE; - - while (slots > 0) { - tmr = tiw->w[ix]; - if (tmr) { - ErtsTWheelTimer *end = tmr; - do { - ERTS_TW_ASSERT(tmr->timeout_pos > skip_to_pos); - tmr = tmr->next; - } while (tmr != end); - } - ix++; - if (ix == ERTS_TW_SOON_WHEEL_END_SLOT) - ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; - slots--; - } - - ix = later_slot(tiw->later.pos); - tmp = skip_to_pos; - tmp &= ERTS_TW_LATER_WHEEL_POS_MASK; - if (tmp >= tiw->later.pos) { - tmp -= tiw->later.pos; - tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - tmp++; - ERTS_TW_ASSERT(tmp >= 0); - if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE) - slots = (int) tmp; - else - slots = ERTS_TW_LATER_WHEEL_SIZE; - - while (slots > 0) { - tmr = tiw->w[ix]; - if (tmr) { - ErtsMonotonicTime tpos = tmr->timeout_pos; - ErtsTWheelTimer *end = tmr; - do { - tpos &= ERTS_TW_LATER_WHEEL_POS_MASK; - tpos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - ERTS_TW_ASSERT(tpos > skip_to_pos); - tmr = tmr->next; - } while (tmr != end); - } - ix++; - if (ix == ERTS_TW_LATER_WHEEL_END_SLOT) - ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; - slots--; - } - } -} -#endif - static ERTS_INLINE void timeout_timer(ErtsTWheelTimer *p) { @@ -758,16 +793,23 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) restarted = 0; bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); + tiw->true_next_timeout_time = 1; + tiw->next_timeout_pos = bump_to; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); while (1) { ErtsTWheelTimer *p; if (tiw->nto == 0) { empty_wheel: - ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to); + ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(tiw, bump_to); + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, bump_to); tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK; + tiw->next_timeout_pos = bump_to + ERTS_CLKTCKS_WEEK; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->next_timeout_pos);; tiw->pos = bump_to; + tiw->later.pos = bump_to + ERTS_TW_SOON_WHEEL_SIZE; + tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK; tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; ERTS_MSACC_POP_STATE_M_X(); return; @@ -811,8 +853,6 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) ERTS_TW_ASSERT(tiw->nto > 0); ERTS_TW_ASSERT(tiw->at_once.nto > 0); tiw->yield_slot = ERTS_TW_SLOT_AT_ONCE; - tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); ERTS_MSACC_POP_STATE_M_X(); return; /* Yield! */ } @@ -847,28 +887,22 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) sys_memcpy((void *) bump_scnt, (void *) scnt, sizeof(Sint) * ERTS_TW_SCNT_SIZE); - if (tiw->true_next_timeout_time) { - ErtsMonotonicTime skip_until_pos; + if (tiw->soon.min_tpos > tiw->pos) { + ErtsMonotonicTime skip_until_pos = tiw->soon.min_tpos; + /* * No need inspecting slots where we know no timeouts * to trigger should reside. */ - skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); if (skip_until_pos > bump_to) skip_until_pos = bump_to; skip_until_pos--; if (skip_until_pos > tiw->pos) { - ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos); - + ERTS_TW_DBG_VERIFY_EMPTY_SOON_SLOTS(tiw, skip_until_pos); tiw->pos = skip_until_pos; - - skip_until_pos++; - skip_until_pos &= ERTS_TW_LATER_WHEEL_POS_MASK; - if (tiw->later.pos < skip_until_pos) - tiw->later.pos = skip_until_pos; } } @@ -884,6 +918,9 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) slot = soon_slot(tiw->pos+1); tiw->pos = bump_to; + tiw->next_timeout_pos = bump_to; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); + scnt_ix = scnt_get_ix(slot); /* Timeout timers in soon wheel */ @@ -910,7 +947,8 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= p->slot && p->slot < ERTS_TW_SOON_WHEEL_END_SLOT); - tiw->soon.nto--; + if (--tiw->soon.nto == 0) + tiw->soon.min_tpos = ERTS_MAX_CLKTCKS; scnt_ix_dec(scnt, scnt_ix); if (p->timeout_pos <= bump_to) { timeout_timer(p); @@ -933,8 +971,6 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) } if (yield_count <= 0) { - tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to); tiw->yield_slot = slot; tiw->yield_slots_left = slots; ERTS_MSACC_POP_STATE_M_X(); @@ -959,9 +995,9 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) } while (restarted); tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK; + ERTS_TW_ASSERT(tiw->next_timeout_pos == bump_to); - (void) find_next_timeout(NULL, tiw, 0); + (void) find_next_timeout(NULL, tiw); ERTS_MSACC_POP_STATE_M_X(); } @@ -988,15 +1024,31 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) goto restart_yielded_slot; } else { - ErtsMonotonicTime tmp_slots = cpos; - tmp_slots += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + ErtsMonotonicTime end_later_pos, tmp_slots, min_tpos; + + min_tpos = tiw->later.min_tpos & ERTS_TW_LATER_WHEEL_POS_MASK; + end_later_pos = cpos + ERTS_TW_SOON_WHEEL_SIZE; + end_later_pos &= ERTS_TW_LATER_WHEEL_POS_MASK; + + /* Skip known empty slots... */ + if (min_tpos > later_pos) { + if (min_tpos > end_later_pos) { + later_pos = end_later_pos; + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, later_pos); + goto done; + } + later_pos = min_tpos; + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, later_pos); + } + + tmp_slots = end_later_pos; tmp_slots -= later_pos; tmp_slots /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; - tmp_slots++; if (tmp_slots < ERTS_TW_LATER_WHEEL_SIZE) slots = (int) tmp_slots; else slots = ERTS_TW_LATER_WHEEL_SIZE; + fslot = later_slot(later_pos); scnt_ix = scnt_get_ix(fslot); } @@ -1028,6 +1080,10 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) ERTS_TW_ASSERT(tpos >= later_pos); ERTS_TW_ASSERT(p->slot == fslot); + if (--tiw->later.nto == 0) { + tiw->later.min_tpos = ERTS_MAX_CLKTCKS; + tiw->later.min_tpos_slot = ERTS_TW_LATER_WHEEL_END_SLOT; + } scnt_ix_dec(scnt, scnt_ix); if (tpos >= later_pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) { @@ -1061,8 +1117,6 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) if (ycount < 0) { tiw->later.pos = later_pos; - tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(cpos); tiw->yield_slot = fslot; tiw->yield_slots_left = slots; *ycount_p = 0; @@ -1078,6 +1132,8 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) scnt_later_wheel_next(&fslot, &slots, &later_pos, &scnt_ix, bump_scnt); } +done: + ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); tiw->later.pos = later_pos; @@ -1129,13 +1185,17 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp) tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime); tiw->nto = 0; tiw->at_once.nto = 0; + tiw->soon.min_tpos = ERTS_MAX_CLKTCKS; tiw->soon.nto = 0; - tiw->later.pos = tiw->pos; + tiw->later.min_tpos = ERTS_MAX_CLKTCKS; + tiw->later.min_tpos_slot = ERTS_TW_LATER_WHEEL_END_SLOT; + tiw->later.pos = tiw->pos + ERTS_TW_SOON_WHEEL_SIZE; tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK; - tiw->later.pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE; + tiw->later.nto = 0; tiw->yield_slot = ERTS_TW_SLOT_INACTIVE; tiw->true_next_timeout_time = 0; - tiw->next_timeout_time = mtime + ERTS_MONOTONIC_WEEK; + tiw->next_timeout_pos = tiw->pos + ERTS_CLKTCKS_WEEK; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->next_timeout_pos); tiw->sentinel.next = &tiw->sentinel; tiw->sentinel.prev = &tiw->sentinel; tiw->sentinel.timeout = NULL; @@ -1175,7 +1235,6 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, void *arg, ErtsMonotonicTime timeout_pos) { int slot; - ErtsMonotonicTime timeout_time; ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS); p->timeout = timeout; @@ -1195,6 +1254,8 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, /* soon wheel */ p->timeout_pos = timeout_pos; slot = soon_slot(timeout_pos); + if (tiw->soon.min_tpos > timeout_pos) + tiw->soon.min_tpos = timeout_pos; } else { /* later wheel */ @@ -1214,11 +1275,13 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw, } insert_timer_into_slot(tiw, slot, p); - timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); - if (timeout_time < tiw->next_timeout_time) { + if (timeout_pos <= tiw->next_timeout_pos) { tiw->true_next_timeout_time = 1; - tiw->next_timeout_time = timeout_time; + if (timeout_pos < tiw->next_timeout_pos) { + tiw->next_timeout_pos = timeout_pos; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); + } } ERTS_MSACC_POP_STATE_M_X(); } @@ -1264,6 +1327,73 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw, } } +#ifdef ERTS_TW_DEBUG + +void +dbg_verify_empty_soon_slots(ErtsTimerWheel *tiw, ErtsMonotonicTime to_pos) +{ + int ix; + ErtsMonotonicTime tmp; + + ix = soon_slot(tiw->pos); + tmp = to_pos; + if (tmp > tiw->pos) { + int slots; + tmp -= tiw->pos; + ERTS_TW_ASSERT(tmp > 0); + if (tmp < (ErtsMonotonicTime) ERTS_TW_SOON_WHEEL_SIZE) + slots = (int) tmp; + else + slots = ERTS_TW_SOON_WHEEL_SIZE; + + while (slots > 0) { + ERTS_TW_ASSERT(!tiw->w[ix]); + ix++; + if (ix == ERTS_TW_SOON_WHEEL_END_SLOT) + ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT; + slots--; + } + } +} + +void +dbg_verify_empty_later_slots(ErtsTimerWheel *tiw, ErtsMonotonicTime to_pos) +{ + int ix; + ErtsMonotonicTime tmp; + + ix = later_slot(tiw->later.pos); + tmp = to_pos; + tmp &= ERTS_TW_LATER_WHEEL_POS_MASK; + if (tmp > tiw->later.pos) { + int slots; + tmp -= tiw->later.pos; + tmp /= ERTS_TW_LATER_WHEEL_SLOT_SIZE; + ERTS_TW_ASSERT(tmp > 0); + if (tmp < (ErtsMonotonicTime) ERTS_TW_LATER_WHEEL_SIZE) + slots = (int) tmp; + else + slots = ERTS_TW_LATER_WHEEL_SIZE; + + while (slots > 0) { + ErtsTWheelTimer *tmr = tiw->w[ix]; + if (tmr) { + ErtsTWheelTimer *end = tmr; + do { + ERTS_TW_ASSERT(tmr->timeout_pos > to_pos); + tmr = tmr->next; + } while (tmr != end); + } + ix++; + if (ix == ERTS_TW_LATER_WHEEL_END_SLOT) + ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT; + slots--; + } + } +} + +#endif /* ERTS_TW_DEBUG */ + #ifdef ERTS_TW_HARD_DEBUG static void @@ -1385,10 +1515,11 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos) ERTS_TW_ASSERT(tiw->at_once.nto == at_once_tmo); ERTS_TW_ASSERT(tiw->soon.nto == soon_tmo); + ERTS_TW_ASSERT(tiw->later.nto == later_tmo); ERTS_TW_ASSERT(tiw->nto == soon_tmo + later_tmo + at_once_tmo); for (six = 0; six < ERTS_TW_SCNT_SIZE; six++) ERTS_TW_ASSERT(scnt[six] == tiw->scnt[six]); } -#endif +#endif /* ERTS_TW_HARD_DEBUG */ diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index c689d495e6..643cfbf323 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -2871,6 +2871,81 @@ define etp-disasm end end +############################################################################ +# +# Timer Wheel +# + +define etp-timer-wheel +# Args: TimerWheel + if (!erts_initialized) + printf "System not initialized!\n" + else + set $tiw = $arg0 + printf "Number of timers: %d\n", $tiw->nto + printf "Min timeout pos: %d\n", $tiw->next_timeout_pos + printf "\n--- Soon Wheel ---\n" + set $ix = $tiw->pos & etp_tw_soon_wheel_mask + printf "Position: %ld (%d)\n", $tiw->pos, $ix + printf "Min timeout position: %ld (%d)\n", $tiw->soon.min_tpos, $tiw->soon.min_tpos & etp_tw_soon_wheel_mask + printf "Number of timers: %d\n", $tiw->soon.nto + set $slots = etp_tw_soon_wheel_size + while $slots > 0 + set $tmr = $tiw->w[$ix] + if ($tmr != (ErtsTWheelTimer *) 0x0) + printf "---\n" + printf "Slot: %d\n", $ix + printf "\n" + while 1 + printf "- Timeout pos: %ld\n", $tmr->timeout_pos + printf " Pointer: (ErtsTWheelTimer *) %p\n", $tmr + set $tmr = $tmr->next + if ($tmr == $tiw->w[$ix]) + loop_break + end + end + end + set $ix++ + if ($ix == (etp_tw_soon_wheel_first_slot + etp_tw_soon_wheel_size)) + set $ix = etp_tw_soon_wheel_first_slot + end + set $slots-- + end + printf "\n--- Later Wheel ---\n" + set $ix = (($tiw->later.pos >> etp_tw_later_wheel_shift) & etp_tw_later_wheel_mask) + etp_tw_later_wheel_first_slot + printf "Position: %ld (%d)\n", $tiw->later.pos, $ix + printf "Min timeout position: %ld (%d)\n", $tiw->later.min_tpos, (($tiw->later.min_tpos >> etp_tw_later_wheel_shift) & etp_tw_later_wheel_mask) + etp_tw_later_wheel_first_slot + printf "Number of timers: %d\n", $tiw->later.nto + set $slots = etp_tw_later_wheel_size + set $slot_pos = $tiw->later.pos + while $slots > 0 + set $tmr = $tiw->w[$ix] + if ($tmr != (ErtsTWheelTimer *) 0x0) + printf "---\n" + printf "Slot: %d\n", $ix + printf "Slot Range: [%ld, %ld]\n", $slot_pos, $slot_pos + etp_tw_later_wheel_slot_size + printf "Pre timeout pos: %ld\n", $slot_pos - etp_tw_later_wheel_slot_size + printf "\n" + while 1 + printf "- Timeout pos: %ld\n", $tmr->timeout_pos + printf " Pointer: (ErtsTWheelTimer *) %p\n", $tmr + set $tmr = $tmr->next + if ($tmr == $tiw->w[$ix]) + loop_break + end + end + end + set $ix++ + if ($ix == (etp_tw_later_wheel_first_slot + etp_tw_later_wheel_size)) + set $ix = etp_tw_later_wheel_first_slot + end + set $slot_pos = $slot_pos + etp_tw_later_wheel_slot_size + set $slots-- + end + end + printf "---\n" +end + document etp-disasm %--------------------------------------------------------------------------- % etp-disasm StartI EndI -- cgit v1.2.3 From 02da78a8cce79a07f00ff0816967e84ead8a3bfa Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 18 Apr 2017 16:37:44 +0200 Subject: Fix of later timer wheel --- erts/emulator/beam/time.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'erts') diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index f530ee5de0..cee3cb619f 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -1033,8 +1033,8 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) /* Skip known empty slots... */ if (min_tpos > later_pos) { if (min_tpos > end_later_pos) { - later_pos = end_later_pos; - ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, later_pos); + ERTS_TW_DBG_VERIFY_EMPTY_LATER_SLOTS(tiw, end_later_pos); + tiw->later.pos = end_later_pos; goto done; } later_pos = min_tpos; @@ -1051,6 +1051,8 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) fslot = later_slot(later_pos); scnt_ix = scnt_get_ix(fslot); + + tiw->later.pos = end_later_pos; } while (slots > 0) { @@ -1077,7 +1079,6 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) while (1) { ErtsMonotonicTime tpos = p->timeout_pos; - ERTS_TW_ASSERT(tpos >= later_pos); ERTS_TW_ASSERT(p->slot == fslot); if (--tiw->later.nto == 0) { @@ -1086,7 +1087,7 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) } scnt_ix_dec(scnt, scnt_ix); - if (tpos >= later_pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) { + if (tpos >= tiw->later.pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE) { /* keep in later slot; very uncommon... */ insert_timer_into_slot(tiw, fslot, p); ycount -= ERTS_TW_COST_SLOT_MOVE; @@ -1116,7 +1117,6 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) } if (ycount < 0) { - tiw->later.pos = later_pos; tiw->yield_slot = fslot; tiw->yield_slots_left = slots; *ycount_p = 0; @@ -1129,15 +1129,13 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p) } } - scnt_later_wheel_next(&fslot, &slots, &later_pos, &scnt_ix, bump_scnt); + scnt_later_wheel_next(&fslot, &slots, NULL, &scnt_ix, bump_scnt); } done: ERTS_HARD_DBG_CHK_WHEELS(tiw, 0); - tiw->later.pos = later_pos; - *ycount_p = ycount; return 0; -- cgit v1.2.3