From 9c78f149517dc02457d4c59e90bc9b03d411e28c Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 5 May 2015 20:05:00 +0200 Subject: Optimized timer implementation --- erts/emulator/beam/time.c | 699 ++++++++++++++++++++++------------------------ 1 file changed, 340 insertions(+), 359 deletions(-) (limited to 'erts/emulator/beam/time.c') diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 1fffb5f357..ea19d8b362 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -76,6 +76,11 @@ #include "sys.h" #include "erl_vm.h" #include "global.h" +#define ERTS_WANT_TIMER_WHEEL_API +#include "erl_time.h" + +#define ERTS_MONOTONIC_DAY ERTS_SEC_TO_MONOTONIC(60*60*24) +#define ERTS_CLKTCKS_DAY ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY) #ifdef ERTS_ENABLE_LOCK_CHECK #define ASSERT_NO_LOCKED_LOCKS erts_lc_check_exact(NULL, 0) @@ -83,20 +88,24 @@ #define ASSERT_NO_LOCKED_LOCKS #endif -#define ERTS_MONOTONIC_DAY ERTS_SEC_TO_MONOTONIC(60*60*24) -#define ERTS_CLKTCKS_DAY ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY) - +#if 0 +# define ERTS_TW_DEBUG +#endif +#if defined(DEBUG) && !defined(ERTS_TW_DEBUG) +# define ERTS_TW_DEBUG +#endif -/* BEGIN tiw_lock protected variables -** -** The individual timer cells in tiw are also protected by the same mutex. -*/ +#undef ERTS_TW_ASSERT +#if defined(ERTS_TW_DEBUG) +# define ERTS_TW_ASSERT(E) ERTS_ASSERT(E) +#else +# define ERTS_TW_ASSERT(E) ((void) 1) +#endif -/* timing wheel size NEED to be a power of 2 */ -#ifdef SMALL_MEMORY -#define TIW_SIZE (1 << 13) +#ifdef ERTS_TW_DEBUG +# define ERTS_TWHEEL_BUMP_YIELD_LIMIT 5 #else -#define TIW_SIZE (1 << 20) +# define ERTS_TWHEEL_BUMP_YIELD_LIMIT 100 #endif /* Actual interval time chosen by sys_init_time() */ @@ -110,54 +119,22 @@ static int tiw_itime; /* Constant after init */ #endif struct ErtsTimerWheel_ { - ErlTimer *w[TIW_SIZE]; + ErtsTWheelTimer *w[ERTS_TIW_SIZE]; ErtsMonotonicTime pos; Uint nto; struct { - ErlTimer *head; - ErlTimer *tail; + ErtsTWheelTimer *head; + ErtsTWheelTimer *tail; Uint nto; } at_once; + int yield_slot; + int yield_slots_left; + int yield_start_pos; + ErtsTWheelTimer sentinel; int true_next_timeout_time; ErtsMonotonicTime next_timeout_time; - erts_atomic64_t next_timeout; - erts_smp_atomic32_t is_bumping; - erts_smp_mtx_t lock; }; -ErtsTimerWheel *erts_default_timer_wheel; /* managed by aux thread */ - -static ERTS_INLINE ErtsTimerWheel * -get_timer_wheel(ErlTimer *p) -{ - return (ErtsTimerWheel *) erts_smp_atomic_read_acqb(&p->wheel); -} - -static ERTS_INLINE void -set_timer_wheel(ErlTimer *p, ErtsTimerWheel *tiw) -{ - erts_smp_atomic_set_relb(&p->wheel, (erts_aint_t) tiw); -} - -static ERTS_INLINE void -init_next_timeout(ErtsTimerWheel *tiw, - ErtsMonotonicTime time) -{ - erts_atomic64_init_nob(&tiw->next_timeout, - (erts_aint64_t) time); -} - -static ERTS_INLINE void -set_next_timeout(ErtsTimerWheel *tiw, - ErtsMonotonicTime time, - int true_timeout) -{ - tiw->true_next_timeout_time = true_timeout; - tiw->next_timeout_time = time; - erts_atomic64_set_relb(&tiw->next_timeout, - (erts_aint64_t) time); -} - /* get the time (in units of TIW_ITIME) to the next timeout, or -1 if there are no timeouts */ @@ -167,35 +144,23 @@ find_next_timeout(ErtsTimerWheel *tiw, ErtsMonotonicTime max_search_time) { int start_ix, tiw_pos_ix; - ErlTimer *p; + ErtsTWheelTimer *p; int true_min_timeout; - ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos, timeout_limit; - - ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&tiw->lock)); + ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos; if (tiw->true_next_timeout_time) return tiw->next_timeout_time; - /* We never set next timeout beyond timeout_limit */ - timeout_limit = curr_time + ERTS_MONOTONIC_DAY; - if (tiw->nto == 0) { /* no timeouts in wheel */ - true_min_timeout = tiw->true_next_timeout_time = 0; - min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(timeout_limit); + true_min_timeout = 0; + min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + ERTS_MONOTONIC_DAY); goto found_next; } - /* - * Don't want others entering trying to bump - * timers while we are checking... - */ - set_next_timeout(tiw, timeout_limit, 0); - - true_min_timeout = 1; slot_timeout_pos = tiw->pos; min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time); - start_ix = tiw_pos_ix = (int) (tiw->pos & (TIW_SIZE-1)); + start_ix = tiw_pos_ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1)); do { slot_timeout_pos++; @@ -206,93 +171,125 @@ find_next_timeout(ErtsTimerWheel *tiw, p = tiw->w[tiw_pos_ix]; - while (p) { - ErtsMonotonicTime timeout_pos; - ASSERT(p != p->next); - timeout_pos = p->timeout_pos; - if (min_timeout_pos > timeout_pos) { - min_timeout_pos = timeout_pos; - if (min_timeout_pos <= slot_timeout_pos) - goto found_next; - } - p = p->next; + if (p) { + ErtsTWheelTimer *end = p; + + do { + ErtsMonotonicTime timeout_pos; + timeout_pos = p->timeout_pos; + if (min_timeout_pos > timeout_pos) { + true_min_timeout = 1; + min_timeout_pos = timeout_pos; + if (min_timeout_pos <= slot_timeout_pos) + goto found_next; + } + p = p->next; + } while (p != end); } tiw_pos_ix++; - if (tiw_pos_ix == TIW_SIZE) + if (tiw_pos_ix == ERTS_TIW_SIZE) tiw_pos_ix = 0; } while (start_ix != tiw_pos_ix); found_next: min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos); - if (min_timeout != tiw->next_timeout_time) - set_next_timeout(tiw, min_timeout, true_min_timeout); + tiw->next_timeout_time = min_timeout; + tiw->true_next_timeout_time = true_min_timeout; return min_timeout; } -static void -remove_timer(ErtsTimerWheel *tiw, ErlTimer *p) +static ERTS_INLINE void +insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p) { - if (p->slot >= 0) { - /* Timer in wheel... */ - ASSERT(p->slot < TIW_SIZE); - if (p->prev) - p->prev->next = p->next; - else { - ASSERT(tiw->w[p->slot] == p); - tiw->w[p->slot] = p->next; + ERTS_TW_ASSERT(slot >= 0); + ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE); + p->slot = slot; + if (!tiw->w[slot]) { + tiw->w[slot] = p; + p->next = p; + p->prev = p; + } + else { + ErtsTWheelTimer *next, *prev; + next = tiw->w[slot]; + prev = next->prev; + p->next = next; + p->prev = prev; + prev->next = p; + next->prev = p; + } +} + +static ERTS_INLINE void +remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) +{ + int slot = p->slot; + ERTS_TW_ASSERT(slot != ERTS_TWHEEL_SLOT_INACTIVE); + + if (slot >= 0) { + /* + * Timer in wheel or in circular + * list of timers currently beeing + * triggered (referred by sentinel). + */ + ERTS_TW_ASSERT(slot < ERTS_TIW_SIZE); + + if (p->next == p) { + ERTS_TW_ASSERT(tiw->w[slot] == p); + tiw->w[slot] = NULL; } - if(p->next) + else { + if (tiw->w[slot] == p) + tiw->w[slot] = p->next; + p->prev->next = p->next; p->next->prev = p->prev; + } } else { /* Timer in "at once" queue... */ - ASSERT(p->slot == -1); + ERTS_TW_ASSERT(slot == ERTS_TWHEEL_SLOT_AT_ONCE); if (p->prev) p->prev->next = p->next; else { - ASSERT(tiw->at_once.head == p); + ERTS_TW_ASSERT(tiw->at_once.head == p); tiw->at_once.head = p->next; } if (p->next) p->next->prev = p->prev; else { - ASSERT(tiw->at_once.tail == p); + ERTS_TW_ASSERT(tiw->at_once.tail == p); tiw->at_once.tail = p->prev; } - ASSERT(tiw->at_once.nto > 0); + ERTS_TW_ASSERT(tiw->at_once.nto > 0); tiw->at_once.nto--; } - p->slot = -2; + p->slot = ERTS_TWHEEL_SLOT_INACTIVE; +#if 0 p->next = NULL; p->prev = NULL; +#endif - set_timer_wheel(p, NULL); tiw->nto--; } ErtsMonotonicTime -erts_check_next_timeout_time(ErtsTimerWheel *tiw, - ErtsMonotonicTime max_search_time) +erts_check_next_timeout_time(ErtsSchedulerData *esdp) { - ErtsMonotonicTime next, curr; - - curr = erts_get_monotonic_time(); - - erts_smp_mtx_lock(&tiw->lock); - - next = find_next_timeout(tiw, curr, max_search_time); - - erts_smp_mtx_unlock(&tiw->lock); - - return next; + /* + * Called before a scheduler is about to wait. We wont + * check more than 10 minutes into the future. + */ + return find_next_timeout(esdp->timer_wheel, + erts_get_monotonic_time(esdp), + ERTS_SEC_TO_MONOTONIC(10*60)); } -#ifndef DEBUG +#ifndef ERTS_TW_DEBUG #define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) ((void) 0) #else #define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) debug_check_safe_to_skip_to((TIW), (TO)) @@ -300,198 +297,255 @@ static void debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos) { int slots, ix; - ErlTimer *tmr; + ErtsTWheelTimer *tmr; ErtsMonotonicTime tmp; - ix = (int) (tiw->pos & (TIW_SIZE-1)); + ix = (int) (tiw->pos & (ERTS_TIW_SIZE-1)); tmp = skip_to_pos - tiw->pos; - ASSERT(tmp >= 0); - if (tmp < (ErtsMonotonicTime) TIW_SIZE) + ERTS_TW_ASSERT(tmp >= 0); + if (tmp < (ErtsMonotonicTime) ERTS_TIW_SIZE) slots = (int) tmp; else - slots = TIW_SIZE; + slots = ERTS_TIW_SIZE; while (slots > 0) { - tmr = tiw->w[ix]; - while (tmr) { - ASSERT(tmr->timeout_pos > skip_to_pos); - tmr = tmr->next; - } - ix++; - if (ix == TIW_SIZE) - ix = 0; - slots--; + tmr = tiw->w[ix]; + if (tmr) { + ErtsTWheelTimer *end = tmr; + do { + ERTS_TW_ASSERT(tmr->timeout_pos > skip_to_pos); + tmr = tmr->next; + } while (tmr != end); + } + ix++; + if (ix == ERTS_TIW_SIZE) + ix = 0; + slots--; } } #endif +static ERTS_INLINE void +timeout_timer(ErtsTWheelTimer *p) +{ + ErlTimeoutProc timeout; + void *arg; +#if 0 + p->next = NULL; + p->prev = NULL; +#endif + p->slot = ERTS_TWHEEL_SLOT_INACTIVE; + timeout = p->u.func.timeout; + arg = p->u.func.arg; + (*timeout)(arg); +} + void erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time) { - int tiw_pos_ix, slots; - ErlTimer *p, *timeout_head, **timeout_tail; - ErtsMonotonicTime bump_to, tmp_slots; - - if (erts_smp_atomic32_cmpxchg_nob(&tiw->is_bumping, 1, 0) != 0) - return; /* Another thread is currently bumping... */ + int tiw_pos_ix, slots, yielded_slot_restarted, yield_count; + ErtsMonotonicTime bump_to, tmp_slots, old_pos; - bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); + yield_count = ERTS_TWHEEL_BUMP_YIELD_LIMIT; - erts_smp_mtx_lock(&tiw->lock); + /* + * In order to be fair we always continue with work + * where we left off when restarting after a yield. + */ - if (tiw->pos >= bump_to) { - timeout_head = NULL; - goto done; + if (tiw->yield_slot >= 0) { + yielded_slot_restarted = 1; + tiw_pos_ix = tiw->yield_slot; + slots = tiw->yield_slots_left; + bump_to = tiw->pos; + old_pos = tiw->yield_start_pos; + goto restart_yielded_slot; } - /* Don't want others here while we are bumping... */ - set_next_timeout(tiw, curr_time + ERTS_MONOTONIC_DAY, 0); + do { + + yielded_slot_restarted = 0; + + bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); - if (!tiw->at_once.head) { - timeout_head = NULL; - timeout_tail = &timeout_head; - } - else { - ASSERT(tiw->nto >= tiw->at_once.nto); - p = timeout_head = tiw->at_once.head; while (1) { - set_timer_wheel(p, NULL); - if (!p->next) { - timeout_tail = &p->next; - break; + ErtsTWheelTimer *p; + + old_pos = tiw->pos; + + if (tiw->nto == 0) { + empty_wheel: + ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to); + tiw->true_next_timeout_time = 0; + tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; + tiw->pos = bump_to; + tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + return; } - } - tiw->nto -= tiw->at_once.nto; - tiw->at_once.head = NULL; - tiw->at_once.tail = NULL; - tiw->at_once.nto = 0; - } - if (tiw->nto == 0) { - ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to); - tiw->pos = bump_to; - goto done; - } + p = tiw->at_once.head; + while (p) { + if (--yield_count <= 0) { + ERTS_TW_ASSERT(tiw->nto > 0); + ERTS_TW_ASSERT(tiw->at_once.nto > 0); + tiw->yield_slot = ERTS_TWHEEL_SLOT_AT_ONCE; + tiw->true_next_timeout_time = 1; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos); + return; + } + + ERTS_TW_ASSERT(tiw->nto > 0); + ERTS_TW_ASSERT(tiw->at_once.nto > 0); + tiw->nto--; + tiw->at_once.nto--; + tiw->at_once.head = p->next; + if (p->next) + p->next->prev = NULL; + else + tiw->at_once.tail = NULL; + + timeout_timer(p); + + p = tiw->at_once.head; + } - if (tiw->true_next_timeout_time) { - ErtsMonotonicTime skip_until_pos; - /* - * No need inspecting slots where we know no timeouts - * to trigger should reside. - */ + if (tiw->pos >= bump_to) + break; - skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); - if (skip_until_pos > bump_to) - skip_until_pos = bump_to; + if (tiw->nto == 0) + goto empty_wheel; - ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos); - ASSERT(skip_until_pos > tiw->pos); + if (tiw->true_next_timeout_time) { + ErtsMonotonicTime skip_until_pos; + /* + * No need inspecting slots where we know no timeouts + * to trigger should reside. + */ - tiw->pos = skip_until_pos - 1; - } + skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time); + if (skip_until_pos > bump_to) + skip_until_pos = bump_to; - tiw_pos_ix = (int) ((tiw->pos+1) & (TIW_SIZE-1)); - tmp_slots = (bump_to - tiw->pos); - if (tmp_slots < (ErtsMonotonicTime) TIW_SIZE) - slots = (int) tmp_slots; - else - slots = TIW_SIZE; - - while (slots > 0) { - p = tiw->w[tiw_pos_ix]; - while (p) { - ErlTimer *next = p->next; - ASSERT(p != next); - if (p->timeout_pos <= bump_to) { /* we have a timeout */ - /* Remove from list */ - remove_timer(tiw, p); - *timeout_tail = p; /* Insert in timeout queue */ - timeout_tail = &p->next; + skip_until_pos--; + + if (skip_until_pos > tiw->pos) { + ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos); + + tiw->pos = skip_until_pos; + } + } + + tiw_pos_ix = (int) ((tiw->pos+1) & (ERTS_TIW_SIZE-1)); + tmp_slots = (bump_to - tiw->pos); + if (tmp_slots < (ErtsMonotonicTime) ERTS_TIW_SIZE) + slots = (int) tmp_slots; + else + slots = ERTS_TIW_SIZE; + + tiw->pos = bump_to; + + while (slots > 0) { + + p = tiw->w[tiw_pos_ix]; + if (p) { + if (p->next == p) { + ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel); + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + } + else { + tiw->sentinel.next = p->next; + tiw->sentinel.prev = p->prev; + tiw->sentinel.next->prev = &tiw->sentinel; + tiw->sentinel.prev->next = &tiw->sentinel; + } + tiw->w[tiw_pos_ix] = NULL; + + while (1) { + + if (p->timeout_pos > bump_to) { + /* Very unusual case... */ + ++yield_count; + insert_timer_into_slot(tiw, tiw_pos_ix, p); + } + else { + /* Normal case... */ + timeout_timer(p); + tiw->nto--; + } + + restart_yielded_slot: + + p = tiw->sentinel.next; + if (p == &tiw->sentinel) { + ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel); + break; + } + + if (--yield_count <= 0) { + tiw->true_next_timeout_time = 1; + tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(old_pos); + tiw->yield_slot = tiw_pos_ix; + tiw->yield_slots_left = slots; + tiw->yield_start_pos = old_pos; + return; /* Yield! */ + } + + tiw->sentinel.next = p->next; + p->next->prev = &tiw->sentinel; + } + } + tiw_pos_ix++; + if (tiw_pos_ix == ERTS_TIW_SIZE) + tiw_pos_ix = 0; + slots--; } - p = next; } - tiw_pos_ix++; - if (tiw_pos_ix == TIW_SIZE) - tiw_pos_ix = 0; - slots--; - } - ASSERT(tmp_slots >= (ErtsMonotonicTime) TIW_SIZE - || tiw_pos_ix == (int) ((bump_to+1) & (TIW_SIZE-1))); + } while (yielded_slot_restarted); - tiw->pos = bump_to; + tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; + tiw->true_next_timeout_time = 0; + tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_DAY; /* Search at most two seconds ahead... */ (void) find_next_timeout(tiw, curr_time, ERTS_SEC_TO_MONOTONIC(2)); - -done: - - erts_smp_mtx_unlock(&tiw->lock); - - erts_smp_atomic32_set_nob(&tiw->is_bumping, 0); - - /* Call timedout timers callbacks */ - while (timeout_head) { - ErlTimeoutProc timeout; - void *arg; - p = timeout_head; - timeout_head = p->next; - /* Here comes hairy use of the timer fields! - * They are reset without having the lock. - * It is assumed that no code but this will - * accesses any field until the ->timeout - * callback is called. - */ - ASSERT(p->timeout_pos <= bump_to); - p->next = NULL; - p->prev = NULL; - p->slot = 0; - timeout = p->timeout; - arg = p->arg; - (*timeout)(arg); - } } Uint erts_timer_wheel_memory_size(void) { -#ifdef ERTS_SMP - return sizeof(ErtsTimerWheel)*(1 + erts_no_schedulers); -#else - return sizeof(ErtsTimerWheel); -#endif + return sizeof(ErtsTimerWheel)*erts_no_schedulers; } ErtsTimerWheel * -erts_create_timer_wheel(int no) +erts_create_timer_wheel(ErtsSchedulerData *esdp) { ErtsMonotonicTime mtime; int i; ErtsTimerWheel *tiw; - tiw = (ErtsTimerWheel *) erts_alloc(ERTS_ALC_T_TIMER_WHEEL, - sizeof(ErtsTimerWheel)); - for(i = 0; i < TIW_SIZE; i++) + tiw = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_WHEEL, + sizeof(ErtsTimerWheel)); + for(i = 0; i < ERTS_TIW_SIZE; i++) tiw->w[i] = NULL; - erts_smp_atomic32_init_nob(&tiw->is_bumping, 0); - erts_smp_mtx_init_x(&tiw->lock, "timer_wheel", make_small(no)); - - mtime = erts_get_monotonic_time(); + mtime = erts_get_monotonic_time(esdp); tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime); tiw->nto = 0; tiw->at_once.head = NULL; tiw->at_once.tail = NULL; tiw->at_once.nto = 0; + tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE; tiw->true_next_timeout_time = 0; tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY; - init_next_timeout(tiw, mtime + ERTS_MONOTONIC_DAY); + tiw->sentinel.next = &tiw->sentinel; + tiw->sentinel.prev = &tiw->sentinel; return tiw; } ErtsNextTimeoutRef erts_get_next_timeout_reference(ErtsTimerWheel *tiw) { - return (ErtsNextTimeoutRef) &tiw->next_timeout; + return (ErtsNextTimeoutRef) &tiw->next_timeout_time; } @@ -512,155 +566,83 @@ erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode) #else tiw_itime = itime; #endif - - erts_default_timer_wheel = erts_create_timer_wheel(0); } void -erts_set_timer(ErlTimer *p, ErlTimeoutProc timeout, - ErlCancelProc cancel, void *arg, Uint to) +erts_twheel_set_timer(ErtsTimerWheel *tiw, + ErtsTWheelTimer *p, ErlTimeoutProc timeout, + ErlCancelProc cancel, void *arg, + ErtsMonotonicTime timeout_pos) { - ErtsMonotonicTime timeout_time, timeout_pos; - ErtsMonotonicTime curr_time; - ErtsTimerWheel *tiw; - ErtsSchedulerData *esdp; - - curr_time = erts_get_monotonic_time(); - esdp = erts_get_scheduler_data(); - if (esdp) - tiw = esdp->timer_wheel; - else - tiw = erts_default_timer_wheel; + ErtsMonotonicTime timeout_time; - erts_smp_mtx_lock(&tiw->lock); + p->u.func.timeout = timeout; + p->u.func.cancel = cancel; + p->u.func.arg = arg; - if (get_timer_wheel(p)) - ERTS_INTERNAL_ERROR("Double set timer"); + ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE); - p->timeout = timeout; - p->cancel = cancel; - p->arg = arg; - - if (to == 0) { - timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time); + if (timeout_pos <= tiw->pos) { tiw->nto++; tiw->at_once.nto++; p->next = NULL; p->prev = tiw->at_once.tail; - tiw->at_once.tail = p; - if (!tiw->at_once.head) + if (tiw->at_once.tail) { + ERTS_TW_ASSERT(tiw->at_once.head); + tiw->at_once.tail->next = p; + } + else { + ERTS_TW_ASSERT(!tiw->at_once.head); tiw->at_once.head = p; - p->timeout_pos = timeout_pos; - p->slot = -1; - timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); + } + tiw->at_once.tail = p; + p->timeout_pos = tiw->pos; + p->slot = ERTS_TWHEEL_SLOT_AT_ONCE; + timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->pos); } else { - int tm; - ErtsMonotonicTime ticks; - - ticks = ERTS_MSEC_TO_CLKTCKS(to); - timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time - 1) + 1 + ticks; + int slot; /* calculate slot */ - tm = (int) (timeout_pos & (TIW_SIZE-1)); - p->slot = tm; - - /* insert at head of list at slot */ - p->next = tiw->w[tm]; - p->prev = NULL; - if (p->next != NULL) - p->next->prev = p; - tiw->w[tm] = p; + slot = (int) (timeout_pos & (ERTS_TIW_SIZE-1)); + + insert_timer_into_slot(tiw, slot, p); tiw->nto++; timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos); p->timeout_pos = timeout_pos; - - ASSERT(ERTS_MSEC_TO_MONOTONIC(to) <= timeout_time - curr_time); - ASSERT(timeout_time - curr_time - < ERTS_MSEC_TO_MONOTONIC(to) + ERTS_CLKTCKS_TO_MONOTONIC(1)); } - if (timeout_time < tiw->next_timeout_time) - set_next_timeout(tiw, timeout_time, 1); - - set_timer_wheel(p, tiw); - - erts_smp_mtx_unlock(&tiw->lock); - -#if defined(ERTS_SMP) - if (tiw == erts_default_timer_wheel) - erts_interupt_aux_thread_timed(timeout_time); -#endif - + if (timeout_time < tiw->next_timeout_time) { + tiw->true_next_timeout_time = 1; + tiw->next_timeout_time = timeout_time; + } } void -erts_cancel_timer(ErlTimer *p) +erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p) { - ErtsTimerWheel *tiw; - ErlCancelProc cancel; - void *arg = NULL; /* Shut up faulty warning... */ - - tiw = get_timer_wheel(p); - if (!tiw) - return; - - erts_smp_mtx_lock(&tiw->lock); - if (tiw != get_timer_wheel(p)) - cancel = NULL; - else { + if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) { + ErlCancelProc cancel; + void *arg; remove_timer(tiw, p); - - cancel = p->cancel; - arg = p->arg; + cancel = p->u.func.cancel; + arg = p->u.func.arg; + if (cancel) + (*cancel)(arg); } - erts_smp_mtx_unlock(&tiw->lock); - - if (cancel) - (*cancel)(arg); } -/* - Returns the amount of time left in ms until the timer 'p' is triggered. - 0 is returned if 'p' isn't active. - 0 is returned also if the timer is overdue (i.e., would have triggered - immediately if it hadn't been cancelled). -*/ -Uint -erts_time_left(ErlTimer *p) -{ - ErtsTimerWheel *tiw; - ErtsMonotonicTime current_time, timeout_time; - - tiw = get_timer_wheel(p); - if (!tiw) - return 0; - - erts_smp_mtx_lock(&tiw->lock); - if (tiw != get_timer_wheel(p)) - timeout_time = ERTS_MONOTONIC_TIME_MIN; - else - timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos); - erts_smp_mtx_unlock(&tiw->lock); - - current_time = erts_get_monotonic_time(); - if (timeout_time <= current_time) - return 0; - return (Uint) ERTS_MONOTONIC_TO_MSEC(timeout_time - current_time); -} - -#ifdef DEBUG +#ifdef ERTS_TW_DEBUG void erts_p_slpq(void) { - ErtsTimerWheel *tiw = erts_default_timer_wheel; - ErtsMonotonicTime current_time = erts_get_monotonic_time(); + erts_printf("Not yet implemented...\n"); +#if 0 + ErtsMonotonicTime current_time = erts_get_monotonic_time(NULL); int i; - ErlTimer* p; + ErtsTWheelTimer* p; - erts_smp_mtx_lock(&tiw->lock); - /* print the whole wheel, starting at the current position */ erts_printf("\ncurrent time = %bps tiw_pos = %d tiw_nto %d\n", current_time, tiw->pos, tiw->nto); @@ -673,7 +655,7 @@ void erts_p_slpq(void) p->slot); } } - for(i = ((i+1) & (TIW_SIZE-1)); i != (tiw->pos & (TIW_SIZE-1)); i = ((i+1) & (TIW_SIZE-1))) { + for(i = ((i+1) & (ERTS_TIW_SIZE-1)); i != (tiw->pos & (ERTS_TIW_SIZE-1)); i = ((i+1) & (ERTS_TIW_SIZE-1))) { if (tiw->w[i] != NULL) { erts_printf("%d:\n", i); for(p = tiw->w[i]; p != NULL; p = p->next) { @@ -682,7 +664,6 @@ void erts_p_slpq(void) } } } - - erts_smp_mtx_unlock(&tiw->lock); +#endif } -#endif /* DEBUG */ +#endif /* ERTS_TW_DEBUG */ -- cgit v1.2.3