Manage timers to trigger at once in a slot similar to other timers

2 files changed, 193 insertions, 177 deletions

diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 699e69e991..46d6da6448 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -448,8 +448,7 @@ erts_time_unit_conversion(Uint64 value,
 void erts_sched_init_time_sup(ErtsSchedulerData *esdp);
 
 
-#define ERTS_TWHEEL_SLOT_AT_ONCE -1
-#define ERTS_TWHEEL_SLOT_INACTIVE -2
+#define ERTS_TW_SLOT_INACTIVE (-2)
 
 /*
 ** Timer entry:
@@ -481,7 +480,7 @@ ERTS_GLB_INLINE ErtsMonotonicTime erts_tweel_read_timeout(ErtsTWheelTimer *twt);
 
 ERTS_GLB_INLINE void erts_twheel_init_timer(ErtsTWheelTimer *p)
 {
-    p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
+    p->slot = ERTS_TW_SLOT_INACTIVE;
 }
 
 ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_tmo_ref)
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 1b6ca2b269..16e4328a05 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -125,13 +125,17 @@
  * slot position by just removing it from the circular double
  * linked list that it is in.
  *
- * -- At Once List --
+ * -- At Once Slot --
  *
  * If a timer is set that has a time earlier or equal to 'pos',
  * it is not inserted into the wheel. It is instead inserted,
- * into a list referred to by the 'at_once' field. When the
- * bump operation is performed thise timers will be triggered
- * at once.
+ * into a circular double linked list referred to by the "at
+ * once" slot. When the bump operation is performed these timers
+ * will be triggered at once. The circular list of the slot will
+ * be moved to the 'sentinel' field while bumping these timers
+ * as when bumping an ordinary wheel slot. A yielding bump
+ * operation and cancelation of timers is handled the same way
+ * as if the timer was in a wheel slot.
  *
  * -- Searching for Next Timeout --
  *
@@ -262,14 +266,15 @@ static int tiw_itime; /* Constant after init */
 #endif
 
 struct ErtsTimerWheel_ {
-    ErtsTWheelTimer *w[ERTS_TW_SOON_WHEEL_SIZE + ERTS_TW_LATER_WHEEL_SIZE];
+    ErtsTWheelTimer *slots[1                            /* At Once Slot */
+                           + ERTS_TW_SOON_WHEEL_SIZE    /* Soon Wheel Slots */
+                           + ERTS_TW_LATER_WHEEL_SIZE];   /* Later Wheel Slots */
+    ErtsTWheelTimer **w;
     Sint scnt[ERTS_TW_SCNT_SIZE];
     Sint bump_scnt[ERTS_TW_SCNT_SIZE];
     ErtsMonotonicTime pos;
     Uint nto;
     struct {
-	ErtsTWheelTimer *head;
-	ErtsTWheelTimer *tail;
 	Uint nto;
     } at_once;
     struct {
@@ -285,6 +290,8 @@ struct ErtsTimerWheel_ {
     ErtsMonotonicTime next_timeout_time;
 };
 
+#define ERTS_TW_SLOT_AT_ONCE (-1)
+
 #define ERTS_TW_BUMP_LATER_WHEEL(TIW) \
     ((tiw)->pos + ERTS_TW_LATER_WHEEL_SLOT_SIZE >= (TIW)->later.pos)
 
@@ -435,7 +442,7 @@ find_next_timeout(ErtsSchedulerData *esdp, ErtsTimerWheel *tiw, int thorough)
 
     ERTS_HARD_DBG_CHK_WHEELS(tiw, 0);
 
-    ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TWHEEL_SLOT_INACTIVE);
+    ERTS_TW_ASSERT(tiw->yield_slot == ERTS_TW_SLOT_INACTIVE);
 
     if (tiw->nto == 0) { /* no timeouts in wheel */
 	if (!thorough)
@@ -552,28 +559,10 @@ done:
 }
 
 static ERTS_INLINE void
-insert_timer_into_at_once_list(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
-{
-    tiw->at_once.nto++;
-    p->next = NULL;
-    p->prev = tiw->at_once.tail;
-    if (tiw->at_once.tail) {
-        ERTS_TW_ASSERT(tiw->at_once.head);
-        tiw->at_once.tail->next = p;
-    }
-    else {	
-        ERTS_TW_ASSERT(!tiw->at_once.head);
-        tiw->at_once.head = p;
-    }
-    tiw->at_once.tail = p;
-    p->slot = ERTS_TWHEEL_SLOT_AT_ONCE;
-}
-
-static ERTS_INLINE void
 insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p)
 {
-    ERTS_TW_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT <= slot);
-    ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT);
+    ERTS_TW_ASSERT(ERTS_TW_SLOT_AT_ONCE <= slot
+                   && slot < ERTS_TW_LATER_WHEEL_END_SLOT);
     p->slot = slot;
     if (!tiw->w[slot]) {
 	tiw->w[slot] = p;
@@ -589,60 +578,53 @@ insert_timer_into_slot(ErtsTimerWheel *tiw, int slot, ErtsTWheelTimer *p)
 	prev->next = p;
 	next->prev = p;
     }
-    if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) {
-        ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
-        tiw->soon.nto++;
+    if (slot == ERTS_TW_SLOT_AT_ONCE)
+	tiw->at_once.nto++;
+    else {
+        if (slot < ERTS_TW_SOON_WHEEL_END_SLOT) {
+            ERTS_TW_ASSERT(p->timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
+            tiw->soon.nto++;
+        }
+        scnt_inc(tiw->scnt, slot);
     }
-    scnt_inc(tiw->scnt, slot);
 }
 
 static ERTS_INLINE void
 remove_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
 {
     int slot = p->slot;
-    ERTS_TW_ASSERT(slot != ERTS_TWHEEL_SLOT_INACTIVE);
-
-    if (slot >= ERTS_TW_SOON_WHEEL_FIRST_SLOT) {
-	/*
-	 * Timer in wheel or in circular
-	 * list of timers currently beeing
-	 * triggered (referred by sentinel).
-	 */
-	ERTS_TW_ASSERT(slot < ERTS_TW_LATER_WHEEL_END_SLOT);
-
-	if (p->next == p) {
-	    ERTS_TW_ASSERT(tiw->w[slot] == p);
-	    tiw->w[slot] = NULL;
-	}
-	else {
-	    if (tiw->w[slot] == p)
-		tiw->w[slot] = p->next;
-	    p->prev->next = p->next;
-	    p->next->prev = p->prev;
-	}
-        if (slot < ERTS_TW_SOON_WHEEL_END_SLOT)
-            tiw->soon.nto--;
-        scnt_dec(tiw->scnt, slot);
+    ERTS_TW_ASSERT(slot != ERTS_TW_SLOT_INACTIVE);
+
+    /*
+     * Timer is in circular list either referred to
+     * by at once slot, slot in soon wheel, slot
+     * in later wheel, or by sentinel (timers currently
+     * being triggered).
+     */
+    ERTS_TW_ASSERT(ERTS_TW_SLOT_AT_ONCE <= slot
+                   && slot < ERTS_TW_LATER_WHEEL_END_SLOT);
+
+    if (p->next == p) {
+        /* Cannot be referred by sentinel, i.e. must be referred by slot... */
+        ERTS_TW_ASSERT(tiw->w[slot] == p);
+        tiw->w[slot] = NULL;
     }
     else {
-	/* Timer in "at once" queue... */
-	ERTS_TW_ASSERT(slot == ERTS_TWHEEL_SLOT_AT_ONCE);
-	if (p->prev)
-	    p->prev->next = p->next;
-	else {
-	    ERTS_TW_ASSERT(tiw->at_once.head == p);
-	    tiw->at_once.head = p->next;
-	}
-	if (p->next)
-	    p->next->prev = p->prev;
-	else {
-	    ERTS_TW_ASSERT(tiw->at_once.tail == p);
-	    tiw->at_once.tail = p->prev;
-	}
+        if (tiw->w[slot] == p)
+            tiw->w[slot] = p->next;
+        p->prev->next = p->next;
+        p->next->prev = p->prev;
+    }
+    if (slot == ERTS_TW_SLOT_AT_ONCE) {
 	ERTS_TW_ASSERT(tiw->at_once.nto > 0);
 	tiw->at_once.nto--;
     }
-    p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
+    else {
+        if (slot < ERTS_TW_SOON_WHEEL_END_SLOT)
+            tiw->soon.nto--;
+        scnt_dec(tiw->scnt, slot);
+    }
+    p->slot = ERTS_TW_SLOT_INACTIVE;
 }
 
 ErtsMonotonicTime
@@ -732,7 +714,7 @@ timeout_timer(ErtsTWheelTimer *p)
 {
     ErlTimeoutProc timeout;
     void *arg;
-    p->slot = ERTS_TWHEEL_SLOT_INACTIVE;
+    p->slot = ERTS_TW_SLOT_INACTIVE;
     timeout = p->timeout;
     arg = p->arg;
     (*timeout)(arg);
@@ -742,7 +724,7 @@ timeout_timer(ErtsTWheelTimer *p)
 void
 erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 {
-    int slot, yielded_slot_restarted, yield_count, slots, scnt_ix;
+    int slot, restarted, yield_count, slots, scnt_ix;
     ErtsMonotonicTime bump_to;
     Sint *scnt, *bump_scnt;
     ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
@@ -757,23 +739,24 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
      * where we left off when restarting after a yield.
      */
 
-    if (tiw->yield_slot >= ERTS_TW_SOON_WHEEL_FIRST_SLOT) {
-	yielded_slot_restarted = 1;
+    slot = tiw->yield_slot;
+    restarted = slot != ERTS_TW_SLOT_INACTIVE;
+    if (restarted) {
 	bump_to = tiw->pos;
-        if (tiw->yield_slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT)
+        if (slot >= ERTS_TW_LATER_WHEEL_FIRST_SLOT)
             goto restart_yielded_later_slot;
-	slot = tiw->yield_slot;
+        tiw->yield_slot = ERTS_TW_SLOT_INACTIVE;
+        if (slot == ERTS_TW_SLOT_AT_ONCE)
+            goto restart_yielded_at_once_slot;
         scnt_ix = scnt_get_ix(slot);
 	slots = tiw->yield_slots_left;
         ASSERT(0 <= slots && slots <= ERTS_TW_SOON_WHEEL_SIZE);
-        tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
         goto restart_yielded_soon_slot;
     }
 
     do {
 
-	yielded_slot_restarted = 0;
-
+        restarted = 0;
 	bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
 
 	while (1) {
@@ -785,38 +768,59 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 		tiw->true_next_timeout_time = 0;
 		tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK;
 		tiw->pos = bump_to;
-		tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
+		tiw->yield_slot = ERTS_TW_SLOT_INACTIVE;
                 ERTS_MSACC_POP_STATE_M_X();
 		return;
 	    }
 
-	    p = tiw->at_once.head;
-	    while (p) {
-		if (yield_count <= 0) {
-		    ERTS_TW_ASSERT(tiw->nto > 0);
-		    ERTS_TW_ASSERT(tiw->at_once.nto > 0);
-		    tiw->yield_slot = ERTS_TWHEEL_SLOT_AT_ONCE;
-		    tiw->true_next_timeout_time = 1;
-		    tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to);
-                    ERTS_MSACC_POP_STATE_M_X();
-		    return;
-		}
+	    p = tiw->w[ERTS_TW_SLOT_AT_ONCE];
 
-		ERTS_TW_ASSERT(tiw->nto > 0);
-		ERTS_TW_ASSERT(tiw->at_once.nto > 0);
-		tiw->nto--;
-		tiw->at_once.nto--;
-		tiw->at_once.head = p->next;
-		if (p->next)
-		    p->next->prev = NULL;
-		else
-		    tiw->at_once.tail = NULL;
+	    if (p) {
 
-		timeout_timer(p);
+                if (p->next == p) {
+                    ERTS_TW_ASSERT(tiw->sentinel.next == &tiw->sentinel);
+                    ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel);
+                }
+                else {
+                    tiw->sentinel.next = p->next;
+                    tiw->sentinel.prev = p->prev;
+                    tiw->sentinel.next->prev = &tiw->sentinel;
+                    tiw->sentinel.prev->next = &tiw->sentinel;
+                }
+                tiw->w[ERTS_TW_SLOT_AT_ONCE] = NULL;
+
+                while (1) {
+                    ERTS_TW_ASSERT(tiw->nto > 0);
+                    ERTS_TW_ASSERT(tiw->at_once.nto > 0);
+                    tiw->nto--;
+                    tiw->at_once.nto--;
+
+                    timeout_timer(p);
+
+                    yield_count -= ERTS_TW_COST_TIMEOUT;
+
+                restart_yielded_at_once_slot:
 
-                yield_count -= ERTS_TW_COST_TIMEOUT;
+                    p = tiw->sentinel.next;
+                    if (p == &tiw->sentinel) {
+                        ERTS_TW_ASSERT(tiw->sentinel.prev == &tiw->sentinel);
+                        break;
+                    }
+
+                    if (yield_count <= 0) {
+                        ERTS_TW_ASSERT(tiw->nto > 0);
+                        ERTS_TW_ASSERT(tiw->at_once.nto > 0);
+                        tiw->yield_slot = ERTS_TW_SLOT_AT_ONCE;
+                        tiw->true_next_timeout_time = 1;
+                        tiw->next_timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(bump_to);
+                        ERTS_MSACC_POP_STATE_M_X();
+                        return; /* Yield! */
+                    }
+
+                    tiw->sentinel.next = p->next;
+                    p->next->prev = &tiw->sentinel;
+                }
 
-		p = tiw->at_once.head;
 	    }
 
 	    if (tiw->pos >= bump_to) {
@@ -952,7 +956,7 @@ erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
             }
 	}
 
-    } while (yielded_slot_restarted);
+    } while (restarted);
 
     tiw->true_next_timeout_time = 0;
     tiw->next_timeout_time = curr_time + ERTS_MONOTONIC_WEEK;
@@ -980,7 +984,7 @@ bump_later_wheel(ErtsTimerWheel *tiw, int *ycount_p)
         scnt_ix = scnt_get_ix(fslot);
         slots = tiw->yield_slots_left;
         ASSERT(0 <= slots && slots <= ERTS_TW_LATER_WHEEL_SIZE);
-        tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
+        tiw->yield_slot = ERTS_TW_SLOT_INACTIVE;
         goto restart_yielded_slot;
     }
     else {
@@ -1093,11 +1097,29 @@ ErtsTimerWheel *
 erts_create_timer_wheel(ErtsSchedulerData *esdp)
 {
     ErtsMonotonicTime mtime;
-    int i = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
+    int i;
     ErtsTimerWheel *tiw;
+
+    /* Some compile time sanity checks... */
+
+    /* Slots... */
+    ERTS_CT_ASSERT(ERTS_TW_SLOT_AT_ONCE == -1);
+    ERTS_CT_ASSERT(ERTS_TW_SLOT_INACTIVE < ERTS_TW_SLOT_AT_ONCE);
+    ERTS_CT_ASSERT(ERTS_TW_SLOT_AT_ONCE + 1 == ERTS_TW_SOON_WHEEL_FIRST_SLOT);
+    ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_FIRST_SLOT < ERTS_TW_SOON_WHEEL_END_SLOT);
+    ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_END_SLOT == ERTS_TW_LATER_WHEEL_FIRST_SLOT);    
+    ERTS_CT_ASSERT(ERTS_TW_LATER_WHEEL_FIRST_SLOT < ERTS_TW_LATER_WHEEL_END_SLOT);
+
+    /* Both wheel sizes should be a powers of 2 */
+    ERTS_CT_ASSERT(ERTS_TW_SOON_WHEEL_SIZE
+                   && !(ERTS_TW_SOON_WHEEL_SIZE & (ERTS_TW_SOON_WHEEL_SIZE-1)));
+    ERTS_CT_ASSERT(ERTS_TW_LATER_WHEEL_SIZE
+                   && !(ERTS_TW_LATER_WHEEL_SIZE & (ERTS_TW_LATER_WHEEL_SIZE-1)));
+
     tiw = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_TIMER_WHEEL,
 					     sizeof(ErtsTimerWheel));
-    for(; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++)
+    tiw->w = &tiw->slots[1];
+    for(i = ERTS_TW_SLOT_AT_ONCE; i < ERTS_TW_LATER_WHEEL_END_SLOT; i++)
 	tiw->w[i] = NULL;
 
     for (i = 0; i < ERTS_TW_SCNT_SIZE; i++)
@@ -1106,14 +1128,12 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp)
     mtime = erts_get_monotonic_time(esdp);
     tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime);
     tiw->nto = 0;
-    tiw->at_once.head = NULL;
-    tiw->at_once.tail = NULL;
     tiw->at_once.nto = 0;
     tiw->soon.nto = 0;
     tiw->later.pos = tiw->pos;
     tiw->later.pos &= ERTS_TW_LATER_WHEEL_POS_MASK;
     tiw->later.pos += ERTS_TW_LATER_WHEEL_SLOT_SIZE;
-    tiw->yield_slot = ERTS_TWHEEL_SLOT_INACTIVE;
+    tiw->yield_slot = ERTS_TW_SLOT_INACTIVE;
     tiw->true_next_timeout_time = 0;
     tiw->next_timeout_time = mtime + ERTS_MONOTONIC_WEEK;
     tiw->sentinel.next = &tiw->sentinel;
@@ -1154,50 +1174,48 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw,
 		      ErtsTWheelTimer *p, ErlTimeoutProc timeout,
 		      void *arg, ErtsMonotonicTime timeout_pos)
 {
+    int slot;
     ErtsMonotonicTime timeout_time;
     ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 
     p->timeout = timeout;
     p->arg = arg;
 
-    ERTS_TW_ASSERT(p->slot == ERTS_TWHEEL_SLOT_INACTIVE);
+    ERTS_TW_ASSERT(p->slot == ERTS_TW_SLOT_INACTIVE);
 
     tiw->nto++;
+
+    /* calculate slot */
     if (timeout_pos <= tiw->pos) {
-        p->timeout_pos = tiw->pos;
-        insert_timer_into_at_once_list(tiw, p);
-	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(tiw->pos);
+        /* at once */
+        p->timeout_pos = timeout_pos = tiw->pos;
+        slot = ERTS_TW_SLOT_AT_ONCE;
+    }
+    else if (timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) {
+        /* soon wheel */
+        p->timeout_pos = timeout_pos;
+        slot = soon_slot(timeout_pos);
     }
     else {
-	int slot;
-
-	p->timeout_pos = timeout_pos;
-
-	/* calculate slot */
-        if (timeout_pos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE) {
-            /* soon wheel */
-            slot = soon_slot(timeout_pos);
-        }
-        else {
-            /* later wheel */
-            slot = later_slot(timeout_pos);
-
-            /*
-             * Next timeout due to this timeout
-             * should be in good time before the
-             * actual timeout (one later wheel slot
-             * size). This, in order to move it
-             * from the later wheel to the soon
-             * wheel.
-             */
-            timeout_pos &= ERTS_TW_LATER_WHEEL_POS_MASK;
-            timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
-        }
+        /* later wheel */
+        p->timeout_pos = timeout_pos;
+        slot = later_slot(timeout_pos);
 
-	insert_timer_into_slot(tiw, slot, p);
-        timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
+        /*
+         * Next timeout due to this timeout
+         * should be in good time before the
+         * actual timeout (one later wheel slot
+         * size). This, in order to move it
+         * from the later wheel to the soon
+         * wheel.
+         */
+        timeout_pos &= ERTS_TW_LATER_WHEEL_POS_MASK;
+        timeout_pos -= ERTS_TW_LATER_WHEEL_SLOT_SIZE;
     }
 
+    insert_timer_into_slot(tiw, slot, p);
+    timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
+
     if (timeout_time < tiw->next_timeout_time) {
 	tiw->true_next_timeout_time = 1;
 	tiw->next_timeout_time = timeout_time;
@@ -1208,7 +1226,7 @@ erts_twheel_set_timer(ErtsTimerWheel *tiw,
 void
 erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
 {
-    if (p->slot != ERTS_TWHEEL_SLOT_INACTIVE) {
+    if (p->slot != ERTS_TW_SLOT_INACTIVE) {
         ERTS_MSACC_PUSH_AND_SET_STATE_M_X(ERTS_MSACC_STATE_TIMERS);
 	remove_timer(tiw, p);
         tiw->nto--;
@@ -1234,14 +1252,7 @@ erts_twheel_debug_foreach(ErtsTimerWheel *tiw,
 	tmr = tmr->next;
     }
 
-    for (tmr = tiw->at_once.head; tmr; tmr = tmr->next) {
-	if (tmr->timeout == tclbk)
-	    (*func)(arg, tmr->timeout_pos, tmr->arg);
-    }
-
-    for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
-         ix < ERTS_TW_LATER_WHEEL_END_SLOT;
-         ix++) {
+    for (ix = ERTS_TW_SLOT_AT_ONCE; ix < ERTS_TW_LATER_WHEEL_END_SLOT; ix++) {
 	tmr = tiw->w[ix];
 	if (tmr) {
 	    do {
@@ -1262,12 +1273,27 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos)
         scnt_slot, scnt_slots, scnt_six;
     ErtsMonotonicTime min_tpos;
     Sint scnt[ERTS_TW_SCNT_SIZE];
+    ErtsTWheelTimer *p;
 
     for (six = 0; six < ERTS_TW_SCNT_SIZE; six++)
         scnt[six] = 0;
 
     min_tpos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time);
 
+    at_once_tmo = 0;
+    p = tiw->w[ERTS_TW_SLOT_AT_ONCE];
+    if (p) {
+        ErtsTWheelTimer *first = p;
+        do {
+            at_once_tmo++;
+            ERTS_TW_ASSERT(p->slot == ERTS_TW_SLOT_AT_ONCE);
+            ERTS_TW_ASSERT(p->timeout_pos <= tiw->pos);
+            ERTS_TW_ASSERT(!check_min_tpos || tiw->pos >= min_tpos);
+            ERTS_TW_ASSERT(p->next->prev == p);
+            p = p->next;
+        } while (p != first);
+    }
+
     soon_tmo = 0;
     scnt_slot = ERTS_TW_SOON_WHEEL_END_SLOT-1;
     scnt_slots = ERTS_TW_SOON_WHEEL_SIZE;
@@ -1277,8 +1303,6 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos)
     for (ix = ERTS_TW_SOON_WHEEL_FIRST_SLOT;
          ix < ERTS_TW_SOON_WHEEL_END_SLOT;
          ix++) {
-        ErtsTWheelTimer *p;
-
         p = tiw->w[ix];
         six = scnt_get_ix(ix);
         ERTS_TW_ASSERT(!p || six == scnt_six);
@@ -1310,8 +1334,6 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos)
     for (ix = ERTS_TW_LATER_WHEEL_FIRST_SLOT;
          ix < ERTS_TW_LATER_WHEEL_END_SLOT;
          ix++) {
-        ErtsTWheelTimer *p;
-
         p = tiw->w[ix];
         six = scnt_get_ix(ix);
         ERTS_TW_ASSERT(!p || six == scnt_six);
@@ -1336,35 +1358,30 @@ hrd_dbg_check_wheels(ErtsTimerWheel *tiw, int check_min_tpos)
                                 NULL, &scnt_six, tiw->scnt);
     }
 
-    if (tiw->yield_slot >= 0) {
-        ErtsTWheelTimer *p = tiw->sentinel.next;
+    if (tiw->yield_slot != ERTS_TW_SLOT_INACTIVE) {
+        p = tiw->sentinel.next;
         ix = tiw->yield_slot;
         while (p != &tiw->sentinel) {
             ErtsMonotonicTime tpos = p->timeout_pos;
-            scnt_inc(scnt, ix);
-            if (ix >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) {
-                later_tmo++;
-                ERTS_TW_ASSERT(ix == later_slot(tpos));
-            }
+            ERTS_TW_ASSERT(ix == p->slot);
+            if (ix == ERTS_TW_SLOT_AT_ONCE)
+                at_once_tmo++;
             else {
-                soon_tmo++;
-                ERTS_TW_ASSERT(ix == (tpos & ERTS_TW_SOON_WHEEL_MASK));
-                ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
+                scnt_inc(scnt, ix);
+                if (ix >= ERTS_TW_LATER_WHEEL_FIRST_SLOT) {
+                    later_tmo++;
+                    ERTS_TW_ASSERT(ix == later_slot(tpos));
+                }
+                else {
+                    soon_tmo++;
+                    ERTS_TW_ASSERT(ix == (tpos & ERTS_TW_SOON_WHEEL_MASK));
+                    ERTS_TW_ASSERT(tpos < tiw->pos + ERTS_TW_SOON_WHEEL_SIZE);
+                }
+                p = p->next;
             }
-            p = p->next;
         }
     }
 
-    at_once_tmo = 0;
-    if (tiw->at_once.head) {
-        ErtsTWheelTimer *p = tiw->at_once.head;
-        while (p) {
-            ErtsMonotonicTime tpos = p->timeout_pos;
-            at_once_tmo++;
-            ERTS_TW_ASSERT(tpos <= tiw->pos);
-            p = p->next;
-        }
-    }
 
     ERTS_TW_ASSERT(tiw->at_once.nto == at_once_tmo);
     ERTS_TW_ASSERT(tiw->soon.nto == soon_tmo);