Multiple timer wheels

8 files changed, 424 insertions, 245 deletions

diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c
index 8b444f2b01..0bd8d20c34 100644
--- a/erts/emulator/beam/erl_bif_timer.c
+++ b/erts/emulator/beam/erl_bif_timer.c
@@ -481,7 +481,7 @@ setup_bif_timer(Uint32 xflags,
 
     tab_insert(btm);
     ASSERT(btm == tab_find(ref));
-    btm->tm.active = 0; /* MUST be initalized */
+    erts_init_timer(&btm->tm);
     erts_set_timer(&btm->tm,
 		  (ErlTimeoutProc) bif_timer_timeout,
 		  (ErlCancelProc) bif_timer_cleanup,
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index be2c5ced9e..6708324a29 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -340,6 +340,7 @@ erl_init(int ncpu,
 			 no_dirty_io_schedulers
 #endif
 			 );
+    erts_late_init_time_sup();
     erts_init_cpu_topology(); /* Must be after init_scheduling */
     erts_init_gc(); /* Must be after init_scheduling */
     erts_alloc_late_init();
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 6e562e16c8..45b6fc5fb3 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2187,7 +2187,7 @@ aux_work_timeout_late_init(void)
 {
     aux_work_tmo->initialized = 1;
     if (erts_atomic32_read_nob(&aux_work_tmo->refc)) {
-	aux_work_tmo->timer.data.active = 0;
+	erts_init_timer(&aux_work_tmo->timer.data);
 	erts_set_timer(&aux_work_tmo->timer.data,
 		       aux_work_timeout,
 		       NULL,
@@ -2220,7 +2220,6 @@ aux_work_timeout(void *unused)
     if (refc != 1
 	|| 1 != erts_atomic32_cmpxchg_relb(&aux_work_tmo->refc, 0, 1)) {
 	/* Setup next timeout... */
-	aux_work_tmo->timer.data.active = 0;
 	erts_set_timer(&aux_work_tmo->timer.data,
 		       aux_work_timeout,
 		       NULL,
@@ -2239,7 +2238,7 @@ setup_aux_work_timer(void)
     else
 #endif
     {
-	aux_work_tmo->timer.data.active = 0;
+	erts_init_timer(&aux_work_tmo->timer.data);
 	erts_set_timer(&aux_work_tmo->timer.data,
 		       aux_work_timeout,
 		       NULL,
@@ -2641,6 +2640,13 @@ thr_prgr_fin_wait(void *vssi)
 
 static void init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp);
 
+void
+erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time)
+{
+    /* TODO only poke when needed (based on timeout_time) */
+    erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(-1));
+}
+
 static void *
 aux_thread(void *unused)
 {
@@ -2649,6 +2655,11 @@ aux_thread(void *unused)
     erts_aint32_t aux_work;
     ErtsThrPrgrCallbacks callbacks;
     int thr_prgr_active = 1;
+    ErtsTimerWheel *timer_wheel = erts_default_timer_wheel;
+    ErtsNextTimeoutRef nxt_tmo_ref = erts_get_next_timeout_reference(timer_wheel);
+
+    if (!timer_wheel)
+	ERTS_INTERNAL_ERROR("Missing aux timer wheel");
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
@@ -2672,6 +2683,7 @@ aux_thread(void *unused)
     sched_prep_spin_wait(ssi);
 
     while (1) {
+	ErtsMonotonicTime current_time;
 	erts_aint32_t flgs;
 
 	aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
@@ -2683,28 +2695,56 @@ aux_thread(void *unused)
 		erts_thr_progress_leader_update(NULL);
 	}
 
-	if (!aux_work) {
-	    if (thr_prgr_active)
-		erts_thr_progress_active(NULL, thr_prgr_active = 0);
-	    erts_thr_progress_prepare_wait(NULL);
+	if (aux_work) {
+	    current_time = erts_get_monotonic_time();
+	    if (current_time >= erts_next_timeout_time(nxt_tmo_ref)) {
+		if (!thr_prgr_active)
+		    erts_thr_progress_active(NULL, thr_prgr_active = 1);
+		erts_bump_timers(timer_wheel, current_time);
+	    }
+	}
+	else {
+	    ErtsMonotonicTime timeout_time;
+	    timeout_time = erts_check_next_timeout_time(timer_wheel,
+							ERTS_SEC_TO_MONOTONIC(10*60));
+	    current_time = erts_get_monotonic_time();
+	    if (current_time >= timeout_time) {
+		if (!thr_prgr_active)
+		    erts_thr_progress_active(NULL, thr_prgr_active = 1);
+	    }
+	    else {
+		if (thr_prgr_active)
+		    erts_thr_progress_active(NULL, thr_prgr_active = 0);
+		erts_thr_progress_prepare_wait(NULL);
 
-	    ERTS_SCHED_FAIR_YIELD();
+		ERTS_SCHED_FAIR_YIELD();
 
-	    flgs = sched_spin_wait(ssi, 0);
+		flgs = sched_spin_wait(ssi, 0);
 
-	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
-		ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-		flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
 		if (flgs & ERTS_SSI_FLG_SLEEPING) {
-		    int res;
-		    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
 		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-		    do {
-			res = erts_tse_wait(ssi->event);
-		    } while (res == EINTR);
+		    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
+		    if (flgs & ERTS_SSI_FLG_SLEEPING) {
+			int res;
+			ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
+			ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+			current_time = erts_get_monotonic_time();
+			do {
+			    Sint64 timeout;
+			    if (current_time >= timeout_time)
+				break;
+			    timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
+							     - current_time
+							     - 1) + 1;
+			    res = erts_tse_twait(ssi->event, timeout);
+			    current_time = erts_get_monotonic_time();
+			} while (res == EINTR);
+		    }
 		}
+		erts_thr_progress_finalize_wait(NULL);
 	    }
-	    erts_thr_progress_finalize_wait(NULL);
+	    if (current_time >= timeout_time)
+		erts_bump_timers(timer_wheel, current_time);
 	}
 
 	flgs = sched_prep_spin_wait(ssi);
@@ -2771,6 +2811,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    sched_wall_time_change(esdp, thr_prgr_active);
 
 	while (1) {
+	    ErtsMonotonicTime current_time;
 
 	    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 	    if (aux_work) {
@@ -2784,34 +2825,65 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 		    erts_thr_progress_leader_update(esdp);
 	    }
 
-	    if (aux_work)
+	    if (aux_work) {
 		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
+		current_time = erts_get_monotonic_time();
+		if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
+			erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			sched_wall_time_change(esdp, 1);
+		    }
+		    erts_bump_timers(esdp->timer_wheel, current_time);
+		}
+	    }
 	    else {
-		if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
-		    if (thr_prgr_active) {
-			erts_thr_progress_active(esdp, thr_prgr_active = 0);
-			sched_wall_time_change(esdp, 0);
+		ErtsMonotonicTime timeout_time;
+		timeout_time = erts_check_next_timeout_time(esdp->timer_wheel,
+							    ERTS_SEC_TO_MONOTONIC(10*60));
+		current_time = erts_get_monotonic_time();
+		if (current_time >= timeout_time) {
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
+			erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			sched_wall_time_change(esdp, 1);
 		    }
-		    erts_thr_progress_prepare_wait(esdp);
 		}
+		else {
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+			if (thr_prgr_active) {
+			    erts_thr_progress_active(esdp, thr_prgr_active = 0);
+			    sched_wall_time_change(esdp, 0);
+			}
+			erts_thr_progress_prepare_wait(esdp);
+		    }
 
-		ERTS_SCHED_FAIR_YIELD();
+		    ERTS_SCHED_FAIR_YIELD();
 
-		flgs = sched_spin_wait(ssi, spincount);
-		if (flgs & ERTS_SSI_FLG_SLEEPING) {
-		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-		    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
+		    flgs = sched_spin_wait(ssi, spincount);
 		    if (flgs & ERTS_SSI_FLG_SLEEPING) {
-			int res;
-			ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
 			ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-			do {
-			    res = erts_tse_twait(ssi->event, -1);
-			} while (res == EINTR);
+			flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
+			if (flgs & ERTS_SSI_FLG_SLEEPING) {
+			    int res;
+			    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
+			    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+			    current_time = erts_get_monotonic_time();
+			    do {
+				Sint64 timeout;
+				if (current_time >= timeout_time)
+				    break;
+				timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
+								 - current_time
+								 - 1) + 1;
+				res = erts_tse_twait(ssi->event, timeout);
+				current_time = erts_get_monotonic_time();
+			    } while (res == EINTR);
+			}
 		    }
+		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+			erts_thr_progress_finalize_wait(esdp);
 		}
-		if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
-		    erts_thr_progress_finalize_wait(esdp);
+		if (current_time >= timeout_time)
+		    erts_bump_timers(esdp->timer_wheel, current_time);
 	    }
 
 	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
@@ -2879,8 +2951,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    erl_sys_schedule(1); /* Might give us something to do */
 
 	    current_time = erts_get_monotonic_time();
-	    if (current_time >= erts_next_timeout_time())
-		erts_bump_timers(current_time);
+	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
+		erts_bump_timers(esdp->timer_wheel, current_time);
 
 	sys_aux_work:
 #ifndef ERTS_SMP
@@ -2997,8 +3069,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	{
 	    ErtsMonotonicTime current_time = erts_get_monotonic_time();
-	    if (current_time >= erts_next_timeout_time())
-		erts_bump_timers(current_time);
+	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
+		erts_bump_timers(esdp->timer_wheel, current_time);
 	}
 
 #ifndef ERTS_SMP
@@ -5269,6 +5341,10 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 #else
     esdp->no = (Uint) num;
 #endif
+
+    esdp->timer_wheel = erts_default_timer_wheel;
+    esdp->next_tmo_ref = erts_get_next_timeout_reference(esdp->timer_wheel);
+
     esdp->ssi = ssi;
     esdp->current_process = NULL;
     esdp->current_port = NULL;
@@ -6765,6 +6841,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 	    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 
 	    while (1) {
+		ErtsMonotonicTime current_time;
 		erts_aint32_t qmask;
 		erts_aint32_t flgs;
 
@@ -6789,30 +6866,64 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 		    }
 		}
 
-		if (!aux_work) {
-		    if (thr_prgr_active) {
-			erts_thr_progress_active(esdp, thr_prgr_active = 0);
-			sched_wall_time_change(esdp, 0);
+		if (aux_work) {
+		    current_time = erts_get_monotonic_time();
+		    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
+			if (!thr_prgr_active) {
+			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			    sched_wall_time_change(esdp, 1);
+			}
+			erts_bump_timers(esdp->timer_wheel, current_time);
 		    }
-		    erts_thr_progress_prepare_wait(esdp);
-		    flgs = sched_spin_suspended(ssi,
-						ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT);
-		    if (flgs == (ERTS_SSI_FLG_SLEEPING
-				 | ERTS_SSI_FLG_WAITING
-				 | ERTS_SSI_FLG_SUSPENDED)) {
-			flgs = sched_set_suspended_sleeptype(ssi);
+		}
+		else {
+		    ErtsMonotonicTime timeout_time;
+		    timeout_time = erts_check_next_timeout_time(esdp->timer_wheel,
+								ERTS_SEC_TO_MONOTONIC(60*60));
+		    current_time = erts_get_monotonic_time();
+
+		    if (current_time >= timeout_time) {
+			if (!thr_prgr_active) {
+			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
+			    sched_wall_time_change(esdp, 1);
+			}
+		    }
+		    else {		    
+			if (thr_prgr_active) {
+			    erts_thr_progress_active(esdp, thr_prgr_active = 0);
+			    sched_wall_time_change(esdp, 0);
+			}
+			erts_thr_progress_prepare_wait(esdp);
+			flgs = sched_spin_suspended(ssi,
+						    ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT);
 			if (flgs == (ERTS_SSI_FLG_SLEEPING
-				     | ERTS_SSI_FLG_TSE_SLEEPING
 				     | ERTS_SSI_FLG_WAITING
 				     | ERTS_SSI_FLG_SUSPENDED)) {
-			    int res;
-
-			    do {
-				res = erts_tse_twait(ssi->event, -1);
-			    } while (res == EINTR);
+			    flgs = sched_set_suspended_sleeptype(ssi);
+			    if (flgs == (ERTS_SSI_FLG_SLEEPING
+					 | ERTS_SSI_FLG_TSE_SLEEPING
+					 | ERTS_SSI_FLG_WAITING
+					 | ERTS_SSI_FLG_SUSPENDED)) {
+				int res;
+
+				current_time = erts_get_monotonic_time();
+				do {
+				    Sint64 timeout;
+				    if (current_time >= timeout_time)
+					break;
+				    timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
+								     - current_time
+								     - 1) + 1;
+				    res = erts_tse_twait(ssi->event, timeout);
+				    current_time = erts_get_monotonic_time();
+				} while (res == EINTR);
+			    }
 			}
+			erts_thr_progress_finalize_wait(esdp);
 		    }
-		    erts_thr_progress_finalize_wait(esdp);
+
+		    if (current_time >= timeout_time)
+			erts_bump_timers(esdp->timer_wheel, current_time);
 		}
 
 		flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING
@@ -7631,6 +7742,9 @@ sched_thread_func(void *vesdp)
     ErtsThrPrgrCallbacks callbacks;
     ErtsSchedulerData *esdp = vesdp;
     Uint no = esdp->no;
+
+    esdp->timer_wheel = erts_create_timer_wheel((int) no);
+    esdp->next_tmo_ref = erts_get_next_timeout_reference(esdp->timer_wheel);
 #ifdef ERTS_SMP
     ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch();
     callbacks.arg = (void *) esdp->ssi;
@@ -9048,9 +9162,9 @@ Process *schedule(Process *p, int calls)
 
 	{
 	    ErtsMonotonicTime current_time = erts_get_monotonic_time();
-	    if (current_time >= erts_next_timeout_time()) {
+	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
 		erts_smp_runq_unlock(rq);
-		erts_bump_timers(current_time);
+		erts_bump_timers(esdp->timer_wheel, current_time);
 		erts_smp_runq_lock(rq);
 	    }
 	}
@@ -9213,8 +9327,8 @@ Process *schedule(Process *p, int calls)
 	    erl_sys_schedule(1);
 
 	    current_time = erts_get_monotonic_time();
-	    if (current_time >= erts_next_timeout_time())
-		erts_bump_timers(current_time);
+	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
+		erts_bump_timers(esdp->timer_wheel, current_time);
 
 #ifdef ERTS_SMP
 	    erts_smp_runq_lock(rq);
@@ -10652,7 +10766,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 #ifdef ERTS_SMP
     p->common.u.alive.ptimer = NULL;
 #else
-    sys_memset(&p->common.u.alive.tm, 0, sizeof(ErlTimer));
+    erts_init_timer(&p->common.u.alive.tm);
 #endif
 
     p->common.u.alive.reg = NULL;
@@ -10845,7 +10959,7 @@ void erts_init_empty_process(Process *p)
 #ifdef ERTS_SMP
     p->common.u.alive.ptimer = NULL;
 #else
-    memset(&(p->common.u.alive.tm), 0, sizeof(ErlTimer));
+    erts_init_timer(&p->common.u.alive.tm);
 #endif
     p->next = NULL;
     p->off_heap.first = NULL;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 53a992e115..77a4b45b09 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -562,6 +562,8 @@ struct ErtsSchedulerData_ {
     Eterm* x_reg_array;		/* X registers */
     FloatDef* f_reg_array;	/* Floating point registers. */
 
+    ErtsTimerWheel *timer_wheel;
+    ErtsNextTimeoutRef next_tmo_ref;
 #ifdef ERTS_SMP
     ethr_tid tid;		/* Thread id */
     struct erl_bits_state erl_bits_state; /* erl_bits.c state */
@@ -2237,6 +2239,8 @@ extern int erts_disable_proc_not_running_opt;
 
 void erts_smp_notify_inc_runq(ErtsRunQueue *runq);
 
+void erts_interupt_aux_thread_timed(ErtsMonotonicTime timeout_time);
+
 #ifdef ERTS_SMP
 void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t);
 ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi);
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index e461594e9c..7933f4973d 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -26,6 +26,10 @@
 #define ERTS_TIME_ASSERT(B) ((void) 1)
 #endif
 
+typedef struct ErtsTimerWheel_ ErtsTimerWheel;
+typedef erts_atomic64_t * ErtsNextTimeoutRef;
+extern ErtsTimerWheel *erts_default_timer_wheel;
+
 extern SysTimeval erts_first_emu_time;
 
 /*
@@ -35,8 +39,8 @@ typedef struct erl_timer {
     struct erl_timer* next;	/* next entry tiw slot or chain */
     struct erl_timer* prev;	/* prev entry tiw slot or chain */
     Uint slot;			/* slot in timer wheel */
+    erts_smp_atomic_t wheel;
     ErtsMonotonicTime timeout_pos; /* Timeout in absolute clock ticks */
-    int    active;		/* 1=activated, 0=deactivated */
     /* called when timeout */
     void (*timeout)(void*);
     /* called when cancel (may be NULL) */
@@ -63,7 +67,6 @@ union ErtsSmpPTimer_ {
     ErtsSmpPTimer *next;
 };
 
-
 void erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref,
 			    Eterm id,
 			    ErlTimeoutProc timeout_func,
@@ -79,28 +82,35 @@ void erts_late_init_time_sup(void);
 
 /* timer-wheel api */
 
+ErtsTimerWheel *erts_create_timer_wheel(int);
+ErtsNextTimeoutRef erts_get_next_timeout_reference(ErtsTimerWheel *);
 void erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode);
 void erts_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint);
 void erts_cancel_timer(ErlTimer*);
-void erts_bump_timers(ErtsMonotonicTime);
-Uint erts_timer_wheel_memory_size(void);
 Uint erts_time_left(ErlTimer *);
+void erts_bump_timers(ErtsTimerWheel *, ErtsMonotonicTime);
+Uint erts_timer_wheel_memory_size(void);
 
 #ifdef DEBUG
 void erts_p_slpq(void);
 #endif
 
-ErtsMonotonicTime erts_check_next_timeout_time(ErtsMonotonicTime);
+ErtsMonotonicTime erts_check_next_timeout_time(ErtsTimerWheel *,
+					       ErtsMonotonicTime);
 
-extern erts_atomic64_t erts_next_timeout__;
-
-ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(void);
+ERTS_GLB_INLINE void erts_init_timer(ErlTimer *p);
+ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(void)
+ERTS_GLB_INLINE void erts_init_timer(ErlTimer *p)
+{
+    erts_smp_atomic_init_nob(&p->wheel, (erts_aint_t) NULL);
+}
+
+ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_tmo_ref)
 {
-    return (ErtsMonotonicTime) erts_atomic64_read_acqb(&erts_next_timeout__);
+    return (ErtsMonotonicTime) erts_atomic64_read_acqb((erts_atomic64_t *) nxt_tmo_ref);
 }
 
 #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c
index 7dfa7d8743..d47d1682d7 100644
--- a/erts/emulator/beam/erl_time_sup.c
+++ b/erts/emulator/beam/erl_time_sup.c
@@ -690,6 +690,7 @@ static void
 late_init_time_correction(void)
 {
     if (time_sup.inf.c.finalized_offset) {
+	erts_init_timer(&time_sup.inf.c.parmon.timer);
 	erts_set_timer(&time_sup.inf.c.parmon.timer,
 #ifndef ERTS_HAVE_CORRECTED_OS_MONOTONIC
 		       init_check_time_correction,
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index c9f8b68bca..9f997e1d0b 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -86,9 +86,6 @@
 #define ERTS_MONOTONIC_DAY ERTS_SEC_TO_MONOTONIC(60*60*24)
 #define ERTS_CLKTCKS_DAY ERTS_MONOTONIC_TO_CLKTCKS(ERTS_MONOTONIC_DAY)
 
-static erts_smp_atomic32_t is_bumping;
-static erts_smp_mtx_t tiw_lock;
-
 
 /* BEGIN tiw_lock protected variables 
 **
@@ -101,14 +98,6 @@ static erts_smp_mtx_t tiw_lock;
 #else
 #define TIW_SIZE (1 << 20)
 #endif
-static ErlTimer** tiw;		/* the timing wheel, allocated in init_time() */
-static ErtsMonotonicTime tiw_pos; /* current position in wheel */
-static Uint tiw_nto;		/* number of timeouts in wheel */
-static struct {
-    ErlTimer *head;
-    ErlTimer **tail;
-    Uint nto;
-} tiw_at_once;
 
 /* Actual interval time chosen by sys_init_time() */
 
@@ -120,23 +109,52 @@ static int tiw_itime; /* Constant after init */
 #  define TIW_ITIME tiw_itime
 #endif
 
-static int true_next_timeout_time;
-static ErtsMonotonicTime next_timeout_time;
-erts_atomic64_t erts_next_timeout__;
+struct ErtsTimerWheel_ {
+    ErlTimer *w[TIW_SIZE];
+    ErtsMonotonicTime pos;
+    Uint nto;
+    struct {
+	ErlTimer *head;
+	ErlTimer **tail;
+	Uint nto;
+    } at_once;
+    int true_next_timeout_time;
+    ErtsMonotonicTime next_timeout_time;
+    erts_atomic64_t next_timeout;
+    erts_smp_atomic32_t is_bumping;
+    erts_smp_mtx_t lock;
+};
+
+ErtsTimerWheel *erts_default_timer_wheel; /* managed by aux thread */
+
+static ERTS_INLINE ErtsTimerWheel *
+get_timer_wheel(ErlTimer *p)
+{
+    return (ErtsTimerWheel *) erts_smp_atomic_read_acqb(&p->wheel);
+}
+
+static ERTS_INLINE void
+set_timer_wheel(ErlTimer *p, ErtsTimerWheel *tiw)
+{
+    erts_smp_atomic_set_relb(&p->wheel, (erts_aint_t) tiw);
+}
 
 static ERTS_INLINE void
-init_next_timeout(ErtsMonotonicTime time)
+init_next_timeout(ErtsTimerWheel *tiw,
+		  ErtsMonotonicTime time)
 {
-    erts_atomic64_init_nob(&erts_next_timeout__,
+    erts_atomic64_init_nob(&tiw->next_timeout,
 			   (erts_aint64_t) time);
 }
 
 static ERTS_INLINE void
-set_next_timeout(ErtsMonotonicTime time, int true_timeout)
+set_next_timeout(ErtsTimerWheel *tiw,
+		 ErtsMonotonicTime time,
+		 int true_timeout)
 {
-    true_next_timeout_time = true_timeout;
-    next_timeout_time = time;
-    erts_atomic64_set_relb(&erts_next_timeout__,
+    tiw->true_next_timeout_time = true_timeout;
+    tiw->next_timeout_time = time;
+    erts_atomic64_set_relb(&tiw->next_timeout,
 			   (erts_aint64_t) time);
 }
 
@@ -144,7 +162,8 @@ set_next_timeout(ErtsMonotonicTime time, int true_timeout)
    or -1 if there are no timeouts                     */
 
 static ERTS_INLINE ErtsMonotonicTime
-find_next_timeout(ErtsMonotonicTime curr_time,
+find_next_timeout(ErtsTimerWheel *tiw,
+		  ErtsMonotonicTime curr_time,
 		  ErtsMonotonicTime max_search_time)
 {
     int start_ix, tiw_pos_ix;
@@ -152,16 +171,16 @@ find_next_timeout(ErtsMonotonicTime curr_time,
     int true_min_timeout;
     ErtsMonotonicTime min_timeout, min_timeout_pos, slot_timeout_pos, timeout_limit;
 
-    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&tiw_lock));
+    ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&tiw->lock));
 
-    if (true_next_timeout_time)
-	return next_timeout_time;
+    if (tiw->true_next_timeout_time)
+	return tiw->next_timeout_time;
 
     /* We never set next timeout beyond timeout_limit */
     timeout_limit = curr_time + ERTS_MONOTONIC_DAY;
 
-    if (tiw_nto == 0) { /* no timeouts in wheel */
-	true_min_timeout = true_next_timeout_time = 0;
+    if (tiw->nto == 0) { /* no timeouts in wheel */
+	true_min_timeout = tiw->true_next_timeout_time = 0;
 	min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(timeout_limit);
 	goto found_next;
     }
@@ -170,13 +189,13 @@ find_next_timeout(ErtsMonotonicTime curr_time,
      * Don't want others entering trying to bump
      * timers while we are checking...
      */
-    set_next_timeout(timeout_limit, 0);
+    set_next_timeout(tiw, timeout_limit, 0);
 
     true_min_timeout = 1;
-    slot_timeout_pos = tiw_pos;
+    slot_timeout_pos = tiw->pos;
     min_timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time + max_search_time);
 
-    start_ix = tiw_pos_ix = (int) (tiw_pos & (TIW_SIZE-1));
+    start_ix = tiw_pos_ix = (int) (tiw->pos & (TIW_SIZE-1));
 
     do {
 	slot_timeout_pos++;
@@ -185,7 +204,7 @@ find_next_timeout(ErtsMonotonicTime curr_time,
 	    break;
 	}
 
-	p = tiw[tiw_pos_ix];
+	p = tiw->w[tiw_pos_ix];
 
 	while (p) {
 	    ErtsMonotonicTime timeout_pos;
@@ -207,16 +226,18 @@ find_next_timeout(ErtsMonotonicTime curr_time,
 found_next:
 
     min_timeout = ERTS_CLKTCKS_TO_MONOTONIC(min_timeout_pos);
-    if (min_timeout != next_timeout_time)
-	set_next_timeout(min_timeout, true_min_timeout);
+    if (min_timeout != tiw->next_timeout_time)
+	set_next_timeout(tiw, min_timeout, true_min_timeout);
 
     return min_timeout;
 }
 
-static void remove_timer(ErlTimer *p) {
+static void
+remove_timer(ErtsTimerWheel *tiw, ErlTimer *p)
+{
     /* first */
     if (!p->prev) {
-	tiw[p->slot] = p->next;
+	tiw->w[p->slot] = p->next;
 	if(p->next)
 	    p->next->prev = NULL;
     } else {
@@ -233,40 +254,41 @@ static void remove_timer(ErlTimer *p) {
 
     p->next = NULL;
     p->prev = NULL;
-    /* Make sure cancel callback isn't called */
-    p->active = 0;
-    tiw_nto--;
+
+    set_timer_wheel(p, NULL);
+    tiw->nto--;
 }
 
 ErtsMonotonicTime
-erts_check_next_timeout_time(ErtsMonotonicTime max_search_time)
+erts_check_next_timeout_time(ErtsTimerWheel *tiw,
+			     ErtsMonotonicTime max_search_time)
 {
     ErtsMonotonicTime next, curr;
 
     curr = erts_get_monotonic_time();
 
-    erts_smp_mtx_lock(&tiw_lock);
+    erts_smp_mtx_lock(&tiw->lock);
 
-    next = find_next_timeout(curr, max_search_time);
+    next = find_next_timeout(tiw, curr, max_search_time);
 
-    erts_smp_mtx_unlock(&tiw_lock);
+    erts_smp_mtx_unlock(&tiw->lock);
 
     return next;
 }
 
 #ifndef DEBUG
-#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TO) ((void) 0)
+#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) ((void) 0)
 #else
-#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TO) debug_check_safe_to_skip_to((TO))
+#define ERTS_DBG_CHK_SAFE_TO_SKIP_TO(TIW, TO) debug_check_safe_to_skip_to((TIW), (TO))
 static void
-debug_check_safe_to_skip_to(ErtsMonotonicTime skip_to_pos)
+debug_check_safe_to_skip_to(ErtsTimerWheel *tiw, ErtsMonotonicTime skip_to_pos)
 {
     int slots, ix;
     ErlTimer *tmr;
     ErtsMonotonicTime tmp;
 
-    ix = (int) (tiw_pos & (TIW_SIZE-1));
-    tmp = skip_to_pos - tiw_pos;
+    ix = (int) (tiw->pos & (TIW_SIZE-1));
+    tmp = skip_to_pos - tiw->pos;
     ASSERT(tmp >= 0);
     if (tmp < (ErtsMonotonicTime) TIW_SIZE)
 	slots = (int) tmp;
@@ -274,7 +296,7 @@ debug_check_safe_to_skip_to(ErtsMonotonicTime skip_to_pos)
 	slots = TIW_SIZE;
 
      while (slots > 0) {
-	tmr = tiw[ix];
+	tmr = tiw->w[ix];
 	while (tmr) {
 	    ASSERT(tmr->timeout_pos > skip_to_pos);
 	    tmr = tmr->next;
@@ -287,79 +309,80 @@ debug_check_safe_to_skip_to(ErtsMonotonicTime skip_to_pos)
 }
 #endif
 
-void erts_bump_timers(ErtsMonotonicTime curr_time)
+void
+erts_bump_timers(ErtsTimerWheel *tiw, ErtsMonotonicTime curr_time)
 {
     int tiw_pos_ix, slots;
     ErlTimer *p, *timeout_head, **timeout_tail;
     ErtsMonotonicTime bump_to, tmp_slots;
 
-    if (erts_smp_atomic32_cmpxchg_nob(&is_bumping, 1, 0) != 0)
+    if (erts_smp_atomic32_cmpxchg_nob(&tiw->is_bumping, 1, 0) != 0)
 	return; /* Another thread is currently bumping... */
 
     bump_to = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
 
-    erts_smp_mtx_lock(&tiw_lock);
+    erts_smp_mtx_lock(&tiw->lock);
 
-    if (tiw_pos >= bump_to) {
+    if (tiw->pos >= bump_to) {
 	timeout_head = NULL;
 	goto done;
     }
 
     /* Don't want others here while we are bumping... */
-    set_next_timeout(curr_time + ERTS_MONOTONIC_DAY, 0);
+    set_next_timeout(tiw, curr_time + ERTS_MONOTONIC_DAY, 0);
 
-    if (!tiw_at_once.head) {
+    if (!tiw->at_once.head) {
 	timeout_head = NULL;
 	timeout_tail = &timeout_head;
     }
     else {
-	ASSERT(tiw_nto >= tiw_at_once.nto);
-	timeout_head = tiw_at_once.head;
-	timeout_tail = tiw_at_once.tail;
-	tiw_nto -= tiw_at_once.nto;
-	tiw_at_once.head = NULL;
-	tiw_at_once.tail = &tiw_at_once.head;
-	tiw_at_once.nto = 0;
+	ASSERT(tiw->nto >= tiw->at_once.nto);
+	timeout_head = tiw->at_once.head;
+	timeout_tail = tiw->at_once.tail;
+	tiw->nto -= tiw->at_once.nto;
+	tiw->at_once.head = NULL;
+	tiw->at_once.tail = &tiw->at_once.head;
+	tiw->at_once.nto = 0;
     }
 
-    if (tiw_nto == 0) {
-	ERTS_DBG_CHK_SAFE_TO_SKIP_TO(bump_to);
-	tiw_pos = bump_to;
+    if (tiw->nto == 0) {
+	ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, bump_to);
+	tiw->pos = bump_to;
 	goto done;
     }
 
-    if (true_next_timeout_time) {
+    if (tiw->true_next_timeout_time) {
 	ErtsMonotonicTime skip_until_pos;
 	/*
 	 * No need inspecting slots where we know no timeouts
 	 * to trigger should reside.
 	 */
 
-	skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(next_timeout_time);
+	skip_until_pos = ERTS_MONOTONIC_TO_CLKTCKS(tiw->next_timeout_time);
 	if (skip_until_pos > bump_to)
 	    skip_until_pos = bump_to;
 
-	ERTS_DBG_CHK_SAFE_TO_SKIP_TO(skip_until_pos);
-	ASSERT(skip_until_pos > tiw_pos);
+	ERTS_DBG_CHK_SAFE_TO_SKIP_TO(tiw, skip_until_pos);
+	ASSERT(skip_until_pos > tiw->pos);
 
-	tiw_pos = skip_until_pos - 1;
+	tiw->pos = skip_until_pos - 1;
     }
 
-    tiw_pos_ix = (int) ((tiw_pos+1) & (TIW_SIZE-1));
-    tmp_slots = (bump_to - tiw_pos);
+    tiw_pos_ix = (int) ((tiw->pos+1) & (TIW_SIZE-1));
+    tmp_slots = (bump_to - tiw->pos);
     if (tmp_slots < (ErtsMonotonicTime) TIW_SIZE)
 	slots = (int) tmp_slots;
     else
 	slots = TIW_SIZE;
  
     while (slots > 0) {
-	p = tiw[tiw_pos_ix];
+	p = tiw->w[tiw_pos_ix];
 	while (p) {
 	    ErlTimer *next = p->next;
 	    ASSERT(p != next);
 	    if (p->timeout_pos <= bump_to) { /* we have a timeout */
 		/* Remove from list */
-		remove_timer(p);
+		remove_timer(tiw, p);
 		*timeout_tail = p;	/* Insert in timeout queue */
 		timeout_tail = &p->next;
 	    }
@@ -374,19 +397,21 @@ void erts_bump_timers(ErtsMonotonicTime curr_time)
     ASSERT(tmp_slots >= (ErtsMonotonicTime) TIW_SIZE
 	   || tiw_pos_ix == (int) ((bump_to+1) & (TIW_SIZE-1)));
 
-    tiw_pos = bump_to;
+    tiw->pos = bump_to;
 
     /* Search at most two seconds ahead... */
-    (void) find_next_timeout(curr_time, ERTS_SEC_TO_MONOTONIC(2));
+    (void) find_next_timeout(tiw, curr_time, ERTS_SEC_TO_MONOTONIC(2));
 
 done:
 
-    erts_smp_mtx_unlock(&tiw_lock);
+    erts_smp_mtx_unlock(&tiw->lock);
     
-    erts_smp_atomic32_set_nob(&is_bumping, 0);
+    erts_smp_atomic32_set_nob(&tiw->is_bumping, 0);
 
     /* Call timedout timers callbacks */
     while (timeout_head) {
+	ErlTimeoutProc timeout;
+	void *arg;
 	p = timeout_head;
 	timeout_head = p->next;
 	/* Here comes hairy use of the timer fields!
@@ -399,23 +424,61 @@ done:
 	p->next = NULL;
 	p->prev = NULL;
 	p->slot = 0;
-	(*p->timeout)(p->arg);
+	timeout = p->timeout;
+	arg = p->arg;
+	(*timeout)(arg);
     }
 }
 
 Uint
 erts_timer_wheel_memory_size(void)
 {
-    return (Uint) TIW_SIZE * sizeof(ErlTimer*);
+#ifdef ERTS_SMP
+    return sizeof(ErtsTimerWheel)*(1 + erts_no_schedulers);
+#else
+    return sizeof(ErtsTimerWheel);
+#endif
 }
 
+ErtsTimerWheel *
+erts_create_timer_wheel(int no)
+{
+    ErtsMonotonicTime mtime;
+    int i;
+    ErtsTimerWheel *tiw;
+    tiw = (ErtsTimerWheel *) erts_alloc(ERTS_ALC_T_TIMER_WHEEL,
+					sizeof(ErtsTimerWheel));
+    for(i = 0; i < TIW_SIZE; i++)
+	tiw->w[i] = NULL;
+
+    erts_smp_atomic32_init_nob(&tiw->is_bumping, 0);
+    erts_smp_mtx_init_x(&tiw->lock, "timer_wheel", make_small(no));
+
+    mtime = erts_get_monotonic_time();
+    tiw->pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime);
+    tiw->nto = 0;
+    tiw->at_once.head = NULL;
+    tiw->at_once.tail = &tiw->at_once.head;
+    tiw->at_once.nto = 0;
+    tiw->true_next_timeout_time = 0;
+    tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY;
+    init_next_timeout(tiw, mtime + ERTS_MONOTONIC_DAY);
+    return tiw;
+}
+
+ErtsNextTimeoutRef
+erts_get_next_timeout_reference(ErtsTimerWheel *tiw)
+{
+    return (ErtsNextTimeoutRef) &tiw->next_timeout;
+}
+
+
 /* this routine links the time cells into a free list at the start
    and sets the time queue as empty */
 void
 erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode)
 {
-    ErtsMonotonicTime mtime;
-    int i, itime;
+    int itime;
 
     /* system dependent init; must be done before do_time_init()
        if timer thread is enabled */
@@ -428,41 +491,39 @@ erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode)
     tiw_itime = itime;
 #endif
 
-    erts_smp_atomic32_init_nob(&is_bumping, 0);
-    erts_smp_mtx_init(&tiw_lock, "timer_wheel");
-
-    tiw = (ErlTimer**) erts_alloc(ERTS_ALC_T_TIMER_WHEEL,
-				  TIW_SIZE * sizeof(ErlTimer*));
-    for(i = 0; i < TIW_SIZE; i++)
-	tiw[i] = NULL;
-
-    mtime = erts_get_monotonic_time();
-    tiw_pos = ERTS_MONOTONIC_TO_CLKTCKS(mtime);
-    tiw_nto = 0;
-    tiw_at_once.head = NULL;
-    tiw_at_once.tail = &tiw_at_once.head;
-    tiw_at_once.nto = 0;
-    init_next_timeout(mtime + ERTS_MONOTONIC_DAY);
-
-    erts_late_init_time_sup();
+    erts_default_timer_wheel = erts_create_timer_wheel(0);
 }
 
+void
+erts_set_timer(ErlTimer *p, ErlTimeoutProc timeout,
+	       ErlCancelProc cancel, void *arg, Uint to)
+{
+    ErtsMonotonicTime timeout_time, timeout_pos;
+    ErtsMonotonicTime curr_time;
+    ErtsTimerWheel *tiw;
+    ErtsSchedulerData *esdp;
+
+    curr_time = erts_get_monotonic_time();
+    esdp = erts_get_scheduler_data();
+    if (esdp)
+	tiw = esdp->timer_wheel;
+    else
+	tiw = erts_default_timer_wheel;
 
+    erts_smp_mtx_lock(&tiw->lock);
 
+    if (get_timer_wheel(p))
+	ERTS_INTERNAL_ERROR("Double set timer");
 
-/*
-** Insert a process into the time queue, with a timeout 't'
-*/
-static ErtsMonotonicTime
-insert_timer(ErlTimer* p, ErtsMonotonicTime curr_time, ErtsMonotonicTime to)
-{
-    ErtsMonotonicTime timeout_time, timeout_pos;
+    p->timeout = timeout;
+    p->cancel = cancel;
+    p->arg = arg;
 
     if (to == 0) {
 	timeout_pos = ERTS_MONOTONIC_TO_CLKTCKS(curr_time);
-	tiw_nto++;
-	tiw_at_once.nto++;
-	*tiw_at_once.tail = p;
+	tiw->nto++;
+	tiw->at_once.nto++;
+	*tiw->at_once.tail = p;
 	p->next = NULL;
 	p->timeout_pos = timeout_pos;
 	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
@@ -479,13 +540,13 @@ insert_timer(ErlTimer* p, ErtsMonotonicTime curr_time, ErtsMonotonicTime to)
 	p->slot = (Uint) tm;
   
 	/* insert at head of list at slot */
-	p->next = tiw[tm];
+	p->next = tiw->w[tm];
 	p->prev = NULL;
 	if (p->next != NULL)
 	    p->next->prev = p;
-	tiw[tm] = p;
+	tiw->w[tm] = p;
 
-	tiw_nto++;
+	tiw->nto++;
 
 	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(timeout_pos);
 	p->timeout_pos = timeout_pos;
@@ -495,59 +556,45 @@ insert_timer(ErlTimer* p, ErtsMonotonicTime curr_time, ErtsMonotonicTime to)
 	       < ERTS_MSEC_TO_MONOTONIC(to) + ERTS_CLKTCKS_TO_MONOTONIC(1));
     }
 
-    if (timeout_time < next_timeout_time)
-	set_next_timeout(timeout_time, 1);
+    if (timeout_time < tiw->next_timeout_time)
+	set_next_timeout(tiw, timeout_time, 1);
 
-    return timeout_time;
-}
+    set_timer_wheel(p, tiw);
+
+    erts_smp_mtx_unlock(&tiw->lock);
 
-void
-erts_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel,
-	      void* arg, Uint t)
-{
-#ifdef ERTS_SMP
-    ErtsMonotonicTime timeout_time;
-#endif
-    ErtsMonotonicTime current_time = erts_get_monotonic_time();
-    erts_smp_mtx_lock(&tiw_lock);
-    if (p->active) { /* XXX assert ? */
-	erts_smp_mtx_unlock(&tiw_lock);
-	return;
-    }
-    p->timeout = timeout;
-    p->cancel = cancel;
-    p->arg = arg;
-    p->active = 1;
-#ifdef ERTS_SMP
-    timeout_time =
-#else
-	(void)
-#endif
-	insert_timer(p, current_time, (ErtsMonotonicTime) t);
-    erts_smp_mtx_unlock(&tiw_lock);
 #if defined(ERTS_SMP)
-    erts_sys_schedule_interrupt_timed(1, timeout_time);
+    if (tiw == erts_default_timer_wheel)
+	erts_interupt_aux_thread_timed(timeout_time);
 #endif
+
 }
 
 void
-erts_cancel_timer(ErlTimer* p)
+erts_cancel_timer(ErlTimer *p)
 {
-    erts_smp_mtx_lock(&tiw_lock);
-    if (!p->active) { /* allow repeated cancel (drivers) */
-	erts_smp_mtx_unlock(&tiw_lock);
-	return;
-    }
-
-    remove_timer(p);
-    p->slot = 0;
+    ErtsTimerWheel *tiw;
+    ErlCancelProc cancel;
+    void *arg;
 
-    if (p->cancel != NULL) {
-	erts_smp_mtx_unlock(&tiw_lock);
-	(*p->cancel)(p->arg);
+    tiw = get_timer_wheel(p);
+    if (!tiw)
 	return;
+    
+    erts_smp_mtx_lock(&tiw->lock);
+    if (tiw != get_timer_wheel(p))
+	cancel = NULL;
+    else {
+	remove_timer(tiw, p);
+	p->slot = 0;
+
+	cancel = p->cancel;
+	arg = p->arg;
     }
-    erts_smp_mtx_unlock(&tiw_lock);
+    erts_smp_mtx_unlock(&tiw->lock);
+
+    if (cancel)
+	(*cancel)(arg);
 }
 
 /*
@@ -559,18 +606,19 @@ erts_cancel_timer(ErlTimer* p)
 Uint
 erts_time_left(ErlTimer *p)
 {
+    ErtsTimerWheel *tiw;
     ErtsMonotonicTime current_time, timeout_time;
 
-    erts_smp_mtx_lock(&tiw_lock);
-
-    if (!p->active) {
-	erts_smp_mtx_unlock(&tiw_lock);
+    tiw = get_timer_wheel(p);
+    if (!tiw)
 	return 0;
-    }
 
-    timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos);
-
-    erts_smp_mtx_unlock(&tiw_lock);
+    erts_smp_mtx_lock(&tiw->lock);
+    if (tiw != get_timer_wheel(p))
+	timeout_time = ERTS_MONOTONIC_TIME_MIN;
+    else
+	timeout_time = ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos);
+    erts_smp_mtx_unlock(&tiw->lock);
 
     current_time = erts_get_monotonic_time();
     if (timeout_time <= current_time)
@@ -581,34 +629,35 @@ erts_time_left(ErlTimer *p)
 #ifdef DEBUG
 void erts_p_slpq(void)
 {
+    ErtsTimerWheel *tiw = erts_default_timer_wheel;
     ErtsMonotonicTime current_time = erts_get_monotonic_time();
     int i;
     ErlTimer* p;
   
-    erts_smp_mtx_lock(&tiw_lock);
+    erts_smp_mtx_lock(&tiw->lock);
 
     /* print the whole wheel, starting at the current position */
     erts_printf("\ncurrent time = %bps tiw_pos = %d tiw_nto %d\n",
-		current_time, tiw_pos, tiw_nto);
-    i = tiw_pos;
-    if (tiw[i] != NULL) {
+		current_time, tiw->pos, tiw->nto);
+    i = tiw->pos;
+    if (tiw->w[i] != NULL) {
 	erts_printf("%d:\n", i);
-	for(p = tiw[i]; p != NULL; p = p->next) {
+	for(p = tiw->w[i]; p != NULL; p = p->next) {
 	    erts_printf(" (timeout time %bps, slot %d)\n",
 			ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos),
 			p->slot);
 	}
     }
-    for(i = ((i+1) & (TIW_SIZE-1)); i != (tiw_pos & (TIW_SIZE-1)); i = ((i+1) & (TIW_SIZE-1))) {
-	if (tiw[i] != NULL) {
+    for(i = ((i+1) & (TIW_SIZE-1)); i != (tiw->pos & (TIW_SIZE-1)); i = ((i+1) & (TIW_SIZE-1))) {
+	if (tiw->w[i] != NULL) {
 	    erts_printf("%d:\n", i);
-	    for(p = tiw[i]; p != NULL; p = p->next) {
+	    for(p = tiw->w[i]; p != NULL; p = p->next) {
 		erts_printf(" (timeout time %bps, slot %d)\n",
 			    ERTS_CLKTCKS_TO_MONOTONIC(p->timeout_pos), p->slot);
 	    }
 	}
     }
 
-    erts_smp_mtx_unlock(&tiw_lock);
+    erts_smp_mtx_unlock(&tiw->lock);
 }
 #endif /* DEBUG */
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 54f1a122c4..da03960b59 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -3777,7 +3777,7 @@ erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref,
     res->timer.timeout_func = timeout_func;
     res->timer.timer_ref = timer_ref;
     res->timer.id = id;
-    res->timer.tm.active = 0; /* MUST be initalized */
+    erts_init_timer(&res->timer.tm);
 
     ASSERT(!*timer_ref);