Merge branch 'rickard/timer-optimization/OTP-12650'

* rickard/timer-optimization/OTP-12650:
  Optimized timer implementation
  Reusable red-black tree implementation

Conflicts:
	erts/emulator/beam/erl_bif_timer.c

1 files changed, 264 insertions, 253 deletions

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 00c7b163c2..af8db519d4 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -44,8 +44,10 @@
 #include "dtrace-wrapper.h"
 #include "erl_ptab.h"
 #include "erl_bif_unique.h"
+#define ERTS_WANT_TIMER_WHEEL_API
+#include "erl_time.h"
 
-
+#define ERTS_CHECK_TIME_REDS CONTEXT_REDS
 #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0)
 #define ERTS_DELAYED_WAKEUP_REDUCTIONS ((Uint64) CONTEXT_REDS/2)
 
@@ -463,7 +465,7 @@ static int stack_element_dump(int to, void *to_arg, Eterm* sp, int yreg);
 static void aux_work_timeout(void *unused);
 static void aux_work_timeout_early_init(int no_schedulers);
 static void aux_work_timeout_late_init(void);
-static void setup_aux_work_timer(void);
+static void setup_aux_work_timer(ErtsSchedulerData *esdp);
 
 static int execute_sys_tasks(Process *c_p,
 			     erts_aint32_t *statep,
@@ -493,6 +495,8 @@ dbg_chk_aux_work_val(erts_aint32_t value)
     valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
     valid |= ERTS_SSI_AUX_WORK_DD;
     valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR;
+    valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS;
+    valid |= ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR;
     valid |= ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP;
 #endif
 #if HAVE_ERTS_MSEG
@@ -610,13 +614,11 @@ erts_pre_init_process(void)
 #endif
 }
 
-#ifdef ERTS_SMP
 static void
 release_process(void *vproc)
 {
-    erts_smp_proc_dec_refc((Process *) vproc);
+    erts_proc_dec_refc((Process *) vproc);
 }
-#endif
 
 /* initialize the scheduler */
 void
@@ -632,16 +634,18 @@ erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab)
 
     erts_ptab_init_table(&erts_proc,
 			 ERTS_ALC_T_PROC_TABLE,
-#ifdef ERTS_SMP
 			 release_process,
-#else
-			 NULL,
-#endif
 			 (ErtsPTabElementCommon *) &erts_invalid_process.common,
 			 proc_tab_size,
 			 sizeof(Process),
 			 "process_table",
-			 legacy_proc_tab);
+			 legacy_proc_tab,
+#ifdef ERTS_SMP
+			 1
+#else
+			 0
+#endif
+	);
 
     last_reductions = 0;
     last_exact_reductions = 0;
@@ -1031,7 +1035,7 @@ reply_sched_wall_time(void *vswtrp)
     if (rp_locks)
 	erts_smp_proc_unlock(rp, rp_locks);
 
-    erts_smp_proc_dec_refc(rp);
+    erts_proc_dec_refc(rp);
 
     if (erts_smp_atomic32_dec_read_nob(&swtrp->refc) == 0)
 	swtreq_free(vswtrp);
@@ -1063,7 +1067,7 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable)
     erts_smp_atomic32_init_nob(&swtrp->refc,
 			       (erts_aint32_t) erts_no_schedulers);
 
-    erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
+    erts_proc_add_refc(c_p, (Sint32) erts_no_schedulers);
 
 #ifdef ERTS_SMP
     if (erts_no_schedulers > 1)
@@ -1124,7 +1128,7 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data)
     xplocks &= ~plocks;
     if (xplocks && erts_smp_proc_trylock(p, xplocks) == EBUSY) {
 	if (xplocks & ERTS_PROC_LOCK_MAIN) {
-	    erts_smp_proc_inc_refc(p);
+	    erts_proc_inc_refc(p);
 	    erts_smp_proc_unlock(p, plocks);
 	    erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL);
 	    refc = 1;
@@ -1140,7 +1144,7 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data)
     if (xplocks)
 	erts_smp_proc_unlock(p, xplocks);
     if (refc)
-	erts_smp_proc_dec_refc(p);
+	erts_proc_dec_refc(p);
     ASSERT(p->psd);
     if (p->psd != psd)
 	erts_free(ERTS_ALC_T_PSD, psd);
@@ -1768,6 +1772,101 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i
 }
 
 /*
+ * Canceled timers
+ */
+
+void
+erts_notify_canceled_timer(ErtsSchedulerData *esdp, int rsid)
+{
+    ASSERT(esdp && esdp == erts_get_scheduler_data());
+    if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp))
+	schedule_aux_work_wakeup(&esdp->aux_work_data,
+				 rsid,
+				 ERTS_SSI_AUX_WORK_CNCLD_TMRS);
+    else
+	set_aux_work_flags_wakeup_relb(ERTS_SCHED_SLEEP_INFO_IX(rsid-1),
+				       ERTS_SSI_AUX_WORK_CNCLD_TMRS);
+}
+
+static ERTS_INLINE erts_aint32_t
+handle_canceled_timers(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
+{
+    ErtsSchedulerSleepInfo *ssi = awdp->ssi;
+    int need_thr_progress = 0;
+    ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
+    int more_work = 0;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
+    unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS);
+    erts_handle_canceled_timers((void *) awdp->esdp,
+				&need_thr_progress,
+				&wakeup,
+				&more_work);
+    if (more_work) {
+	if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS)
+	    & ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR) {
+	    unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR);
+	    aux_work &= ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR;
+	}
+	return aux_work;
+    }
+
+    if (need_thr_progress) {
+	if (wakeup == ERTS_THR_PRGR_INVALID)
+	    wakeup = erts_thr_progress_later(awdp->esdp);
+	awdp->cncld_tmrs.thr_prgr = wakeup;
+	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR);
+	haw_thr_prgr_soft_wakeup(awdp, wakeup);
+    }
+    return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS;
+}
+
+static ERTS_INLINE erts_aint32_t
+handle_canceled_timers_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
+{
+    ErtsSchedulerSleepInfo *ssi;
+    int need_thr_progress;
+    int more_work;
+    ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
+    ErtsThrPrgrVal current = haw_thr_prgr_current(awdp);
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
+    if (!erts_thr_progress_has_reached_this(current, awdp->cncld_tmrs.thr_prgr))
+	return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR;
+
+    ssi = awdp->ssi;
+    need_thr_progress = 0;
+    more_work = 0;
+
+    erts_handle_canceled_timers((void *) awdp->esdp,
+				&need_thr_progress,
+				&wakeup,
+				&more_work);
+    if (more_work) {
+	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS);
+	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR);
+	return ((aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR)
+		| ERTS_SSI_AUX_WORK_CNCLD_TMRS);
+    }
+
+    if (need_thr_progress) {
+	if (wakeup == ERTS_THR_PRGR_INVALID)
+	    wakeup = erts_thr_progress_later(awdp->esdp);
+	awdp->cncld_tmrs.thr_prgr = wakeup;
+	haw_thr_prgr_soft_wakeup(awdp, wakeup);
+    }
+    else {
+	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR);
+    }
+
+    return aux_work & ~ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR;
+}
+
+/*
  * Handle scheduled thread progress later operations.
  */
 #define ERTS_MAX_THR_PRGR_LATER_OPS 50
@@ -1865,7 +1964,7 @@ completed_dealloc(void *vproc)
 {
     if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == 0) {
 	erts_resume((Process *) vproc, (ErtsProcLocks) 0);
-	erts_smp_proc_dec_refc((Process *) vproc);
+	erts_proc_dec_refc((Process *) vproc);
     }
 }
 
@@ -1914,7 +2013,7 @@ erts_debug_wait_deallocations(Process *c_p)
 				      count,
 				      0)) {
 	erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
-	erts_smp_proc_inc_refc(c_p);
+	erts_proc_inc_refc(c_p);
 	/* scheduler threads */
 	erts_schedule_multi_misc_aux_work(0,
 					  erts_no_schedulers,
@@ -2029,7 +2128,7 @@ static ERTS_INLINE erts_aint32_t
 handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 {
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO);
-    setup_aux_work_timer();
+    setup_aux_work_timer(awdp->esdp);
     return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO;
 }
 
@@ -2089,6 +2188,11 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
 #ifdef ERTS_SMP
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP,
 		    handle_thr_prgr_later_op);
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CNCLD_TMRS,
+		    handle_canceled_timers);
+    /* CNCLD_TMRS must be before CNCLD_TMRS_THR_PRGR */
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR,
+		    handle_canceled_timers_thr_prgr);
 #endif
 
 #if ERTS_USE_ASYNC_READY_Q
@@ -2138,8 +2242,8 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
 
 typedef struct {
     union {
-	ErlTimer data;
-	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErlTimer))];
+	ErtsTWheelTimer data;
+	char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsTWheelTimer))];
     } timer;
 
     int initialized;
@@ -2149,6 +2253,22 @@ typedef struct {
 
 static ErtsAuxWorkTmo *aux_work_tmo;
 
+static ERTS_INLINE void
+start_aux_work_timer(ErtsSchedulerData *esdp)
+{
+    ErtsMonotonicTime tmo = erts_get_monotonic_time(esdp);
+    tmo = ERTS_MONOTONIC_TO_CLKTCKS(tmo-1);
+    tmo += ERTS_MSEC_TO_CLKTCKS(1000) + 1;
+    erts_twheel_init_timer(&aux_work_tmo->timer.data);
+    ASSERT(esdp);
+    erts_twheel_set_timer(esdp->timer_wheel,
+			  &aux_work_tmo->timer.data,
+			  aux_work_timeout,
+			  NULL,
+			  (void *) esdp,
+			  tmo);
+}
+
 static void
 aux_work_timeout_early_init(int no_schedulers)
 {
@@ -2181,18 +2301,12 @@ void
 aux_work_timeout_late_init(void)
 {
     aux_work_tmo->initialized = 1;
-    if (erts_atomic32_read_nob(&aux_work_tmo->refc)) {
-	erts_init_timer(&aux_work_tmo->timer.data);
-	erts_set_timer(&aux_work_tmo->timer.data,
-		       aux_work_timeout,
-		       NULL,
-		       NULL,
-		       1000);
-    }
+    if (erts_atomic32_read_nob(&aux_work_tmo->refc))
+	start_aux_work_timer(erts_get_scheduler_data());
 }
 
 static void
-aux_work_timeout(void *unused)
+aux_work_timeout(void *vesdp)
 {
     erts_aint32_t refc;
     int i;
@@ -2215,31 +2329,18 @@ aux_work_timeout(void *unused)
     if (refc != 1
 	|| 1 != erts_atomic32_cmpxchg_relb(&aux_work_tmo->refc, 0, 1)) {
 	/* Setup next timeout... */
-	erts_set_timer(&aux_work_tmo->timer.data,
-		       aux_work_timeout,
-		       NULL,
-		       NULL,
-		       1000);
+	start_aux_work_timer((ErtsSchedulerData *) vesdp);
     }
 }
 
 static void
-setup_aux_work_timer(void)
+setup_aux_work_timer(ErtsSchedulerData *esdp)
 {
-#ifndef ERTS_SMP
-    if (!erts_get_scheduler_data())
+    if (!esdp || !esdp->timer_wheel)
 	set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(0),
 				      ERTS_SSI_AUX_WORK_SET_TMO);
     else
-#endif
-    {
-	erts_init_timer(&aux_work_tmo->timer.data);
-	erts_set_timer(&aux_work_tmo->timer.data,
-		       aux_work_timeout,
-		       NULL,
-		       NULL,
-		       1000);
-    }
+	start_aux_work_timer(esdp);
 }
 
 erts_aint32_t
@@ -2270,7 +2371,7 @@ erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable)
 	if (refc == 1) {
 	    erts_atomic32_inc_acqb(&aux_work_tmo->refc);
 	    if (aux_work_tmo->initialized) 
-		setup_aux_work_timer();
+		setup_aux_work_timer(erts_get_scheduler_data());
 	}
     }
     return old;
@@ -2649,11 +2750,6 @@ aux_thread(void *unused)
     erts_aint32_t aux_work;
     ErtsThrPrgrCallbacks callbacks;
     int thr_prgr_active = 1;
-    ErtsTimerWheel *timer_wheel = erts_default_timer_wheel;
-    ErtsNextTimeoutRef nxt_tmo_ref = erts_get_next_timeout_reference(timer_wheel);
-
-    if (!timer_wheel)
-	ERTS_INTERNAL_ERROR("Missing aux timer wheel");
 
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
@@ -2677,7 +2773,6 @@ aux_thread(void *unused)
     sched_prep_spin_wait(ssi);
 
     while (1) {
-	ErtsMonotonicTime current_time;
 	erts_aint32_t flgs;
 
 	aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
@@ -2689,56 +2784,28 @@ aux_thread(void *unused)
 		erts_thr_progress_leader_update(NULL);
 	}
 
-	if (aux_work) {
-	    current_time = erts_get_monotonic_time();
-	    if (current_time >= erts_next_timeout_time(nxt_tmo_ref)) {
-		if (!thr_prgr_active)
-		    erts_thr_progress_active(NULL, thr_prgr_active = 1);
-		erts_bump_timers(timer_wheel, current_time);
-	    }
-	}
-	else {
-	    ErtsMonotonicTime timeout_time;
-	    timeout_time = erts_check_next_timeout_time(timer_wheel,
-							ERTS_SEC_TO_MONOTONIC(10*60));
-	    current_time = erts_get_monotonic_time();
-	    if (current_time >= timeout_time) {
-		if (!thr_prgr_active)
-		    erts_thr_progress_active(NULL, thr_prgr_active = 1);
-	    }
-	    else {
-		if (thr_prgr_active)
-		    erts_thr_progress_active(NULL, thr_prgr_active = 0);
-		erts_thr_progress_prepare_wait(NULL);
+	if (!aux_work) {
+	    if (thr_prgr_active)
+		erts_thr_progress_active(NULL, thr_prgr_active = 0);
+	    erts_thr_progress_prepare_wait(NULL);
 
-		ERTS_SCHED_FAIR_YIELD();
+	    ERTS_SCHED_FAIR_YIELD();
 
-		flgs = sched_spin_wait(ssi, 0);
+	    flgs = sched_spin_wait(ssi, 0);
 
+	    if (flgs & ERTS_SSI_FLG_SLEEPING) {
+		ASSERT(flgs & ERTS_SSI_FLG_WAITING);
+		flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
 		if (flgs & ERTS_SSI_FLG_SLEEPING) {
+		    int res;
+		    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
 		    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-		    flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING);
-		    if (flgs & ERTS_SSI_FLG_SLEEPING) {
-			int res;
-			ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
-			ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-			current_time = erts_get_monotonic_time();
-			do {
-			    Sint64 timeout;
-			    if (current_time >= timeout_time)
-				break;
-			    timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time
-							     - current_time
-							     - 1) + 1;
-			    res = erts_tse_twait(ssi->event, timeout);
-			    current_time = erts_get_monotonic_time();
-			} while (res == EINTR);
-		    }
+		    do {
+			res = erts_tse_wait(ssi->event);
+		    } while (res == EINTR);
 		}
-		erts_thr_progress_finalize_wait(NULL);
 	    }
-	    if (current_time >= timeout_time)
-		erts_bump_timers(timer_wheel, current_time);
+	    erts_thr_progress_finalize_wait(NULL);
 	}
 
 	flgs = sched_prep_spin_wait(ssi);
@@ -2821,7 +2888,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	    if (aux_work) {
 		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
-		current_time = erts_get_monotonic_time();
+		current_time = erts_get_monotonic_time(esdp);
 		if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
 		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
 			erts_thr_progress_active(esdp, thr_prgr_active = 1);
@@ -2832,9 +2899,8 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    }
 	    else {
 		ErtsMonotonicTime timeout_time;
-		timeout_time = erts_check_next_timeout_time(esdp->timer_wheel,
-							    ERTS_SEC_TO_MONOTONIC(10*60));
-		current_time = erts_get_monotonic_time();
+		timeout_time = erts_check_next_timeout_time(esdp);
+		current_time = erts_get_monotonic_time(esdp);
 		if (current_time >= timeout_time) {
 		    if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
 			erts_thr_progress_active(esdp, thr_prgr_active = 1);
@@ -2860,7 +2926,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 			    int res;
 			    ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING);
 			    ASSERT(flgs & ERTS_SSI_FLG_WAITING);
-			    current_time = erts_get_monotonic_time();
+			    current_time = erts_get_monotonic_time(esdp);
 			    do {
 				Sint64 timeout;
 				if (current_time >= timeout_time)
@@ -2869,7 +2935,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 								 - current_time
 								 - 1) + 1;
 				res = erts_tse_twait(ssi->event, timeout);
-				current_time = erts_get_monotonic_time();
+				current_time = erts_get_monotonic_time(esdp);
 			    } while (res == EINTR);
 			}
 		    }
@@ -2944,7 +3010,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
 	    erl_sys_schedule(1); /* Might give us something to do */
 
-	    current_time = erts_get_monotonic_time();
+	    current_time = erts_get_monotonic_time(esdp);
 	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
 		erts_bump_timers(esdp->timer_wheel, current_time);
 
@@ -3062,7 +3128,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	erl_sys_schedule(0);
 
 	{
-	    ErtsMonotonicTime current_time = erts_get_monotonic_time();
+	    ErtsMonotonicTime current_time = erts_get_monotonic_time(esdp);
 	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
 		erts_bump_timers(esdp->timer_wheel, current_time);
 	}
@@ -5273,6 +5339,7 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
     awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
     awdp->dd.completed_callback = NULL;
     awdp->dd.completed_arg = NULL;
+    awdp->cncld_tmrs.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
     awdp->later_op.thr_prgr = ERTS_THR_PRGR_VAL_FIRST;
     awdp->later_op.size = 0;
     awdp->later_op.first = NULL;
@@ -5338,9 +5405,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     esdp->no = (Uint) num;
 #endif
 
-    esdp->timer_wheel = erts_default_timer_wheel;
-    esdp->next_tmo_ref = erts_get_next_timeout_reference(esdp->timer_wheel);
-
     esdp->ssi = ssi;
     esdp->current_process = NULL;
     esdp->current_port = NULL;
@@ -5353,6 +5417,9 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
     esdp->run_queue = runq;
     esdp->run_queue->scheduler = esdp;
 
+    esdp->last_monotonic_time = 0;
+    esdp->check_time_reds = 0;
+
     esdp->thr_id = (Uint32) num;
     erts_sched_bif_unique_init(esdp);
 
@@ -5916,13 +5983,6 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, Proces
     int check_emigration_need;
 #endif
 
-#ifdef ERTS_SMP
-    if ((p->static_flags & ERTS_STC_FLG_PREFER_SCHED)
-	&& p->preferred_run_queue != RUNQ_READ_RQ(&p->run_queue)) {
-	RUNQ_SET_RQ(&p->run_queue, p->preferred_run_queue);
-    }
-#endif
-
     a = state;
 
     while (1) {
@@ -6730,6 +6790,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 	}
     }
 
+    (void) erts_get_monotonic_time(esdp);
     erts_smp_runq_lock(esdp->run_queue);
     non_empty_runq(esdp->run_queue);
 
@@ -6863,7 +6924,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 		}
 
 		if (aux_work) {
-		    current_time = erts_get_monotonic_time();
+		    current_time = erts_get_monotonic_time(esdp);
 		    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
 			if (!thr_prgr_active) {
 			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
@@ -6874,9 +6935,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 		}
 		else {
 		    ErtsMonotonicTime timeout_time;
-		    timeout_time = erts_check_next_timeout_time(esdp->timer_wheel,
-								ERTS_SEC_TO_MONOTONIC(60*60));
-		    current_time = erts_get_monotonic_time();
+		    timeout_time = erts_check_next_timeout_time(esdp);
+		    current_time = erts_get_monotonic_time(esdp);
 
 		    if (current_time >= timeout_time) {
 			if (!thr_prgr_active) {
@@ -6902,7 +6962,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 					 | ERTS_SSI_FLG_SUSPENDED)) {
 				int res;
 
-				current_time = erts_get_monotonic_time();
+				current_time = erts_get_monotonic_time(esdp);
 				do {
 				    Sint64 timeout;
 				    if (current_time >= timeout_time)
@@ -6911,7 +6971,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 								     - current_time
 								     - 1) + 1;
 				    res = erts_tse_twait(ssi->event, timeout);
-				    current_time = erts_get_monotonic_time();
+				    current_time = erts_get_monotonic_time(esdp);
 				} while (res == EINTR);
 			    }
 			}
@@ -7739,8 +7799,8 @@ sched_thread_func(void *vesdp)
     ErtsSchedulerData *esdp = vesdp;
     Uint no = esdp->no;
 
-    esdp->timer_wheel = erts_create_timer_wheel((int) no);
-    esdp->next_tmo_ref = erts_get_next_timeout_reference(esdp->timer_wheel);
+    erts_sched_init_time_sup(esdp);
+
 #ifdef ERTS_SMP
     ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch();
     callbacks.arg = (void *) esdp->ssi;
@@ -9109,7 +9169,7 @@ Process *schedule(Process *p, int calls)
 	schedule_out_process(rq, state, p, proxy_p); /* Returns with rq locked! */
 	proxy_p = NULL;
 
-	ERTS_PROC_REDUCTIONS_EXECUTED(rq,
+	ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq,
 				      (int) ERTS_PSFLGS_GET_USR_PRIO(state),
 				      reds,
 				      actual_reds);
@@ -9126,12 +9186,9 @@ Process *schedule(Process *p, int calls)
 	    ASSERT(esdp->free_process == p);
 	    esdp->free_process = NULL;
 #else
-	    state = erts_smp_atomic32_read_nob(&p->state);
-	    if (!(state & ERTS_PSFLG_IN_RUNQ))
-		erts_free_proc(p);
+	    erts_proc_dec_refc(p);
 #endif
 	}
-
 #ifdef ERTS_SMP
 	ASSERT(!esdp->free_process);
 #endif
@@ -9139,13 +9196,13 @@ Process *schedule(Process *p, int calls)
 
 	ERTS_SMP_CHK_NO_PROC_LOCKS;
 
-	{
-	    ErtsMonotonicTime current_time = erts_get_monotonic_time();
-	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
-		erts_smp_runq_unlock(rq);
-		erts_bump_timers(esdp->timer_wheel, current_time);
-		erts_smp_runq_lock(rq);
-	    }
+	if (esdp->check_time_reds >= ERTS_CHECK_TIME_REDS)
+	    (void) erts_get_monotonic_time(esdp);
+
+	if (esdp->last_monotonic_time >= erts_next_timeout_time(esdp->next_tmo_ref)) {
+	    erts_smp_runq_unlock(rq);
+	    erts_bump_timers(esdp->timer_wheel, esdp->last_monotonic_time);
+	    erts_smp_runq_lock(rq);
 	}
 	BM_STOP_TIMER(system);
 
@@ -9305,7 +9362,7 @@ Process *schedule(Process *p, int calls)
 	    erts_smp_runq_unlock(rq);
 	    erl_sys_schedule(1);
 
-	    current_time = erts_get_monotonic_time();
+	    current_time = erts_get_monotonic_time(esdp);
 	    if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref))
 		erts_bump_timers(esdp->timer_wheel, current_time);
 
@@ -9446,13 +9503,8 @@ Process *schedule(Process *p, int calls)
 				      | ERTS_PSFLG_PENDING_EXIT
 				      | ERTS_PSFLG_ACTIVE_SYS))
 			    == ERTS_PSFLG_SUSPENDED)) {
-			if (state & ERTS_PSFLG_FREE) {
-#ifdef ERTS_SMP
-			    erts_smp_proc_dec_refc(p);
-#else
-			    erts_free_proc(p);
-#endif
-			}
+			if (state & ERTS_PSFLG_FREE)
+			    erts_proc_dec_refc(p);
 			if (proxy_p) {
 			    free_proxy_proc(proxy_p);
 			    proxy_p = NULL;
@@ -9595,6 +9647,20 @@ Process *schedule(Process *p, int calls)
 	/* Never run a suspended process */
 	ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state)));
 
+	ASSERT(erts_proc_read_refc(p) > 0);
+
+	if (ERTS_PTMR_IS_TIMED_OUT(p)) {
+	    BeamInstr** pi;
+#ifdef ERTS_SMP
+	    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
+#endif
+	    pi = (BeamInstr **) p->def_arg_reg;
+	    p->i = *pi;
+	    p->flags &= ~F_INSLPQUEUE;
+	    p->flags |= F_TIMO;
+	    ERTS_PTMR_CLEAR(p);
+	}
+
 	return p;
     }
 }
@@ -10503,6 +10569,8 @@ erts_free_proc(Process *p)
 #ifdef ERTS_SMP
     erts_proc_lock_fin(p);
 #endif
+    ASSERT(erts_smp_atomic32_read_nob(&p->state) & ERTS_PSFLG_FREE);
+    ASSERT(0 == erts_proc_read_refc(p));
     erts_free(ERTS_ALC_T_PROC, (void *) p);
 }
 
@@ -10555,6 +10623,8 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state)
 	return NULL;
     }
 
+    ASSERT(erts_proc_read_refc(p) > 0);
+
     ASSERT(internal_pid_serial(p->common.id) <= ERTS_MAX_PID_SERIAL);
     
     p->approx_started = erts_get_approx_time();
@@ -10604,10 +10674,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	    int ix = so->scheduler-1;
 	    ASSERT(0 <= ix && ix < erts_no_run_queues);
 	    rq = ERTS_RUNQ_IX(ix);
-	    if (!(so->flags & SPO_PREFER_SCHED)) {
-		/* Unsupported feature... */
-		state |= ERTS_PSFLG_BOUND;
-	    }
+	    /* Unsupported feature... */
+	    state |= ERTS_PSFLG_BOUND;
 	}
 	prio = (erts_aint32_t) so->priority;
     }
@@ -10615,9 +10683,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     state |= (((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_ACT_PRIO_OFFSET)
 	      | ((prio & ERTS_PSFLGS_PRIO_MASK) << ERTS_PSFLGS_USR_PRIO_OFFSET));
 
-    if (so->flags & SPO_OFF_HEAP_MSGS)
-	state |= ERTS_PSFLG_OFF_HEAP_MSGS;
-
     if (!rq)
 	rq = erts_get_runq_proc(parent);
 
@@ -10641,12 +10706,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     heap_need = arg_size;
 
     p->flags = erts_default_process_flags;
-    if (so->flags & SPO_OFF_HEAP_MSGS)
-	p->flags |= F_OFF_HEAP_MSGS;
 
-#ifdef ERTS_SMP
-    p->preferred_run_queue = NULL;
-#endif
     p->static_flags = 0;
     if (so->flags & SPO_SYSTEM_PROC)
 	p->static_flags |= ERTS_STC_FLG_SYSTEM_PROC;
@@ -10654,12 +10714,6 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 	p->min_heap_size  = so->min_heap_size;
 	p->min_vheap_size = so->min_vheap_size;
 	p->max_gen_gcs    = so->max_gen_gcs;
-	if (so->flags & SPO_PREFER_SCHED) {
-#ifdef ERTS_SMP
-	    p->preferred_run_queue = rq;
-#endif
-	    p->static_flags |= ERTS_STC_FLG_PREFER_SCHED;
-	}
     } else {
 	p->min_heap_size  = H_MIN_SIZE;
 	p->min_vheap_size = BIN_VH_MIN_SIZE;
@@ -10668,9 +10722,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->schedule_count = 0;
     ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0));
     
-    p->initial[INITIAL_MOD] = mod;
-    p->initial[INITIAL_FUN] = func;
-    p->initial[INITIAL_ARI] = (Uint) arity;
+    p->u.initial[INITIAL_MOD] = mod;
+    p->u.initial[INITIAL_FUN] = func;
+    p->u.initial[INITIAL_ARI] = (Uint) arity;
 
     /*
      * Must initialize binary lists here before copying binaries to process.
@@ -10711,7 +10765,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
 
     /* No need to initialize p->fcalls. */
 
-    p->current = p->initial+INITIAL_MOD;
+    p->current = p->u.initial+INITIAL_MOD;
 
     p->i = (BeamInstr *) beam_apply;
     p->cp = (BeamInstr *) beam_apply+1;
@@ -10734,11 +10788,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->ftrace = NIL;
     p->reds = 0;
 
-#ifdef ERTS_SMP
-    p->common.u.alive.ptimer = NULL;
-#else
-    erts_init_timer(&p->common.u.alive.tm);
-#endif
+    ERTS_PTMR_INIT(p);
 
     p->common.u.alive.reg = NULL;
     ERTS_P_LINKS(p) = NULL;
@@ -10769,7 +10819,8 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->msg_inq.last = &p->msg_inq.first;
     p->msg_inq.len = 0;
 #endif
-    p->u.bif_timers = NULL;
+    p->bif_timers = NULL;
+    p->accessor_bif_timers = NULL;
     p->mbuf = NULL;
     p->mbuf_sz = 0;
     p->psd = NULL;
@@ -10927,11 +10978,7 @@ void erts_init_empty_process(Process *p)
     p->bin_old_vheap = 0;
     p->sys_task_qs = NULL;
     p->bin_vheap_mature = 0;
-#ifdef ERTS_SMP
-    p->common.u.alive.ptimer = NULL;
-#else
-    erts_init_timer(&p->common.u.alive.tm);
-#endif
+    ERTS_PTMR_INIT(p);
     p->next = NULL;
     p->off_heap.first = NULL;
     p->off_heap.overhead = 0;
@@ -10952,14 +10999,15 @@ void erts_init_empty_process(Process *p)
     p->msg.last = &p->msg.first;
     p->msg.save = &p->msg.first;
     p->msg.len = 0;
-    p->u.bif_timers = NULL;
+    p->bif_timers = NULL;
+    p->accessor_bif_timers = NULL;
     p->dictionary = NULL;
     p->seq_trace_clock = 0;
     p->seq_trace_lastcnt = 0;
     p->seq_trace_token = NIL;
-    p->initial[0] = 0;
-    p->initial[1] = 0;
-    p->initial[2] = 0;
+    p->u.initial[0] = 0;
+    p->u.initial[1] = 0;
+    p->u.initial[2] = 0;
     p->catches = 0;
     p->cp = NULL;
     p->i = NULL;
@@ -11007,7 +11055,6 @@ void erts_init_empty_process(Process *p)
     p->pending_suspenders = NULL;
     p->pending_exit.reason = THE_NON_VALUE;
     p->pending_exit.bp = NULL;
-    p->preferred_run_queue = NULL;
     erts_proc_lock_init(p);
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL);
     RUNQ_SET_RQ(&p->run_queue, ERTS_RUNQ_IX(0));
@@ -11047,7 +11094,8 @@ erts_debug_verify_clean_empty_process(Process* p)
     ASSERT(p->suspend_monitors == NULL);
     ASSERT(p->msg.first == NULL);
     ASSERT(p->msg.len == 0);
-    ASSERT(p->u.bif_timers == NULL);
+    ASSERT(p->bif_timers == NULL);
+    ASSERT(p->accessor_bif_timers == NULL);
     ASSERT(p->dictionary == NULL);
     ASSERT(p->catches == 0);
     ASSERT(p->cp == NULL);
@@ -11211,7 +11259,6 @@ set_proc_exiting(Process *p,
      */
     p->freason = EXTAG_EXIT;
     KILL_CATCHES(p);
-    cancel_timer(p);
     p->i = (BeamInstr *) beam_exit;
 
     if (enqueue)
@@ -11901,6 +11948,7 @@ erts_do_exit_process(Process* p, Eterm reason)
 {
     p->arity = 0;		/* No live registers */
     p->fvalue = reason;
+    
 
 #ifdef USE_VM_PROBES
     if (DTRACE_ENABLED(process_exit)) {
@@ -11955,20 +12003,20 @@ erts_do_exit_process(Process* p, Eterm reason)
     ASSERT((ERTS_TRACE_FLAGS(p) & F_INITIAL_TRACE_FLAGS)
 	   == F_INITIAL_TRACE_FLAGS);
 
-    cancel_timer(p);		/* Always cancel timer just in case */
-
-    if (p->u.bif_timers)
-	erts_cancel_bif_timers(p, ERTS_PROC_LOCKS_ALL);
+    ASSERT(erts_proc_read_refc(p) > 0);
+    if (ERTS_PTMR_IS_SET(p)) {
+	erts_cancel_proc_timer(p);
+	ASSERT(erts_proc_read_refc(p) > 0);
+    }
 
     erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL_MINOR);
 
     /*
-     * The p->u.bif_timers of this process can *not* be used anymore;
+     * p->u.initial of this process can *not* be used anymore;
      * will be overwritten by misc termination data.
      */
     p->u.terminate = NULL;
 
-
     erts_continue_exit_process(p);
 }
 
@@ -11993,6 +12041,27 @@ erts_continue_exit_process(Process *p)
 
     ASSERT(ERTS_PROC_IS_EXITING(p));
 
+    ASSERT(erts_proc_read_refc(p) > 0);
+    if (p->bif_timers) {
+	if (erts_cancel_bif_timers(p, p->bif_timers, &p->u.terminate)) {
+	    ASSERT(erts_proc_read_refc(p) > 0);
+	    goto yield;
+	}
+	ASSERT(erts_proc_read_refc(p) > 0);
+	p->bif_timers = NULL;
+    }
+
+    if (p->accessor_bif_timers) {
+	if (erts_detach_accessor_bif_timers(p,
+					    p->accessor_bif_timers,
+					    &p->u.terminate)) {
+	    ASSERT(erts_proc_read_refc(p) > 0);
+	    goto yield;
+	}
+	ASSERT(erts_proc_read_refc(p) > 0);
+	p->accessor_bif_timers = NULL;
+    }
+
 #ifdef ERTS_SMP
     if (p->flags & F_HAVE_BLCKD_MSCHED) {
 	ErtsSchedSuspendResult ssr;
@@ -12086,6 +12155,8 @@ erts_continue_exit_process(Process *p)
 
 	p->scheduler_data->current_process = NULL;
 	p->scheduler_data->free_process = p;
+#else
+	erts_proc_inc_refc(p); /* Decremented in schedule() */
 #endif
 
 	/* Time of death! */
@@ -12104,29 +12175,23 @@ erts_continue_exit_process(Process *p)
     {
 	/* Inactivate and notify free */
 	erts_aint32_t n, e, a = erts_smp_atomic32_read_nob(&p->state);
-#ifdef ERTS_SMP
 	int refc_inced = 0;
-#endif
 	while (1) {
 	    n = e = a;
 	    ASSERT(a & ERTS_PSFLG_EXITING);
 	    n |= ERTS_PSFLG_FREE;
 	    n &= ~ERTS_PSFLG_ACTIVE;
-#ifdef ERTS_SMP
 	    if ((n & ERTS_PSFLG_IN_RUNQ) && !refc_inced) {
-		erts_smp_proc_inc_refc(p);
+		erts_proc_inc_refc(p);
 		refc_inced = 1;
 	    }
-#endif
 	    a = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
 	    if (a == e)
 		break;
 	}
 
-#ifdef ERTS_SMP
 	if (refc_inced && !(n & ERTS_PSFLG_IN_RUNQ))
-	    erts_smp_proc_dec_refc(p);
-#endif
+	    erts_proc_dec_refc(p);
     }
     
     dep = ((p->flags & F_DISTRIBUTION)
@@ -12217,64 +12282,6 @@ erts_continue_exit_process(Process *p)
 
 }
 
-/* Callback for process timeout */
-static void
-timeout_proc(Process* p)
-{
-    erts_aint32_t state;
-    BeamInstr** pi = (BeamInstr **) p->def_arg_reg;
-    p->i = *pi;
-    p->flags |= F_TIMO;
-    p->flags &= ~F_INSLPQUEUE;
-
-    state = erts_smp_atomic32_read_acqb(&p->state);
-    if (!(state & ERTS_PSFLG_ACTIVE))
-	schedule_process(p, state, ERTS_PROC_LOCK_MAIN|ERTS_PROC_LOCK_STATUS);
-}
-
-
-void
-cancel_timer(Process* p)
-{
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));
-    p->flags &= ~(F_INSLPQUEUE|F_TIMO);
-#ifdef ERTS_SMP
-    erts_cancel_smp_ptimer(p->common.u.alive.ptimer);
-#else
-    erts_cancel_timer(&p->common.u.alive.tm);
-#endif
-}
-
-/*
- * Insert a process into the time queue, with a timeout 'timeout' in ms.
- */
-void
-set_timer(Process* p, Uint timeout)
-{
-    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p));
-
-    /* check for special case timeout=0 DONT ADD TO time queue */
-    if (timeout == 0) {
-	p->flags |= F_TIMO;
-	return;
-    }
-    p->flags |= F_INSLPQUEUE;
-    p->flags &= ~F_TIMO;
-
-#ifdef ERTS_SMP
-    erts_create_smp_ptimer(&p->common.u.alive.ptimer,
-			   p->common.id,
-			   (ErlTimeoutProc) timeout_proc,
-			   timeout);
-#else
-    erts_set_timer(&p->common.u.alive.tm,
-		  (ErlTimeoutProc) timeout_proc,
-		  NULL,
-		  (void*) p,
-		  timeout);
-#endif
-}
-
 /*
  * Stack dump functions follow.
  */
@@ -12432,6 +12439,10 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) {
                 erts_print(to, to_arg, "FIX_ALLOC_LOWER_LIM"); break;
             case ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP:
                 erts_print(to, to_arg, "THR_PRGR_LATER_OP"); break;
+            case ERTS_SSI_AUX_WORK_CNCLD_TMRS:
+                erts_print(to, to_arg, "CANCELED_TIMERS"); break;
+            case ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR:
+                erts_print(to, to_arg, "CANCELED_TIMERS_THR_PRGR"); break;
             case ERTS_SSI_AUX_WORK_ASYNC_READY:
                 erts_print(to, to_arg, "ASYNC_READY"); break;
             case ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN: