Merge branch 'rickard/timer-improvement/OTP-12650'

* rickard/timer-improvement/OTP-12650:
  Callback timer
  Disable accessor timer option
  Aux work flag descriptions
  Fix test cases

9 files changed, 652 insertions, 171 deletions

diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index dcae5509ec..d11f24220a 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -581,8 +581,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
 	= erts_timer_type_size(ERTS_ALC_T_HL_PTIMER);
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_BIF_TIMER)]
 	= erts_timer_type_size(ERTS_ALC_T_BIF_TIMER);
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_ABIF_TIMER)]
 	= erts_timer_type_size(ERTS_ALC_T_ABIF_TIMER);
+#endif
 
 #ifdef HARD_DEBUG
     hdbg_init();
@@ -2343,10 +2345,12 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		       &size.processes_used,
 		       fi,
 		       ERTS_ALC_T_BIF_TIMER);
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 	add_fix_values(&size.processes,
 		       &size.processes_used,
 		       fi,
 		       ERTS_ALC_T_ABIF_TIMER);
+#endif
     }
 
     if (want.atom || want.atom_used) {
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 57c506458c..2721e13250 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -167,7 +167,7 @@ type	TIMER_SERVICE	LONG_LIVED	SYSTEM		timer_service
 type    LL_PTIMER	FIXED_SIZE	PROCESSES	ll_ptimer
 type    HL_PTIMER	FIXED_SIZE	PROCESSES	hl_ptimer
 type    BIF_TIMER	FIXED_SIZE	PROCESSES	bif_timer
-type    ABIF_TIMER	FIXED_SIZE	PROCESSES	accessor_bif_timer
+# type    ABIF_TIMER	FIXED_SIZE	PROCESSES	accessor_bif_timer
 type    TIMER_REQUEST	SHORT_LIVED	PROCESSES	timer_request
 type    BTM_YIELD_STATE	SHORT_LIVED	PROCESSES	btm_yield_state
 type	REG_TABLE	STANDARD	SYSTEM		reg_tab
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index f74aea80a7..ae8a5c266a 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4050,7 +4050,14 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 	}
 	else if (ERTS_IS_ATOM_STR("wait", BIF_ARG_1)) {
 	    if (ERTS_IS_ATOM_STR("deallocations", BIF_ARG_2)) {
-		if (erts_debug_wait_deallocations(BIF_P)) {
+		int flag = ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS;
+		if (erts_debug_wait_completed(BIF_P, flag)) {
+		    ERTS_BIF_YIELD_RETURN(BIF_P, am_ok);
+		}
+	    }
+	    if (ERTS_IS_ATOM_STR("timer_cancellations", BIF_ARG_2)) {
+		int flag = ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS;
+		if (erts_debug_wait_completed(BIF_P, flag)) {
 		    ERTS_BIF_YIELD_RETURN(BIF_P, am_ok);
 		}
 	    }
diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index 51cd843935..8eacb921fe 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -81,6 +81,13 @@ static void hdbg_chk_srv(ErtsHLTimerService *srv);
 #error "ERTS_REF_NUMBERS changed. Update me..."
 #endif
 
+typedef enum {
+    ERTS_TMR_BIF,
+    ERTS_TMR_PROC,
+    ERTS_TMR_PORT,
+    ERTS_TMR_CALLBACK
+} ErtsTmrType;
+
 #define ERTS_BIF_TIMER_SHORT_TIME 5000
 
 #ifdef ERTS_SMP
@@ -93,11 +100,14 @@ static void hdbg_chk_srv(ErtsHLTimerService *srv);
 /* Bit 0 to 9 contains scheduler id (see mask below) */
 #define ERTS_TMR_ROFLG_HLT		(((Uint32) 1) << 10)
 #define ERTS_TMR_ROFLG_BIF_TMR		(((Uint32) 1) << 11)
-#define ERTS_TMR_ROFLG_ABIF_TMR		(((Uint32) 1) << 12)
-#define ERTS_TMR_ROFLG_PRE_ALC		(((Uint32) 1) << 13)
-#define ERTS_TMR_ROFLG_REG_NAME		(((Uint32) 1) << 14)
-#define ERTS_TMR_ROFLG_PROC		(((Uint32) 1) << 15)
-#define ERTS_TMR_ROFLG_PORT		(((Uint32) 1) << 16)
+#define ERTS_TMR_ROFLG_PRE_ALC		(((Uint32) 1) << 12)
+#define ERTS_TMR_ROFLG_REG_NAME		(((Uint32) 1) << 13)
+#define ERTS_TMR_ROFLG_PROC		(((Uint32) 1) << 14)
+#define ERTS_TMR_ROFLG_PORT		(((Uint32) 1) << 15)
+#define ERTS_TMR_ROFLG_CALLBACK		(((Uint32) 1) << 16)
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
+#define ERTS_TMR_ROFLG_ABIF_TMR		(((Uint32) 1) << 17)
+#endif
 
 #define ERTS_TMR_ROFLG_SID_MASK	\
     (ERTS_TMR_ROFLG_HLT - (Uint32) 1)
@@ -141,6 +151,7 @@ typedef struct {
     Uint32 roflgs;
     erts_smp_atomic32_t refc;
     union {
+	void *arg;
 	erts_atomic_t next;
     } u;
 } ErtsTmrHead;
@@ -156,6 +167,7 @@ struct ErtsHLTimer_ {
 	Process *proc;
 	Port *port;
 	Eterm name;
+	void (*callback)(void *);
     } receiver;
 
 #ifdef ERTS_HLT_HARD_DEBUG
@@ -172,19 +184,28 @@ struct ErtsHLTimer_ {
 	Eterm message;
 	ErlHeapFragment *bp;
     } btm;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     struct {
 	Eterm accessor;
 	ErtsHLTimerTree tree;
     } abtm;
+#endif
 };
 
 #define ERTS_HL_PTIMER_SIZE offsetof(ErtsHLTimer, btm)
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 #define ERTS_BIF_TIMER_SIZE offsetof(ErtsHLTimer, abtm)
 #define ERTS_ABIF_TIMER_SIZE sizeof(ErtsHLTimer)
+#else
+#define ERTS_BIF_TIMER_SIZE sizeof(ErtsHLTimer)
+#endif
 
 typedef struct {
     ErtsTmrHead head; /* NEED to be first! */
-    void *p;
+    union {
+	void *p;
+	void (*callback)(void *);
+    } u;
     ErtsTWheelTimer tw_tmr;
 } ErtsTWTimer;
 
@@ -340,8 +361,8 @@ refn_is_lt(Uint32 *x, Uint32 *y)
 #define ERTS_RBT_WANT_SMALLEST
 #define ERTS_RBT_WANT_LOOKUP_INSERT
 #define ERTS_RBT_WANT_REPLACE
+#define ERTS_RBT_WANT_FOREACH
 #ifdef ERTS_HLT_HARD_DEBUG
-#  define ERTS_RBT_WANT_FOREACH
 #  define ERTS_RBT_WANT_LOOKUP
 #endif
 #define ERTS_RBT_UNDEF
@@ -452,8 +473,6 @@ same_time_list_foreach_destroy_yielding(ErtsHLTimer **root,
     }    
 }
 
-#ifdef ERTS_HLT_HARD_DEBUG
-
 static ERTS_INLINE void
 same_time_list_foreach(ErtsHLTimer *root,
 		       void (*op)(ErtsHLTimer *, void *),
@@ -468,6 +487,8 @@ same_time_list_foreach(ErtsHLTimer *root,
     }
 }
 
+#ifdef ERTS_HLT_HARD_DEBUG
+
 static ERTS_INLINE ErtsHLTimer *
 same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x)
 {
@@ -584,6 +605,8 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x)
 
 #include "erl_rbtree.h"
 
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
+
 #define ERTS_RBT_PREFIX abtm
 #define ERTS_RBT_T ErtsHLTimer
 #define ERTS_RBT_KEY_T Uint32 *
@@ -634,6 +657,8 @@ same_time_list_lookup(ErtsHLTimer *root, ErtsHLTimer *x)
 
 #include "erl_rbtree.h"
 
+#endif /* ERTS_BTM_ACCESSOR_SUPPORT */
+
 #ifdef ERTS_SMP
 static void init_canceled_queue(ErtsHLTCncldTmrQ *cq);
 #endif
@@ -673,7 +698,9 @@ erts_timer_type_size(ErtsAlcType_t type)
     case ERTS_ALC_T_LL_PTIMER: return sizeof(ErtsTWTimer);
     case ERTS_ALC_T_HL_PTIMER: return ERTS_HL_PTIMER_SIZE;
     case ERTS_ALC_T_BIF_TIMER: return ERTS_BIF_TIMER_SIZE;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     case ERTS_ALC_T_ABIF_TIMER: return ERTS_ABIF_TIMER_SIZE;
+#endif
     default: ERTS_INTERNAL_ERROR("Unknown type");
     }
     return 0;
@@ -747,9 +774,9 @@ schedule_tw_timer_destroy(ErtsTWTimer *tmr)
      * dropped at once...
      */
     if (tmr->head.roflgs & ERTS_TMR_ROFLG_PROC)
-	erts_proc_dec_refc((Process *) tmr->p);
-    else
-	erts_port_dec_refc((Port *) tmr->p);
+	erts_proc_dec_refc((Process *) tmr->u.p);
+    else if (tmr->head.roflgs & ERTS_TMR_ROFLG_PORT)
+	erts_port_dec_refc((Port *) tmr->u.p);
 
     erts_schedule_thr_prgr_later_cleanup_op(
 	scheduled_tw_timer_destroy,
@@ -771,7 +798,7 @@ static void
 tw_proc_timeout(void *vtwtp)
 {
     ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
-    Process *proc = (Process *) twtp->p;
+    Process *proc = (Process *) twtp->u.p;
     if (proc_timeout_common(proc, vtwtp))
 	tw_timer_dec_refc(twtp);
     tw_timer_dec_refc(twtp);
@@ -781,7 +808,7 @@ static void
 tw_port_timeout(void *vtwtp)
 {
     ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
-    Port *port = (Port *) twtp->p;
+    Port *port = (Port *) twtp->u.p;
     if (port_timeout_common(port, vtwtp))
 	tw_timer_dec_refc(twtp);
     tw_timer_dec_refc(twtp);
@@ -801,13 +828,26 @@ cancel_tw_timer(ErtsSchedulerData *esdp, ErtsTWTimer *tmr)
     erts_twheel_cancel_timer(esdp->timer_wheel, &tmr->tw_tmr);
 }
 
+static void
+tw_callback_timeout(void *vtwtp)
+{
+    ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
+    void (*callback)(void *) = twtp->u.callback;
+    void *arg = twtp->head.u.arg;
+    tw_timer_dec_refc(twtp);
+    (*callback)(arg);
+}
+
 static ErtsTWTimer *
 create_tw_timer(ErtsSchedulerData *esdp, 
-		void *p, int is_proc,
+		ErtsTmrType type, void *p,
+		void (*callback)(void *), void *arg,
 		ErtsMonotonicTime timeout_pos)
 {
     ErtsTWTimer *tmr;
     void (*timeout_func)(void *);
+    void (*cancel_func)(void *);
+    erts_aint32_t refc;
 
     tmr = tw_timer_alloc();
     erts_twheel_init_timer(&tmr->tw_tmr);
@@ -815,24 +855,48 @@ create_tw_timer(ErtsSchedulerData *esdp,
     tmr->head.roflgs = (Uint32) esdp->no;
     ERTS_HLT_ASSERT((tmr->head.roflgs
 		     & ~ERTS_TMR_ROFLG_SID_MASK) == 0);
-    tmr->p = p;
-    if (is_proc) {
+
+    switch (type) {
+
+    case ERTS_TMR_PROC:
+	tmr->u.p = p;
 	tmr->head.roflgs |= ERTS_TMR_ROFLG_PROC;
 	timeout_func = tw_proc_timeout;
+	cancel_func = tw_ptimer_cancel;
 	erts_proc_inc_refc((Process *) p);
-    }
-    else {
+	refc = 2;
+	break;
+
+    case ERTS_TMR_PORT:
+	tmr->u.p = p;
 	tmr->head.roflgs |= ERTS_TMR_ROFLG_PORT;
 	timeout_func = tw_port_timeout;
+	cancel_func = tw_ptimer_cancel;
 	erts_port_inc_refc((Port *) p);
+	refc = 2;
+	break;
+
+    case ERTS_TMR_CALLBACK:
+	tmr->head.u.arg = arg;
+	tmr->u.callback = callback;
+
+	tmr->head.roflgs |= ERTS_TMR_ROFLG_CALLBACK;
+	timeout_func = tw_callback_timeout;
+	cancel_func = NULL;
+	refc = 1;
+	break;
+
+    default:
+	ERTS_INTERNAL_ERROR("Unsupported timer type");
+	return NULL;
     }
 
-    erts_smp_atomic32_init_nob(&tmr->head.refc, 2);
+    erts_smp_atomic32_init_nob(&tmr->head.refc, refc);
 
     erts_twheel_set_timer(esdp->timer_wheel,
 			  &tmr->tw_tmr,
 			  timeout_func,
-			  tw_ptimer_cancel,
+			  cancel_func,
 			  tmr,
 			  timeout_pos);
 
@@ -852,8 +916,10 @@ hl_timer_destroy(ErtsHLTimer *tmr)
     else {
 	if (roflgs & ERTS_TMR_ROFLG_PRE_ALC)
 	    bif_timer_pre_free(tmr);
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 	else if (roflgs & ERTS_TMR_ROFLG_ABIF_TMR)
 	    erts_free(ERTS_ALC_T_ABIF_TIMER, tmr);
+#endif
 	else
 	    erts_free(ERTS_ALC_T_BIF_TIMER, tmr);
     }
@@ -948,6 +1014,8 @@ check_canceled_queue(ErtsSchedulerData *esdp, ErtsHLTimerService *srv)
 #endif
 }
 
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
+
 static void
 hlt_delete_abtm(ErtsHLTimer *tmr)
 {
@@ -971,18 +1039,20 @@ hlt_delete_abtm(ErtsHLTimer *tmr)
     }
 }
 
+#endif
+
 static ErtsHLTimer *
 create_hl_timer(ErtsSchedulerData *esdp,
 		ErtsMonotonicTime timeout_pos,
-		int short_time, int is_bif_tmr,
+		int short_time, ErtsTmrType type,
 		void *rcvrp, Eterm rcvr, Eterm acsr,
-		Eterm msg, Uint32 *refn)
+		Eterm msg, Uint32 *refn,
+		void (*callback)(void *), void *arg)
 {
     ErtsHLTimerService *srv = esdp->timer_service;
     ErtsHLTimer *tmr, *st_tmr;
     erts_aint32_t refc;
     Uint32 roflgs;
-    int is_abif_tmr = is_bif_tmr && is_value(acsr) && acsr != rcvr;
 
     check_canceled_queue(esdp, srv);
 
@@ -990,43 +1060,69 @@ create_hl_timer(ErtsSchedulerData *esdp,
 
     roflgs = ((Uint32) esdp->no) | ERTS_TMR_ROFLG_HLT;
 
-    if (!is_bif_tmr)
+    if (type != ERTS_TMR_BIF) {
+
 	tmr = erts_alloc(ERTS_ALC_T_HL_PTIMER,
 			 ERTS_HL_PTIMER_SIZE);
-    else if (short_time) {
-	tmr = bif_timer_pre_alloc();
-	if (!tmr)
-	    goto alloc_bif_timer;
-	roflgs |= ERTS_TMR_ROFLG_PRE_ALC;
-    }
-    else {
-    alloc_bif_timer:
-	if (is_abif_tmr)
-	    tmr = erts_alloc(ERTS_ALC_T_ABIF_TIMER,
-			     ERTS_ABIF_TIMER_SIZE);
-	else
-	    tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER,
-			     ERTS_BIF_TIMER_SIZE);
-    }
+	tmr->timeout = timeout_pos;
 
-    tmr->timeout = timeout_pos;
+	switch (type) {
+
+	case ERTS_TMR_PROC:
+	    ERTS_HLT_ASSERT(is_internal_pid(rcvr));
 
-    if (!is_bif_tmr) {
-	if (is_internal_pid(rcvr)) {
 	    erts_proc_inc_refc((Process *) rcvrp);
 	    tmr->receiver.proc = (Process *) rcvrp;
 	    roflgs |= ERTS_TMR_ROFLG_PROC;
-	}
-	else {
-	    erts_port_inc_refc((Port *) rcvrp);
+	    refc = 2;
+	    break;
+
+	case ERTS_TMR_PORT:
 	    ERTS_HLT_ASSERT(is_internal_port(rcvr));
+	    erts_port_inc_refc((Port *) rcvrp);
 	    tmr->receiver.port = (Port *) rcvrp;
 	    roflgs |= ERTS_TMR_ROFLG_PORT;
+	    refc = 2;
+	    break;
+
+	case ERTS_TMR_CALLBACK:
+	    roflgs |= ERTS_TMR_ROFLG_CALLBACK;
+	    tmr->receiver.callback = callback;
+	    tmr->head.u.arg = arg;
+	    refc = 1;
+	    break;
+
+	default:
+	    ERTS_INTERNAL_ERROR("Unsupported timer type");
+	    return NULL;
 	}
-	refc = 2;
+
     }
-    else {
+    else { /* ERTS_TMR_BIF */
 	Uint hsz;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
+	int is_abif_tmr = is_value(acsr) && acsr != rcvr;
+#endif
+
+	if (short_time) {
+	    tmr = bif_timer_pre_alloc();
+	    if (!tmr)
+		goto alloc_bif_timer;
+	    roflgs |= ERTS_TMR_ROFLG_PRE_ALC;
+	}
+	else {
+	alloc_bif_timer:
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
+	    if (is_abif_tmr)
+		tmr = erts_alloc(ERTS_ALC_T_ABIF_TIMER,
+				 ERTS_ABIF_TIMER_SIZE);
+	    else
+#endif
+		tmr = erts_alloc(ERTS_ALC_T_BIF_TIMER,
+				 ERTS_BIF_TIMER_SIZE);
+	}
+
+	tmr->timeout = timeout_pos;
 
 	roflgs |= ERTS_TMR_ROFLG_BIF_TMR;
 	if (is_internal_pid(rcvr)) {
@@ -1057,6 +1153,8 @@ create_hl_timer(ErtsSchedulerData *esdp,
 	tmr->btm.refn[2] = refn[2];
 
 	tmr->btm.proc_tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
+
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 	if (is_abif_tmr) {
 	    Process *aproc;
 	    roflgs |= ERTS_TMR_ROFLG_ABIF_TMR;
@@ -1071,6 +1169,9 @@ create_hl_timer(ErtsSchedulerData *esdp,
 		erts_smp_proc_unlock(aproc, ERTS_PROC_LOCK_BTM);
 	    }
 	}
+#endif
+
+	btm_rbt_insert(&srv->btm_tree, tmr);
     }
 
     tmr->head.roflgs = roflgs;
@@ -1098,9 +1199,6 @@ create_hl_timer(ErtsSchedulerData *esdp,
     if (st_tmr)
 	same_time_list_insert(&st_tmr->time.tree.same_time, tmr);
 
-    if (is_bif_tmr)
-	btm_rbt_insert(&srv->btm_tree, tmr);
-
 #ifdef ERTS_HLT_HARD_DEBUG
     tmr->pending_timeout = 0;
 #endif
@@ -1120,8 +1218,10 @@ hlt_bif_timer_timeout(ErtsHLTimer *tmr, Uint32 roflgs)
     Uint32 is_reg_name = (roflgs & ERTS_TMR_ROFLG_REG_NAME);
     ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_BIF_TMR);
 
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR)
 	hlt_delete_abtm(tmr);
+#endif
 
     if (is_reg_name) {
 	Eterm pid;
@@ -1203,10 +1303,13 @@ static void hlt_timeout(ErtsHLTimer *tmr, void *vsrv)
 	    hlt_bif_timer_timeout(tmr, roflgs);
 	else if (roflgs & ERTS_TMR_ROFLG_PROC)
 	    hlt_proc_timeout(tmr);
-	else {
-	    ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_PORT);
+	else if (roflgs & ERTS_TMR_ROFLG_PORT)
 	    hlt_port_timeout(tmr);
+	else {
+	    ERTS_HLT_ASSERT(roflgs & ERTS_TMR_ROFLG_CALLBACK);
+	    (*tmr->receiver.callback)(tmr->head.u.arg);
 	}
+
     }
 
     tmr->time.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
@@ -1300,8 +1403,10 @@ hlt_delete_timer(ErtsSchedulerData *esdp, ErtsHLTimer *tmr)
 	    tmr->btm.tree.parent = ERTS_HLT_PFIELD_NOT_IN_TABLE;
 	}
 
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 	if (tmr->head.roflgs & ERTS_TMR_ROFLG_ABIF_TMR)
 	    hlt_delete_abtm(tmr);
+#endif
     }
 
     if (tmr->time.tree.parent == ERTS_HLT_PFIELD_NOT_IN_TABLE) {
@@ -1670,8 +1775,9 @@ setup_bif_timer(Process *c_p, ErtsMonotonicTime timeout_pos,
 
     tmo_msg = wrap ? TUPLE3(tmp_hp, am_timeout, ref, msg) : msg;
 
-    tmr = create_hl_timer(esdp, timeout_pos, short_time, 1, NULL, 
-			  rcvr, acsr, tmo_msg, internal_ref_numbers(ref));
+    tmr = create_hl_timer(esdp, timeout_pos, short_time,
+			  ERTS_TMR_BIF, NULL,  rcvr, acsr, tmo_msg,
+			  internal_ref_numbers(ref), NULL, NULL);
 
     UnUseTmpHeap(4, c_p);
 
@@ -1943,8 +2049,10 @@ try_access_sched_remote_btm(ErtsSchedulerData *esdp,
      */
     erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_BTM);
     tmr = proc_btm_rbt_lookup(c_p->bif_timers, trefn);
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     if (!tmr)
 	tmr = abtm_rbt_lookup(c_p->accessor_bif_timers, trefn);
+#endif
     erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_BTM);
     if (!tmr)
 	return 0;
@@ -2184,11 +2292,13 @@ parse_bif_timer_options(Eterm option_list, int *async, int *info,
 	    if (!abs || !bool_arg(tp[2], abs))
 		return 0;
 	    break;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 	case am_accessor:
 	    if (!accessor || is_not_internal_pid(tp[2]))
 		return 0;
 	    *accessor = tp[2];
 	    break;
+#endif
 	default:
 	    return 0;
 	}
@@ -2250,7 +2360,9 @@ typedef struct {
     ErtsBifTimers *bif_timers;
     union {
 	proc_btm_rbt_yield_state_t proc_btm_yield_state;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
 	abtm_rbt_yield_state_t abtm_yield_state;
+#endif
     } u;
 } ErtsBifTimerYieldState;
 
@@ -2291,6 +2403,8 @@ int erts_cancel_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp)
     return res;
 }
 
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
+
 static void
 detach_bif_timer(ErtsHLTimer *tmr, void *vesdp)
 {
@@ -2335,6 +2449,8 @@ int erts_detach_accessor_bif_timers(Process *p, ErtsBifTimers *btm, void **vyspp
     return res;
 }
 
+#endif /* ERTS_BTM_ACCESSOR_SUPPORT */
+
 static ERTS_INLINE int
 parse_timeout_pos(ErtsSchedulerData *esdp, Eterm arg,
 		  ErtsMonotonicTime *conv_arg, int abs,
@@ -2498,6 +2614,80 @@ BIF_RETTYPE read_timer_2(BIF_ALIST_2)
     return ret;
 }
 
+static void
+start_callback_timer(ErtsSchedulerData *esdp,
+		     int twt,
+		     ErtsMonotonicTime timeout_pos,
+		     void (*callback)(void *),
+		     void *arg)
+		     
+{
+    if (twt)
+	create_tw_timer(esdp, ERTS_TMR_CALLBACK, NULL,
+			callback, arg, timeout_pos);
+    else
+	create_hl_timer(esdp, timeout_pos, 0,
+			ERTS_TMR_CALLBACK, NULL,
+			NIL, THE_NON_VALUE, NIL,
+			NULL, callback, arg);
+}
+
+typedef struct {
+    int twt;
+    ErtsMonotonicTime timeout_pos;
+    void (*callback)(void *);
+    void *arg;
+} ErtsStartCallbackTimerRequest;
+
+static void
+scheduled_start_callback_timer(void *vsctr)
+{
+    ErtsStartCallbackTimerRequest *sctr
+	= (ErtsStartCallbackTimerRequest *) vsctr;
+
+    start_callback_timer(erts_get_scheduler_data(),
+			 sctr->twt,
+			 sctr->timeout_pos,
+			 sctr->callback,	
+			 sctr->arg);
+
+    erts_free(ERTS_ALC_T_TIMER_REQUEST, vsctr);
+}
+
+void
+erts_start_timer_callback(ErtsMonotonicTime tmo,
+			  void (*callback)(void *),
+			  void *arg)
+{
+    ErtsSchedulerData *esdp;
+    ErtsMonotonicTime timeout_pos;
+    int twt;
+
+    esdp = erts_get_scheduler_data();
+    timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp),
+				  tmo);
+    twt = tmo < ERTS_TIMER_WHEEL_MSEC;
+
+    if (esdp)
+	start_callback_timer(esdp,
+			     twt,
+			     timeout_pos,
+			     callback,
+			     arg);
+    else {
+	ErtsStartCallbackTimerRequest *sctr;
+	sctr = erts_alloc(ERTS_ALC_T_TIMER_REQUEST,
+			  sizeof(ErtsStartCallbackTimerRequest));
+	sctr->twt = twt;
+	sctr->timeout_pos = timeout_pos;
+	sctr->callback = callback;
+	sctr->arg = arg;
+	erts_schedule_misc_aux_work(1,
+				    scheduled_start_callback_timer,
+				    (void *) sctr);
+    }
+}
+
 /*
  * Process and Port timer functionality.
  *
@@ -2521,11 +2711,13 @@ set_proc_timer_common(Process *c_p, ErtsSchedulerData *esdp, Sint64 tmo,
 	c_p->flags &= ~F_TIMO;
 
 	if (tmo < ERTS_TIMER_WHEEL_MSEC)
-	    tmr = (void *) create_tw_timer(esdp, (void *) c_p, 1, timeout_pos);
+	    tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PROC, (void *) c_p,
+					   NULL, NULL, timeout_pos);
 	else
-	    tmr = (void *) create_hl_timer(esdp, timeout_pos,
-					   short_time, 0, (void *) c_p,
-					   c_p->common.id, NIL, NIL, NULL);
+	    tmr = (void *) create_hl_timer(esdp, timeout_pos, short_time,
+					   ERTS_TMR_PROC, (void *) c_p,
+					   c_p->common.id, THE_NON_VALUE,
+					   NIL, NULL, NULL, NULL);
 	erts_smp_atomic_set_relb(&c_p->common.timer, (erts_aint_t) tmr);
     }
 }
@@ -2608,13 +2800,12 @@ erts_set_port_timer(Port *c_prt, Sint64 tmo)
     timeout_pos = get_timeout_pos(erts_get_monotonic_time(esdp), tmo);
 
     if (tmo < ERTS_TIMER_WHEEL_MSEC)
-	tmr = (void *) create_tw_timer(esdp, (void *) c_prt, 0,
-				       timeout_pos);
+	tmr = (void *) create_tw_timer(esdp, ERTS_TMR_PORT, (void *) c_prt,
+				       NULL, NULL, timeout_pos);
     else
-	tmr = (void *) create_hl_timer(esdp, timeout_pos, 0, 0,
-				       (void *) c_prt,
-				       c_prt->common.id, NIL, NIL,
-				       NULL);
+	tmr = (void *) create_hl_timer(esdp, timeout_pos, 0, ERTS_TMR_PORT,
+				       (void *) c_prt, c_prt->common.id,
+				       THE_NON_VALUE, NIL, NULL, NULL, NULL);
     erts_smp_atomic_set_relb(&c_prt->common.timer, (erts_aint_t) tmr);
 }
 
@@ -2759,6 +2950,98 @@ erts_debug_bif_timer_foreach(void (*func)(Eterm,
     }
 }
 
+typedef struct {
+    void (*tclbk)(void *);
+    void (*func)(void *,
+		 ErtsMonotonicTime,
+		 void *);
+    void *arg;
+} ErtsDebugForeachCallbackTimer;
+
+static void
+debug_callback_timer_foreach_list(ErtsHLTimer *tmr, void *vdfct)
+{
+    ErtsDebugForeachCallbackTimer *dfct
+	= (ErtsDebugForeachCallbackTimer *) vdfct;
+
+    if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK)
+	&& (tmr->receiver.callback && dfct->tclbk))
+	(*dfct->func)(dfct->arg,
+		      tmr->timeout,
+		      tmr->head.u.arg);
+}
+
+static void
+debug_callback_timer_foreach(ErtsHLTimer *tmr, void *vdfct)
+{
+    ErtsDebugForeachCallbackTimer *dfct
+	= (ErtsDebugForeachCallbackTimer *) vdfct;
+
+    if (tmr->time.tree.same_time)
+	same_time_list_foreach(tmr->time.tree.same_time,
+			       debug_callback_timer_foreach_list,
+			       vdfct);
+
+    if ((tmr->head.roflgs & ERTS_TMR_ROFLG_CALLBACK)
+	&& (tmr->receiver.callback && dfct->tclbk))
+	(*dfct->func)(dfct->arg,
+		      tmr->timeout,
+		      tmr->head.u.arg);
+}
+
+static void
+debug_tw_callback_timer(void *vdfct,
+			ErtsMonotonicTime timeout_pos,
+			void *vtwtp)
+{
+    ErtsTWTimer *twtp = (ErtsTWTimer *) vtwtp;
+    ErtsDebugForeachCallbackTimer *dfct
+	= (ErtsDebugForeachCallbackTimer *) vdfct;
+
+    if (twtp->u.callback == dfct->tclbk)
+	(*dfct->func)(dfct->arg,
+		      timeout_pos,
+		      twtp->head.u.arg);
+}
+
+void
+erts_debug_callback_timer_foreach(void (*tclbk)(void *),
+				  void (*func)(void *,
+					       ErtsMonotonicTime,
+					       void *),
+				  void *arg)
+{
+    int six;
+    ErtsDebugForeachCallbackTimer dfct;
+
+    dfct.tclbk = tclbk;
+    dfct.func = func;
+    dfct.arg = arg;
+
+    if (!erts_smp_thr_progress_is_blocking())
+	ERTS_INTERNAL_ERROR("Not blocking thread progress");
+
+    for (six = 0; six < erts_no_schedulers; six++) {
+	ErtsHLTimerService *srv =
+	    erts_aligned_scheduler_data[six].esd.timer_service;
+	ErtsTimerWheel *twheel =
+	    erts_aligned_scheduler_data[six].esd.timer_wheel;
+
+	erts_twheel_debug_foreach(twheel,
+				  tw_callback_timeout,
+				  debug_tw_callback_timer,
+				  (void *) &dfct);
+
+	if (srv->yield.root)
+	    debug_callback_timer_foreach(srv->yield.root,
+					 (void *) &dfct);
+
+	time_rbt_foreach(srv->btm_tree,
+			 debug_callback_timer_foreach,
+			 (void *) &dfct);
+    }
+}
+
 #ifdef ERTS_HLT_HARD_DEBUG
 
 typedef struct {
diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h
index 30889a71da..24c57fc873 100644
--- a/erts/emulator/beam/erl_hl_timer.h
+++ b/erts/emulator/beam/erl_hl_timer.h
@@ -59,7 +59,9 @@ int erts_cancel_bif_timers(Process *, ErtsBifTimers *, void **);
 int erts_detach_accessor_bif_timers(Process *, ErtsBifTimers *, void **);
 ErtsHLTimerService *erts_create_timer_service(void);
 void erts_hl_timer_init(void);
-
+void erts_start_timer_callback(ErtsMonotonicTime,
+			       void (*)(void *),
+			       void *);
 #ifdef ERTS_SMP
 void
 erts_handle_canceled_timers(void *vesdp,
@@ -76,5 +78,10 @@ void erts_debug_bif_timer_foreach(void (*func)(Eterm,
 					       ErlHeapFragment *,
 					       void *),
 				  void *arg);
-
+void
+erts_debug_callback_timer_foreach(void (*tclbk)(void *),
+				  void (*func)(void *,
+					       ErtsMonotonicTime,
+					       void *),
+				  void *arg);
 #endif /* ERL_HL_TIMER_H__ */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1e7747c509..43e84a1be1 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -165,6 +165,9 @@ Uint erts_no_dirty_cpu_schedulers;
 Uint erts_no_dirty_io_schedulers;
 #endif
 
+static char *erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_NO_FLAGS] = {0};
+int erts_aux_work_no_flags = ERTS_SSI_AUX_WORK_NO_FLAGS;
+
 #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_LAZY		(4*1024*1024)
 #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_LAZY			(512*1024)
 #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_MEDIUM			(64*1024)
@@ -508,6 +511,7 @@ dbg_chk_aux_work_val(erts_aint32_t value)
 #ifdef ERTS_SSI_AUX_WORK_REAP_PORTS
     valid |= ERTS_SSI_AUX_WORK_REAP_PORTS;
 #endif
+    valid |= ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
 
     if (~valid & value)
 	erl_exit(ERTS_ABORT_EXIT,
@@ -566,6 +570,41 @@ erts_pre_init_process(void)
     erts_tsd_key_create(&sched_data_key, "erts_sched_data_key");
 #endif
 
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX]
+	= "DELAYED_AW_WAKEUP";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DD_IX]
+	= "DD";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX]
+	= "DD_THR_PRGR";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX]
+	= "FIX_ALLOC_DEALLOC";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX]
+	= "FIX_ALLOC_LOWER_LIM";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX]
+	= "THR_PRGR_LATER_OP";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX]
+	= "CNCLD_TMRS";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX]
+	= "CNCLD_TMRS_THR_PRGR";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_ASYNC_READY_IX]
+	= "ASYNC_READY";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX]
+	= "ASYNC_READY_CLEAN";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX]
+	= "MISC_THR_PRGR";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_IX]
+	= "MISC";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX]
+	= "CHECK_CHILDREN";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_SET_TMO_IX]
+	= "SET_TMO";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX]
+	= "MSEG_CACHE_CHECK";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_REAP_PORTS_IX]
+	= "REAP_PORTS";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX]
+	= "DEBUG_WAIT_COMPLETED";
+
 #ifdef ERTS_ENABLE_LOCK_CHECK
  {
      int ix;
@@ -1715,11 +1754,6 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin
 	awdp->dd.thr_prgr = wakeup;
 	haw_thr_prgr_soft_wakeup(awdp, wakeup);
     }
-    else if (awdp->dd.completed_callback) {
-	awdp->dd.completed_callback(awdp->dd.completed_arg);
-	awdp->dd.completed_callback = NULL;
-	awdp->dd.completed_arg = NULL;
-    }
     return aux_work & ~ERTS_SSI_AUX_WORK_DD;
 }
 
@@ -1761,11 +1795,6 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i
     }
     else {
 	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
-	if (awdp->dd.completed_callback) {
-	    awdp->dd.completed_callback(awdp->dd.completed_arg);
-	    awdp->dd.completed_callback = NULL;
-	    awdp->dd.completed_arg = NULL;
-	}
     }
 
     return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR;
@@ -1955,78 +1984,142 @@ erts_schedule_thr_prgr_later_cleanup_op(void (*later_func)(void *),
 #endif
 }
 
-#ifdef ERTS_SMP
+static ERTS_INLINE erts_aint32_t
+handle_debug_wait_completed(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
+{
+    ErtsSchedulerSleepInfo *ssi = awdp->ssi;
+    erts_aint32_t saved_aux_work, flags;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
+
+    flags = awdp->debug.wait_completed.flags;
+
+    if (aux_work & flags)
+	return aux_work;
+
+    saved_aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 
-static erts_atomic32_t completed_dealloc_count;
+    if (saved_aux_work & flags)
+	return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
+
+    awdp->debug.wait_completed.callback(awdp->debug.wait_completed.arg);
+
+    awdp->debug.wait_completed.flags = 0;
+    awdp->debug.wait_completed.callback = NULL;
+    awdp->debug.wait_completed.arg = NULL;
+
+    unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED);
+
+    return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
+}
+
+static erts_atomic32_t debug_wait_completed_count;
+static int debug_wait_completed_flags;
 
 static void
-completed_dealloc(void *vproc)
+thr_debug_wait_completed(void *vproc)
 {
-    if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == 0) {
+    if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == 0) {
 	erts_resume((Process *) vproc, (ErtsProcLocks) 0);
 	erts_proc_dec_refc((Process *) vproc);
     }
 }
 
 static void
-setup_completed_dealloc(void *vproc)
+setup_thr_debug_wait_completed(void *vproc)
 {
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
-    ErtsAuxWorkData *awdp = (esdp
-			     ? &esdp->aux_work_data
-			     : aux_thread_aux_work_data);
-    erts_alloc_fix_alloc_shrink(awdp->sched_id, 0);
-    set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_DD);
-    awdp->dd.completed_callback = completed_dealloc;
-    awdp->dd.completed_arg = vproc;
+    ErtsAuxWorkData *awdp;
+    erts_aint32_t wait_flags, aux_work_flags;
+#ifdef ERTS_SMP
+    awdp = esdp ? &esdp->aux_work_data : aux_thread_aux_work_data;
+#else
+    awdp = &esdp->aux_work_data;
+#endif
+
+    wait_flags = 0;
+    aux_work_flags = ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
+
+    if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS) {
+	erts_alloc_fix_alloc_shrink(awdp->sched_id, 0);
+	wait_flags |= (ERTS_SSI_AUX_WORK_DD
+		       | ERTS_SSI_AUX_WORK_DD_THR_PRGR
+		       | ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP);
+#ifdef ERTS_SMP
+	aux_work_flags |= ERTS_SSI_AUX_WORK_DD;
+#endif
+    }
+
+    if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS) {
+	wait_flags |= (ERTS_SSI_AUX_WORK_CNCLD_TMRS
+		       | ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR
+		       | ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP);
+#ifdef ERTS_SMP
+	if (awdp->esdp && !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp))
+	    aux_work_flags |= ERTS_SSI_AUX_WORK_CNCLD_TMRS;
+#endif
+    }
+
+    set_aux_work_flags_wakeup_nob(awdp->ssi, aux_work_flags);
+
+    awdp->debug.wait_completed.flags = wait_flags;
+    awdp->debug.wait_completed.callback = thr_debug_wait_completed;
+    awdp->debug.wait_completed.arg = vproc;
 }
 
 static void
-prep_setup_completed_dealloc(void *vproc)
+prep_setup_thr_debug_wait_completed(void *vproc)
 {
-    erts_aint32_t count = (erts_aint32_t) (erts_no_schedulers+1);
-    if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == count) {
+    erts_aint32_t count = (erts_aint32_t) erts_no_schedulers;
+#ifdef ERTS_SMP
+    count += 1; /* aux thread */
+#endif
+    if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == count) {
 	/* scheduler threads */
 	erts_schedule_multi_misc_aux_work(0,
 					  erts_no_schedulers,
-					  setup_completed_dealloc,
+					  setup_thr_debug_wait_completed,
 					  vproc);
+#ifdef ERTS_SMP
 	/* aux_thread */
 	erts_schedule_misc_aux_work(0,
-				    setup_completed_dealloc,
+				    setup_thr_debug_wait_completed,
 				    vproc);
+#endif
     }
 }
 
-#endif /* ERTS_SMP */
 
 int
-erts_debug_wait_deallocations(Process *c_p)
+erts_debug_wait_completed(Process *c_p, int flags)
 {
-#ifndef ERTS_SMP
-    erts_alloc_fix_alloc_shrink(1, 0);
-    return 1;
-#else
     /* Only one process at a time can do this */
-    erts_aint32_t count = (erts_aint32_t) (2*(erts_no_schedulers+1));
-    if (0 == erts_atomic32_cmpxchg_mb(&completed_dealloc_count,
+    erts_aint32_t count = (erts_aint32_t) (2*erts_no_schedulers);
+#ifdef ERTS_SMP
+    count += 2; /* aux thread */
+#endif
+    if (0 == erts_atomic32_cmpxchg_mb(&debug_wait_completed_count,
 				      count,
 				      0)) {
+	debug_wait_completed_flags = flags;
 	erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
 	erts_proc_inc_refc(c_p);
 	/* scheduler threads */
 	erts_schedule_multi_misc_aux_work(0,
 					  erts_no_schedulers,
-					  prep_setup_completed_dealloc,
+					  prep_setup_thr_debug_wait_completed,
 					  (void *) c_p);
+#ifdef ERTS_SMP
 	/* aux_thread */
 	erts_schedule_misc_aux_work(0,
-				    prep_setup_completed_dealloc,
+				    prep_setup_thr_debug_wait_completed,
 				    (void *) c_p);
+#endif
 	return 1;
     }
     return 0;
-#endif
 }
 
 
@@ -2227,6 +2320,14 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_REAP_PORTS,
 		    handle_reap_ports);
 
+    /*
+     * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED *need* to be
+     * the last flag checked!
+     */
+
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED,
+		    handle_debug_wait_completed);
+
     ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
 
 #ifdef ERTS_SMP
@@ -5354,8 +5455,6 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
     awdp->latest_wakeup = ERTS_THR_PRGR_VAL_FIRST;
     awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
     awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
-    awdp->dd.completed_callback = NULL;
-    awdp->dd.completed_arg = NULL;
     awdp->cncld_tmrs.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
     awdp->later_op.thr_prgr = ERTS_THR_PRGR_VAL_FIRST;
     awdp->later_op.size = 0;
@@ -5386,6 +5485,9 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
 	    awdp->delayed_wakeup.sched2jix[i] = -1;
     }
 #endif
+    awdp->debug.wait_completed.flags = 0;
+    awdp->debug.wait_completed.callback = NULL;
+    awdp->debug.wait_completed.arg = NULL;
 }
 
 static void
@@ -5694,11 +5796,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     init_swtreq_alloc();
 #endif
 
+    erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */
+    debug_wait_completed_flags = 0;
 
 #ifdef ERTS_SMP
 
-    erts_atomic32_init_nob(&completed_dealloc_count, 0); /* debug only */
-
     aux_thread_aux_work_data =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
 					   sizeof(ErtsAuxWorkData));
@@ -10855,7 +10957,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->msg_inq.len = 0;
 #endif
     p->bif_timers = NULL;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     p->accessor_bif_timers = NULL;
+#endif
     p->mbuf = NULL;
     p->mbuf_sz = 0;
     p->psd = NULL;
@@ -11035,7 +11139,9 @@ void erts_init_empty_process(Process *p)
     p->msg.save = &p->msg.first;
     p->msg.len = 0;
     p->bif_timers = NULL;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     p->accessor_bif_timers = NULL;
+#endif
     p->dictionary = NULL;
     p->seq_trace_clock = 0;
     p->seq_trace_lastcnt = 0;
@@ -11130,7 +11236,9 @@ erts_debug_verify_clean_empty_process(Process* p)
     ASSERT(p->msg.first == NULL);
     ASSERT(p->msg.len == 0);
     ASSERT(p->bif_timers == NULL);
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     ASSERT(p->accessor_bif_timers == NULL);
+#endif
     ASSERT(p->dictionary == NULL);
     ASSERT(p->catches == 0);
     ASSERT(p->cp == NULL);
@@ -12086,6 +12194,7 @@ erts_continue_exit_process(Process *p)
 	p->bif_timers = NULL;
     }
 
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     if (p->accessor_bif_timers) {
 	if (erts_detach_accessor_bif_timers(p,
 					    p->accessor_bif_timers,
@@ -12096,6 +12205,7 @@ erts_continue_exit_process(Process *p)
 	ASSERT(erts_proc_read_refc(p) > 0);
 	p->accessor_bif_timers = NULL;
     }
+#endif
 
 #ifdef ERTS_SMP
     if (p->flags & F_HAVE_BLCKD_MSCHED) {
@@ -12458,45 +12568,13 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) {
 
     flg = erts_atomic32_read_dirty(&esdp->ssi->aux_work);
     erts_print(to, to_arg, "Scheduler Sleep Info Aux Work: ");
-    for (i = 0; i < ERTS_SSI_AUX_WORK_MAX && flg; i++) {
+    for (i = 0; i < ERTS_SSI_AUX_WORK_NO_FLAGS && flg; i++) {
         erts_aint32_t chk = (1 << i);
         if (flg & chk) {
-            switch (chk) {
-            case ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP:
-                erts_print(to, to_arg, "DELAYED_AW_WAKEUP"); break;
-            case ERTS_SSI_AUX_WORK_DD:
-                erts_print(to, to_arg, "DELAYED_DEALLOC"); break;
-            case ERTS_SSI_AUX_WORK_DD_THR_PRGR:
-                erts_print(to, to_arg, "DELAYED_DEALLOC_THR_PRGR"); break;
-            case ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC:
-                erts_print(to, to_arg, "FIX_ALLOC_DEALLOC"); break;
-            case ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM:
-                erts_print(to, to_arg, "FIX_ALLOC_LOWER_LIM"); break;
-            case ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP:
-                erts_print(to, to_arg, "THR_PRGR_LATER_OP"); break;
-            case ERTS_SSI_AUX_WORK_CNCLD_TMRS:
-                erts_print(to, to_arg, "CANCELED_TIMERS"); break;
-            case ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR:
-                erts_print(to, to_arg, "CANCELED_TIMERS_THR_PRGR"); break;
-            case ERTS_SSI_AUX_WORK_ASYNC_READY:
-                erts_print(to, to_arg, "ASYNC_READY"); break;
-            case ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN:
-                erts_print(to, to_arg, "ASYNC_READY_CLEAN"); break;
-            case ERTS_SSI_AUX_WORK_MISC_THR_PRGR:
-                erts_print(to, to_arg, "MISC_THR_PRGR"); break;
-            case ERTS_SSI_AUX_WORK_MISC:
-                erts_print(to, to_arg, "MISC"); break;
-            case ERTS_SSI_AUX_WORK_CHECK_CHILDREN:
-                erts_print(to, to_arg, "CHECK_CHILDREN"); break;
-            case ERTS_SSI_AUX_WORK_SET_TMO:
-                erts_print(to, to_arg, "SET_TMO"); break;
-            case ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK:
-                erts_print(to, to_arg, "MSEG_CACHE_CHECK"); break;
-            case ERTS_SSI_AUX_WORK_REAP_PORTS:
-                erts_print(to, to_arg, "REAP_PORTS"); break;
-            default:
-                erts_print(to, to_arg, "UNKNOWN(%d)", flg); break;
-            }
+	    if (erts_aux_work_flag_descr[i])
+                erts_print(to, to_arg, "%s", erts_aux_work_flag_descr[i]);
+	    else
+                erts_print(to, to_arg, "1<<%d", i);
             if (flg > chk)
                 erts_print(to, to_arg, " | ");
             flg -= chk;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index b1c30e7652..1da1239609 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -268,28 +268,73 @@ typedef enum {
   | ERTS_SSI_FLG_SUSPENDED)
 
 /*
- * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative
- * eachother. Most frequent - lowest bit number.
+ * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
+ * order relative eachother. Most frequent at lowest at lowest
+ * index.
+ *
+ * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX *need* to be
+ * highest index...
+ *
+ * Remember to update description in erts_pre_init_process()
+ * when adding new flags...
  */
 
-#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP	(((erts_aint32_t) 1) << 0)
-#define ERTS_SSI_AUX_WORK_DD			(((erts_aint32_t) 1) << 1)
-#define ERTS_SSI_AUX_WORK_DD_THR_PRGR		(((erts_aint32_t) 1) << 2)
-#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC	(((erts_aint32_t) 1) << 3)
-#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM	(((erts_aint32_t) 1) << 4)
-#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP	(((erts_aint32_t) 1) << 5)
-#define ERTS_SSI_AUX_WORK_CNCLD_TMRS		(((erts_aint32_t) 1) << 6)
-#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR	(((erts_aint32_t) 1) << 7)
-#define ERTS_SSI_AUX_WORK_ASYNC_READY		(((erts_aint32_t) 1) << 8)
-#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN	(((erts_aint32_t) 1) << 9)
-#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR		(((erts_aint32_t) 1) << 10)
-#define ERTS_SSI_AUX_WORK_MISC			(((erts_aint32_t) 1) << 11)
-#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN	(((erts_aint32_t) 1) << 12)
-#define ERTS_SSI_AUX_WORK_SET_TMO		(((erts_aint32_t) 1) << 13)
-#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK	(((erts_aint32_t) 1) << 14)
-#define ERTS_SSI_AUX_WORK_REAP_PORTS		(((erts_aint32_t) 1) << 15)
-
-#define ERTS_SSI_AUX_WORK_MAX                                           16
+typedef enum {
+    ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX,
+    ERTS_SSI_AUX_WORK_DD_IX,
+    ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX,
+    ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX,
+    ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX,
+    ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX,
+    ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX,
+    ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX,
+    ERTS_SSI_AUX_WORK_ASYNC_READY_IX,
+    ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX,
+    ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX,
+    ERTS_SSI_AUX_WORK_MISC_IX,
+    ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX,
+    ERTS_SSI_AUX_WORK_SET_TMO_IX,
+    ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX,
+    ERTS_SSI_AUX_WORK_REAP_PORTS_IX,
+    ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX, /* SHOULD be last flag index */
+
+    ERTS_SSI_AUX_WORK_NO_FLAGS /* Not a flag index... */
+} ErtsSsiAuxWorkFlagIndex;
+
+#define ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX)
+#define ERTS_SSI_AUX_WORK_DD \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_IX)
+#define ERTS_SSI_AUX_WORK_DD_THR_PRGR \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX)
+#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX)
+#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX)
+#define ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX)
+#define ERTS_SSI_AUX_WORK_CNCLD_TMRS \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX)
+#define ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX)
+#define ERTS_SSI_AUX_WORK_ASYNC_READY \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_IX)
+#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX)
+#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX)
+#define ERTS_SSI_AUX_WORK_MISC \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MISC_IX)
+#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX)
+#define ERTS_SSI_AUX_WORK_SET_TMO \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_SET_TMO_IX)
+#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX)
+#define ERTS_SSI_AUX_WORK_REAP_PORTS \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_REAP_PORTS_IX)
+#define ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED \
+    (((erts_aint32_t) 1) << ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX)
 
 typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
 
@@ -515,8 +560,6 @@ typedef struct {
 #ifdef ERTS_SMP
     struct {
 	ErtsThrPrgrVal thr_prgr;
-	void (*completed_callback)(void *);
-	void (*completed_arg)(void *);
     } dd;
     struct {
 	ErtsThrPrgrVal thr_prgr;
@@ -545,6 +588,13 @@ typedef struct {
 	ErtsDelayedAuxWorkWakeupJob *job;
     } delayed_wakeup;
 #endif
+    struct {
+	struct {
+	    erts_aint32_t flags;
+	    void (*callback)(void *);
+	    void *arg;
+	} wait_completed;
+    } debug;
 } ErtsAuxWorkData;
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -925,7 +975,9 @@ struct process {
     ErlMessageQueue msg;	/* Message queue */
 
     ErtsBifTimers *bif_timers;	/* Bif timers aiming at this process */
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     ErtsBifTimers *accessor_bif_timers;	/* Accessor bif timers */
+#endif
 
     ProcDict  *dictionary;       /* Process dictionary, may be NULL */
 
@@ -1692,7 +1744,11 @@ Eterm erts_get_reader_groups_map(Process *c_p);
 Eterm erts_debug_reader_groups_map(Process *c_p, int groups);
 
 Uint erts_debug_nbalance(void);
-int erts_debug_wait_deallocations(Process *c_p);
+
+#define ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS		(1 << 0)
+#define ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS	(1 << 1)
+
+int erts_debug_wait_completed(Process *c_p, int flags);
 
 Uint erts_process_memory(Process *c_p);
 
diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h
index 4560cd23af..e64b86496a 100644
--- a/erts/emulator/beam/erl_time.h
+++ b/erts/emulator/beam/erl_time.h
@@ -453,4 +453,12 @@ ERTS_GLB_INLINE ErtsMonotonicTime erts_next_timeout_time(ErtsNextTimeoutRef nxt_
 
 #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
 
+void
+erts_twheel_debug_foreach(ErtsTimerWheel *tiw,
+			  void (*tclbk)(void *),
+			  void (*func)(void *,
+				       ErtsMonotonicTime,
+				       void *),
+			  void *arg);
+
 #endif /* timer wheel api */
diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c
index 8bffdedb2b..fe7c5826f5 100644
--- a/erts/emulator/beam/time.c
+++ b/erts/emulator/beam/time.c
@@ -529,6 +529,9 @@ erts_create_timer_wheel(ErtsSchedulerData *esdp)
     tiw->next_timeout_time = mtime + ERTS_MONOTONIC_DAY;
     tiw->sentinel.next = &tiw->sentinel;
     tiw->sentinel.prev = &tiw->sentinel;
+    tiw->sentinel.u.func.timeout = NULL;
+    tiw->sentinel.u.func.cancel = NULL;
+    tiw->sentinel.u.func.arg = NULL;
     return tiw;
 }
 
@@ -624,6 +627,41 @@ erts_twheel_cancel_timer(ErtsTimerWheel *tiw, ErtsTWheelTimer *p)
     }
 }
 
+void
+erts_twheel_debug_foreach(ErtsTimerWheel *tiw,
+			  void (*tclbk)(void *),
+			  void (*func)(void *,
+				       ErtsMonotonicTime,
+				       void *),
+			  void *arg)
+{
+    ErtsTWheelTimer *tmr;
+    int ix;
+
+    tmr = tiw->sentinel.next;
+    while (tmr != &tiw->sentinel) {
+	if (tmr->u.func.timeout == tclbk)
+	    (*func)(arg, tmr->timeout_pos, tmr->u.func.arg);
+	tmr = tmr->next;
+    }
+
+    for (tmr = tiw->at_once.head; tmr; tmr = tmr->next) {
+	if (tmr->u.func.timeout == tclbk)
+	    (*func)(arg, tmr->timeout_pos, tmr->u.func.arg);
+    }
+
+    for (ix = 0; ix < ERTS_TIW_SIZE; ix++) {
+	tmr = tiw->w[ix];
+	if (tmr) {
+	    do {
+		if (tmr->u.func.timeout == tclbk)
+		    (*func)(arg, tmr->timeout_pos, tmr->u.func.arg);
+		tmr = tmr->next;
+	    } while (tmr != tiw->w[ix]);
+	}
+    }
+}
+
 #ifdef ERTS_TW_DEBUG
 void erts_p_slpq(void)
 {