Merge branch 'rickard/timer-improvement/OTP-12650'

* rickard/timer-improvement/OTP-12650:
  Callback timer
  Disable accessor timer option
  Aux work flag descriptions
  Fix test cases

1 files changed, 157 insertions, 79 deletions

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 1e7747c509..43e84a1be1 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -165,6 +165,9 @@ Uint erts_no_dirty_cpu_schedulers;
 Uint erts_no_dirty_io_schedulers;
 #endif
 
+static char *erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_NO_FLAGS] = {0};
+int erts_aux_work_no_flags = ERTS_SSI_AUX_WORK_NO_FLAGS;
+
 #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_VERY_LAZY		(4*1024*1024)
 #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_LAZY			(512*1024)
 #define ERTS_THR_PRGR_LATER_CLEANUP_OP_THRESHOLD_MEDIUM			(64*1024)
@@ -508,6 +511,7 @@ dbg_chk_aux_work_val(erts_aint32_t value)
 #ifdef ERTS_SSI_AUX_WORK_REAP_PORTS
     valid |= ERTS_SSI_AUX_WORK_REAP_PORTS;
 #endif
+    valid |= ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
 
     if (~valid & value)
 	erl_exit(ERTS_ABORT_EXIT,
@@ -566,6 +570,41 @@ erts_pre_init_process(void)
     erts_tsd_key_create(&sched_data_key, "erts_sched_data_key");
 #endif
 
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP_IX]
+	= "DELAYED_AW_WAKEUP";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DD_IX]
+	= "DD";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DD_THR_PRGR_IX]
+	= "DD_THR_PRGR";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC_IX]
+	= "FIX_ALLOC_DEALLOC";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM_IX]
+	= "FIX_ALLOC_LOWER_LIM";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP_IX]
+	= "THR_PRGR_LATER_OP";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CNCLD_TMRS_IX]
+	= "CNCLD_TMRS";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR_IX]
+	= "CNCLD_TMRS_THR_PRGR";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_ASYNC_READY_IX]
+	= "ASYNC_READY";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN_IX]
+	= "ASYNC_READY_CLEAN";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_THR_PRGR_IX]
+	= "MISC_THR_PRGR";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MISC_IX]
+	= "MISC";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_CHECK_CHILDREN_IX]
+	= "CHECK_CHILDREN";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_SET_TMO_IX]
+	= "SET_TMO";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK_IX]
+	= "MSEG_CACHE_CHECK";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_REAP_PORTS_IX]
+	= "REAP_PORTS";
+    erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED_IX]
+	= "DEBUG_WAIT_COMPLETED";
+
 #ifdef ERTS_ENABLE_LOCK_CHECK
  {
      int ix;
@@ -1715,11 +1754,6 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waitin
 	awdp->dd.thr_prgr = wakeup;
 	haw_thr_prgr_soft_wakeup(awdp, wakeup);
     }
-    else if (awdp->dd.completed_callback) {
-	awdp->dd.completed_callback(awdp->dd.completed_arg);
-	awdp->dd.completed_callback = NULL;
-	awdp->dd.completed_arg = NULL;
-    }
     return aux_work & ~ERTS_SSI_AUX_WORK_DD;
 }
 
@@ -1761,11 +1795,6 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, i
     }
     else {
 	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
-	if (awdp->dd.completed_callback) {
-	    awdp->dd.completed_callback(awdp->dd.completed_arg);
-	    awdp->dd.completed_callback = NULL;
-	    awdp->dd.completed_arg = NULL;
-	}
     }
 
     return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR;
@@ -1955,78 +1984,142 @@ erts_schedule_thr_prgr_later_cleanup_op(void (*later_func)(void *),
 #endif
 }
 
-#ifdef ERTS_SMP
+static ERTS_INLINE erts_aint32_t
+handle_debug_wait_completed(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
+{
+    ErtsSchedulerSleepInfo *ssi = awdp->ssi;
+    erts_aint32_t saved_aux_work, flags;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ASSERT(!awdp->esdp || !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp));
+#endif
+
+    flags = awdp->debug.wait_completed.flags;
+
+    if (aux_work & flags)
+	return aux_work;
+
+    saved_aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 
-static erts_atomic32_t completed_dealloc_count;
+    if (saved_aux_work & flags)
+	return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
+
+    awdp->debug.wait_completed.callback(awdp->debug.wait_completed.arg);
+
+    awdp->debug.wait_completed.flags = 0;
+    awdp->debug.wait_completed.callback = NULL;
+    awdp->debug.wait_completed.arg = NULL;
+
+    unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED);
+
+    return aux_work & ~ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
+}
+
+static erts_atomic32_t debug_wait_completed_count;
+static int debug_wait_completed_flags;
 
 static void
-completed_dealloc(void *vproc)
+thr_debug_wait_completed(void *vproc)
 {
-    if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == 0) {
+    if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == 0) {
 	erts_resume((Process *) vproc, (ErtsProcLocks) 0);
 	erts_proc_dec_refc((Process *) vproc);
     }
 }
 
 static void
-setup_completed_dealloc(void *vproc)
+setup_thr_debug_wait_completed(void *vproc)
 {
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
-    ErtsAuxWorkData *awdp = (esdp
-			     ? &esdp->aux_work_data
-			     : aux_thread_aux_work_data);
-    erts_alloc_fix_alloc_shrink(awdp->sched_id, 0);
-    set_aux_work_flags_wakeup_nob(awdp->ssi, ERTS_SSI_AUX_WORK_DD);
-    awdp->dd.completed_callback = completed_dealloc;
-    awdp->dd.completed_arg = vproc;
+    ErtsAuxWorkData *awdp;
+    erts_aint32_t wait_flags, aux_work_flags;
+#ifdef ERTS_SMP
+    awdp = esdp ? &esdp->aux_work_data : aux_thread_aux_work_data;
+#else
+    awdp = &esdp->aux_work_data;
+#endif
+
+    wait_flags = 0;
+    aux_work_flags = ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED;
+
+    if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_DEALLOCATIONS) {
+	erts_alloc_fix_alloc_shrink(awdp->sched_id, 0);
+	wait_flags |= (ERTS_SSI_AUX_WORK_DD
+		       | ERTS_SSI_AUX_WORK_DD_THR_PRGR
+		       | ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP);
+#ifdef ERTS_SMP
+	aux_work_flags |= ERTS_SSI_AUX_WORK_DD;
+#endif
+    }
+
+    if (debug_wait_completed_flags & ERTS_DEBUG_WAIT_COMPLETED_TIMER_CANCELLATIONS) {
+	wait_flags |= (ERTS_SSI_AUX_WORK_CNCLD_TMRS
+		       | ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR
+		       | ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP);
+#ifdef ERTS_SMP
+	if (awdp->esdp && !ERTS_SCHEDULER_IS_DIRTY(awdp->esdp))
+	    aux_work_flags |= ERTS_SSI_AUX_WORK_CNCLD_TMRS;
+#endif
+    }
+
+    set_aux_work_flags_wakeup_nob(awdp->ssi, aux_work_flags);
+
+    awdp->debug.wait_completed.flags = wait_flags;
+    awdp->debug.wait_completed.callback = thr_debug_wait_completed;
+    awdp->debug.wait_completed.arg = vproc;
 }
 
 static void
-prep_setup_completed_dealloc(void *vproc)
+prep_setup_thr_debug_wait_completed(void *vproc)
 {
-    erts_aint32_t count = (erts_aint32_t) (erts_no_schedulers+1);
-    if (erts_atomic32_dec_read_mb(&completed_dealloc_count) == count) {
+    erts_aint32_t count = (erts_aint32_t) erts_no_schedulers;
+#ifdef ERTS_SMP
+    count += 1; /* aux thread */
+#endif
+    if (erts_atomic32_dec_read_mb(&debug_wait_completed_count) == count) {
 	/* scheduler threads */
 	erts_schedule_multi_misc_aux_work(0,
 					  erts_no_schedulers,
-					  setup_completed_dealloc,
+					  setup_thr_debug_wait_completed,
 					  vproc);
+#ifdef ERTS_SMP
 	/* aux_thread */
 	erts_schedule_misc_aux_work(0,
-				    setup_completed_dealloc,
+				    setup_thr_debug_wait_completed,
 				    vproc);
+#endif
     }
 }
 
-#endif /* ERTS_SMP */
 
 int
-erts_debug_wait_deallocations(Process *c_p)
+erts_debug_wait_completed(Process *c_p, int flags)
 {
-#ifndef ERTS_SMP
-    erts_alloc_fix_alloc_shrink(1, 0);
-    return 1;
-#else
     /* Only one process at a time can do this */
-    erts_aint32_t count = (erts_aint32_t) (2*(erts_no_schedulers+1));
-    if (0 == erts_atomic32_cmpxchg_mb(&completed_dealloc_count,
+    erts_aint32_t count = (erts_aint32_t) (2*erts_no_schedulers);
+#ifdef ERTS_SMP
+    count += 2; /* aux thread */
+#endif
+    if (0 == erts_atomic32_cmpxchg_mb(&debug_wait_completed_count,
 				      count,
 				      0)) {
+	debug_wait_completed_flags = flags;
 	erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL);
 	erts_proc_inc_refc(c_p);
 	/* scheduler threads */
 	erts_schedule_multi_misc_aux_work(0,
 					  erts_no_schedulers,
-					  prep_setup_completed_dealloc,
+					  prep_setup_thr_debug_wait_completed,
 					  (void *) c_p);
+#ifdef ERTS_SMP
 	/* aux_thread */
 	erts_schedule_misc_aux_work(0,
-				    prep_setup_completed_dealloc,
+				    prep_setup_thr_debug_wait_completed,
 				    (void *) c_p);
+#endif
 	return 1;
     }
     return 0;
-#endif
 }
 
 
@@ -2227,6 +2320,14 @@ handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work, int waiting)
     HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_REAP_PORTS,
 		    handle_reap_ports);
 
+    /*
+     * ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED *need* to be
+     * the last flag checked!
+     */
+
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED,
+		    handle_debug_wait_completed);
+
     ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
 
 #ifdef ERTS_SMP
@@ -5354,8 +5455,6 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
     awdp->latest_wakeup = ERTS_THR_PRGR_VAL_FIRST;
     awdp->misc.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
     awdp->dd.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
-    awdp->dd.completed_callback = NULL;
-    awdp->dd.completed_arg = NULL;
     awdp->cncld_tmrs.thr_prgr = ERTS_THR_PRGR_VAL_WAITING;
     awdp->later_op.thr_prgr = ERTS_THR_PRGR_VAL_FIRST;
     awdp->later_op.size = 0;
@@ -5386,6 +5485,9 @@ init_aux_work_data(ErtsAuxWorkData *awdp, ErtsSchedulerData *esdp, char *dawwp)
 	    awdp->delayed_wakeup.sched2jix[i] = -1;
     }
 #endif
+    awdp->debug.wait_completed.flags = 0;
+    awdp->debug.wait_completed.callback = NULL;
+    awdp->debug.wait_completed.arg = NULL;
 }
 
 static void
@@ -5694,11 +5796,11 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
     init_swtreq_alloc();
 #endif
 
+    erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */
+    debug_wait_completed_flags = 0;
 
 #ifdef ERTS_SMP
 
-    erts_atomic32_init_nob(&completed_dealloc_count, 0); /* debug only */
-
     aux_thread_aux_work_data =
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
 					   sizeof(ErtsAuxWorkData));
@@ -10855,7 +10957,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader).
     p->msg_inq.len = 0;
 #endif
     p->bif_timers = NULL;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     p->accessor_bif_timers = NULL;
+#endif
     p->mbuf = NULL;
     p->mbuf_sz = 0;
     p->psd = NULL;
@@ -11035,7 +11139,9 @@ void erts_init_empty_process(Process *p)
     p->msg.save = &p->msg.first;
     p->msg.len = 0;
     p->bif_timers = NULL;
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     p->accessor_bif_timers = NULL;
+#endif
     p->dictionary = NULL;
     p->seq_trace_clock = 0;
     p->seq_trace_lastcnt = 0;
@@ -11130,7 +11236,9 @@ erts_debug_verify_clean_empty_process(Process* p)
     ASSERT(p->msg.first == NULL);
     ASSERT(p->msg.len == 0);
     ASSERT(p->bif_timers == NULL);
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     ASSERT(p->accessor_bif_timers == NULL);
+#endif
     ASSERT(p->dictionary == NULL);
     ASSERT(p->catches == 0);
     ASSERT(p->cp == NULL);
@@ -12086,6 +12194,7 @@ erts_continue_exit_process(Process *p)
 	p->bif_timers = NULL;
     }
 
+#ifdef ERTS_BTM_ACCESSOR_SUPPORT
     if (p->accessor_bif_timers) {
 	if (erts_detach_accessor_bif_timers(p,
 					    p->accessor_bif_timers,
@@ -12096,6 +12205,7 @@ erts_continue_exit_process(Process *p)
 	ASSERT(erts_proc_read_refc(p) > 0);
 	p->accessor_bif_timers = NULL;
     }
+#endif
 
 #ifdef ERTS_SMP
     if (p->flags & F_HAVE_BLCKD_MSCHED) {
@@ -12458,45 +12568,13 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) {
 
     flg = erts_atomic32_read_dirty(&esdp->ssi->aux_work);
     erts_print(to, to_arg, "Scheduler Sleep Info Aux Work: ");
-    for (i = 0; i < ERTS_SSI_AUX_WORK_MAX && flg; i++) {
+    for (i = 0; i < ERTS_SSI_AUX_WORK_NO_FLAGS && flg; i++) {
         erts_aint32_t chk = (1 << i);
         if (flg & chk) {
-            switch (chk) {
-            case ERTS_SSI_AUX_WORK_DELAYED_AW_WAKEUP:
-                erts_print(to, to_arg, "DELAYED_AW_WAKEUP"); break;
-            case ERTS_SSI_AUX_WORK_DD:
-                erts_print(to, to_arg, "DELAYED_DEALLOC"); break;
-            case ERTS_SSI_AUX_WORK_DD_THR_PRGR:
-                erts_print(to, to_arg, "DELAYED_DEALLOC_THR_PRGR"); break;
-            case ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC:
-                erts_print(to, to_arg, "FIX_ALLOC_DEALLOC"); break;
-            case ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM:
-                erts_print(to, to_arg, "FIX_ALLOC_LOWER_LIM"); break;
-            case ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP:
-                erts_print(to, to_arg, "THR_PRGR_LATER_OP"); break;
-            case ERTS_SSI_AUX_WORK_CNCLD_TMRS:
-                erts_print(to, to_arg, "CANCELED_TIMERS"); break;
-            case ERTS_SSI_AUX_WORK_CNCLD_TMRS_THR_PRGR:
-                erts_print(to, to_arg, "CANCELED_TIMERS_THR_PRGR"); break;
-            case ERTS_SSI_AUX_WORK_ASYNC_READY:
-                erts_print(to, to_arg, "ASYNC_READY"); break;
-            case ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN:
-                erts_print(to, to_arg, "ASYNC_READY_CLEAN"); break;
-            case ERTS_SSI_AUX_WORK_MISC_THR_PRGR:
-                erts_print(to, to_arg, "MISC_THR_PRGR"); break;
-            case ERTS_SSI_AUX_WORK_MISC:
-                erts_print(to, to_arg, "MISC"); break;
-            case ERTS_SSI_AUX_WORK_CHECK_CHILDREN:
-                erts_print(to, to_arg, "CHECK_CHILDREN"); break;
-            case ERTS_SSI_AUX_WORK_SET_TMO:
-                erts_print(to, to_arg, "SET_TMO"); break;
-            case ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK:
-                erts_print(to, to_arg, "MSEG_CACHE_CHECK"); break;
-            case ERTS_SSI_AUX_WORK_REAP_PORTS:
-                erts_print(to, to_arg, "REAP_PORTS"); break;
-            default:
-                erts_print(to, to_arg, "UNKNOWN(%d)", flg); break;
-            }
+	    if (erts_aux_work_flag_descr[i])
+                erts_print(to, to_arg, "%s", erts_aux_work_flag_descr[i]);
+	    else
+                erts_print(to, to_arg, "1<<%d", i);
             if (flg > chk)
                 erts_print(to, to_arg, " | ");
             flg -= chk;