Merge branch 'rickard/barriers/OTP-9922' into maint

* rickard/barriers/OTP-9922:
  Reduce thread progress read operations in handle_aux_work()
  Misc memory barrier fixes

1 files changed, 132 insertions, 91 deletions

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index eedbb35486..138acfeb2c 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -347,40 +347,25 @@ dbg_chk_aux_work_val(erts_aint32_t value)
 {
     erts_aint32_t valid = 0;
 
-#ifdef ERTS_SSI_AUX_WORK_SET_TMO
     valid |= ERTS_SSI_AUX_WORK_SET_TMO;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_CHECK_CHILDREN
-    valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_MISC
     valid |= ERTS_SSI_AUX_WORK_MISC;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_MISC_THR_PRGR
-    valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY
-    valid |= ERTS_SSI_AUX_WORK_ASYNC_READY;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN
-    valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
-#endif
-
-#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
     valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC
     valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC;
+#if ERTS_USE_ASYNC_READY_Q
+    valid |= ERTS_SSI_AUX_WORK_ASYNC_READY;
+    valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
 #endif
-#ifdef ERTS_SSI_AUX_WORK_DD
+#ifdef ERTS_SMP
+    valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
     valid |= ERTS_SSI_AUX_WORK_DD;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_DD
     valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR;
 #endif
-#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK
+#if HAVE_ERTS_MSEG
     valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK;
 #endif
+#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
+    valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
+#endif
 
     if (~valid & value)
 	erl_exit(ERTS_ABORT_EXIT,
@@ -911,6 +896,27 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs)
     return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs);
 }
 
+#ifdef ERTS_SMP
+
+static ERTS_INLINE void
+thr_prgr_current_reset(ErtsAuxWorkData *awdp)
+{
+    awdp->current_thr_prgr = ERTS_THR_PRGR_INVALID;
+}
+
+static ERTS_INLINE ErtsThrPrgrVal
+thr_prgr_current(ErtsAuxWorkData *awdp)
+{
+    ErtsThrPrgrVal current = awdp->current_thr_prgr;
+    if (current == ERTS_THR_PRGR_INVALID) {
+	current = erts_thr_progress_current();
+	awdp->current_thr_prgr = current;
+    }
+    return current;
+}
+
+#endif
+
 typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t;
 struct erts_misc_aux_work_t_ {
     void (*func)(void *);
@@ -983,7 +989,7 @@ misc_aux_work_clean(ErtsThrQ_t *q,
     return aux_work;
 }
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_misc_aux_work(ErtsAuxWorkData *awdp,
 		     erts_aint32_t aux_work)
 {
@@ -1003,12 +1009,13 @@ handle_misc_aux_work(ErtsAuxWorkData *awdp,
 
 #ifdef ERTS_SMP
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp,
 			      erts_aint32_t aux_work)
 {
-    if (!erts_thr_progress_has_reached(awdp->misc.thr_prgr))
-	return aux_work;
+    if (!erts_thr_progress_has_reached_this(thr_prgr_current(awdp),
+					    awdp->misc.thr_prgr))
+	return aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
 
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR);
 
@@ -1081,7 +1088,7 @@ erts_notify_check_async_ready_queue(void *vno)
 				  ERTS_SSI_AUX_WORK_ASYNC_READY);
 }
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_async_ready(ErtsAuxWorkData *awdp,
 		   erts_aint32_t aux_work)
 {
@@ -1103,7 +1110,7 @@ handle_async_ready(ErtsAuxWorkData *awdp,
 	    | ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN);
 }
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_async_ready_clean(ErtsAuxWorkData *awdp,
 			 erts_aint32_t aux_work)
 {
@@ -1111,7 +1118,8 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp,
 
 #ifdef ERTS_SMP
     if (awdp->async_ready.need_thr_prgr
-	&& !erts_thr_progress_has_reached(awdp->async_ready.thr_prgr)) {
+	&& !erts_thr_progress_has_reached_this(thr_prgr_current(awdp),
+					       awdp->async_ready.thr_prgr)) {
 	return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
     }
 
@@ -1139,7 +1147,7 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp,
 
 #endif
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 {
     ErtsSchedulerSleepInfo *ssi = awdp->ssi;
@@ -1167,16 +1175,18 @@ erts_alloc_notify_delayed_dealloc(int ix)
 				  ERTS_SSI_AUX_WORK_DD);
 }
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 {
     ErtsSchedulerSleepInfo *ssi = awdp->ssi;
     int need_thr_progress = 0;
+    ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
     int more_work = 0;
 
     unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
     erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
 						&need_thr_progress,
+						&wakeup,
 						&more_work);
     if (more_work) {
 	if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD)
@@ -1188,9 +1198,12 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
     }
 
     if (need_thr_progress) {
+	if (wakeup == ERTS_THR_PRGR_INVALID)
+	    wakeup = erts_thr_progress_later_than(thr_prgr_current(awdp));
+	awdp->dd.thr_prgr = wakeup;
 	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
-	awdp->dd.thr_prgr = erts_thr_progress_later();
-	erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr);
+	awdp->dd.thr_prgr = wakeup;
+	erts_thr_progress_wakeup(awdp->esdp, wakeup);
     }
     else if (awdp->dd.completed_callback) {
 	awdp->dd.completed_callback(awdp->dd.completed_arg);
@@ -1200,14 +1213,16 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
     return aux_work & ~ERTS_SSI_AUX_WORK_DD;
 }
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 {
     ErtsSchedulerSleepInfo *ssi;
     int need_thr_progress;
     int more_work;
+    ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
+    ErtsThrPrgrVal current = thr_prgr_current(awdp);
 
-    if (!erts_thr_progress_has_reached(awdp->dd.thr_prgr))
+    if (!erts_thr_progress_has_reached_this(current, awdp->dd.thr_prgr))
 	return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR;
 
     ssi = awdp->ssi;
@@ -1216,6 +1231,7 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 
     erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
 						&need_thr_progress,
+						&wakeup,
 						&more_work);
     if (more_work) {
 	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
@@ -1225,8 +1241,10 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
     }
 
     if (need_thr_progress) {
-	awdp->dd.thr_prgr = erts_thr_progress_later();
-	erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr);
+	if (wakeup == ERTS_THR_PRGR_INVALID)
+	    wakeup = erts_thr_progress_later_than(current);
+	awdp->dd.thr_prgr = wakeup;
+	erts_thr_progress_wakeup(awdp->esdp, wakeup);
     }
     else {
 	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
@@ -1323,7 +1341,7 @@ erts_smp_notify_check_children_needed(void)
 				      ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
 }
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 {
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
@@ -1333,9 +1351,9 @@ handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 
 #endif
 
-#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK
+#if HAVE_ERTS_MSEG
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 {
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK);
@@ -1345,7 +1363,7 @@ handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 
 #endif
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 {
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO);
@@ -1353,69 +1371,92 @@ handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
     return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO;
 }
 
-static ERTS_INLINE erts_aint32_t
-handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
+static erts_aint32_t
+handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work)
 {
+#undef HANDLE_AUX_WORK
+#define HANDLE_AUX_WORK(FLG, HNDLR) \
+    ignore |= FLG; \
+    if (aux_work & FLG) { \
+	aux_work = HNDLR(awdp, aux_work); \
+	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \
+	if (!(aux_work & ~ignore)) { \
+	    ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \
+	    return aux_work; \
+	} \
+    }
+
+    erts_aint32_t aux_work = orig_aux_work;
+    erts_aint32_t ignore = 0;
+
+#ifdef ERTS_SMP
+    thr_prgr_current_reset(awdp);
+#endif
+
+    ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
+    ASSERT(aux_work);
+
     /*
      * Handlers are *only* allowed to modify flags in return value
      * and ssi flags that are explicity handled by the handler.
      * Handlers are, e.g., not allowed to read the ssi flag field and
      * then unconditionally return that value.
+     *
+     * Flag field returned should only contain flags for work that
+     * can continue immediately.
+     */
+
+    /*
+     * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative
+     * eachother. Most frequent first.
      */
-    ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    if (aux_work & ERTS_SSI_AUX_WORK_SET_TMO) {
-	aux_work = handle_setup_aux_work_timer(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
 #ifdef ERTS_SMP
-    if (aux_work & ERTS_SSI_AUX_WORK_MISC_THR_PRGR) {
-	aux_work = handle_misc_aux_work_thr_prgr(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD,
+		    handle_delayed_dealloc);
+    /* DD must be before DD_THR_PRGR */
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD_THR_PRGR,
+		    handle_delayed_dealloc_thr_prgr);
 #endif
-    if (aux_work & ERTS_SSI_AUX_WORK_MISC) {
-	aux_work = handle_misc_aux_work(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
+
+    HANDLE_AUX_WORK((ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
+		     | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
+		    handle_fix_alloc);
+
 #if ERTS_USE_ASYNC_READY_Q
-    if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY) {
-	aux_work = handle_async_ready(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
-    if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN) {
-	aux_work = handle_async_ready_clean(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY,
+		    handle_async_ready);
+    /* ASYNC_READY must be before ASYNC_READY_CLEAN */
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN,
+		    handle_async_ready_clean);
 #endif
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-    if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) {
-	aux_work = handle_check_children(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
-#endif
-    if (aux_work & (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
-		    | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)) {
-	aux_work = handle_fix_alloc(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
+
 #ifdef ERTS_SMP
-    if (aux_work & ERTS_SSI_AUX_WORK_DD) {
-	aux_work = handle_delayed_dealloc(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
-    if (aux_work & ERTS_SSI_AUX_WORK_DD_THR_PRGR) {
-	aux_work = handle_delayed_dealloc_thr_prgr(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC_THR_PRGR,
+		    handle_misc_aux_work_thr_prgr);
 #endif
-#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK
-    if (aux_work & ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK) {
-	aux_work = handle_mseg_cache_check(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
+    /* MISC_THR_PRGR must be before MISC */
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC,
+		    handle_misc_aux_work);
+
+#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CHECK_CHILDREN,
+		    handle_check_children);
+#endif
+
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO,
+		    handle_setup_aux_work_timer);
+
+#if HAVE_ERTS_MSEG
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK,
+		    handle_mseg_cache_check);
 #endif
+
     ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
+
     return aux_work;
+
+#undef HANDLE_AUX_WORK
+
 }
 
 typedef struct {