Merge branch 'maint'

* maint:
  Reduce thread progress read operations in handle_aux_work()
  Misc memory barrier fixes

15 files changed, 704 insertions, 212 deletions

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 0d3b6a4dba..6f5020dc14 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -646,6 +646,9 @@ bin_check(void)
 void
 erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 {
+#ifdef ERTS_SMP
+    ErtsThrPrgrData tpd_buf; /* in case we aren't a managed thread... */
+#endif
     int fd;
     time_t now;
     size_t dumpnamebufsize = MAXPATHLEN;
@@ -663,7 +666,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
      * We do not release system again. We expect an exit() or abort() after
      * dump has been written.
      */
-    erts_thr_progress_fatal_error_block(60000);
+    erts_thr_progress_fatal_error_block(60000, &tpd_buf);
     /* Either worked or not... */
 
     /* Allow us to pass certain places without locking... */
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 4d02a67d54..df27186680 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -1577,9 +1577,11 @@ erts_alloc_register_scheduler(void *vesdp)
     }
 }
 
+#ifdef ERTS_SMP
 void
 erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
 					    int *need_thr_progress,
+					    ErtsThrPrgrVal *thr_prgr_p,
 					    int *more_work)
 {
     ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
@@ -1599,10 +1601,12 @@ erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
 	    erts_alcu_check_delayed_dealloc(allctr,
 					    1,
 					    need_thr_progress,
+					    thr_prgr_p,
 					    more_work);
 	}
     }
 }
+#endif
 
 erts_aint32_t
 erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs)
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 991061c48e..e475f9d8a2 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -21,6 +21,10 @@
 #define ERL_ALLOC_H__
 
 #include "erl_alloc_types.h"
+#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
+#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
+#include "erl_thr_progress.h"
+#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
 #include "erl_alloc_util.h"
 #ifdef USE_THREADS
 #include "erl_threads.h"
@@ -132,9 +136,12 @@ typedef struct {
 extern ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1];
 
 void erts_alloc_register_scheduler(void *vesdp);
+#ifdef ERTS_SMP
 void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
 						 int *need_thr_progress,
+						 ErtsThrPrgrVal *thr_prgr_p,
 						 int *more_work);
+#endif
 erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs);
 
 __decl_noreturn void erts_alloc_enomem(ErtsAlcType_t,Uint)		
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 2e1a92f61d..e0d525bdde 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -949,7 +949,6 @@ ddq_check_incoming(ErtsAllctrDDQueue_t *ddq)
 		ERTS_THR_MEMORY_BARRIER;
 	    else {
 		ddq->head.next.unref_end = (ErtsAllctrDDBlock_t *) ilast;
-		ERTS_THR_MEMORY_BARRIER;
 		ddq->head.next.thr_progress = erts_thr_progress_later();
 		erts_atomic32_set_relb(&ddq->tail.data.um_refc_ix,
 				       um_refc_ix);
@@ -961,12 +960,24 @@ ddq_check_incoming(ErtsAllctrDDQueue_t *ddq)
     return 1;
 }
 
+static ERTS_INLINE void
+store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsAllctrDDQueue_t *ddq)
+{
+    if (!ddq->head.next.thr_progress_reached
+	&& (*prev_val == ERTS_THR_PRGR_INVALID
+	    || erts_thr_progress_cmp(ddq->head.next.thr_progress,
+				     *prev_val) < 0)) {
+	*prev_val = ddq->head.next.thr_progress;
+    }
+}
+
 static ERTS_INLINE int
 handle_delayed_dealloc(Allctr_t *allctr,
 		       int allctr_locked,
 		       int use_limit,
 		       int ops_limit,
 		       int *need_thr_progress,
+		       ErtsThrPrgrVal *thr_prgr_p,
 		       int *need_more_work)
 {
     int need_thr_prgr = 0;
@@ -1008,8 +1019,12 @@ handle_delayed_dealloc(Allctr_t *allctr,
 	    if (have_checked_incoming)
 		break;
 	    need_thr_prgr = ddq_check_incoming(ddq);
-	    if (need_thr_progress)
+	    if (need_thr_progress) {
 		*need_thr_progress |= need_thr_prgr;
+		if (need_thr_prgr)
+		    store_earliest_thr_prgr(thr_prgr_p, ddq);
+
+	    }
 	    have_checked_incoming = 1;
 	    goto dequeue;
 	}
@@ -1067,6 +1082,8 @@ handle_delayed_dealloc(Allctr_t *allctr,
     if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) {
 	need_thr_prgr = ddq_check_incoming(ddq);
 	*need_thr_progress |= need_thr_prgr;
+	if (need_thr_prgr)
+	    store_earliest_thr_prgr(thr_prgr_p, ddq);
     }
 
     if (allctr->thread_safe && !allctr_locked)
@@ -1086,25 +1103,27 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, Allctr_t *allctr, void *ptr)
 
 #endif
 
+#ifdef ERTS_SMP
 void
 erts_alcu_check_delayed_dealloc(Allctr_t *allctr,
 				int limit,
 				int *need_thr_progress,
+				ErtsThrPrgrVal *thr_prgr_p,
 				int *more_work)
 {
-#ifdef ERTS_SMP
     handle_delayed_dealloc(allctr,
 			   0,
 			   limit,
 			   ERTS_ALCU_DD_OPS_LIM_HIGH,
 			   need_thr_progress,
+			   thr_prgr_p,
 			   more_work);
-#endif
 }
+#endif
 
 #define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \
     handle_delayed_dealloc((Allctr), (Locked), 1, \
-			 ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL)
+			   ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL)
 
 /* Multi block carrier alloc/realloc/free ... */
 
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index fc1eddb116..cedf4ccf85 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -170,7 +170,9 @@ Eterm	erts_alcu_info(Allctr_t *, int, int *, void *, Uint **, Uint *);
 void	erts_alcu_init(AlcUInit_t *);
 void    erts_alcu_current_size(Allctr_t *, AllctrSize_t *,
 			       ErtsAlcUFixInfo_t *, int);
-void    erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, int *);
+#ifdef ERTS_SMP
+void    erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *, int *);
+#endif
 erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
 
 #endif
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index eb89baf1c9..51bdf53823 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -251,7 +251,7 @@ free_dbtable(DbTable* tb)
 	ASSERT(is_immed(tb->common.heir_data));
 	erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
 	ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable));
-	ERTS_THR_MEMORY_BARRIER;
+	ERTS_SMP_MEMORY_BARRIER;
 }
 
 #ifdef ERTS_SMP
@@ -3816,7 +3816,7 @@ void db_info(int to, void *to_arg, int show)    /* Called by break handler */
 Uint
 erts_get_ets_misc_mem_size(void)
 {
-    ERTS_THR_MEMORY_BARRIER;
+    ERTS_SMP_MEMORY_BARRIER;
     /* Memory not allocated in ets_alloc */
     return (Uint) erts_smp_atomic_read_nob(&erts_ets_misc_mem_size);
 }
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 038a667b06..c726be5fb4 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -105,11 +105,7 @@
 #define NSEG_2   256 /* Size of second segment table */
 #define NSEG_INC 128 /* Number of segments to grow after that */
 
-#ifdef ETHR_ORDERED_READ_DEPEND
-#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_nob(&(tb)->segtab))
-#else
-#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_rb(&(tb)->segtab))
-#endif
+#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_ddrb(&(tb)->segtab))
 #define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive))
 #define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems))
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index eedbb35486..138acfeb2c 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -347,40 +347,25 @@ dbg_chk_aux_work_val(erts_aint32_t value)
 {
     erts_aint32_t valid = 0;
 
-#ifdef ERTS_SSI_AUX_WORK_SET_TMO
     valid |= ERTS_SSI_AUX_WORK_SET_TMO;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_CHECK_CHILDREN
-    valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_MISC
     valid |= ERTS_SSI_AUX_WORK_MISC;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_MISC_THR_PRGR
-    valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY
-    valid |= ERTS_SSI_AUX_WORK_ASYNC_READY;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN
-    valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
-#endif
-
-#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
     valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC
     valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC;
+#if ERTS_USE_ASYNC_READY_Q
+    valid |= ERTS_SSI_AUX_WORK_ASYNC_READY;
+    valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
 #endif
-#ifdef ERTS_SSI_AUX_WORK_DD
+#ifdef ERTS_SMP
+    valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
     valid |= ERTS_SSI_AUX_WORK_DD;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_DD
     valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR;
 #endif
-#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK
+#if HAVE_ERTS_MSEG
     valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK;
 #endif
+#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
+    valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
+#endif
 
     if (~valid & value)
 	erl_exit(ERTS_ABORT_EXIT,
@@ -911,6 +896,27 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs)
     return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs);
 }
 
+#ifdef ERTS_SMP
+
+static ERTS_INLINE void
+thr_prgr_current_reset(ErtsAuxWorkData *awdp)
+{
+    awdp->current_thr_prgr = ERTS_THR_PRGR_INVALID;
+}
+
+static ERTS_INLINE ErtsThrPrgrVal
+thr_prgr_current(ErtsAuxWorkData *awdp)
+{
+    ErtsThrPrgrVal current = awdp->current_thr_prgr;
+    if (current == ERTS_THR_PRGR_INVALID) {
+	current = erts_thr_progress_current();
+	awdp->current_thr_prgr = current;
+    }
+    return current;
+}
+
+#endif
+
 typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t;
 struct erts_misc_aux_work_t_ {
     void (*func)(void *);
@@ -983,7 +989,7 @@ misc_aux_work_clean(ErtsThrQ_t *q,
     return aux_work;
 }
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_misc_aux_work(ErtsAuxWorkData *awdp,
 		     erts_aint32_t aux_work)
 {
@@ -1003,12 +1009,13 @@ handle_misc_aux_work(ErtsAuxWorkData *awdp,
 
 #ifdef ERTS_SMP
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp,
 			      erts_aint32_t aux_work)
 {
-    if (!erts_thr_progress_has_reached(awdp->misc.thr_prgr))
-	return aux_work;
+    if (!erts_thr_progress_has_reached_this(thr_prgr_current(awdp),
+					    awdp->misc.thr_prgr))
+	return aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
 
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR);
 
@@ -1081,7 +1088,7 @@ erts_notify_check_async_ready_queue(void *vno)
 				  ERTS_SSI_AUX_WORK_ASYNC_READY);
 }
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_async_ready(ErtsAuxWorkData *awdp,
 		   erts_aint32_t aux_work)
 {
@@ -1103,7 +1110,7 @@ handle_async_ready(ErtsAuxWorkData *awdp,
 	    | ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN);
 }
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_async_ready_clean(ErtsAuxWorkData *awdp,
 			 erts_aint32_t aux_work)
 {
@@ -1111,7 +1118,8 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp,
 
 #ifdef ERTS_SMP
     if (awdp->async_ready.need_thr_prgr
-	&& !erts_thr_progress_has_reached(awdp->async_ready.thr_prgr)) {
+	&& !erts_thr_progress_has_reached_this(thr_prgr_current(awdp),
+					       awdp->async_ready.thr_prgr)) {
 	return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
     }
 
@@ -1139,7 +1147,7 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp,
 
 #endif
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 {
     ErtsSchedulerSleepInfo *ssi = awdp->ssi;
@@ -1167,16 +1175,18 @@ erts_alloc_notify_delayed_dealloc(int ix)
 				  ERTS_SSI_AUX_WORK_DD);
 }
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 {
     ErtsSchedulerSleepInfo *ssi = awdp->ssi;
     int need_thr_progress = 0;
+    ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
     int more_work = 0;
 
     unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
     erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
 						&need_thr_progress,
+						&wakeup,
 						&more_work);
     if (more_work) {
 	if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD)
@@ -1188,9 +1198,12 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
     }
 
     if (need_thr_progress) {
+	if (wakeup == ERTS_THR_PRGR_INVALID)
+	    wakeup = erts_thr_progress_later_than(thr_prgr_current(awdp));
+	awdp->dd.thr_prgr = wakeup;
 	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
-	awdp->dd.thr_prgr = erts_thr_progress_later();
-	erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr);
+	awdp->dd.thr_prgr = wakeup;
+	erts_thr_progress_wakeup(awdp->esdp, wakeup);
     }
     else if (awdp->dd.completed_callback) {
 	awdp->dd.completed_callback(awdp->dd.completed_arg);
@@ -1200,14 +1213,16 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
     return aux_work & ~ERTS_SSI_AUX_WORK_DD;
 }
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 {
     ErtsSchedulerSleepInfo *ssi;
     int need_thr_progress;
     int more_work;
+    ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
+    ErtsThrPrgrVal current = thr_prgr_current(awdp);
 
-    if (!erts_thr_progress_has_reached(awdp->dd.thr_prgr))
+    if (!erts_thr_progress_has_reached_this(current, awdp->dd.thr_prgr))
 	return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR;
 
     ssi = awdp->ssi;
@@ -1216,6 +1231,7 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 
     erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
 						&need_thr_progress,
+						&wakeup,
 						&more_work);
     if (more_work) {
 	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
@@ -1225,8 +1241,10 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
     }
 
     if (need_thr_progress) {
-	awdp->dd.thr_prgr = erts_thr_progress_later();
-	erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr);
+	if (wakeup == ERTS_THR_PRGR_INVALID)
+	    wakeup = erts_thr_progress_later_than(current);
+	awdp->dd.thr_prgr = wakeup;
+	erts_thr_progress_wakeup(awdp->esdp, wakeup);
     }
     else {
 	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
@@ -1323,7 +1341,7 @@ erts_smp_notify_check_children_needed(void)
 				      ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
 }
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 {
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
@@ -1333,9 +1351,9 @@ handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 
 #endif
 
-#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK
+#if HAVE_ERTS_MSEG
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 {
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK);
@@ -1345,7 +1363,7 @@ handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 
 #endif
 
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
 handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 {
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO);
@@ -1353,69 +1371,92 @@ handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
     return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO;
 }
 
-static ERTS_INLINE erts_aint32_t
-handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
+static erts_aint32_t
+handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work)
 {
+#undef HANDLE_AUX_WORK
+#define HANDLE_AUX_WORK(FLG, HNDLR) \
+    ignore |= FLG; \
+    if (aux_work & FLG) { \
+	aux_work = HNDLR(awdp, aux_work); \
+	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \
+	if (!(aux_work & ~ignore)) { \
+	    ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \
+	    return aux_work; \
+	} \
+    }
+
+    erts_aint32_t aux_work = orig_aux_work;
+    erts_aint32_t ignore = 0;
+
+#ifdef ERTS_SMP
+    thr_prgr_current_reset(awdp);
+#endif
+
+    ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
+    ASSERT(aux_work);
+
     /*
      * Handlers are *only* allowed to modify flags in return value
      * and ssi flags that are explicity handled by the handler.
      * Handlers are, e.g., not allowed to read the ssi flag field and
      * then unconditionally return that value.
+     *
+     * Flag field returned should only contain flags for work that
+     * can continue immediately.
+     */
+
+    /*
+     * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative
+     * eachother. Most frequent first.
      */
-    ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    if (aux_work & ERTS_SSI_AUX_WORK_SET_TMO) {
-	aux_work = handle_setup_aux_work_timer(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
 #ifdef ERTS_SMP
-    if (aux_work & ERTS_SSI_AUX_WORK_MISC_THR_PRGR) {
-	aux_work = handle_misc_aux_work_thr_prgr(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD,
+		    handle_delayed_dealloc);
+    /* DD must be before DD_THR_PRGR */
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD_THR_PRGR,
+		    handle_delayed_dealloc_thr_prgr);
 #endif
-    if (aux_work & ERTS_SSI_AUX_WORK_MISC) {
-	aux_work = handle_misc_aux_work(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
+
+    HANDLE_AUX_WORK((ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
+		     | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
+		    handle_fix_alloc);
+
 #if ERTS_USE_ASYNC_READY_Q
-    if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY) {
-	aux_work = handle_async_ready(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
-    if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN) {
-	aux_work = handle_async_ready_clean(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY,
+		    handle_async_ready);
+    /* ASYNC_READY must be before ASYNC_READY_CLEAN */
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN,
+		    handle_async_ready_clean);
 #endif
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
-    if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) {
-	aux_work = handle_check_children(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
-#endif
-    if (aux_work & (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
-		    | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)) {
-	aux_work = handle_fix_alloc(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
+
 #ifdef ERTS_SMP
-    if (aux_work & ERTS_SSI_AUX_WORK_DD) {
-	aux_work = handle_delayed_dealloc(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
-    if (aux_work & ERTS_SSI_AUX_WORK_DD_THR_PRGR) {
-	aux_work = handle_delayed_dealloc_thr_prgr(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC_THR_PRGR,
+		    handle_misc_aux_work_thr_prgr);
 #endif
-#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK
-    if (aux_work & ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK) {
-	aux_work = handle_mseg_cache_check(awdp, aux_work);
-	ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
-    }
+    /* MISC_THR_PRGR must be before MISC */
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC,
+		    handle_misc_aux_work);
+
+#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CHECK_CHILDREN,
+		    handle_check_children);
+#endif
+
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO,
+		    handle_setup_aux_work_timer);
+
+#if HAVE_ERTS_MSEG
+    HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK,
+		    handle_mseg_cache_check);
 #endif
+
     ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
+
     return aux_work;
+
+#undef HANDLE_AUX_WORK
+
 }
 
 typedef struct {
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 173b7df69d..c23810f15a 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -248,25 +248,22 @@ typedef enum {
   | ERTS_SSI_FLG_WAITING				\
   | ERTS_SSI_FLG_SUSPENDED)
 
-#define ERTS_SSI_AUX_WORK_SET_TMO		(((erts_aint32_t) 1) << 0)
-#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN	(((erts_aint32_t) 1) << 1)
-#define ERTS_SSI_AUX_WORK_MISC			(((erts_aint32_t) 1) << 2)
-#ifdef ERTS_SMP
-#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR		(((erts_aint32_t) 1) << 3)
-#endif
-#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM	(((erts_aint32_t) 1) << 4)
-#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC	(((erts_aint32_t) 1) << 5)
-#define ERTS_SSI_AUX_WORK_ASYNC_READY		(((erts_aint32_t) 1) << 6)
-#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN	(((erts_aint32_t) 1) << 7)
-#ifdef ERTS_SMP
-#define ERTS_SSI_AUX_WORK_DD			(((erts_aint32_t) 1) << 8)
-#define ERTS_SSI_AUX_WORK_DD_THR_PRGR		(((erts_aint32_t) 1) << 9)
-#endif
-#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK	(((erts_aint32_t) 1) << 10)
+/*
+ * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative
+ * eachother. Most frequent - lowest bit number.
+ */
 
-#if !HAVE_ERTS_MSEG
-#  undef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK
-#endif
+#define ERTS_SSI_AUX_WORK_DD			(((erts_aint32_t) 1) << 0)
+#define ERTS_SSI_AUX_WORK_DD_THR_PRGR		(((erts_aint32_t) 1) << 1)
+#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC	(((erts_aint32_t) 1) << 2)
+#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM	(((erts_aint32_t) 1) << 3)
+#define ERTS_SSI_AUX_WORK_ASYNC_READY		(((erts_aint32_t) 1) << 4)
+#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN	(((erts_aint32_t) 1) << 5)
+#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR		(((erts_aint32_t) 1) << 6)
+#define ERTS_SSI_AUX_WORK_MISC			(((erts_aint32_t) 1) << 7)
+#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN	(((erts_aint32_t) 1) << 8)
+#define ERTS_SSI_AUX_WORK_SET_TMO		(((erts_aint32_t) 1) << 9)
+#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK	(((erts_aint32_t) 1) << 10)
 
 typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
 
@@ -407,6 +404,9 @@ typedef struct {
     int sched_id;
     ErtsSchedulerData *esdp;
     ErtsSchedulerSleepInfo *ssi;
+#ifdef ERTS_SMP
+    ErtsThrPrgrVal current_thr_prgr;
+#endif
     struct {
 	int ix;
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
index a7ccea7403..bff9d246a3 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -227,7 +227,6 @@ fetch_remote(erts_sspa_chunk_header_t *chdr, int max)
 		    ERTS_THR_MEMORY_BARRIER;
 		else {
 		    chdr->head.next.unref_end = (erts_sspa_blk_t *) ilast;
-		    ERTS_THR_MEMORY_BARRIER;
 		    chdr->head.next.thr_progress = erts_thr_progress_later();
 		    erts_atomic32_set_relb(&chdr->tail.data.um_refc_ix,
 					   um_refc_ix);
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 63179dfad4..a32e9d9d7c 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -61,6 +61,11 @@ typedef erts_spinlock_t erts_smp_spinlock_t;
 typedef erts_rwlock_t erts_smp_rwlock_t;
 void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */
 
+#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER
+#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER
+#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER
+#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
+
 #else /* #ifdef ERTS_SMP */
 
 #define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0}
@@ -95,6 +100,11 @@ typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t;
 typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t;
 #endif
 
+#define ERTS_SMP_MEMORY_BARRIER
+#define ERTS_SMP_WRITE_MEMORY_BARRIER
+#define ERTS_SMP_READ_MEMORY_BARRIER
+#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER
+
 #endif /* #ifdef ERTS_SMP */
 
 ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id);
@@ -206,13 +216,8 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
 
 /*
- * Functions implementing atomic operations with with no (nob),
- * full (mb), acquire (acqb), release (relb), read (rb), and
- * write (wb) memory barriers.
- *
- * If SMP support has been disabled, they are mapped to functions
- * that performs the same operation, but aren't atomic and don't
- * imply memory barriers.
+ * See "Documentation of atomics and memory barriers" at the top
+ * of erl_threads.h for info on atomics.
  */
 
 #ifdef ERTS_SMP
@@ -239,6 +244,11 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb
 #define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb
 
+#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb
+#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb
+#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb
+#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb
+
 #define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb
 #define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb
 #define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb
@@ -307,6 +317,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
 #define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
 
+#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb
+#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb
+#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb
+#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb
+#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb
+#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb
+#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb
+#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb
+#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb
+#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb
+#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb
+#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb
+#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb
+
 #define erts_smp_atomic_init_rb erts_atomic_init_rb
 #define erts_smp_atomic_set_rb erts_atomic_set_rb
 #define erts_smp_atomic_read_rb erts_atomic_read_rb
@@ -393,6 +417,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
 #define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
 
+#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb
+#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb
+#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb
+#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb
+#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb
+#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb
+#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb
+#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb
+#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb
+#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb
+#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb
+#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb
+#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb
+
 #define erts_smp_atomic32_init_rb erts_atomic32_init_rb
 #define erts_smp_atomic32_set_rb erts_atomic32_set_rb
 #define erts_smp_atomic32_read_rb erts_atomic32_read_rb
@@ -445,6 +483,11 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read
 #define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
 
+#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg
+
 #define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init
 #define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set
 #define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read
@@ -513,6 +556,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
 #define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
 
+#define erts_smp_atomic_init_ddrb erts_no_atomic_set
+#define erts_smp_atomic_set_ddrb erts_no_atomic_set
+#define erts_smp_atomic_read_ddrb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc
+#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read
+#define erts_smp_atomic_add_ddrb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
+
 #define erts_smp_atomic_init_rb erts_no_atomic_set
 #define erts_smp_atomic_set_rb erts_no_atomic_set
 #define erts_smp_atomic_read_rb erts_no_atomic_read
@@ -599,6 +656,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
 #define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
 
+#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set
+#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set
+#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
+
 #define erts_smp_atomic32_init_rb erts_no_atomic32_set
 #define erts_smp_atomic32_set_rb erts_no_atomic32_set
 #define erts_smp_atomic32_read_rb erts_no_atomic32_read
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 75f8209c3b..9ef83746c5 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -33,10 +33,11 @@
  * This module keeps track of the progress of a set of managed threads. Only
  * threads that behave well can be allowed to be managed. A managed thread
  * should update its thread progress frequently. Currently only scheduler
- * threads and the aux_thread are managed threads. We typically do not want
- * any async threads as managed threads since they cannot guarantee a
- * frequent update of thread progress, since they execute user implemented
- * driver code.
+ * threads, the system-message-dispatcher threads, and the aux-thread are
+ * managed threads. We typically do not want any async threads as managed
+ * threads since they cannot guarantee a frequent update of thread progress,
+ * since they execute user implemented driver code that is assumed to be
+ * time consuming.
  *
  * erts_thr_progress_current() returns the global current thread progress
  * value of managed threads. I.e., the latest progress value that all
@@ -112,8 +113,10 @@
  *
  * On 32-bit systems we therefore need a double word atomic.
  */
-
+#undef read_acqb
 #define read_acqb erts_thr_prgr_read_acqb__
+#undef read_nob
+#define read_nob erts_thr_prgr_read_nob__
 
 #ifdef ARCH_64
 
@@ -129,12 +132,6 @@ set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
     erts_atomic_set_nob(atmc, val);
 }
 
-static ERTS_INLINE ErtsThrPrgrVal
-read_nob(ERTS_THR_PRGR_ATOMIC *atmc)
-{
-    return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc);
-}
-
 static ERTS_INLINE void
 init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 {
@@ -143,52 +140,44 @@ init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 
 #else
 
-#undef dw_sint_to_val
-#define dw_sint_to_val erts_thr_prgr_dw_sint_to_val__
+#undef dw_aint_to_val
+#define dw_aint_to_val erts_thr_prgr_dw_aint_to_val__
 
 static void
-val_to_dw_sint(ethr_dw_sint_t *dw_sint, ErtsThrPrgrVal val)
+val_to_dw_aint(erts_dw_aint_t *dw_aint, ErtsThrPrgrVal val)
 {
 #ifdef ETHR_SU_DW_NAINT_T__
-    dw_sint->dw_sint = (ETHR_SU_DW_NAINT_T__) val;
+    dw_aint->dw_sint = (ETHR_SU_DW_NAINT_T__) val;
 #else
-    dw_sint->sint[ETHR_DW_SINT_LOW_WORD]
-	= (ethr_sint_t) (val & 0xffffffff);
-    dw_sint->sint[ETHR_DW_SINT_HIGH_WORD]
-	= (ethr_sint_t) ((val >> 32) & 0xffffffff);
+    dw_aint->sint[ERTS_DW_AINT_LOW_WORD]
+	= (erts_aint_t) (val & 0xffffffff);
+    dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]
+	= (erts_aint_t) ((val >> 32) & 0xffffffff);
 #endif
 }
 
 static ERTS_INLINE void
 set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 {
-    ethr_dw_sint_t dw_sint;
-    val_to_dw_sint(&dw_sint, val);
-    erts_dw_atomic_set_mb(atmc, &dw_sint);
+    erts_dw_aint_t dw_aint;
+    val_to_dw_aint(&dw_aint, val);
+    erts_dw_atomic_set_mb(atmc, &dw_aint);
 }
 
 static ERTS_INLINE void
 set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 {
-    ethr_dw_sint_t dw_sint;
-    val_to_dw_sint(&dw_sint, val);
-    erts_dw_atomic_set_nob(atmc, &dw_sint);
-}
-
-static ERTS_INLINE ErtsThrPrgrVal
-read_nob(ERTS_THR_PRGR_ATOMIC *atmc)
-{
-    ethr_dw_sint_t dw_sint;
-    erts_dw_atomic_read_nob(atmc, &dw_sint);
-    return erts_thr_prgr_dw_sint_to_val__(&dw_sint);
+    erts_dw_aint_t dw_aint;
+    val_to_dw_aint(&dw_aint, val);
+    erts_dw_atomic_set_nob(atmc, &dw_aint);
 }
 
 static ERTS_INLINE void
 init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 {
-    ethr_dw_sint_t dw_sint;
-    val_to_dw_sint(&dw_sint, val);
-    erts_dw_atomic_init_nob(atmc, &dw_sint);
+    erts_dw_aint_t dw_aint;
+    val_to_dw_aint(&dw_aint, val);
+    erts_dw_atomic_init_nob(atmc, &dw_aint);
 }
 
 #endif
@@ -1222,9 +1211,9 @@ erts_thr_progress_block(void)
 }
 
 void
-erts_thr_progress_fatal_error_block(SWord timeout)
+erts_thr_progress_fatal_error_block(SWord timeout,
+				    ErtsThrPrgrData *tmp_tpd_bufp)
 {
-    ErtsThrPrgrData tpd_buf;
     ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL);
     erts_aint32_t bc;
     SWord time_left = timeout;
@@ -1248,7 +1237,7 @@ erts_thr_progress_fatal_error_block(SWord timeout)
 	 * since we never complete an unblock after a fatal error
 	 * block.
 	 */
-	tpd = &tpd_buf;
+	tpd = tmp_tpd_bufp;
 	init_tmp_thr_prgr_data(tpd);
     }
 
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index 68d14174b9..1bbc49993e 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -44,7 +44,6 @@
 #define erts_smp_thr_progress_unblock erts_thr_progress_unblock
 #define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking
 
-void erts_thr_progress_fatal_error_block(SWord timeout);
 void erts_thr_progress_block(void);
 void erts_thr_progress_unblock(void);
 int erts_thr_progress_is_blocking(void);
@@ -73,6 +72,10 @@ typedef struct {
 	ErtsThrPrgrVal current;
     } previous;
 } ErtsThrPrgrData;
+
+void erts_thr_progress_fatal_error_block(SWord timeout,
+					 ErtsThrPrgrData *tmp_tpd_bufp);
+
 #endif /* ERTS_SMP */
 
 #endif
@@ -86,6 +89,7 @@ typedef struct {
 #ifdef ERTS_SMP
 
 #define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0))
+#define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0))
 
 extern erts_tsd_key_t erts_thr_prgr_data_key__;
 
@@ -127,14 +131,19 @@ void erts_thr_progress_dbg_print_state(void);
 
 #ifdef ARCH_32
 #define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t
-ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint);
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint);
 #endif
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc);
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc);
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc);
 
 ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void);
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later_than(ErtsThrPrgrVal val);
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(void);
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void);
 ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
+ERTS_GLB_INLINE int erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val);
+ERTS_GLB_INLINE int erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
 ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -142,33 +151,61 @@ ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);
 #ifdef ARCH_64
 
 ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+    return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
 erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
 {
     return (ErtsThrPrgrVal) erts_atomic_read_acqb(atmc);
 }
 
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+    return (ErtsThrPrgrVal) erts_atomic_read_mb(atmc);
+}
+
 #else /* ARCH_32 */
 
 ERTS_GLB_INLINE ErtsThrPrgrVal
-erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint)
+erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint)
 {
 #ifdef ETHR_SU_DW_NAINT_T__
-    return (ErtsThrPrgrVal) dw_sint->dw_sint;
+    return (ErtsThrPrgrVal) dw_aint->dw_sint;
 #else
     ErtsThrPrgrVal res;
-    res = (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_HIGH_WORD]);
+    res = (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]);
     res <<= 32;
-    res |= (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_LOW_WORD]);
+    res |= (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_LOW_WORD]);
     return res;
 #endif
 }
 
 ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+    erts_dw_aint_t dw_aint;
+    erts_dw_atomic_read_nob(atmc, &dw_aint);
+    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
 erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
 {
-    ethr_dw_sint_t dw_sint;
-    erts_dw_atomic_read_acqb(atmc, &dw_sint);
-    return erts_thr_prgr_dw_sint_to_val__(&dw_sint);
+    erts_dw_aint_t dw_aint;
+    erts_dw_atomic_read_acqb(atmc, &dw_aint);
+    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+    erts_dw_aint_t dw_aint;
+    erts_dw_atomic_read_mb(atmc, &dw_aint);
+    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
 }
 
 #endif
@@ -181,9 +218,9 @@ erts_thr_progress_is_managed_thread(void)
 }
 
 ERTS_GLB_INLINE ErtsThrPrgrVal
-erts_thr_progress_later(void)
+erts_thr_progress_later_than(ErtsThrPrgrVal val)
 {
-    ErtsThrPrgrVal val = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
+    ERTS_THR_MEMORY_BARRIER;
     if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2)))
 	return ((ErtsThrPrgrVal) 0);
     else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1)))
@@ -193,9 +230,19 @@ erts_thr_progress_later(void)
 }
 
 ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_progress_later(void)
+{
+    ErtsThrPrgrVal val = erts_thr_prgr_read_mb__(&erts_thr_prgr__.current);
+    return erts_thr_progress_later_than(val);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
 erts_thr_progress_current(void)
 {
-    return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
+    if (erts_thr_progress_is_managed_thread())
+	return erts_thr_prgr_read_nob__(&erts_thr_prgr__.current);
+    else
+	return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
 }
 
 ERTS_GLB_INLINE int
@@ -217,13 +264,29 @@ erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0)
 }
 
 ERTS_GLB_INLINE int
-erts_thr_progress_has_reached(ErtsThrPrgrVal val)
+erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val)
+{
+    if (this == val)
+	return 1;
+    return erts_thr_progress_has_passed__(this, val);
+}
+
+ERTS_GLB_INLINE int
+erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
 {
-    ErtsThrPrgrVal current;
-    current = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
-    if (current == val)
+    if (val1 == val2)
+	return 0;
+    if (erts_thr_progress_has_passed__(val1, val2))
 	return 1;
-    return erts_thr_progress_has_passed__(current, val);
+    else
+	return -1;
+}	
+
+ERTS_GLB_INLINE int
+erts_thr_progress_has_reached(ErtsThrPrgrVal val)
+{
+    ErtsThrPrgrVal current = erts_thr_progress_current();
+    return erts_thr_progress_has_reached_this(current, val);
 }
 
 #endif
diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c
index efb8c635d7..70949ece76 100644
--- a/erts/emulator/beam/erl_thr_queue.c
+++ b/erts/emulator/beam/erl_thr_queue.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -418,10 +418,9 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify)
 	    }
 
 	    if (q->head.unref_end == (ErtsThrQElement_t *) ilast)
-		ERTS_THR_MEMORY_BARRIER;
+		ERTS_SMP_MEMORY_BARRIER;
 	    else {
 		q->head.next.unref_end = (ErtsThrQElement_t *) ilast;
-		ERTS_THR_MEMORY_BARRIER;
 #ifdef ERTS_SMP
 		q->head.next.thr_progress = erts_thr_progress_later();
 #endif
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 065e7077c0..ee47c98009 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -25,6 +25,235 @@
 #ifndef ERL_THREAD_H__
 #define ERL_THREAD_H__
 
+/*
+ * --- Documentation of atomics and memory barriers --------------------------
+ *
+ * The following explicit memory barriers exist:
+ *   
+ * - ERTS_THR_MEMORY_BARRIER
+ *      Full memory barrier. Orders both loads, and stores. No
+ *      load or store is allowed to be reordered over the
+ *      barrier.
+ * - ERTS_THR_WRITE_MEMORY_BARRIER
+ *      Write barrier. Orders *only* stores. These are not
+ *      allowed to be reordered over the barrier.
+ * - ERTS_THR_READ_MEMORY_BARRIER
+ *      Read barrier. Orders *only* loads. These are not
+ *      allowed to be reordered over the barrier.
+ * - ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
+ *      Data dependency read barrier. Orders *only* loads
+ *      according to data dependency across the barrier.
+ *
+ * If thread support has been disabled, these barriers will become no-ops.
+ *
+ * If the prefix ERTS_THR_ is replaced with ERTS_SMP_, the barriers will
+ * be enabled only in the SMP enabled runtime system.
+ *
+ * --- Atomic operations ---
+ *
+ * Atomics operations exist for 32-bit, word size, and double word size
+ * integers. Function prototypes are listed below.
+ *
+ * Each function implementing an atomic operation exist with the following
+ * implied memory barrier semantics. Not all combinations are useful, but
+ * all of them exist for simplicity. <B> is suffix in function name:
+ *
+ * - <B>  - Description
+ *
+ * - mb   - Full memory barrier. Orders both loads, and
+ *          stores before, and after the atomic operation.
+ *          No load or store is allowed to be reordered
+ *          over the atomic operation.
+ * - relb - Release barrier. Orders both loads, and
+ *          stores appearing *before* the atomic
+ *          operation. These are not allowed to be
+ *          reordered over the atomic operation.
+ * - acqb - Acquire barrier. Orders both loads, and stores
+ *          appearing *after* the atomic operation. These
+ *          are not allowed to be reordered over the
+ *          atomic operation.
+ * - wb   - Write barrier. Orders *only* stores. These are
+ *          not allowed to be reordered over the barrier.
+ *          Store in atomic operation is ordered *after*
+ *          the barrier.
+ * - rb   - Read barrier. Orders *only* loads. These are
+ *          not allowed to be reordered over the barrier.
+ *          Load in atomic operation is ordered *before*
+ *          the barrier. 
+ * - ddrb - Data dependency read barrier. Orders *only*
+ *          loads according to data dependency across the
+ *          barrier. Load in atomic operation is ordered
+ *          before the barrier.
+ *
+ * If thread support has been disabled, these functions are mapped to
+ * functions that performs the same operation, but aren't atomic
+ * and don't imply any memory barriers.
+ *
+ * If the atomic operations are prefixed with erts_smp_ instead of only
+ * erts_ the atomic operations will only be atomic in the SMP enabled
+ * runtime system, and will be mapped to non-atomic operations without
+ * memory barriers in the runtime system without SMP support. Atomic
+ * operations with erts_smp_ prefix should use the atomic types
+ * erts_smp_atomic32_t, erts_smp_atomic_t, and erts_smp_dw_atomic_t
+ * instead of erts_atomic32_t, erts_atomic_t, and erts_dw_atomic_t. The
+ * integer data types erts_aint32_t, erts_aint_t, and erts_dw_atomic_t
+ * are the same.
+ *
+ * --- 32-bit atomic operations ---
+ *
+ * The following 32-bit atomic operations exist. <B> should be
+ * replaced with a supported memory barrier (see above). Note
+ * that sizeof(erts_atomic32_t) might be larger than 4!
+ *
+ *
+ * Initialize (not necessarily the same as the set operation):
+ *   void erts_atomic32_init_<B>(erts_atomic32_t *atmc,
+ *                               erts_aint32_t val);
+ *
+ * Set value:
+ *   void erts_atomic32_set_<B>(erts_atomic32_t *atmc,
+ *                              erts_aint32_t val);
+ *
+ * Read; returns current value:
+ *   erts_aint32_t erts_atomic32_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Increment; returns resulting value:
+ *   erts_aint32_t erts_atomic32_inc_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Decrement; returns resulting value:
+ *   erts_aint32_t erts_atomic32_dec_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Increment:
+ *   void erts_atomic32_inc_<B>(erts_atomic32_t *atmc);
+ *
+ * Decrement:
+ *   void erts_atomic32_dec_<B>(erts_atomic32_t *atmc);
+ *
+ * Add value; returns resulting value:
+ *   erts_aint32_t erts_atomic32_add_read_<B>(erts_atomic32_t *atmc,
+ *                                            erts_aint32_t val);
+ *
+ * Add value:
+ *   void erts_atomic32_add_<B>(erts_atomic32_t *atmc,
+ *                              erts_aint32_t val);
+ *
+ * Bitwise-or; returns previous value:
+ *   erts_aint32_t erts_atomic32_read_bor_<B>(erts_atomic32_t *atmc,
+ *                                            erts_aint32_t val);
+ *
+ * Bitwise-and; returns previous value:
+ *   erts_aint32_t erts_atomic32_read_band_<B>(erts_atomic32_t *atmc,
+ *                                             erts_aint32_t val);
+ *
+ * Exchange; returns previous value:
+ *   erts_aint32_t erts_atomic32_xchg_<B>(erts_atomic32_t *atmc,
+ *                                        erts_aint32_t val);
+ *
+ * Compare and exchange; returns previous or current value. If
+ * returned value equals 'exp' the value was changed to 'new';
+ * otherwise not:
+ *   erts_aint32_t erts_atomic32_cmpxchg_<B>(erts_atomic32_t *a,
+ *                                           erts_aint32_t new,
+ *                                           erts_aint32_t exp);
+ *
+ * --- Word size atomic operations ---
+ *
+ * The following word size (same size as sizeof(void *)) atomic
+ * operations exist. <B> should be replaced with a supported
+ * memory barrier (see above). Note that sizeof(erts_atomic_t)
+ * might be larger than sizeof(void *)!
+ *
+ * Initialize (not necessarily the same as the set operation):
+ *   void erts_atomic_init_<B>(erts_atomic_t *atmc,
+ *                             erts_aint_t val);
+ *
+ * Set value;
+ *   void erts_atomic_set_<B>(erts_atomic_t *atmc,
+ *                            erts_aint_t val);
+ *
+ * Read; returns current value:
+ *   erts_aint_t erts_atomic_read_<B>(erts_atomic_t *atmc);
+ *
+ * Increment; returns resulting value:
+ *   erts_aint_t erts_atomic_inc_read_<B>(erts_atomic_t *atmc);
+ *
+ * Decrement; returns resulting value:
+ *   erts_aint_t erts_atomic_dec_read_<B>(erts_atomic_t *atmc);
+ *
+ * Increment:
+ *   void erts_atomic_inc_<B>(erts_atomic_t *atmc);
+ *
+ * Decrement:
+ *   void erts_atomic_dec_<B>(erts_atomic_t *atmc);
+ *
+ * Add value; returns resulting value:
+ *   erts_aint_t erts_atomic_add_read_<B>(erts_atomic_t *atmc,
+ *                                        erts_aint_t val);
+ *
+ * Add value:
+ *   void erts_atomic_add_<B>(erts_atomic_t *atmc,
+ *                            erts_aint_t val);
+ *
+ * Bitwise-or; returns previous value:
+ *   erts_aint_t erts_atomic_read_bor_<B>(erts_atomic_t *atmc,
+ *                                        erts_aint_t val);
+ *
+ * Bitwise-and; returns previous value:
+ *   erts_aint_t erts_atomic_read_band_<B>(erts_atomic_t *atmc,
+ *                                         erts_aint_t val);
+ *
+ * Exchange; returns previous value:
+ *   erts_aint_t erts_atomic_xchg_<B>(erts_atomic_t *atmc,
+ *                                    erts_aint_t val);
+ *
+ * Compare and exchange; returns previous or current value. If
+ * returned value equals 'exp' the value was changed to 'new';
+ * otherwise not:
+ *   erts_aint_t erts_atomic_cmpxchg_<B>(erts_atomic_t *a,
+ *                                       erts_aint_t new,
+ *                                       erts_aint_t exp);
+ *
+ * --- Double word size atomic operations ---
+ *
+ * The following double word atomic operations exist. <B> should be
+ * replaced with a supported memory barrier (see above).
+ *
+ * Note that sizeof(erts_dw_atomic_t) usually is larger than
+ * 2*sizeof(void *)!
+ * 
+ * The erts_dw_aint_t data type should be accessed as if it was defined
+ * like this:
+ *
+ *     typedef struct {
+ *         erts_aint_t sint[2];
+ *     } erts_dw_aint_t;
+ *
+ *     Most significant word is 'sint[ERTS_DW_AINT_HIGH_WORD]' and least
+ *     significant word is 'sint[ERTS_DW_AINT_LOW_WORD]'.
+ *
+ *
+ * Initialize (not necessarily the same as the set operation):
+ *   void erts_dw_atomic_init_<B>(erts_dw_atomic_t *var,
+ *                           erts_dw_aint_t *val);
+ *
+ * Set; value is written into 'val':
+ *   void erts_dw_atomic_set_<B>(erts_dw_atomic_t *var,
+ *                               erts_dw_aint_t *val);
+ *
+ * Read; value is written into 'val':
+ *   void erts_dw_atomic_read_<B>(erts_dw_atomic_t *var,
+ *                                erts_dw_aint_t *val);
+ *
+ * Compare and exchange; returns a value != 0 if exchange was
+ * made; otherwise 0. 'new_val' contains new value to set. If 'exp_act'
+ * contains the same value as in memory when the function is called,
+ * 'new' is written to memory; otherwise, not. If exchange was not
+ * made, 'exp_act' contains the actual value in memory:
+ *   int erts_dw_atomic_cmpxchg_<B>(erts_dw_atomic_t *var,
+ *                                  erts_dw_aint_t *new,
+ *                                  erts_dw_aint_t *exp_act);
+ */
+
 #define ERTS_SPIN_BODY ETHR_SPIN_BODY
 
 #include "sys.h"
@@ -52,6 +281,9 @@ typedef Sint32 erts_no_atomic32_t;
 #endif
 
 #define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER
+#define ERTS_THR_WRITE_MEMORY_BARRIER ETHR_WRITE_MEMORY_BARRIER
+#define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
+#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
 #define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__)
@@ -113,6 +345,9 @@ typedef ethr_ts_event erts_tse_t;
 #define erts_aint32_t ethr_sint32_t 
 #define erts_atomic32_t ethr_atomic32_t
 
+#define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD
+#define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD
+
 /* spinlock */
 typedef struct {
     ethr_spinlock_t slck;
@@ -149,6 +384,9 @@ __decl_noreturn void  __noreturn erts_thr_fatal_error(int, char *);
 #else /* #ifdef USE_THREADS */
 
 #define ERTS_THR_MEMORY_BARRIER
+#define ERTS_THR_WRITE_MEMORY_BARRIER
+#define ERTS_THR_READ_MEMORY_BARRIER
+#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
 
 #define ERTS_THR_OPTS_DEFAULT_INITER 0
 typedef int erts_thr_opts_t;
@@ -361,18 +599,13 @@ ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set,
 ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
 
+#ifdef USE_THREADS
+
 /*
- * Functions implementing atomic operations with with no (nob),
- * full (mb), acquire (acqb), release (relb), read (rb), and
- * write (wb) memory barriers.
- *
- * If thread support has been disabled, they are mapped to
- * functions that performs the same operation, but aren't atomic
- * and don't imply memory barriers.
+ * See "Documentation of atomics and memory barriers" at the top
+ * of this file for info on atomics.
  */
 
-#ifdef USE_THREADS
-
 /* Double word size atomics */
 
 #define erts_dw_atomic_init_nob ethr_dw_atomic_init
@@ -395,6 +628,11 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb
 #define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb
 
+#define erts_dw_atomic_init_ddrb ethr_dw_atomic_init_ddrb
+#define erts_dw_atomic_set_ddrb ethr_dw_atomic_set_ddrb
+#define erts_dw_atomic_read_ddrb ethr_dw_atomic_read_ddrb
+#define erts_dw_atomic_cmpxchg_ddrb ethr_dw_atomic_cmpxchg_ddrb
+
 #define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb
 #define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb
 #define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb
@@ -463,6 +701,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_atomic_xchg_relb ethr_atomic_xchg_relb
 #define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb
 
+#define erts_atomic_init_ddrb ethr_atomic_init_ddrb
+#define erts_atomic_set_ddrb ethr_atomic_set_ddrb
+#define erts_atomic_read_ddrb ethr_atomic_read_ddrb
+#define erts_atomic_inc_read_ddrb ethr_atomic_inc_read_ddrb
+#define erts_atomic_dec_read_ddrb ethr_atomic_dec_read_ddrb
+#define erts_atomic_inc_ddrb ethr_atomic_inc_ddrb
+#define erts_atomic_dec_ddrb ethr_atomic_dec_ddrb
+#define erts_atomic_add_read_ddrb ethr_atomic_add_read_ddrb
+#define erts_atomic_add_ddrb ethr_atomic_add_ddrb
+#define erts_atomic_read_bor_ddrb ethr_atomic_read_bor_ddrb
+#define erts_atomic_read_band_ddrb ethr_atomic_read_band_ddrb
+#define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb
+#define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb
+
 #define erts_atomic_init_rb ethr_atomic_init_rb
 #define erts_atomic_set_rb ethr_atomic_set_rb
 #define erts_atomic_read_rb ethr_atomic_read_rb
@@ -549,6 +801,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb
 #define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb
 
+#define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb
+#define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb
+#define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb
+#define erts_atomic32_inc_read_ddrb ethr_atomic32_inc_read_ddrb
+#define erts_atomic32_dec_read_ddrb ethr_atomic32_dec_read_ddrb
+#define erts_atomic32_inc_ddrb ethr_atomic32_inc_ddrb
+#define erts_atomic32_dec_ddrb ethr_atomic32_dec_ddrb
+#define erts_atomic32_add_read_ddrb ethr_atomic32_add_read_ddrb
+#define erts_atomic32_add_ddrb ethr_atomic32_add_ddrb
+#define erts_atomic32_read_bor_ddrb ethr_atomic32_read_bor_ddrb
+#define erts_atomic32_read_band_ddrb ethr_atomic32_read_band_ddrb
+#define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb
+#define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb
+
 #define erts_atomic32_init_rb ethr_atomic32_init_rb
 #define erts_atomic32_set_rb ethr_atomic32_set_rb
 #define erts_atomic32_read_rb ethr_atomic32_read_rb
@@ -601,6 +867,11 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_dw_atomic_read_relb erts_no_dw_atomic_read
 #define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
 
+#define erts_dw_atomic_init_ddrb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_ddrb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_ddrb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg
+
 #define erts_dw_atomic_init_rb erts_no_dw_atomic_init
 #define erts_dw_atomic_set_rb erts_no_dw_atomic_set
 #define erts_dw_atomic_read_rb erts_no_dw_atomic_read
@@ -669,6 +940,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_atomic_xchg_relb erts_no_atomic_xchg
 #define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
 
+#define erts_atomic_init_ddrb erts_no_atomic_set
+#define erts_atomic_set_ddrb erts_no_atomic_set
+#define erts_atomic_read_ddrb erts_no_atomic_read
+#define erts_atomic_inc_read_ddrb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_ddrb erts_no_atomic_dec_read
+#define erts_atomic_inc_ddrb erts_no_atomic_inc
+#define erts_atomic_dec_ddrb erts_no_atomic_dec
+#define erts_atomic_add_read_ddrb erts_no_atomic_add_read
+#define erts_atomic_add_ddrb erts_no_atomic_add
+#define erts_atomic_read_bor_ddrb erts_no_atomic_read_bor
+#define erts_atomic_read_band_ddrb erts_no_atomic_read_band
+#define erts_atomic_xchg_ddrb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
+
 #define erts_atomic_init_rb erts_no_atomic_set
 #define erts_atomic_set_rb erts_no_atomic_set
 #define erts_atomic_read_rb erts_no_atomic_read
@@ -755,6 +1040,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_atomic32_xchg_relb erts_no_atomic32_xchg
 #define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
 
+#define erts_atomic32_init_ddrb erts_no_atomic32_set
+#define erts_atomic32_set_ddrb erts_no_atomic32_set
+#define erts_atomic32_read_ddrb erts_no_atomic32_read
+#define erts_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_ddrb erts_no_atomic32_inc
+#define erts_atomic32_dec_ddrb erts_no_atomic32_dec
+#define erts_atomic32_add_read_ddrb erts_no_atomic32_add_read
+#define erts_atomic32_add_ddrb erts_no_atomic32_add
+#define erts_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
+
 #define erts_atomic32_init_rb erts_no_atomic32_set
 #define erts_atomic32_set_rb erts_no_atomic32_set
 #define erts_atomic32_read_rb erts_no_atomic32_read