Misc memory barrier fixes

- Document barrier semantics
- Introduce ddrb suffix on atomic ops
- Barrier macros for both non-SMP and SMP case
- Make the thread progress API a bit more intuitive

18 files changed, 1408 insertions, 229 deletions

diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 0d3b6a4dba..6f5020dc14 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -646,6 +646,9 @@ bin_check(void)
 void
 erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
 {
+#ifdef ERTS_SMP
+    ErtsThrPrgrData tpd_buf; /* in case we aren't a managed thread... */
+#endif
     int fd;
     time_t now;
     size_t dumpnamebufsize = MAXPATHLEN;
@@ -663,7 +666,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
      * We do not release system again. We expect an exit() or abort() after
      * dump has been written.
      */
-    erts_thr_progress_fatal_error_block(60000);
+    erts_thr_progress_fatal_error_block(60000, &tpd_buf);
     /* Either worked or not... */
 
     /* Allow us to pass certain places without locking... */
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 4d02a67d54..df27186680 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -1577,9 +1577,11 @@ erts_alloc_register_scheduler(void *vesdp)
     }
 }
 
+#ifdef ERTS_SMP
 void
 erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
 					    int *need_thr_progress,
+					    ErtsThrPrgrVal *thr_prgr_p,
 					    int *more_work)
 {
     ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
@@ -1599,10 +1601,12 @@ erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
 	    erts_alcu_check_delayed_dealloc(allctr,
 					    1,
 					    need_thr_progress,
+					    thr_prgr_p,
 					    more_work);
 	}
     }
 }
+#endif
 
 erts_aint32_t
 erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs)
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 991061c48e..e475f9d8a2 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -21,6 +21,10 @@
 #define ERL_ALLOC_H__
 
 #include "erl_alloc_types.h"
+#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
+#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
+#include "erl_thr_progress.h"
+#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
 #include "erl_alloc_util.h"
 #ifdef USE_THREADS
 #include "erl_threads.h"
@@ -132,9 +136,12 @@ typedef struct {
 extern ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1];
 
 void erts_alloc_register_scheduler(void *vesdp);
+#ifdef ERTS_SMP
 void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
 						 int *need_thr_progress,
+						 ErtsThrPrgrVal *thr_prgr_p,
 						 int *more_work);
+#endif
 erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs);
 
 __decl_noreturn void erts_alloc_enomem(ErtsAlcType_t,Uint)		
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index c32938bdff..0e719f4a5a 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -949,7 +949,6 @@ ddq_check_incoming(ErtsAllctrDDQueue_t *ddq)
 		ERTS_THR_MEMORY_BARRIER;
 	    else {
 		ddq->head.next.unref_end = (ErtsAllctrDDBlock_t *) ilast;
-		ERTS_THR_MEMORY_BARRIER;
 		ddq->head.next.thr_progress = erts_thr_progress_later();
 		erts_atomic32_set_relb(&ddq->tail.data.um_refc_ix,
 				       um_refc_ix);
@@ -961,12 +960,24 @@ ddq_check_incoming(ErtsAllctrDDQueue_t *ddq)
     return 1;
 }
 
+static ERTS_INLINE void
+store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsAllctrDDQueue_t *ddq)
+{
+    if (!ddq->head.next.thr_progress_reached
+	&& (*prev_val == ERTS_THR_PRGR_INVALID
+	    || erts_thr_progress_cmp(ddq->head.next.thr_progress,
+				     *prev_val) < 0)) {
+	*prev_val = ddq->head.next.thr_progress;
+    }
+}
+
 static ERTS_INLINE int
 handle_delayed_dealloc(Allctr_t *allctr,
 		       int allctr_locked,
 		       int use_limit,
 		       int ops_limit,
 		       int *need_thr_progress,
+		       ErtsThrPrgrVal *thr_prgr_p,
 		       int *need_more_work)
 {
     int need_thr_prgr = 0;
@@ -1008,8 +1019,12 @@ handle_delayed_dealloc(Allctr_t *allctr,
 	    if (have_checked_incoming)
 		break;
 	    need_thr_prgr = ddq_check_incoming(ddq);
-	    if (need_thr_progress)
+	    if (need_thr_progress) {
 		*need_thr_progress |= need_thr_prgr;
+		if (need_thr_prgr)
+		    store_earliest_thr_prgr(thr_prgr_p, ddq);
+
+	    }
 	    have_checked_incoming = 1;
 	    goto dequeue;
 	}
@@ -1067,6 +1082,8 @@ handle_delayed_dealloc(Allctr_t *allctr,
     if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) {
 	need_thr_prgr = ddq_check_incoming(ddq);
 	*need_thr_progress |= need_thr_prgr;
+	if (need_thr_prgr)
+	    store_earliest_thr_prgr(thr_prgr_p, ddq);
     }
 
     if (allctr->thread_safe && !allctr_locked)
@@ -1086,25 +1103,27 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, Allctr_t *allctr, void *ptr)
 
 #endif
 
+#ifdef ERTS_SMP
 void
 erts_alcu_check_delayed_dealloc(Allctr_t *allctr,
 				int limit,
 				int *need_thr_progress,
+				ErtsThrPrgrVal *thr_prgr_p,
 				int *more_work)
 {
-#ifdef ERTS_SMP
     handle_delayed_dealloc(allctr,
 			   0,
 			   limit,
 			   ERTS_ALCU_DD_OPS_LIM_HIGH,
 			   need_thr_progress,
+			   thr_prgr_p,
 			   more_work);
-#endif
 }
+#endif
 
 #define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \
     handle_delayed_dealloc((Allctr), (Locked), 1, \
-			 ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL)
+			   ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL)
 
 /* Multi block carrier alloc/realloc/free ... */
 
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index fc1eddb116..cedf4ccf85 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -170,7 +170,9 @@ Eterm	erts_alcu_info(Allctr_t *, int, int *, void *, Uint **, Uint *);
 void	erts_alcu_init(AlcUInit_t *);
 void    erts_alcu_current_size(Allctr_t *, AllctrSize_t *,
 			       ErtsAlcUFixInfo_t *, int);
-void    erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, int *);
+#ifdef ERTS_SMP
+void    erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *, int *);
+#endif
 erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
 
 #endif
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index eb89baf1c9..51bdf53823 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -251,7 +251,7 @@ free_dbtable(DbTable* tb)
 	ASSERT(is_immed(tb->common.heir_data));
 	erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
 	ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable));
-	ERTS_THR_MEMORY_BARRIER;
+	ERTS_SMP_MEMORY_BARRIER;
 }
 
 #ifdef ERTS_SMP
@@ -3816,7 +3816,7 @@ void db_info(int to, void *to_arg, int show)    /* Called by break handler */
 Uint
 erts_get_ets_misc_mem_size(void)
 {
-    ERTS_THR_MEMORY_BARRIER;
+    ERTS_SMP_MEMORY_BARRIER;
     /* Memory not allocated in ets_alloc */
     return (Uint) erts_smp_atomic_read_nob(&erts_ets_misc_mem_size);
 }
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 038a667b06..c726be5fb4 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1998-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -105,11 +105,7 @@
 #define NSEG_2   256 /* Size of second segment table */
 #define NSEG_INC 128 /* Number of segments to grow after that */
 
-#ifdef ETHR_ORDERED_READ_DEPEND
-#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_nob(&(tb)->segtab))
-#else
-#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_rb(&(tb)->segtab))
-#endif
+#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_ddrb(&(tb)->segtab))
 #define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive))
 #define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems))
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b8c6b64fc0..57ef7bb8b8 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -709,6 +709,27 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs)
     return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs);
 }
 
+#ifdef ERTS_SMP
+
+static ERTS_INLINE void
+thr_prgr_current_reset(ErtsAuxWorkData *awdp)
+{
+    awdp->current_thr_prgr = ERTS_THR_PRGR_INVALID;
+}
+
+static ERTS_INLINE ErtsThrPrgrVal
+thr_prgr_current(ErtsAuxWorkData *awdp)
+{
+    ErtsThrPrgrVal current = awdp->current_thr_prgr;
+    if (current == ERTS_THR_PRGR_INVALID) {
+	current = erts_thr_progress_current();
+	awdp->current_thr_prgr = current;
+    }
+    return current;
+}
+
+#endif
+
 typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t;
 struct erts_misc_aux_work_t_ {
     void (*func)(void *);
@@ -805,7 +826,8 @@ static erts_aint32_t
 handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp,
 			      erts_aint32_t aux_work)
 {
-    if (!erts_thr_progress_has_reached(awdp->misc.thr_prgr))
+    if (!erts_thr_progress_has_reached_this(thr_prgr_current(awdp),
+					    awdp->misc.thr_prgr))
 	return aux_work;
 
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR);
@@ -909,7 +931,8 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp,
 
 #ifdef ERTS_SMP
     if (awdp->async_ready.need_thr_prgr
-	&& !erts_thr_progress_has_reached(awdp->async_ready.thr_prgr)) {
+	&& !erts_thr_progress_has_reached_this(thr_prgr_current(awdp),
+					       awdp->async_ready.thr_prgr)) {
 	return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
     }
 
@@ -970,11 +993,13 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 {
     ErtsSchedulerSleepInfo *ssi = awdp->ssi;
     int need_thr_progress = 0;
+    ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
     int more_work = 0;
 
     unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
     erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
 						&need_thr_progress,
+						&wakeup,
 						&more_work);
     if (more_work) {
 	if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD)
@@ -986,9 +1011,12 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
     }
 
     if (need_thr_progress) {
+	if (wakeup == ERTS_THR_PRGR_INVALID)
+	    wakeup = erts_thr_progress_later_than(thr_prgr_current(awdp));
+	awdp->dd.thr_prgr = wakeup;
 	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
-	awdp->dd.thr_prgr = erts_thr_progress_later();
-	erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr);
+	awdp->dd.thr_prgr = wakeup;
+	erts_thr_progress_wakeup(awdp->esdp, wakeup);
     }
     else if (awdp->dd.completed_callback) {
 	awdp->dd.completed_callback(awdp->dd.completed_arg);
@@ -1004,8 +1032,10 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
     ErtsSchedulerSleepInfo *ssi;
     int need_thr_progress;
     int more_work;
+    ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
+    ErtsThrPrgrVal current = thr_prgr_current(awdp);
 
-    if (!erts_thr_progress_has_reached(awdp->dd.thr_prgr))
+    if (!erts_thr_progress_has_reached_this(current, awdp->dd.thr_prgr))
 	return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR;
 
     ssi = awdp->ssi;
@@ -1014,6 +1044,7 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 
     erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
 						&need_thr_progress,
+						&wakeup,
 						&more_work);
     if (more_work) {
 	set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
@@ -1023,8 +1054,10 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
     }
 
     if (need_thr_progress) {
-	awdp->dd.thr_prgr = erts_thr_progress_later();
-	erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr);
+	if (wakeup == ERTS_THR_PRGR_INVALID)
+	    wakeup = erts_thr_progress_later_than(current);
+	awdp->dd.thr_prgr = wakeup;
+	erts_thr_progress_wakeup(awdp->esdp, wakeup);
     }
     else {
 	unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
@@ -1154,6 +1187,9 @@ handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 static ERTS_INLINE erts_aint32_t
 handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
 {
+#ifdef ERTS_SMP
+    thr_prgr_current_reset(awdp);
+#endif
     /*
      * Handlers are *only* allowed to modify flags in return value
      * and ssi flags that are explicity handled by the handler.
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index a51b380bb0..ad32dc11f9 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -397,6 +397,9 @@ typedef struct {
     int sched_id;
     ErtsSchedulerData *esdp;
     ErtsSchedulerSleepInfo *ssi;
+#ifdef ERTS_SMP
+    ErtsThrPrgrVal current_thr_prgr;
+#endif
     struct {
 	int ix;
 #ifdef ERTS_SMP
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
index a7ccea7403..bff9d246a3 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -227,7 +227,6 @@ fetch_remote(erts_sspa_chunk_header_t *chdr, int max)
 		    ERTS_THR_MEMORY_BARRIER;
 		else {
 		    chdr->head.next.unref_end = (erts_sspa_blk_t *) ilast;
-		    ERTS_THR_MEMORY_BARRIER;
 		    chdr->head.next.thr_progress = erts_thr_progress_later();
 		    erts_atomic32_set_relb(&chdr->tail.data.um_refc_ix,
 					   um_refc_ix);
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 63179dfad4..a32e9d9d7c 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -61,6 +61,11 @@ typedef erts_spinlock_t erts_smp_spinlock_t;
 typedef erts_rwlock_t erts_smp_rwlock_t;
 void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */
 
+#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER
+#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER
+#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER
+#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
+
 #else /* #ifdef ERTS_SMP */
 
 #define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0}
@@ -95,6 +100,11 @@ typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t;
 typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t;
 #endif
 
+#define ERTS_SMP_MEMORY_BARRIER
+#define ERTS_SMP_WRITE_MEMORY_BARRIER
+#define ERTS_SMP_READ_MEMORY_BARRIER
+#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER
+
 #endif /* #ifdef ERTS_SMP */
 
 ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id);
@@ -206,13 +216,8 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
 
 /*
- * Functions implementing atomic operations with with no (nob),
- * full (mb), acquire (acqb), release (relb), read (rb), and
- * write (wb) memory barriers.
- *
- * If SMP support has been disabled, they are mapped to functions
- * that performs the same operation, but aren't atomic and don't
- * imply memory barriers.
+ * See "Documentation of atomics and memory barriers" at the top
+ * of erl_threads.h for info on atomics.
  */
 
 #ifdef ERTS_SMP
@@ -239,6 +244,11 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb
 #define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb
 
+#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb
+#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb
+#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb
+#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb
+
 #define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb
 #define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb
 #define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb
@@ -307,6 +317,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
 #define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
 
+#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb
+#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb
+#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb
+#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb
+#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb
+#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb
+#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb
+#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb
+#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb
+#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb
+#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb
+#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb
+#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb
+
 #define erts_smp_atomic_init_rb erts_atomic_init_rb
 #define erts_smp_atomic_set_rb erts_atomic_set_rb
 #define erts_smp_atomic_read_rb erts_atomic_read_rb
@@ -393,6 +417,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
 #define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
 
+#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb
+#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb
+#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb
+#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb
+#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb
+#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb
+#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb
+#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb
+#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb
+#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb
+#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb
+#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb
+#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb
+
 #define erts_smp_atomic32_init_rb erts_atomic32_init_rb
 #define erts_smp_atomic32_set_rb erts_atomic32_set_rb
 #define erts_smp_atomic32_read_rb erts_atomic32_read_rb
@@ -445,6 +483,11 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read
 #define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
 
+#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg
+
 #define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init
 #define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set
 #define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read
@@ -513,6 +556,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
 #define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
 
+#define erts_smp_atomic_init_ddrb erts_no_atomic_set
+#define erts_smp_atomic_set_ddrb erts_no_atomic_set
+#define erts_smp_atomic_read_ddrb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc
+#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read
+#define erts_smp_atomic_add_ddrb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
+
 #define erts_smp_atomic_init_rb erts_no_atomic_set
 #define erts_smp_atomic_set_rb erts_no_atomic_set
 #define erts_smp_atomic_read_rb erts_no_atomic_read
@@ -599,6 +656,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
 #define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
 
+#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set
+#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set
+#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
+
 #define erts_smp_atomic32_init_rb erts_no_atomic32_set
 #define erts_smp_atomic32_set_rb erts_no_atomic32_set
 #define erts_smp_atomic32_read_rb erts_no_atomic32_read
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 75f8209c3b..9ef83746c5 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -33,10 +33,11 @@
  * This module keeps track of the progress of a set of managed threads. Only
  * threads that behave well can be allowed to be managed. A managed thread
  * should update its thread progress frequently. Currently only scheduler
- * threads and the aux_thread are managed threads. We typically do not want
- * any async threads as managed threads since they cannot guarantee a
- * frequent update of thread progress, since they execute user implemented
- * driver code.
+ * threads, the system-message-dispatcher threads, and the aux-thread are
+ * managed threads. We typically do not want any async threads as managed
+ * threads since they cannot guarantee a frequent update of thread progress,
+ * since they execute user implemented driver code that is assumed to be
+ * time consuming.
  *
  * erts_thr_progress_current() returns the global current thread progress
  * value of managed threads. I.e., the latest progress value that all
@@ -112,8 +113,10 @@
  *
  * On 32-bit systems we therefore need a double word atomic.
  */
-
+#undef read_acqb
 #define read_acqb erts_thr_prgr_read_acqb__
+#undef read_nob
+#define read_nob erts_thr_prgr_read_nob__
 
 #ifdef ARCH_64
 
@@ -129,12 +132,6 @@ set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
     erts_atomic_set_nob(atmc, val);
 }
 
-static ERTS_INLINE ErtsThrPrgrVal
-read_nob(ERTS_THR_PRGR_ATOMIC *atmc)
-{
-    return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc);
-}
-
 static ERTS_INLINE void
 init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 {
@@ -143,52 +140,44 @@ init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 
 #else
 
-#undef dw_sint_to_val
-#define dw_sint_to_val erts_thr_prgr_dw_sint_to_val__
+#undef dw_aint_to_val
+#define dw_aint_to_val erts_thr_prgr_dw_aint_to_val__
 
 static void
-val_to_dw_sint(ethr_dw_sint_t *dw_sint, ErtsThrPrgrVal val)
+val_to_dw_aint(erts_dw_aint_t *dw_aint, ErtsThrPrgrVal val)
 {
 #ifdef ETHR_SU_DW_NAINT_T__
-    dw_sint->dw_sint = (ETHR_SU_DW_NAINT_T__) val;
+    dw_aint->dw_sint = (ETHR_SU_DW_NAINT_T__) val;
 #else
-    dw_sint->sint[ETHR_DW_SINT_LOW_WORD]
-	= (ethr_sint_t) (val & 0xffffffff);
-    dw_sint->sint[ETHR_DW_SINT_HIGH_WORD]
-	= (ethr_sint_t) ((val >> 32) & 0xffffffff);
+    dw_aint->sint[ERTS_DW_AINT_LOW_WORD]
+	= (erts_aint_t) (val & 0xffffffff);
+    dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]
+	= (erts_aint_t) ((val >> 32) & 0xffffffff);
 #endif
 }
 
 static ERTS_INLINE void
 set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 {
-    ethr_dw_sint_t dw_sint;
-    val_to_dw_sint(&dw_sint, val);
-    erts_dw_atomic_set_mb(atmc, &dw_sint);
+    erts_dw_aint_t dw_aint;
+    val_to_dw_aint(&dw_aint, val);
+    erts_dw_atomic_set_mb(atmc, &dw_aint);
 }
 
 static ERTS_INLINE void
 set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 {
-    ethr_dw_sint_t dw_sint;
-    val_to_dw_sint(&dw_sint, val);
-    erts_dw_atomic_set_nob(atmc, &dw_sint);
-}
-
-static ERTS_INLINE ErtsThrPrgrVal
-read_nob(ERTS_THR_PRGR_ATOMIC *atmc)
-{
-    ethr_dw_sint_t dw_sint;
-    erts_dw_atomic_read_nob(atmc, &dw_sint);
-    return erts_thr_prgr_dw_sint_to_val__(&dw_sint);
+    erts_dw_aint_t dw_aint;
+    val_to_dw_aint(&dw_aint, val);
+    erts_dw_atomic_set_nob(atmc, &dw_aint);
 }
 
 static ERTS_INLINE void
 init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 {
-    ethr_dw_sint_t dw_sint;
-    val_to_dw_sint(&dw_sint, val);
-    erts_dw_atomic_init_nob(atmc, &dw_sint);
+    erts_dw_aint_t dw_aint;
+    val_to_dw_aint(&dw_aint, val);
+    erts_dw_atomic_init_nob(atmc, &dw_aint);
 }
 
 #endif
@@ -1222,9 +1211,9 @@ erts_thr_progress_block(void)
 }
 
 void
-erts_thr_progress_fatal_error_block(SWord timeout)
+erts_thr_progress_fatal_error_block(SWord timeout,
+				    ErtsThrPrgrData *tmp_tpd_bufp)
 {
-    ErtsThrPrgrData tpd_buf;
     ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL);
     erts_aint32_t bc;
     SWord time_left = timeout;
@@ -1248,7 +1237,7 @@ erts_thr_progress_fatal_error_block(SWord timeout)
 	 * since we never complete an unblock after a fatal error
 	 * block.
 	 */
-	tpd = &tpd_buf;
+	tpd = tmp_tpd_bufp;
 	init_tmp_thr_prgr_data(tpd);
     }
 
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index 68d14174b9..1bbc49993e 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -44,7 +44,6 @@
 #define erts_smp_thr_progress_unblock erts_thr_progress_unblock
 #define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking
 
-void erts_thr_progress_fatal_error_block(SWord timeout);
 void erts_thr_progress_block(void);
 void erts_thr_progress_unblock(void);
 int erts_thr_progress_is_blocking(void);
@@ -73,6 +72,10 @@ typedef struct {
 	ErtsThrPrgrVal current;
     } previous;
 } ErtsThrPrgrData;
+
+void erts_thr_progress_fatal_error_block(SWord timeout,
+					 ErtsThrPrgrData *tmp_tpd_bufp);
+
 #endif /* ERTS_SMP */
 
 #endif
@@ -86,6 +89,7 @@ typedef struct {
 #ifdef ERTS_SMP
 
 #define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0))
+#define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0))
 
 extern erts_tsd_key_t erts_thr_prgr_data_key__;
 
@@ -127,14 +131,19 @@ void erts_thr_progress_dbg_print_state(void);
 
 #ifdef ARCH_32
 #define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t
-ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint);
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint);
 #endif
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc);
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc);
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc);
 
 ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void);
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later_than(ErtsThrPrgrVal val);
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(void);
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void);
 ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
+ERTS_GLB_INLINE int erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val);
+ERTS_GLB_INLINE int erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
 ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -142,33 +151,61 @@ ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);
 #ifdef ARCH_64
 
 ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+    return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
 erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
 {
     return (ErtsThrPrgrVal) erts_atomic_read_acqb(atmc);
 }
 
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+    return (ErtsThrPrgrVal) erts_atomic_read_mb(atmc);
+}
+
 #else /* ARCH_32 */
 
 ERTS_GLB_INLINE ErtsThrPrgrVal
-erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint)
+erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint)
 {
 #ifdef ETHR_SU_DW_NAINT_T__
-    return (ErtsThrPrgrVal) dw_sint->dw_sint;
+    return (ErtsThrPrgrVal) dw_aint->dw_sint;
 #else
     ErtsThrPrgrVal res;
-    res = (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_HIGH_WORD]);
+    res = (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]);
     res <<= 32;
-    res |= (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_LOW_WORD]);
+    res |= (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_LOW_WORD]);
     return res;
 #endif
 }
 
 ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+    erts_dw_aint_t dw_aint;
+    erts_dw_atomic_read_nob(atmc, &dw_aint);
+    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
 erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
 {
-    ethr_dw_sint_t dw_sint;
-    erts_dw_atomic_read_acqb(atmc, &dw_sint);
-    return erts_thr_prgr_dw_sint_to_val__(&dw_sint);
+    erts_dw_aint_t dw_aint;
+    erts_dw_atomic_read_acqb(atmc, &dw_aint);
+    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+    erts_dw_aint_t dw_aint;
+    erts_dw_atomic_read_mb(atmc, &dw_aint);
+    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
 }
 
 #endif
@@ -181,9 +218,9 @@ erts_thr_progress_is_managed_thread(void)
 }
 
 ERTS_GLB_INLINE ErtsThrPrgrVal
-erts_thr_progress_later(void)
+erts_thr_progress_later_than(ErtsThrPrgrVal val)
 {
-    ErtsThrPrgrVal val = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
+    ERTS_THR_MEMORY_BARRIER;
     if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2)))
 	return ((ErtsThrPrgrVal) 0);
     else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1)))
@@ -193,9 +230,19 @@ erts_thr_progress_later(void)
 }
 
 ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_progress_later(void)
+{
+    ErtsThrPrgrVal val = erts_thr_prgr_read_mb__(&erts_thr_prgr__.current);
+    return erts_thr_progress_later_than(val);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
 erts_thr_progress_current(void)
 {
-    return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
+    if (erts_thr_progress_is_managed_thread())
+	return erts_thr_prgr_read_nob__(&erts_thr_prgr__.current);
+    else
+	return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
 }
 
 ERTS_GLB_INLINE int
@@ -217,13 +264,29 @@ erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0)
 }
 
 ERTS_GLB_INLINE int
-erts_thr_progress_has_reached(ErtsThrPrgrVal val)
+erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val)
+{
+    if (this == val)
+	return 1;
+    return erts_thr_progress_has_passed__(this, val);
+}
+
+ERTS_GLB_INLINE int
+erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
 {
-    ErtsThrPrgrVal current;
-    current = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
-    if (current == val)
+    if (val1 == val2)
+	return 0;
+    if (erts_thr_progress_has_passed__(val1, val2))
 	return 1;
-    return erts_thr_progress_has_passed__(current, val);
+    else
+	return -1;
+}	
+
+ERTS_GLB_INLINE int
+erts_thr_progress_has_reached(ErtsThrPrgrVal val)
+{
+    ErtsThrPrgrVal current = erts_thr_progress_current();
+    return erts_thr_progress_has_reached_this(current, val);
 }
 
 #endif
diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c
index efb8c635d7..70949ece76 100644
--- a/erts/emulator/beam/erl_thr_queue.c
+++ b/erts/emulator/beam/erl_thr_queue.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -418,10 +418,9 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify)
 	    }
 
 	    if (q->head.unref_end == (ErtsThrQElement_t *) ilast)
-		ERTS_THR_MEMORY_BARRIER;
+		ERTS_SMP_MEMORY_BARRIER;
 	    else {
 		q->head.next.unref_end = (ErtsThrQElement_t *) ilast;
-		ERTS_THR_MEMORY_BARRIER;
 #ifdef ERTS_SMP
 		q->head.next.thr_progress = erts_thr_progress_later();
 #endif
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 065e7077c0..ee47c98009 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -25,6 +25,235 @@
 #ifndef ERL_THREAD_H__
 #define ERL_THREAD_H__
 
+/*
+ * --- Documentation of atomics and memory barriers --------------------------
+ *
+ * The following explicit memory barriers exist:
+ *   
+ * - ERTS_THR_MEMORY_BARRIER
+ *      Full memory barrier. Orders both loads, and stores. No
+ *      load or store is allowed to be reordered over the
+ *      barrier.
+ * - ERTS_THR_WRITE_MEMORY_BARRIER
+ *      Write barrier. Orders *only* stores. These are not
+ *      allowed to be reordered over the barrier.
+ * - ERTS_THR_READ_MEMORY_BARRIER
+ *      Read barrier. Orders *only* loads. These are not
+ *      allowed to be reordered over the barrier.
+ * - ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
+ *      Data dependency read barrier. Orders *only* loads
+ *      according to data dependency across the barrier.
+ *
+ * If thread support has been disabled, these barriers will become no-ops.
+ *
+ * If the prefix ERTS_THR_ is replaced with ERTS_SMP_, the barriers will
+ * be enabled only in the SMP enabled runtime system.
+ *
+ * --- Atomic operations ---
+ *
+ * Atomics operations exist for 32-bit, word size, and double word size
+ * integers. Function prototypes are listed below.
+ *
+ * Each function implementing an atomic operation exist with the following
+ * implied memory barrier semantics. Not all combinations are useful, but
+ * all of them exist for simplicity. <B> is suffix in function name:
+ *
+ * - <B>  - Description
+ *
+ * - mb   - Full memory barrier. Orders both loads, and
+ *          stores before, and after the atomic operation.
+ *          No load or store is allowed to be reordered
+ *          over the atomic operation.
+ * - relb - Release barrier. Orders both loads, and
+ *          stores appearing *before* the atomic
+ *          operation. These are not allowed to be
+ *          reordered over the atomic operation.
+ * - acqb - Acquire barrier. Orders both loads, and stores
+ *          appearing *after* the atomic operation. These
+ *          are not allowed to be reordered over the
+ *          atomic operation.
+ * - wb   - Write barrier. Orders *only* stores. These are
+ *          not allowed to be reordered over the barrier.
+ *          Store in atomic operation is ordered *after*
+ *          the barrier.
+ * - rb   - Read barrier. Orders *only* loads. These are
+ *          not allowed to be reordered over the barrier.
+ *          Load in atomic operation is ordered *before*
+ *          the barrier. 
+ * - ddrb - Data dependency read barrier. Orders *only*
+ *          loads according to data dependency across the
+ *          barrier. Load in atomic operation is ordered
+ *          before the barrier.
+ *
+ * If thread support has been disabled, these functions are mapped to
+ * functions that performs the same operation, but aren't atomic
+ * and don't imply any memory barriers.
+ *
+ * If the atomic operations are prefixed with erts_smp_ instead of only
+ * erts_ the atomic operations will only be atomic in the SMP enabled
+ * runtime system, and will be mapped to non-atomic operations without
+ * memory barriers in the runtime system without SMP support. Atomic
+ * operations with erts_smp_ prefix should use the atomic types
+ * erts_smp_atomic32_t, erts_smp_atomic_t, and erts_smp_dw_atomic_t
+ * instead of erts_atomic32_t, erts_atomic_t, and erts_dw_atomic_t. The
+ * integer data types erts_aint32_t, erts_aint_t, and erts_dw_atomic_t
+ * are the same.
+ *
+ * --- 32-bit atomic operations ---
+ *
+ * The following 32-bit atomic operations exist. <B> should be
+ * replaced with a supported memory barrier (see above). Note
+ * that sizeof(erts_atomic32_t) might be larger than 4!
+ *
+ *
+ * Initialize (not necessarily the same as the set operation):
+ *   void erts_atomic32_init_<B>(erts_atomic32_t *atmc,
+ *                               erts_aint32_t val);
+ *
+ * Set value:
+ *   void erts_atomic32_set_<B>(erts_atomic32_t *atmc,
+ *                              erts_aint32_t val);
+ *
+ * Read; returns current value:
+ *   erts_aint32_t erts_atomic32_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Increment; returns resulting value:
+ *   erts_aint32_t erts_atomic32_inc_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Decrement; returns resulting value:
+ *   erts_aint32_t erts_atomic32_dec_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Increment:
+ *   void erts_atomic32_inc_<B>(erts_atomic32_t *atmc);
+ *
+ * Decrement:
+ *   void erts_atomic32_dec_<B>(erts_atomic32_t *atmc);
+ *
+ * Add value; returns resulting value:
+ *   erts_aint32_t erts_atomic32_add_read_<B>(erts_atomic32_t *atmc,
+ *                                            erts_aint32_t val);
+ *
+ * Add value:
+ *   void erts_atomic32_add_<B>(erts_atomic32_t *atmc,
+ *                              erts_aint32_t val);
+ *
+ * Bitwise-or; returns previous value:
+ *   erts_aint32_t erts_atomic32_read_bor_<B>(erts_atomic32_t *atmc,
+ *                                            erts_aint32_t val);
+ *
+ * Bitwise-and; returns previous value:
+ *   erts_aint32_t erts_atomic32_read_band_<B>(erts_atomic32_t *atmc,
+ *                                             erts_aint32_t val);
+ *
+ * Exchange; returns previous value:
+ *   erts_aint32_t erts_atomic32_xchg_<B>(erts_atomic32_t *atmc,
+ *                                        erts_aint32_t val);
+ *
+ * Compare and exchange; returns previous or current value. If
+ * returned value equals 'exp' the value was changed to 'new';
+ * otherwise not:
+ *   erts_aint32_t erts_atomic32_cmpxchg_<B>(erts_atomic32_t *a,
+ *                                           erts_aint32_t new,
+ *                                           erts_aint32_t exp);
+ *
+ * --- Word size atomic operations ---
+ *
+ * The following word size (same size as sizeof(void *)) atomic
+ * operations exist. <B> should be replaced with a supported
+ * memory barrier (see above). Note that sizeof(erts_atomic_t)
+ * might be larger than sizeof(void *)!
+ *
+ * Initialize (not necessarily the same as the set operation):
+ *   void erts_atomic_init_<B>(erts_atomic_t *atmc,
+ *                             erts_aint_t val);
+ *
+ * Set value;
+ *   void erts_atomic_set_<B>(erts_atomic_t *atmc,
+ *                            erts_aint_t val);
+ *
+ * Read; returns current value:
+ *   erts_aint_t erts_atomic_read_<B>(erts_atomic_t *atmc);
+ *
+ * Increment; returns resulting value:
+ *   erts_aint_t erts_atomic_inc_read_<B>(erts_atomic_t *atmc);
+ *
+ * Decrement; returns resulting value:
+ *   erts_aint_t erts_atomic_dec_read_<B>(erts_atomic_t *atmc);
+ *
+ * Increment:
+ *   void erts_atomic_inc_<B>(erts_atomic_t *atmc);
+ *
+ * Decrement:
+ *   void erts_atomic_dec_<B>(erts_atomic_t *atmc);
+ *
+ * Add value; returns resulting value:
+ *   erts_aint_t erts_atomic_add_read_<B>(erts_atomic_t *atmc,
+ *                                        erts_aint_t val);
+ *
+ * Add value:
+ *   void erts_atomic_add_<B>(erts_atomic_t *atmc,
+ *                            erts_aint_t val);
+ *
+ * Bitwise-or; returns previous value:
+ *   erts_aint_t erts_atomic_read_bor_<B>(erts_atomic_t *atmc,
+ *                                        erts_aint_t val);
+ *
+ * Bitwise-and; returns previous value:
+ *   erts_aint_t erts_atomic_read_band_<B>(erts_atomic_t *atmc,
+ *                                         erts_aint_t val);
+ *
+ * Exchange; returns previous value:
+ *   erts_aint_t erts_atomic_xchg_<B>(erts_atomic_t *atmc,
+ *                                    erts_aint_t val);
+ *
+ * Compare and exchange; returns previous or current value. If
+ * returned value equals 'exp' the value was changed to 'new';
+ * otherwise not:
+ *   erts_aint_t erts_atomic_cmpxchg_<B>(erts_atomic_t *a,
+ *                                       erts_aint_t new,
+ *                                       erts_aint_t exp);
+ *
+ * --- Double word size atomic operations ---
+ *
+ * The following double word atomic operations exist. <B> should be
+ * replaced with a supported memory barrier (see above).
+ *
+ * Note that sizeof(erts_dw_atomic_t) usually is larger than
+ * 2*sizeof(void *)!
+ * 
+ * The erts_dw_aint_t data type should be accessed as if it was defined
+ * like this:
+ *
+ *     typedef struct {
+ *         erts_aint_t sint[2];
+ *     } erts_dw_aint_t;
+ *
+ *     Most significant word is 'sint[ERTS_DW_AINT_HIGH_WORD]' and least
+ *     significant word is 'sint[ERTS_DW_AINT_LOW_WORD]'.
+ *
+ *
+ * Initialize (not necessarily the same as the set operation):
+ *   void erts_dw_atomic_init_<B>(erts_dw_atomic_t *var,
+ *                           erts_dw_aint_t *val);
+ *
+ * Set; value is written into 'val':
+ *   void erts_dw_atomic_set_<B>(erts_dw_atomic_t *var,
+ *                               erts_dw_aint_t *val);
+ *
+ * Read; value is written into 'val':
+ *   void erts_dw_atomic_read_<B>(erts_dw_atomic_t *var,
+ *                                erts_dw_aint_t *val);
+ *
+ * Compare and exchange; returns a value != 0 if exchange was
+ * made; otherwise 0. 'new_val' contains new value to set. If 'exp_act'
+ * contains the same value as in memory when the function is called,
+ * 'new' is written to memory; otherwise, not. If exchange was not
+ * made, 'exp_act' contains the actual value in memory:
+ *   int erts_dw_atomic_cmpxchg_<B>(erts_dw_atomic_t *var,
+ *                                  erts_dw_aint_t *new,
+ *                                  erts_dw_aint_t *exp_act);
+ */
+
 #define ERTS_SPIN_BODY ETHR_SPIN_BODY
 
 #include "sys.h"
@@ -52,6 +281,9 @@ typedef Sint32 erts_no_atomic32_t;
 #endif
 
 #define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER
+#define ERTS_THR_WRITE_MEMORY_BARRIER ETHR_WRITE_MEMORY_BARRIER
+#define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
+#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
 #define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__)
@@ -113,6 +345,9 @@ typedef ethr_ts_event erts_tse_t;
 #define erts_aint32_t ethr_sint32_t 
 #define erts_atomic32_t ethr_atomic32_t
 
+#define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD
+#define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD
+
 /* spinlock */
 typedef struct {
     ethr_spinlock_t slck;
@@ -149,6 +384,9 @@ __decl_noreturn void  __noreturn erts_thr_fatal_error(int, char *);
 #else /* #ifdef USE_THREADS */
 
 #define ERTS_THR_MEMORY_BARRIER
+#define ERTS_THR_WRITE_MEMORY_BARRIER
+#define ERTS_THR_READ_MEMORY_BARRIER
+#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
 
 #define ERTS_THR_OPTS_DEFAULT_INITER 0
 typedef int erts_thr_opts_t;
@@ -361,18 +599,13 @@ ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set,
 ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
 
+#ifdef USE_THREADS
+
 /*
- * Functions implementing atomic operations with with no (nob),
- * full (mb), acquire (acqb), release (relb), read (rb), and
- * write (wb) memory barriers.
- *
- * If thread support has been disabled, they are mapped to
- * functions that performs the same operation, but aren't atomic
- * and don't imply memory barriers.
+ * See "Documentation of atomics and memory barriers" at the top
+ * of this file for info on atomics.
  */
 
-#ifdef USE_THREADS
-
 /* Double word size atomics */
 
 #define erts_dw_atomic_init_nob ethr_dw_atomic_init
@@ -395,6 +628,11 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb
 #define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb
 
+#define erts_dw_atomic_init_ddrb ethr_dw_atomic_init_ddrb
+#define erts_dw_atomic_set_ddrb ethr_dw_atomic_set_ddrb
+#define erts_dw_atomic_read_ddrb ethr_dw_atomic_read_ddrb
+#define erts_dw_atomic_cmpxchg_ddrb ethr_dw_atomic_cmpxchg_ddrb
+
 #define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb
 #define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb
 #define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb
@@ -463,6 +701,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_atomic_xchg_relb ethr_atomic_xchg_relb
 #define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb
 
+#define erts_atomic_init_ddrb ethr_atomic_init_ddrb
+#define erts_atomic_set_ddrb ethr_atomic_set_ddrb
+#define erts_atomic_read_ddrb ethr_atomic_read_ddrb
+#define erts_atomic_inc_read_ddrb ethr_atomic_inc_read_ddrb
+#define erts_atomic_dec_read_ddrb ethr_atomic_dec_read_ddrb
+#define erts_atomic_inc_ddrb ethr_atomic_inc_ddrb
+#define erts_atomic_dec_ddrb ethr_atomic_dec_ddrb
+#define erts_atomic_add_read_ddrb ethr_atomic_add_read_ddrb
+#define erts_atomic_add_ddrb ethr_atomic_add_ddrb
+#define erts_atomic_read_bor_ddrb ethr_atomic_read_bor_ddrb
+#define erts_atomic_read_band_ddrb ethr_atomic_read_band_ddrb
+#define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb
+#define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb
+
 #define erts_atomic_init_rb ethr_atomic_init_rb
 #define erts_atomic_set_rb ethr_atomic_set_rb
 #define erts_atomic_read_rb ethr_atomic_read_rb
@@ -549,6 +801,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb
 #define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb
 
+#define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb
+#define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb
+#define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb
+#define erts_atomic32_inc_read_ddrb ethr_atomic32_inc_read_ddrb
+#define erts_atomic32_dec_read_ddrb ethr_atomic32_dec_read_ddrb
+#define erts_atomic32_inc_ddrb ethr_atomic32_inc_ddrb
+#define erts_atomic32_dec_ddrb ethr_atomic32_dec_ddrb
+#define erts_atomic32_add_read_ddrb ethr_atomic32_add_read_ddrb
+#define erts_atomic32_add_ddrb ethr_atomic32_add_ddrb
+#define erts_atomic32_read_bor_ddrb ethr_atomic32_read_bor_ddrb
+#define erts_atomic32_read_band_ddrb ethr_atomic32_read_band_ddrb
+#define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb
+#define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb
+
 #define erts_atomic32_init_rb ethr_atomic32_init_rb
 #define erts_atomic32_set_rb ethr_atomic32_set_rb
 #define erts_atomic32_read_rb ethr_atomic32_read_rb
@@ -601,6 +867,11 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_dw_atomic_read_relb erts_no_dw_atomic_read
 #define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
 
+#define erts_dw_atomic_init_ddrb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_ddrb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_ddrb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg
+
 #define erts_dw_atomic_init_rb erts_no_dw_atomic_init
 #define erts_dw_atomic_set_rb erts_no_dw_atomic_set
 #define erts_dw_atomic_read_rb erts_no_dw_atomic_read
@@ -669,6 +940,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_atomic_xchg_relb erts_no_atomic_xchg
 #define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
 
+#define erts_atomic_init_ddrb erts_no_atomic_set
+#define erts_atomic_set_ddrb erts_no_atomic_set
+#define erts_atomic_read_ddrb erts_no_atomic_read
+#define erts_atomic_inc_read_ddrb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_ddrb erts_no_atomic_dec_read
+#define erts_atomic_inc_ddrb erts_no_atomic_inc
+#define erts_atomic_dec_ddrb erts_no_atomic_dec
+#define erts_atomic_add_read_ddrb erts_no_atomic_add_read
+#define erts_atomic_add_ddrb erts_no_atomic_add
+#define erts_atomic_read_bor_ddrb erts_no_atomic_read_bor
+#define erts_atomic_read_band_ddrb erts_no_atomic_read_band
+#define erts_atomic_xchg_ddrb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
+
 #define erts_atomic_init_rb erts_no_atomic_set
 #define erts_atomic_set_rb erts_no_atomic_set
 #define erts_atomic_read_rb erts_no_atomic_read
@@ -755,6 +1040,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_atomic32_xchg_relb erts_no_atomic32_xchg
 #define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
 
+#define erts_atomic32_init_ddrb erts_no_atomic32_set
+#define erts_atomic32_set_ddrb erts_no_atomic32_set
+#define erts_atomic32_read_ddrb erts_no_atomic32_read
+#define erts_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_ddrb erts_no_atomic32_inc
+#define erts_atomic32_dec_ddrb erts_no_atomic32_dec
+#define erts_atomic32_add_read_ddrb erts_no_atomic32_add_read
+#define erts_atomic32_add_ddrb erts_no_atomic32_add
+#define erts_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
+
 #define erts_atomic32_init_rb erts_no_atomic32_set
 #define erts_atomic32_set_rb erts_no_atomic32_set
 #define erts_atomic32_read_rb erts_no_atomic32_read
diff --git a/erts/include/internal/ethr_atomics.h b/erts/include/internal/ethr_atomics.h
index 0f3c26f1df..612894b8c1 100644
--- a/erts/include/internal/ethr_atomics.h
+++ b/erts/include/internal/ethr_atomics.h
@@ -10,7 +10,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -64,12 +64,31 @@
  * Appart from a function implementing the atomic operation
  * with unspecified memory barrier semantics, there are
  * functions implementing each operation with the following
- * memory barrier semantics:
- * - rb (read barrier)
- * - wb (write barrier)
- * - acqb (acquire barrier)
- * - relb (release barrier)
- * - mb (full memory barrier)
+ * implied memory barrier semantics:
+ * - mb   - Full memory barrier. Orders both loads, and
+ *          stores before, and after the atomic operation.
+ *          No load or store is allowed to be reordered
+ *          over the atomic operation.
+ * - relb - Release barrier. Orders both loads, and
+ *          stores appearing *before* the atomic
+ *          operation. These are not allowed to be
+ *          reordered over the atomic operation.
+ * - acqb - Acquire barrier. Orders both loads, and stores
+ *          appearing *after* the atomic operation. These
+ *          are not allowed to be reordered over the
+ *          atomic operation.
+ * - wb   - Write barrier. Orders *only* stores. These are
+ *          not allowed to be reordered over the barrier.
+ *          Store in atomic operation is ordered *after*
+ *          the barrier.
+ * - rb   - Read barrier. Orders *only* loads. These are
+ *          not allowed to be reordered over the barrier.
+ *          Load in atomic operation is ordered *before*
+ *          the barrier. 
+ * - ddrb - Data dependency read barrier. Orders *only*
+ *          loads according to data dependency across the
+ *          barrier. Load in atomic operation is ordered
+ *          before the barrier.
  *
  * We implement all of these operation/barrier
  * combinations, regardless of whether they are useful
@@ -535,24 +554,28 @@ char **ethr_native_su_dw_atomic_ops(void);
 #ifdef ETHR_NEED_DW_ATMC_PROTOTYPES__
 ethr_sint_t *ethr_dw_atomic_addr(ethr_dw_atomic_t *var);
 int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ethr_dw_atomic_cmpxchg_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
 int ethr_dw_atomic_cmpxchg_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
 int ethr_dw_atomic_cmpxchg_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
 int ethr_dw_atomic_cmpxchg_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
 int ethr_dw_atomic_cmpxchg_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
 int ethr_dw_atomic_cmpxchg_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
 void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_set_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_set_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_set_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_set_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_set_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_set_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_read_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_read_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_read_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_read_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_read_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_read_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ethr_dw_atomic_init_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_init_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_init_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ethr_dw_atomic_init_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
@@ -561,24 +584,28 @@ void ethr_dw_atomic_init_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 #if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
 ethr_sint_t *ETHR_DW_ATOMIC_FUNC__(addr)(ethr_dw_atomic_t *var);
 int ETHR_DW_ATOMIC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val);
 void ETHR_DW_ATOMIC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(set_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(read_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
+void ETHR_DW_ATOMIC_FUNC__(init_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
 void ETHR_DW_ATOMIC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val);
@@ -1491,6 +1518,15 @@ static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, et
     return res;
 }
 
+static ETHR_INLINE int ETHR_DW_ATMC_FUNC__(cmpxchg_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_DW_ATMC_FUNC__(cmpxchg)(var, val, old_val);
+#else
+    return ETHR_DW_ATMC_FUNC__(cmpxchg_rb)(var, val, old_val);
+#endif
+}
+
 
 /* --- set() --- */
 
@@ -1981,6 +2017,15 @@ static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_
 
 }
 
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(set_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_DW_ATMC_FUNC__(set)(var, val);
+#else
+    ETHR_DW_ATMC_FUNC__(set_rb)(var, val);
+#endif
+}
+
 
 /* --- read() --- */
 
@@ -2309,6 +2354,15 @@ static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr
 
 }
 
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(read_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_DW_ATMC_FUNC__(read)(var, val);
+#else
+    ETHR_DW_ATMC_FUNC__(read_rb)(var, val);
+#endif
+}
+
 
 /* --- init() --- */
 
@@ -2607,6 +2661,15 @@ static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr
 
 }
 
+static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_DW_ATMC_FUNC__(init)(var, val);
+#else
+    ETHR_DW_ATMC_FUNC__(init_rb)(var, val);
+#endif
+}
+
 #endif /* ETHR_DW_ATMC_INLINE__ */
 
 
@@ -2616,78 +2679,91 @@ static ETHR_INLINE void ETHR_DW_ATMC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr
 #ifdef ETHR_NEED_ATMC_PROTOTYPES__
 ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
+ethr_sint_t ethr_atomic_cmpxchg_ddrb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
 ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
 ethr_sint_t ethr_atomic_cmpxchg_wb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
 ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
 ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
 ethr_sint_t ethr_atomic_cmpxchg_mb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val);
 ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_xchg_ddrb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_xchg_wb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_xchg_acqb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_xchg_relb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_xchg_mb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_set_ddrb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_set_wb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_set_acqb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_set_mb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_init_ddrb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_init_wb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_init_acqb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_init_relb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_init_mb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_add_read_ddrb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_add_read_wb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_add_read_acqb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_add_read_relb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_add_read_mb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_read(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_read_ddrb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_read_wb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_read_relb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_read_mb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_inc_read_ddrb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_inc_read_wb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_inc_read_relb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_inc_read_mb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var);
+ethr_sint_t ethr_atomic_dec_read_ddrb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_dec_read_wb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_dec_read_acqb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_dec_read_mb(ethr_atomic_t *var);
 void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val);
+void ethr_atomic_add_ddrb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_add_wb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_add_acqb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_add_relb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_add_mb(ethr_atomic_t *var, ethr_sint_t val);
 void ethr_atomic_inc(ethr_atomic_t *var);
+void ethr_atomic_inc_ddrb(ethr_atomic_t *var);
 void ethr_atomic_inc_rb(ethr_atomic_t *var);
 void ethr_atomic_inc_wb(ethr_atomic_t *var);
 void ethr_atomic_inc_acqb(ethr_atomic_t *var);
 void ethr_atomic_inc_relb(ethr_atomic_t *var);
 void ethr_atomic_inc_mb(ethr_atomic_t *var);
 void ethr_atomic_dec(ethr_atomic_t *var);
+void ethr_atomic_dec_ddrb(ethr_atomic_t *var);
 void ethr_atomic_dec_rb(ethr_atomic_t *var);
 void ethr_atomic_dec_wb(ethr_atomic_t *var);
 void ethr_atomic_dec_acqb(ethr_atomic_t *var);
 void ethr_atomic_dec_relb(ethr_atomic_t *var);
 void ethr_atomic_dec_mb(ethr_atomic_t *var);
 ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_band_ddrb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_read_band_wb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_read_band_acqb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_read_band_relb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_read_band_mb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val);
+ethr_sint_t ethr_atomic_read_bor_ddrb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_read_bor_wb(ethr_atomic_t *var, ethr_sint_t val);
 ethr_sint_t ethr_atomic_read_bor_acqb(ethr_atomic_t *var, ethr_sint_t val);
@@ -3551,6 +3627,15 @@ static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_mb)(ethr_atomic_t *var,
     return res;
 }
 
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(cmpxchg_ddrb)(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(cmpxchg)(var, val, old_val);
+#else
+    return ETHR_ATMC_FUNC__(cmpxchg_rb)(var, val, old_val);
+#endif
+}
+
 
 /* --- xchg() --- */
 
@@ -3801,6 +3886,15 @@ static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_mb)(ethr_atomic_t *var, eth
     return res;
 }
 
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(xchg_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(xchg)(var, val);
+#else
+    return ETHR_ATMC_FUNC__(xchg_rb)(var, val);
+#endif
+}
+
 
 /* --- set() --- */
 
@@ -3943,6 +4037,15 @@ static ETHR_INLINE void ETHR_ATMC_FUNC__(set_mb)(ethr_atomic_t *var, ethr_sint_t
 #endif
 }
 
+static ETHR_INLINE void ETHR_ATMC_FUNC__(set_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC_FUNC__(set)(var, val);
+#else
+    ETHR_ATMC_FUNC__(set_rb)(var, val);
+#endif
+}
+
 
 /* --- init() --- */
 
@@ -4085,6 +4188,15 @@ static ETHR_INLINE void ETHR_ATMC_FUNC__(init_mb)(ethr_atomic_t *var, ethr_sint_
 #endif
 }
 
+static ETHR_INLINE void ETHR_ATMC_FUNC__(init_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC_FUNC__(init)(var, val);
+#else
+    ETHR_ATMC_FUNC__(init_rb)(var, val);
+#endif
+}
+
 
 /* --- add_read() --- */
 
@@ -4335,6 +4447,15 @@ static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_mb)(ethr_atomic_t *var,
     return res;
 }
 
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(add_read_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(add_read)(var, val);
+#else
+    return ETHR_ATMC_FUNC__(add_read_rb)(var, val);
+#endif
+}
+
 
 /* --- read() --- */
 
@@ -4489,6 +4610,15 @@ static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_mb)(ethr_atomic_t *var)
     return res;
 }
 
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_ddrb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(read)(var);
+#else
+    return ETHR_ATMC_FUNC__(read_rb)(var);
+#endif
+}
+
 
 /* --- inc_read() --- */
 
@@ -4643,6 +4773,15 @@ static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_mb)(ethr_atomic_t *var)
     return res;
 }
 
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(inc_read_ddrb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(inc_read)(var);
+#else
+    return ETHR_ATMC_FUNC__(inc_read_rb)(var);
+#endif
+}
+
 
 /* --- dec_read() --- */
 
@@ -4797,6 +4936,15 @@ static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_mb)(ethr_atomic_t *var)
     return res;
 }
 
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(dec_read_ddrb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(dec_read)(var);
+#else
+    return ETHR_ATMC_FUNC__(dec_read_rb)(var);
+#endif
+}
+
 
 /* --- add() --- */
 
@@ -4939,6 +5087,15 @@ static ETHR_INLINE void ETHR_ATMC_FUNC__(add_mb)(ethr_atomic_t *var, ethr_sint_t
 #endif
 }
 
+static ETHR_INLINE void ETHR_ATMC_FUNC__(add_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC_FUNC__(add)(var, val);
+#else
+    ETHR_ATMC_FUNC__(add_rb)(var, val);
+#endif
+}
+
 
 /* --- inc() --- */
 
@@ -5081,6 +5238,15 @@ static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_mb)(ethr_atomic_t *var)
 #endif
 }
 
+static ETHR_INLINE void ETHR_ATMC_FUNC__(inc_ddrb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC_FUNC__(inc)(var);
+#else
+    ETHR_ATMC_FUNC__(inc_rb)(var);
+#endif
+}
+
 
 /* --- dec() --- */
 
@@ -5223,6 +5389,15 @@ static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_mb)(ethr_atomic_t *var)
 #endif
 }
 
+static ETHR_INLINE void ETHR_ATMC_FUNC__(dec_ddrb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC_FUNC__(dec)(var);
+#else
+    ETHR_ATMC_FUNC__(dec_rb)(var);
+#endif
+}
+
 
 /* --- read_band() --- */
 
@@ -5473,6 +5648,15 @@ static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_mb)(ethr_atomic_t *var
     return res;
 }
 
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_band_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(read_band)(var, val);
+#else
+    return ETHR_ATMC_FUNC__(read_band_rb)(var, val);
+#endif
+}
+
 
 /* --- read_bor() --- */
 
@@ -5723,6 +5907,15 @@ static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_mb)(ethr_atomic_t *var,
     return res;
 }
 
+static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_ddrb)(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC_FUNC__(read_bor)(var, val);
+#else
+    return ETHR_ATMC_FUNC__(read_bor_rb)(var, val);
+#endif
+}
+
 #endif /* ETHR_ATMC_INLINE__ */
 
 
@@ -5732,78 +5925,91 @@ static ETHR_INLINE ethr_sint_t ETHR_ATMC_FUNC__(read_bor_mb)(ethr_atomic_t *var,
 #ifdef ETHR_NEED_ATMC32_PROTOTYPES__
 ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
+ethr_sint32_t ethr_atomic32_cmpxchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
 ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
 ethr_sint32_t ethr_atomic32_cmpxchg_wb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
 ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
 ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
 ethr_sint32_t ethr_atomic32_cmpxchg_mb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val);
 ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_xchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_xchg_wb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_xchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_xchg_relb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_xchg_mb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_set_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_set_wb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_set_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_set_mb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_init_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_init_wb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_init_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_init_relb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_init_mb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_add_read_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_add_read_wb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_add_read_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_add_read_relb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_add_read_mb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_read_ddrb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_read_wb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_read_relb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_read_mb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_inc_read_ddrb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_inc_read_wb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_inc_read_relb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_inc_read_mb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var);
+ethr_sint32_t ethr_atomic32_dec_read_ddrb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_dec_read_wb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_dec_read_acqb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_dec_read_mb(ethr_atomic32_t *var);
 void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val);
+void ethr_atomic32_add_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_add_wb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_add_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_add_relb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_add_mb(ethr_atomic32_t *var, ethr_sint32_t val);
 void ethr_atomic32_inc(ethr_atomic32_t *var);
+void ethr_atomic32_inc_ddrb(ethr_atomic32_t *var);
 void ethr_atomic32_inc_rb(ethr_atomic32_t *var);
 void ethr_atomic32_inc_wb(ethr_atomic32_t *var);
 void ethr_atomic32_inc_acqb(ethr_atomic32_t *var);
 void ethr_atomic32_inc_relb(ethr_atomic32_t *var);
 void ethr_atomic32_inc_mb(ethr_atomic32_t *var);
 void ethr_atomic32_dec(ethr_atomic32_t *var);
+void ethr_atomic32_dec_ddrb(ethr_atomic32_t *var);
 void ethr_atomic32_dec_rb(ethr_atomic32_t *var);
 void ethr_atomic32_dec_wb(ethr_atomic32_t *var);
 void ethr_atomic32_dec_acqb(ethr_atomic32_t *var);
 void ethr_atomic32_dec_relb(ethr_atomic32_t *var);
 void ethr_atomic32_dec_mb(ethr_atomic32_t *var);
 ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_band_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_read_band_wb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_read_band_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_read_band_relb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_read_band_mb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val);
+ethr_sint32_t ethr_atomic32_read_bor_ddrb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_read_bor_wb(ethr_atomic32_t *var, ethr_sint32_t val);
 ethr_sint32_t ethr_atomic32_read_bor_acqb(ethr_atomic32_t *var, ethr_sint32_t val);
@@ -6667,6 +6873,15 @@ static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_mb)(ethr_atomic32_t
     return res;
 }
 
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(cmpxchg_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(cmpxchg)(var, val, old_val);
+#else
+    return ETHR_ATMC32_FUNC__(cmpxchg_rb)(var, val, old_val);
+#endif
+}
+
 
 /* --- xchg() --- */
 
@@ -6917,6 +7132,15 @@ static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_mb)(ethr_atomic32_t *va
     return res;
 }
 
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(xchg_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(xchg)(var, val);
+#else
+    return ETHR_ATMC32_FUNC__(xchg_rb)(var, val);
+#endif
+}
+
 
 /* --- set() --- */
 
@@ -7059,6 +7283,15 @@ static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_mb)(ethr_atomic32_t *var, ethr_si
 #endif
 }
 
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(set_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC32_FUNC__(set)(var, val);
+#else
+    ETHR_ATMC32_FUNC__(set_rb)(var, val);
+#endif
+}
+
 
 /* --- init() --- */
 
@@ -7201,6 +7434,15 @@ static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_mb)(ethr_atomic32_t *var, ethr_s
 #endif
 }
 
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(init_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC32_FUNC__(init)(var, val);
+#else
+    ETHR_ATMC32_FUNC__(init_rb)(var, val);
+#endif
+}
+
 
 /* --- add_read() --- */
 
@@ -7451,6 +7693,15 @@ static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_mb)(ethr_atomic32_t
     return res;
 }
 
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(add_read_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(add_read)(var, val);
+#else
+    return ETHR_ATMC32_FUNC__(add_read_rb)(var, val);
+#endif
+}
+
 
 /* --- read() --- */
 
@@ -7605,6 +7856,15 @@ static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_mb)(ethr_atomic32_t *va
     return res;
 }
 
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_ddrb)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(read)(var);
+#else
+    return ETHR_ATMC32_FUNC__(read_rb)(var);
+#endif
+}
+
 
 /* --- inc_read() --- */
 
@@ -7759,6 +8019,15 @@ static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_mb)(ethr_atomic32_t
     return res;
 }
 
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(inc_read_ddrb)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(inc_read)(var);
+#else
+    return ETHR_ATMC32_FUNC__(inc_read_rb)(var);
+#endif
+}
+
 
 /* --- dec_read() --- */
 
@@ -7913,6 +8182,15 @@ static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_mb)(ethr_atomic32_t
     return res;
 }
 
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(dec_read_ddrb)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(dec_read)(var);
+#else
+    return ETHR_ATMC32_FUNC__(dec_read_rb)(var);
+#endif
+}
+
 
 /* --- add() --- */
 
@@ -8055,6 +8333,15 @@ static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_mb)(ethr_atomic32_t *var, ethr_si
 #endif
 }
 
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(add_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC32_FUNC__(add)(var, val);
+#else
+    ETHR_ATMC32_FUNC__(add_rb)(var, val);
+#endif
+}
+
 
 /* --- inc() --- */
 
@@ -8197,6 +8484,15 @@ static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_mb)(ethr_atomic32_t *var)
 #endif
 }
 
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(inc_ddrb)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC32_FUNC__(inc)(var);
+#else
+    ETHR_ATMC32_FUNC__(inc_rb)(var);
+#endif
+}
+
 
 /* --- dec() --- */
 
@@ -8339,6 +8635,15 @@ static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_mb)(ethr_atomic32_t *var)
 #endif
 }
 
+static ETHR_INLINE void ETHR_ATMC32_FUNC__(dec_ddrb)(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_ATMC32_FUNC__(dec)(var);
+#else
+    ETHR_ATMC32_FUNC__(dec_rb)(var);
+#endif
+}
+
 
 /* --- read_band() --- */
 
@@ -8589,6 +8894,15 @@ static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_mb)(ethr_atomic32_
     return res;
 }
 
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_band_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(read_band)(var, val);
+#else
+    return ETHR_ATMC32_FUNC__(read_band_rb)(var, val);
+#endif
+}
+
 
 /* --- read_bor() --- */
 
@@ -8839,6 +9153,15 @@ static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_mb)(ethr_atomic32_t
     return res;
 }
 
+static ETHR_INLINE ethr_sint32_t ETHR_ATMC32_FUNC__(read_bor_ddrb)(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_ATMC32_FUNC__(read_bor)(var, val);
+#else
+    return ETHR_ATMC32_FUNC__(read_bor_rb)(var, val);
+#endif
+}
+
 #endif /* ETHR_ATMC32_INLINE__ */
 
 #endif /* ETHR_ATOMICS_H__ */
diff --git a/erts/lib_src/common/ethr_atomics.c b/erts/lib_src/common/ethr_atomics.c
index 5796bdc22e..e4213e1eef 100644
--- a/erts/lib_src/common/ethr_atomics.c
+++ b/erts/lib_src/common/ethr_atomics.c
@@ -10,7 +10,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -64,12 +64,31 @@
  * Appart from a function implementing the atomic operation
  * with unspecified memory barrier semantics, there are
  * functions implementing each operation with the following
- * memory barrier semantics:
- * - rb (read barrier)
- * - wb (write barrier)
- * - acqb (acquire barrier)
- * - relb (release barrier)
- * - mb (full memory barrier)
+ * implied memory barrier semantics:
+ * - mb   - Full memory barrier. Orders both loads, and
+ *          stores before, and after the atomic operation.
+ *          No load or store is allowed to be reordered
+ *          over the atomic operation.
+ * - relb - Release barrier. Orders both loads, and
+ *          stores appearing *before* the atomic
+ *          operation. These are not allowed to be
+ *          reordered over the atomic operation.
+ * - acqb - Acquire barrier. Orders both loads, and stores
+ *          appearing *after* the atomic operation. These
+ *          are not allowed to be reordered over the
+ *          atomic operation.
+ * - wb   - Write barrier. Orders *only* stores. These are
+ *          not allowed to be reordered over the barrier.
+ *          Store in atomic operation is ordered *after*
+ *          the barrier.
+ * - rb   - Read barrier. Orders *only* loads. These are
+ *          not allowed to be reordered over the barrier.
+ *          Load in atomic operation is ordered *before*
+ *          the barrier. 
+ * - ddrb - Data dependency read barrier. Orders *only*
+ *          loads according to data dependency across the
+ *          barrier. Load in atomic operation is ordered
+ *          before the barrier.
  *
  * We implement all of these operation/barrier
  * combinations, regardless of whether they are useful
@@ -542,6 +561,15 @@ int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_s
 }
 #endif
 
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_DW_ATOMIC_FUNC__(cmpxchg)(var, val, old_val);
+#else
+    return ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(var, val, old_val);
+#endif
+}
+
 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
 {
     int res;
@@ -756,6 +784,15 @@ void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
 }
 #endif
 
+void ETHR_DW_ATOMIC_FUNC__(set_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_DW_ATOMIC_FUNC__(set)(var, val);
+#else
+    ETHR_DW_ATOMIC_FUNC__(set_rb)(var, val);
+#endif
+}
+
 void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -910,6 +947,15 @@ void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
 }
 #endif
 
+void ETHR_DW_ATOMIC_FUNC__(read_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_DW_ATOMIC_FUNC__(read)(var, val);
+#else
+    ETHR_DW_ATOMIC_FUNC__(read_rb)(var, val);
+#endif
+}
+
 void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -1061,6 +1107,15 @@ void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
 }
 #endif
 
+void ETHR_DW_ATOMIC_FUNC__(init_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_DW_ATOMIC_FUNC__(init)(var, val);
+#else
+    ETHR_DW_ATOMIC_FUNC__(init_rb)(var, val);
+#endif
+}
+
 void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
 {
     ETHR_ASSERT(var);
@@ -1221,6 +1276,15 @@ ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t
     return res;
 }
 
+ethr_sint_t ethr_atomic_cmpxchg_ddrb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_cmpxchg(var, val, old_val);
+#else
+    return ethr_atomic_cmpxchg_rb(var, val, old_val);
+#endif
+}
+
 ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
 {
     ethr_sint_t res;
@@ -1332,6 +1396,15 @@ ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val)
     return res;
 }
 
+ethr_sint_t ethr_atomic_xchg_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_xchg(var, val);
+#else
+    return ethr_atomic_xchg_rb(var, val);
+#endif
+}
+
 ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val)
 {
     ethr_sint_t res;
@@ -1437,6 +1510,15 @@ void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val)
 
 }
 
+void ethr_atomic_set_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic_set(var, val);
+#else
+    ethr_atomic_set_rb(var, val);
+#endif
+}
+
 void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -1536,6 +1618,15 @@ void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val)
 
 }
 
+void ethr_atomic_init_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic_init(var, val);
+#else
+    ethr_atomic_init_rb(var, val);
+#endif
+}
+
 void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val)
 {
     ETHR_ASSERT(var);
@@ -1632,6 +1723,15 @@ ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val)
     return res;
 }
 
+ethr_sint_t ethr_atomic_add_read_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_add_read(var, val);
+#else
+    return ethr_atomic_add_read_rb(var, val);
+#endif
+}
+
 ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val)
 {
     ethr_sint_t res;
@@ -1738,6 +1838,15 @@ ethr_sint_t ethr_atomic_read(ethr_atomic_t *var)
     return res;
 }
 
+ethr_sint_t ethr_atomic_read_ddrb(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_read(var);
+#else
+    return ethr_atomic_read_rb(var);
+#endif
+}
+
 ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var)
 {
     ethr_sint_t res;
@@ -1843,6 +1952,15 @@ ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var)
     return res;
 }
 
+ethr_sint_t ethr_atomic_inc_read_ddrb(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_inc_read(var);
+#else
+    return ethr_atomic_inc_read_rb(var);
+#endif
+}
+
 ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var)
 {
     ethr_sint_t res;
@@ -1949,6 +2067,15 @@ ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var)
     return res;
 }
 
+ethr_sint_t ethr_atomic_dec_read_ddrb(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_dec_read(var);
+#else
+    return ethr_atomic_dec_read_rb(var);
+#endif
+}
+
 ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var)
 {
     ethr_sint_t res;
@@ -2054,6 +2181,15 @@ void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val)
 
 }
 
+void ethr_atomic_add_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic_add(var, val);
+#else
+    ethr_atomic_add_rb(var, val);
+#endif
+}
+
 void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -2154,6 +2290,15 @@ void ethr_atomic_inc(ethr_atomic_t *var)
 
 }
 
+void ethr_atomic_inc_ddrb(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic_inc(var);
+#else
+    ethr_atomic_inc_rb(var);
+#endif
+}
+
 void ethr_atomic_inc_rb(ethr_atomic_t *var)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -2254,6 +2399,15 @@ void ethr_atomic_dec(ethr_atomic_t *var)
 
 }
 
+void ethr_atomic_dec_ddrb(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic_dec(var);
+#else
+    ethr_atomic_dec_rb(var);
+#endif
+}
+
 void ethr_atomic_dec_rb(ethr_atomic_t *var)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -2355,6 +2509,15 @@ ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val)
     return res;
 }
 
+ethr_sint_t ethr_atomic_read_band_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_read_band(var, val);
+#else
+    return ethr_atomic_read_band_rb(var, val);
+#endif
+}
+
 ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val)
 {
     ethr_sint_t res;
@@ -2461,6 +2624,15 @@ ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val)
     return res;
 }
 
+ethr_sint_t ethr_atomic_read_bor_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_read_bor(var, val);
+#else
+    return ethr_atomic_read_bor_rb(var, val);
+#endif
+}
+
 ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val)
 {
     ethr_sint_t res;
@@ -2587,6 +2759,15 @@ ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, eth
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_cmpxchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_cmpxchg(var, val, old_val);
+#else
+    return ethr_atomic32_cmpxchg_rb(var, val, old_val);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
 {
     ethr_sint32_t res;
@@ -2675,6 +2856,15 @@ ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val)
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_xchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_xchg(var, val);
+#else
+    return ethr_atomic32_xchg_rb(var, val);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val)
 {
     ethr_sint32_t res;
@@ -2762,6 +2952,15 @@ void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val)
 
 }
 
+void ethr_atomic32_set_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic32_set(var, val);
+#else
+    ethr_atomic32_set_rb(var, val);
+#endif
+}
+
 void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -2843,6 +3042,15 @@ void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val)
 
 }
 
+void ethr_atomic32_init_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic32_init(var, val);
+#else
+    ethr_atomic32_init_rb(var, val);
+#endif
+}
+
 void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val)
 {
     ETHR_ASSERT(var);
@@ -2921,6 +3129,15 @@ ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val)
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_add_read_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_add_read(var, val);
+#else
+    return ethr_atomic32_add_read_rb(var, val);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val)
 {
     ethr_sint32_t res;
@@ -3009,6 +3226,15 @@ ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var)
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_read_ddrb(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_read(var);
+#else
+    return ethr_atomic32_read_rb(var);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var)
 {
     ethr_sint32_t res;
@@ -3097,6 +3323,15 @@ ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var)
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_inc_read_ddrb(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_inc_read(var);
+#else
+    return ethr_atomic32_inc_read_rb(var);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var)
 {
     ethr_sint32_t res;
@@ -3185,6 +3420,15 @@ ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var)
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_dec_read_ddrb(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_dec_read(var);
+#else
+    return ethr_atomic32_dec_read_rb(var);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var)
 {
     ethr_sint32_t res;
@@ -3272,6 +3516,15 @@ void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val)
 
 }
 
+void ethr_atomic32_add_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic32_add(var, val);
+#else
+    ethr_atomic32_add_rb(var, val);
+#endif
+}
+
 void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -3354,6 +3607,15 @@ void ethr_atomic32_inc(ethr_atomic32_t *var)
 
 }
 
+void ethr_atomic32_inc_ddrb(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic32_inc(var);
+#else
+    ethr_atomic32_inc_rb(var);
+#endif
+}
+
 void ethr_atomic32_inc_rb(ethr_atomic32_t *var)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -3436,6 +3698,15 @@ void ethr_atomic32_dec(ethr_atomic32_t *var)
 
 }
 
+void ethr_atomic32_dec_ddrb(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic32_dec(var);
+#else
+    ethr_atomic32_dec_rb(var);
+#endif
+}
+
 void ethr_atomic32_dec_rb(ethr_atomic32_t *var)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -3519,6 +3790,15 @@ ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val)
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_read_band_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_read_band(var, val);
+#else
+    return ethr_atomic32_read_band_rb(var, val);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val)
 {
     ethr_sint32_t res;
@@ -3607,6 +3887,15 @@ ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val)
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_read_bor_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_read_bor(var, val);
+#else
+    return ethr_atomic32_read_bor_rb(var, val);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val)
 {
     ethr_sint32_t res;
diff --git a/erts/lib_src/utils/make_atomics_api b/erts/lib_src/utils/make_atomics_api
index f4e71c7618..d8b1a56100 100755
--- a/erts/lib_src/utils/make_atomics_api
+++ b/erts/lib_src/utils/make_atomics_api
@@ -4,7 +4,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2011. All Rights Reserved.
+%% Copyright Ericsson AB 2011-2012. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -50,7 +50,9 @@
 -define(DW_RTCHK_MACRO, "ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__").
 
 %% Barrier versions we implement
--define(BARRIERS, [none, rb, wb, acqb, relb, mb]).
+-define(BARRIERS, [none, ddrb, rb, wb, acqb, relb, mb]).
+-define(NON_NATIVE_BARRIERS, [ddrb]).
+-define(NATIVE_BARRIERS, (?BARRIERS -- ?NON_NATIVE_BARRIERS)).
 
 -define(ATOMIC_SIZES, ["dword", "word", "32"]).
 
@@ -381,7 +383,6 @@ do_func_header(#atomic_context{dw = true,
 	       cmpxchg, Inline, Func) ->
     [Inline, "int ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " *", Arg2, ", ", AintT, " *", Arg3, ")"].
 
-
 xbarriers(_Op, none, _NB) ->
     {"", ""};
 
@@ -455,7 +456,7 @@ try_barrier_order_first(mb) ->
 
 try_barrier_order(B) ->
     First = try_barrier_order_first(B),
-    First ++ (?BARRIERS -- First).
+    First ++ (?NATIVE_BARRIERS -- First).
 
 native_barrier_op(#atomic_context{'NATMC' = NATMC} = AC, If, ExtraDecl, Op, B, NB, TypeCasts) ->
     NOpStr = opstr(native(Op)),
@@ -571,12 +572,12 @@ do_cmpxchg_fallback_define(#atomic_context{'NATMC' = NATMC,
     NoneTryBarrierOrder = try_barrier_order(none),
     %% First a sanity check
     ["
-#if (", NotDefCMPXCHG(hd(?BARRIERS)) ,
+#if (", NotDefCMPXCHG(hd(?NATIVE_BARRIERS)) ,
      lists:map(fun (B) ->
 		       [" \\
      && ", NotDefCMPXCHG(B)]
 	       end,
-	       tl(?BARRIERS)), ")
+	       tl(?NATIVE_BARRIERS)), ")
 #  error \"No native cmpxchg() op available\"
 #endif
 
@@ -744,7 +745,7 @@ translate_have_defs(#atomic_context{dw = DW, 'NATMC' = NATMC}) ->
 								      ]
 							     end]
 						    end,
-						    ?BARRIERS)
+						    ?NATIVE_BARRIERS)
 				  end,
 				  case DW of
 				      true -> ?DW_ATOMIC_OPS;
@@ -801,6 +802,46 @@ rtchk_fallback_call(Return, #atomic_context{dw = DW,
 			false -> [RetVar, " ="]
 		    end, [?DW_FUNC_MACRO, "(", opstr(Op), op_barrier_ext(B), ")"], Arg1, Arg2, Arg3, "").
 
+non_native_barrier(B) ->
+    lists:member(B, ?NON_NATIVE_BARRIERS).
+
+non_native_barrier_impl(AC, Type, Macro, Op, B) ->
+    ["
+", func_header(AC, Type, Macro, Op, B), "
+{",
+     case B of
+	 ddrb ->
+	     ["
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ", func_call(AC, Type, Macro, Op, none, true), "
+#else
+    ", func_call(AC, Type, Macro, Op, rb, true), "
+#endif
+"
+	     ]
+     end,
+     "}
+"
+    ].
+
+func_call(#atomic_context{'ATMC' = ATMC} = AC, inline_implementation, _Macro, Op, B, RetStatement) ->
+    func_call(AC, Op, ["ETHR_", ATMC, "_FUNC__(", opstr(Op), op_barrier_ext(B), ")"], RetStatement);
+func_call(#atomic_context{atomic = Atomic} = AC, implementation, false, Op, B, RetStatement) ->
+    func_call(AC, Op, [Atomic, "_", opstr(Op), op_barrier_ext(B)], RetStatement);
+func_call(AC, implementation, Macro, Op, B, RetStatement) ->
+    func_call(AC, Op, [Macro, "(", opstr(Op), op_barrier_ext(B), ")"], RetStatement).
+
+func_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3} = AC, Op, Func, true) ->
+    op_call(Op, DW, case is_return_op(AC, Op) of
+			true -> "return";
+			false -> ""
+		    end, Func, Arg1, Arg2, Arg3, "");
+func_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3, ret_var = RetVar} = AC, Op, Func, false) ->
+    op_call(Op, DW, case is_return_op(AC, Op) of
+			true -> [RetVar, " = "];
+			false -> ""
+		    end, Func, Arg1, Arg2, Arg3, "").
+
 make_implementations(#atomic_context{dw = DW,
 				     ret_type = RetType,
 				     ret_var = RetVar,
@@ -858,73 +899,78 @@ make_implementations(#atomic_context{dw = DW,
 
 ",
 			lists:map(fun (B) ->
-					  TryBarriers = try_barrier_order(B),
-					  ["
+					  case non_native_barrier(B) of
+					      true ->
+						  non_native_barrier_impl(AC, inline_implementation, false, Op, B);
+					      false ->
+						  TryBarriers = try_barrier_order(B),
+						  ["
 ", func_header(AC, inline_implementation, false, Op, B), "
 {
 ",
-					   case is_return_op(AC, Op) of
-					       true ->
-						   ["    ", RetType, " ", RetVar, ";\n"];
-					       _ -> ""
-					   end,
-					   case DW of
-					       true ->
-						   [RtchkBegin,
-						    "\n",
-						    su_dw_native_barrier_op(AC, "#if", Op, B, hd(TryBarriers)),
-						    lists:map(fun (NB) ->
-								      su_dw_native_barrier_op(AC, "#elif", Op, B, NB)
-							      end,
-							      tl(TryBarriers)),
-						    lists:map(fun (NB) ->
-								      dw_native_barrier_op(AC, "#elif", "", Op, B, NB)
-							      end,
-							      TryBarriers),
-						   case simple_fallback(AC, Op, B) of
-						       "" ->
-							   %% No simple fallback available;
-							   %% use cmpxchg() fallbacks...
-							   [cmpxchg_fallbacks(AC#atomic_context{'NATMC' = ["SU_", NATMC]}, true, Op, B),
-							    cmpxchg_fallbacks(AC, false, Op, B),
-							    "#else
+						   case is_return_op(AC, Op) of
+						       true ->
+							   ["    ", RetType, " ", RetVar, ";\n"];
+						       _ -> ""
+						   end,
+						   case DW of
+						       true ->
+							   [RtchkBegin,
+							    "\n",
+							    su_dw_native_barrier_op(AC, "#if", Op, B, hd(TryBarriers)),
+							    lists:map(fun (NB) ->
+									      su_dw_native_barrier_op(AC, "#elif", Op, B, NB)
+								      end,
+								      tl(TryBarriers)),
+							    lists:map(fun (NB) ->
+									      dw_native_barrier_op(AC, "#elif", "", Op, B, NB)
+								      end,
+								      TryBarriers),
+							    case simple_fallback(AC, Op, B) of
+								"" ->
+								    %% No simple fallback available;
+								    %% use cmpxchg() fallbacks...
+								    [cmpxchg_fallbacks(AC#atomic_context{'NATMC' = ["SU_", NATMC]}, true, Op, B),
+								     cmpxchg_fallbacks(AC, false, Op, B),
+								     "#else
 #error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\"
 #endif
 "
-							   ];
-						       SimpleFallback ->
-							   ["#else\n", SimpleFallback, "#endif\n"]
-						   end,
-						    RtchkEnd(false, Op, B), "\n"];
-					       false ->
-						   [native_barrier_op(AC, "#if", "", Op, B, hd(TryBarriers), true),
-						    lists:map(fun (NB) ->
-								      native_barrier_op(AC, "#elif", "", Op, B, NB, true)
-							      end,
-							      tl(TryBarriers)),
-						    case simple_fallback(AC, Op, B) of
-							"" ->
-							    %% No simple fallback available;
-							    %% use cmpxchg() fallbacks...
-							    [cmpxchg_fallbacks(AC, false, Op, B),
-							     "#else
+								    ];
+								SimpleFallback ->
+								    ["#else\n", SimpleFallback, "#endif\n"]
+							    end,
+							    RtchkEnd(false, Op, B), "\n"];
+						       false ->
+							   [native_barrier_op(AC, "#if", "", Op, B, hd(TryBarriers), true),
+							    lists:map(fun (NB) ->
+									      native_barrier_op(AC, "#elif", "", Op, B, NB, true)
+								      end,
+								      tl(TryBarriers)),
+							    case simple_fallback(AC, Op, B) of
+								"" ->
+								    %% No simple fallback available;
+								    %% use cmpxchg() fallbacks...
+								    [cmpxchg_fallbacks(AC, false, Op, B),
+								     "#else
 #error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\"
 #endif
 "
-							    ];
-							SimpleFallback ->
-							    ["#else\n", SimpleFallback, "#endif\n"]
-						    end]
-					   end,
-					   case is_return_op(AC, Op) of
-					       true ->
-						   ["    return ", RetVar, ";\n"];
-					       false ->
-						   ""
-					   end,
-					   "}\n"]
+								    ];
+								SimpleFallback ->
+								    ["#else\n", SimpleFallback, "#endif\n"]
+							    end]
+						   end,
+						   case is_return_op(AC, Op) of
+						       true ->
+							   ["    return ", RetVar, ";\n"];
+						       false ->
+							   ""
+						   end,
+						   "}\n"]
+					  end
 				  end,
-				  ?BARRIERS)]
+				  ?NATIVE_BARRIERS ++ ?NON_NATIVE_BARRIERS)] %% non-native needs to be after native...
 	       end,
 	       case DW of
 		   true -> ?DW_ATOMIC_OPS;
@@ -1159,33 +1205,38 @@ int ethr_have_native_dw_atomic(void)
 
 ",
 			lists:map(fun (B) ->
-					  ["\n",
-					   func_header(AC, implementation,
-						       case DW of
-							   true -> ?DW_FUNC_MACRO;
-							   false -> false
-						       end,  Op, B),
-					   "\n{\n",
-					   case is_return_op(AC, Op) of
-					       true -> ["    ", RetType, " ", RetVar, ";\n"];
-					       false -> ""
-					   end,
-					   case Op of
-					       init -> "";
-					       _ -> ["    ETHR_ASSERT(!ethr_not_inited__);\n"]
-					   end,
-					   ["    ETHR_ASSERT(", Arg1, ");\n"],
-					   make_native_impl_op(AC, Op, B),
-					   make_amc_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), 
-					   make_locked_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B),
-					   case is_return_op(AC, Op) of
-					       true -> ["    return ", RetVar, ";"
-						       ];
-					       false ->
-						   ""
-					   end,
-					   "\n}\n",
-					   make_symbol_to_fallback_impl(AC, Op, B)]
+					Macro = case DW of
+						   true -> ?DW_FUNC_MACRO;
+						   false -> false
+						end,
+					case non_native_barrier(B) of
+					    true ->
+						non_native_barrier_impl(AC, implementation, Macro, Op, B);
+					    false ->
+						["\n",
+					   	func_header(AC, implementation, Macro,  Op, B),
+						   "\n{\n",
+						   case is_return_op(AC, Op) of
+						       true -> ["    ", RetType, " ", RetVar, ";\n"];
+						       false -> ""
+						   end,
+						   case Op of
+						       init -> "";
+						       _ -> ["    ETHR_ASSERT(!ethr_not_inited__);\n"]
+						   end,
+						   ["    ETHR_ASSERT(", Arg1, ");\n"],
+						   make_native_impl_op(AC, Op, B),
+						   make_amc_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), 
+						   make_locked_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B),
+						   case is_return_op(AC, Op) of
+						       true -> ["    return ", RetVar, ";"
+							       ];
+						       false ->
+							   ""
+						   end,
+						   "\n}\n",
+						   make_symbol_to_fallback_impl(AC, Op, B)]
+					  end
 				  end,
 				  ?BARRIERS)]
 	       end,
@@ -1233,7 +1284,7 @@ static char *native_", DW, "atomic", Bits, "_ops[] = {",
 #endif"
 							     ]
 						     end,
-						     ?BARRIERS)
+						     ?NATIVE_BARRIERS)
 				   end,
 				   case NBits of
 				       "dw" -> ?DW_ATOMIC_OPS;
@@ -1390,25 +1441,51 @@ comments() ->
  * Appart from a function implementing the atomic operation
  * with unspecified memory barrier semantics, there are
  * functions implementing each operation with the following
- * memory barrier semantics:
-",
+ * implied memory barrier semantics:",
      lists:map(fun (none) ->
 		       "";
-		   (rb) ->
-		       [" * - rb (read barrier)\n"];
-		   (wb) ->
-		       [" * - wb (write barrier)\n"];
+		   (mb) ->
+		       ["
+ * - mb   - Full memory barrier. Orders both loads, and
+ *          stores before, and after the atomic operation.
+ *          No load or store is allowed to be reordered
+ *          over the atomic operation."];
 		   (acqb) ->
-		       [" * - acqb (acquire barrier)\n"];
+		       ["
+ * - acqb - Acquire barrier. Orders both loads, and stores
+ *          appearing *after* the atomic operation. These
+ *          are not allowed to be reordered over the
+ *          atomic operation."];
 		   (relb) ->
-		       [" * - relb (release barrier)\n"];
-		   (mb) ->
-		       [" * - mb (full memory barrier)\n"];
+		       ["
+ * - relb - Release barrier. Orders both loads, and
+ *          stores appearing *before* the atomic
+ *          operation. These are not allowed to be
+ *          reordered over the atomic operation."];
+		   (rb) ->
+		       ["
+ * - rb   - Read barrier. Orders *only* loads. These are
+ *          not allowed to be reordered over the barrier.
+ *          Load in atomic operation is ordered *before*
+ *          the barrier. "];
+		   (ddrb) ->
+		       ["
+ * - ddrb - Data dependency read barrier. Orders *only*
+ *          loads according to data dependency across the
+ *          barrier. Load in atomic operation is ordered
+ *          before the barrier."];
+		   (wb) ->
+		       ["
+ * - wb   - Write barrier. Orders *only* stores. These are
+ *          not allowed to be reordered over the barrier.
+ *          Store in atomic operation is ordered *after*
+ *          the barrier."];
 		   (B) ->
 		       [" * - ", a2l(B), "\n"]
 	       end,
-	       ?BARRIERS),
-     " *
+	       lists:reverse(?BARRIERS)),
+     "
+ *
  * We implement all of these operation/barrier
  * combinations, regardless of whether they are useful
  * or not (some of them are useless).