Misc memory barrier fixes

- Document barrier semantics
- Introduce ddrb suffix on atomic ops
- Barrier macros for both non-SMP and SMP case
- Make the thread progress API a bit more intuitive

2 files changed, 477 insertions, 111 deletions

diff --git a/erts/lib_src/common/ethr_atomics.c b/erts/lib_src/common/ethr_atomics.c
index 5796bdc22e..e4213e1eef 100644
--- a/erts/lib_src/common/ethr_atomics.c
+++ b/erts/lib_src/common/ethr_atomics.c
@@ -10,7 +10,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -64,12 +64,31 @@
  * Appart from a function implementing the atomic operation
  * with unspecified memory barrier semantics, there are
  * functions implementing each operation with the following
- * memory barrier semantics:
- * - rb (read barrier)
- * - wb (write barrier)
- * - acqb (acquire barrier)
- * - relb (release barrier)
- * - mb (full memory barrier)
+ * implied memory barrier semantics:
+ * - mb   - Full memory barrier. Orders both loads, and
+ *          stores before, and after the atomic operation.
+ *          No load or store is allowed to be reordered
+ *          over the atomic operation.
+ * - relb - Release barrier. Orders both loads, and
+ *          stores appearing *before* the atomic
+ *          operation. These are not allowed to be
+ *          reordered over the atomic operation.
+ * - acqb - Acquire barrier. Orders both loads, and stores
+ *          appearing *after* the atomic operation. These
+ *          are not allowed to be reordered over the
+ *          atomic operation.
+ * - wb   - Write barrier. Orders *only* stores. These are
+ *          not allowed to be reordered over the barrier.
+ *          Store in atomic operation is ordered *after*
+ *          the barrier.
+ * - rb   - Read barrier. Orders *only* loads. These are
+ *          not allowed to be reordered over the barrier.
+ *          Load in atomic operation is ordered *before*
+ *          the barrier. 
+ * - ddrb - Data dependency read barrier. Orders *only*
+ *          loads according to data dependency across the
+ *          barrier. Load in atomic operation is ordered
+ *          before the barrier.
  *
  * We implement all of these operation/barrier
  * combinations, regardless of whether they are useful
@@ -542,6 +561,15 @@ int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_s
 }
 #endif
 
+int ETHR_DW_ATOMIC_FUNC__(cmpxchg_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ETHR_DW_ATOMIC_FUNC__(cmpxchg)(var, val, old_val);
+#else
+    return ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(var, val, old_val);
+#endif
+}
+
 int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
 {
     int res;
@@ -756,6 +784,15 @@ void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
 }
 #endif
 
+void ETHR_DW_ATOMIC_FUNC__(set_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_DW_ATOMIC_FUNC__(set)(var, val);
+#else
+    ETHR_DW_ATOMIC_FUNC__(set_rb)(var, val);
+#endif
+}
+
 void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -910,6 +947,15 @@ void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
 }
 #endif
 
+void ETHR_DW_ATOMIC_FUNC__(read_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_DW_ATOMIC_FUNC__(read)(var, val);
+#else
+    ETHR_DW_ATOMIC_FUNC__(read_rb)(var, val);
+#endif
+}
+
 void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -1061,6 +1107,15 @@ void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
 }
 #endif
 
+void ETHR_DW_ATOMIC_FUNC__(init_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ETHR_DW_ATOMIC_FUNC__(init)(var, val);
+#else
+    ETHR_DW_ATOMIC_FUNC__(init_rb)(var, val);
+#endif
+}
+
 void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
 {
     ETHR_ASSERT(var);
@@ -1221,6 +1276,15 @@ ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t
     return res;
 }
 
+ethr_sint_t ethr_atomic_cmpxchg_ddrb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_cmpxchg(var, val, old_val);
+#else
+    return ethr_atomic_cmpxchg_rb(var, val, old_val);
+#endif
+}
+
 ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
 {
     ethr_sint_t res;
@@ -1332,6 +1396,15 @@ ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val)
     return res;
 }
 
+ethr_sint_t ethr_atomic_xchg_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_xchg(var, val);
+#else
+    return ethr_atomic_xchg_rb(var, val);
+#endif
+}
+
 ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val)
 {
     ethr_sint_t res;
@@ -1437,6 +1510,15 @@ void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val)
 
 }
 
+void ethr_atomic_set_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic_set(var, val);
+#else
+    ethr_atomic_set_rb(var, val);
+#endif
+}
+
 void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -1536,6 +1618,15 @@ void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val)
 
 }
 
+void ethr_atomic_init_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic_init(var, val);
+#else
+    ethr_atomic_init_rb(var, val);
+#endif
+}
+
 void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val)
 {
     ETHR_ASSERT(var);
@@ -1632,6 +1723,15 @@ ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val)
     return res;
 }
 
+ethr_sint_t ethr_atomic_add_read_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_add_read(var, val);
+#else
+    return ethr_atomic_add_read_rb(var, val);
+#endif
+}
+
 ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val)
 {
     ethr_sint_t res;
@@ -1738,6 +1838,15 @@ ethr_sint_t ethr_atomic_read(ethr_atomic_t *var)
     return res;
 }
 
+ethr_sint_t ethr_atomic_read_ddrb(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_read(var);
+#else
+    return ethr_atomic_read_rb(var);
+#endif
+}
+
 ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var)
 {
     ethr_sint_t res;
@@ -1843,6 +1952,15 @@ ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var)
     return res;
 }
 
+ethr_sint_t ethr_atomic_inc_read_ddrb(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_inc_read(var);
+#else
+    return ethr_atomic_inc_read_rb(var);
+#endif
+}
+
 ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var)
 {
     ethr_sint_t res;
@@ -1949,6 +2067,15 @@ ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var)
     return res;
 }
 
+ethr_sint_t ethr_atomic_dec_read_ddrb(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_dec_read(var);
+#else
+    return ethr_atomic_dec_read_rb(var);
+#endif
+}
+
 ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var)
 {
     ethr_sint_t res;
@@ -2054,6 +2181,15 @@ void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val)
 
 }
 
+void ethr_atomic_add_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic_add(var, val);
+#else
+    ethr_atomic_add_rb(var, val);
+#endif
+}
+
 void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -2154,6 +2290,15 @@ void ethr_atomic_inc(ethr_atomic_t *var)
 
 }
 
+void ethr_atomic_inc_ddrb(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic_inc(var);
+#else
+    ethr_atomic_inc_rb(var);
+#endif
+}
+
 void ethr_atomic_inc_rb(ethr_atomic_t *var)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -2254,6 +2399,15 @@ void ethr_atomic_dec(ethr_atomic_t *var)
 
 }
 
+void ethr_atomic_dec_ddrb(ethr_atomic_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic_dec(var);
+#else
+    ethr_atomic_dec_rb(var);
+#endif
+}
+
 void ethr_atomic_dec_rb(ethr_atomic_t *var)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -2355,6 +2509,15 @@ ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val)
     return res;
 }
 
+ethr_sint_t ethr_atomic_read_band_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_read_band(var, val);
+#else
+    return ethr_atomic_read_band_rb(var, val);
+#endif
+}
+
 ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val)
 {
     ethr_sint_t res;
@@ -2461,6 +2624,15 @@ ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val)
     return res;
 }
 
+ethr_sint_t ethr_atomic_read_bor_ddrb(ethr_atomic_t *var, ethr_sint_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic_read_bor(var, val);
+#else
+    return ethr_atomic_read_bor_rb(var, val);
+#endif
+}
+
 ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val)
 {
     ethr_sint_t res;
@@ -2587,6 +2759,15 @@ ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, eth
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_cmpxchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_cmpxchg(var, val, old_val);
+#else
+    return ethr_atomic32_cmpxchg_rb(var, val, old_val);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
 {
     ethr_sint32_t res;
@@ -2675,6 +2856,15 @@ ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val)
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_xchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_xchg(var, val);
+#else
+    return ethr_atomic32_xchg_rb(var, val);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val)
 {
     ethr_sint32_t res;
@@ -2762,6 +2952,15 @@ void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val)
 
 }
 
+void ethr_atomic32_set_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic32_set(var, val);
+#else
+    ethr_atomic32_set_rb(var, val);
+#endif
+}
+
 void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -2843,6 +3042,15 @@ void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val)
 
 }
 
+void ethr_atomic32_init_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic32_init(var, val);
+#else
+    ethr_atomic32_init_rb(var, val);
+#endif
+}
+
 void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val)
 {
     ETHR_ASSERT(var);
@@ -2921,6 +3129,15 @@ ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val)
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_add_read_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_add_read(var, val);
+#else
+    return ethr_atomic32_add_read_rb(var, val);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val)
 {
     ethr_sint32_t res;
@@ -3009,6 +3226,15 @@ ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var)
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_read_ddrb(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_read(var);
+#else
+    return ethr_atomic32_read_rb(var);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var)
 {
     ethr_sint32_t res;
@@ -3097,6 +3323,15 @@ ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var)
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_inc_read_ddrb(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_inc_read(var);
+#else
+    return ethr_atomic32_inc_read_rb(var);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var)
 {
     ethr_sint32_t res;
@@ -3185,6 +3420,15 @@ ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var)
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_dec_read_ddrb(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_dec_read(var);
+#else
+    return ethr_atomic32_dec_read_rb(var);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var)
 {
     ethr_sint32_t res;
@@ -3272,6 +3516,15 @@ void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val)
 
 }
 
+void ethr_atomic32_add_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic32_add(var, val);
+#else
+    ethr_atomic32_add_rb(var, val);
+#endif
+}
+
 void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -3354,6 +3607,15 @@ void ethr_atomic32_inc(ethr_atomic32_t *var)
 
 }
 
+void ethr_atomic32_inc_ddrb(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic32_inc(var);
+#else
+    ethr_atomic32_inc_rb(var);
+#endif
+}
+
 void ethr_atomic32_inc_rb(ethr_atomic32_t *var)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -3436,6 +3698,15 @@ void ethr_atomic32_dec(ethr_atomic32_t *var)
 
 }
 
+void ethr_atomic32_dec_ddrb(ethr_atomic32_t *var)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ethr_atomic32_dec(var);
+#else
+    ethr_atomic32_dec_rb(var);
+#endif
+}
+
 void ethr_atomic32_dec_rb(ethr_atomic32_t *var)
 {
     ETHR_ASSERT(!ethr_not_inited__);
@@ -3519,6 +3790,15 @@ ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val)
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_read_band_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_read_band(var, val);
+#else
+    return ethr_atomic32_read_band_rb(var, val);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val)
 {
     ethr_sint32_t res;
@@ -3607,6 +3887,15 @@ ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val)
     return res;
 }
 
+ethr_sint32_t ethr_atomic32_read_bor_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
+{
+#ifdef ETHR_ORDERED_READ_DEPEND
+    return ethr_atomic32_read_bor(var, val);
+#else
+    return ethr_atomic32_read_bor_rb(var, val);
+#endif
+}
+
 ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val)
 {
     ethr_sint32_t res;
diff --git a/erts/lib_src/utils/make_atomics_api b/erts/lib_src/utils/make_atomics_api
index f4e71c7618..d8b1a56100 100755
--- a/erts/lib_src/utils/make_atomics_api
+++ b/erts/lib_src/utils/make_atomics_api
@@ -4,7 +4,7 @@
 %%
 %% %CopyrightBegin%
 %%
-%% Copyright Ericsson AB 2011. All Rights Reserved.
+%% Copyright Ericsson AB 2011-2012. All Rights Reserved.
 %%
 %% The contents of this file are subject to the Erlang Public License,
 %% Version 1.1, (the "License"); you may not use this file except in
@@ -50,7 +50,9 @@
 -define(DW_RTCHK_MACRO, "ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__").
 
 %% Barrier versions we implement
--define(BARRIERS, [none, rb, wb, acqb, relb, mb]).
+-define(BARRIERS, [none, ddrb, rb, wb, acqb, relb, mb]).
+-define(NON_NATIVE_BARRIERS, [ddrb]).
+-define(NATIVE_BARRIERS, (?BARRIERS -- ?NON_NATIVE_BARRIERS)).
 
 -define(ATOMIC_SIZES, ["dword", "word", "32"]).
 
@@ -381,7 +383,6 @@ do_func_header(#atomic_context{dw = true,
 	       cmpxchg, Inline, Func) ->
     [Inline, "int ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " *", Arg2, ", ", AintT, " *", Arg3, ")"].
 
-
 xbarriers(_Op, none, _NB) ->
     {"", ""};
 
@@ -455,7 +456,7 @@ try_barrier_order_first(mb) ->
 
 try_barrier_order(B) ->
     First = try_barrier_order_first(B),
-    First ++ (?BARRIERS -- First).
+    First ++ (?NATIVE_BARRIERS -- First).
 
 native_barrier_op(#atomic_context{'NATMC' = NATMC} = AC, If, ExtraDecl, Op, B, NB, TypeCasts) ->
     NOpStr = opstr(native(Op)),
@@ -571,12 +572,12 @@ do_cmpxchg_fallback_define(#atomic_context{'NATMC' = NATMC,
     NoneTryBarrierOrder = try_barrier_order(none),
     %% First a sanity check
     ["
-#if (", NotDefCMPXCHG(hd(?BARRIERS)) ,
+#if (", NotDefCMPXCHG(hd(?NATIVE_BARRIERS)) ,
      lists:map(fun (B) ->
 		       [" \\
      && ", NotDefCMPXCHG(B)]
 	       end,
-	       tl(?BARRIERS)), ")
+	       tl(?NATIVE_BARRIERS)), ")
 #  error \"No native cmpxchg() op available\"
 #endif
 
@@ -744,7 +745,7 @@ translate_have_defs(#atomic_context{dw = DW, 'NATMC' = NATMC}) ->
 								      ]
 							     end]
 						    end,
-						    ?BARRIERS)
+						    ?NATIVE_BARRIERS)
 				  end,
 				  case DW of
 				      true -> ?DW_ATOMIC_OPS;
@@ -801,6 +802,46 @@ rtchk_fallback_call(Return, #atomic_context{dw = DW,
 			false -> [RetVar, " ="]
 		    end, [?DW_FUNC_MACRO, "(", opstr(Op), op_barrier_ext(B), ")"], Arg1, Arg2, Arg3, "").
 
+non_native_barrier(B) ->
+    lists:member(B, ?NON_NATIVE_BARRIERS).
+
+non_native_barrier_impl(AC, Type, Macro, Op, B) ->
+    ["
+", func_header(AC, Type, Macro, Op, B), "
+{",
+     case B of
+	 ddrb ->
+	     ["
+#ifdef ETHR_ORDERED_READ_DEPEND
+    ", func_call(AC, Type, Macro, Op, none, true), "
+#else
+    ", func_call(AC, Type, Macro, Op, rb, true), "
+#endif
+"
+	     ]
+     end,
+     "}
+"
+    ].
+
+func_call(#atomic_context{'ATMC' = ATMC} = AC, inline_implementation, _Macro, Op, B, RetStatement) ->
+    func_call(AC, Op, ["ETHR_", ATMC, "_FUNC__(", opstr(Op), op_barrier_ext(B), ")"], RetStatement);
+func_call(#atomic_context{atomic = Atomic} = AC, implementation, false, Op, B, RetStatement) ->
+    func_call(AC, Op, [Atomic, "_", opstr(Op), op_barrier_ext(B)], RetStatement);
+func_call(AC, implementation, Macro, Op, B, RetStatement) ->
+    func_call(AC, Op, [Macro, "(", opstr(Op), op_barrier_ext(B), ")"], RetStatement).
+
+func_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3} = AC, Op, Func, true) ->
+    op_call(Op, DW, case is_return_op(AC, Op) of
+			true -> "return";
+			false -> ""
+		    end, Func, Arg1, Arg2, Arg3, "");
+func_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3, ret_var = RetVar} = AC, Op, Func, false) ->
+    op_call(Op, DW, case is_return_op(AC, Op) of
+			true -> [RetVar, " = "];
+			false -> ""
+		    end, Func, Arg1, Arg2, Arg3, "").
+
 make_implementations(#atomic_context{dw = DW,
 				     ret_type = RetType,
 				     ret_var = RetVar,
@@ -858,73 +899,78 @@ make_implementations(#atomic_context{dw = DW,
 
 ",
 			lists:map(fun (B) ->
-					  TryBarriers = try_barrier_order(B),
-					  ["
+					  case non_native_barrier(B) of
+					      true ->
+						  non_native_barrier_impl(AC, inline_implementation, false, Op, B);
+					      false ->
+						  TryBarriers = try_barrier_order(B),
+						  ["
 ", func_header(AC, inline_implementation, false, Op, B), "
 {
 ",
-					   case is_return_op(AC, Op) of
-					       true ->
-						   ["    ", RetType, " ", RetVar, ";\n"];
-					       _ -> ""
-					   end,
-					   case DW of
-					       true ->
-						   [RtchkBegin,
-						    "\n",
-						    su_dw_native_barrier_op(AC, "#if", Op, B, hd(TryBarriers)),
-						    lists:map(fun (NB) ->
-								      su_dw_native_barrier_op(AC, "#elif", Op, B, NB)
-							      end,
-							      tl(TryBarriers)),
-						    lists:map(fun (NB) ->
-								      dw_native_barrier_op(AC, "#elif", "", Op, B, NB)
-							      end,
-							      TryBarriers),
-						   case simple_fallback(AC, Op, B) of
-						       "" ->
-							   %% No simple fallback available;
-							   %% use cmpxchg() fallbacks...
-							   [cmpxchg_fallbacks(AC#atomic_context{'NATMC' = ["SU_", NATMC]}, true, Op, B),
-							    cmpxchg_fallbacks(AC, false, Op, B),
-							    "#else
+						   case is_return_op(AC, Op) of
+						       true ->
+							   ["    ", RetType, " ", RetVar, ";\n"];
+						       _ -> ""
+						   end,
+						   case DW of
+						       true ->
+							   [RtchkBegin,
+							    "\n",
+							    su_dw_native_barrier_op(AC, "#if", Op, B, hd(TryBarriers)),
+							    lists:map(fun (NB) ->
+									      su_dw_native_barrier_op(AC, "#elif", Op, B, NB)
+								      end,
+								      tl(TryBarriers)),
+							    lists:map(fun (NB) ->
+									      dw_native_barrier_op(AC, "#elif", "", Op, B, NB)
+								      end,
+								      TryBarriers),
+							    case simple_fallback(AC, Op, B) of
+								"" ->
+								    %% No simple fallback available;
+								    %% use cmpxchg() fallbacks...
+								    [cmpxchg_fallbacks(AC#atomic_context{'NATMC' = ["SU_", NATMC]}, true, Op, B),
+								     cmpxchg_fallbacks(AC, false, Op, B),
+								     "#else
 #error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\"
 #endif
 "
-							   ];
-						       SimpleFallback ->
-							   ["#else\n", SimpleFallback, "#endif\n"]
-						   end,
-						    RtchkEnd(false, Op, B), "\n"];
-					       false ->
-						   [native_barrier_op(AC, "#if", "", Op, B, hd(TryBarriers), true),
-						    lists:map(fun (NB) ->
-								      native_barrier_op(AC, "#elif", "", Op, B, NB, true)
-							      end,
-							      tl(TryBarriers)),
-						    case simple_fallback(AC, Op, B) of
-							"" ->
-							    %% No simple fallback available;
-							    %% use cmpxchg() fallbacks...
-							    [cmpxchg_fallbacks(AC, false, Op, B),
-							     "#else
+								    ];
+								SimpleFallback ->
+								    ["#else\n", SimpleFallback, "#endif\n"]
+							    end,
+							    RtchkEnd(false, Op, B), "\n"];
+						       false ->
+							   [native_barrier_op(AC, "#if", "", Op, B, hd(TryBarriers), true),
+							    lists:map(fun (NB) ->
+									      native_barrier_op(AC, "#elif", "", Op, B, NB, true)
+								      end,
+								      tl(TryBarriers)),
+							    case simple_fallback(AC, Op, B) of
+								"" ->
+								    %% No simple fallback available;
+								    %% use cmpxchg() fallbacks...
+								    [cmpxchg_fallbacks(AC, false, Op, B),
+								     "#else
 #error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\"
 #endif
 "
-							    ];
-							SimpleFallback ->
-							    ["#else\n", SimpleFallback, "#endif\n"]
-						    end]
-					   end,
-					   case is_return_op(AC, Op) of
-					       true ->
-						   ["    return ", RetVar, ";\n"];
-					       false ->
-						   ""
-					   end,
-					   "}\n"]
+								    ];
+								SimpleFallback ->
+								    ["#else\n", SimpleFallback, "#endif\n"]
+							    end]
+						   end,
+						   case is_return_op(AC, Op) of
+						       true ->
+							   ["    return ", RetVar, ";\n"];
+						       false ->
+							   ""
+						   end,
+						   "}\n"]
+					  end
 				  end,
-				  ?BARRIERS)]
+				  ?NATIVE_BARRIERS ++ ?NON_NATIVE_BARRIERS)] %% non-native needs to be after native...
 	       end,
 	       case DW of
 		   true -> ?DW_ATOMIC_OPS;
@@ -1159,33 +1205,38 @@ int ethr_have_native_dw_atomic(void)
 
 ",
 			lists:map(fun (B) ->
-					  ["\n",
-					   func_header(AC, implementation,
-						       case DW of
-							   true -> ?DW_FUNC_MACRO;
-							   false -> false
-						       end,  Op, B),
-					   "\n{\n",
-					   case is_return_op(AC, Op) of
-					       true -> ["    ", RetType, " ", RetVar, ";\n"];
-					       false -> ""
-					   end,
-					   case Op of
-					       init -> "";
-					       _ -> ["    ETHR_ASSERT(!ethr_not_inited__);\n"]
-					   end,
-					   ["    ETHR_ASSERT(", Arg1, ");\n"],
-					   make_native_impl_op(AC, Op, B),
-					   make_amc_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), 
-					   make_locked_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B),
-					   case is_return_op(AC, Op) of
-					       true -> ["    return ", RetVar, ";"
-						       ];
-					       false ->
-						   ""
-					   end,
-					   "\n}\n",
-					   make_symbol_to_fallback_impl(AC, Op, B)]
+					Macro = case DW of
+						   true -> ?DW_FUNC_MACRO;
+						   false -> false
+						end,
+					case non_native_barrier(B) of
+					    true ->
+						non_native_barrier_impl(AC, implementation, Macro, Op, B);
+					    false ->
+						["\n",
+					   	func_header(AC, implementation, Macro,  Op, B),
+						   "\n{\n",
+						   case is_return_op(AC, Op) of
+						       true -> ["    ", RetType, " ", RetVar, ";\n"];
+						       false -> ""
+						   end,
+						   case Op of
+						       init -> "";
+						       _ -> ["    ETHR_ASSERT(!ethr_not_inited__);\n"]
+						   end,
+						   ["    ETHR_ASSERT(", Arg1, ");\n"],
+						   make_native_impl_op(AC, Op, B),
+						   make_amc_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), 
+						   make_locked_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B),
+						   case is_return_op(AC, Op) of
+						       true -> ["    return ", RetVar, ";"
+							       ];
+						       false ->
+							   ""
+						   end,
+						   "\n}\n",
+						   make_symbol_to_fallback_impl(AC, Op, B)]
+					  end
 				  end,
 				  ?BARRIERS)]
 	       end,
@@ -1233,7 +1284,7 @@ static char *native_", DW, "atomic", Bits, "_ops[] = {",
 #endif"
 							     ]
 						     end,
-						     ?BARRIERS)
+						     ?NATIVE_BARRIERS)
 				   end,
 				   case NBits of
 				       "dw" -> ?DW_ATOMIC_OPS;
@@ -1390,25 +1441,51 @@ comments() ->
  * Appart from a function implementing the atomic operation
  * with unspecified memory barrier semantics, there are
  * functions implementing each operation with the following
- * memory barrier semantics:
-",
+ * implied memory barrier semantics:",
      lists:map(fun (none) ->
 		       "";
-		   (rb) ->
-		       [" * - rb (read barrier)\n"];
-		   (wb) ->
-		       [" * - wb (write barrier)\n"];
+		   (mb) ->
+		       ["
+ * - mb   - Full memory barrier. Orders both loads, and
+ *          stores before, and after the atomic operation.
+ *          No load or store is allowed to be reordered
+ *          over the atomic operation."];
 		   (acqb) ->
-		       [" * - acqb (acquire barrier)\n"];
+		       ["
+ * - acqb - Acquire barrier. Orders both loads, and stores
+ *          appearing *after* the atomic operation. These
+ *          are not allowed to be reordered over the
+ *          atomic operation."];
 		   (relb) ->
-		       [" * - relb (release barrier)\n"];
-		   (mb) ->
-		       [" * - mb (full memory barrier)\n"];
+		       ["
+ * - relb - Release barrier. Orders both loads, and
+ *          stores appearing *before* the atomic
+ *          operation. These are not allowed to be
+ *          reordered over the atomic operation."];
+		   (rb) ->
+		       ["
+ * - rb   - Read barrier. Orders *only* loads. These are
+ *          not allowed to be reordered over the barrier.
+ *          Load in atomic operation is ordered *before*
+ *          the barrier. "];
+		   (ddrb) ->
+		       ["
+ * - ddrb - Data dependency read barrier. Orders *only*
+ *          loads according to data dependency across the
+ *          barrier. Load in atomic operation is ordered
+ *          before the barrier."];
+		   (wb) ->
+		       ["
+ * - wb   - Write barrier. Orders *only* stores. These are
+ *          not allowed to be reordered over the barrier.
+ *          Store in atomic operation is ordered *after*
+ *          the barrier."];
 		   (B) ->
 		       [" * - ", a2l(B), "\n"]
 	       end,
-	       ?BARRIERS),
-     " *
+	       lists:reverse(?BARRIERS)),
+     "
+ *
  * We implement all of these operation/barrier
  * combinations, regardless of whether they are useful
  * or not (some of them are useless).