Improve usage of libatomic_ops for word size atomics

Use AO_fetch_compare_and_swap*() when present

3 files changed, 337 insertions, 10 deletions

diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h
index 72c054b588..ad5d05704c 100644
--- a/erts/include/internal/ethread.h
+++ b/erts/include/internal/ethread.h
@@ -364,8 +364,8 @@ extern ethr_runtime_t ethr_runtime__;
 #          include "sparc64/ethread.h"
 #        endif
 #      endif
-#      include "gcc/ethread.h"
 #      include "libatomic_ops/ethread.h"
+#      include "gcc/ethread.h"
 #    endif
 #  elif defined(ETHR_HAVE_LIBATOMIC_OPS)
 #    include "libatomic_ops/ethread.h"
diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h
index fb1288c330..734cdf0890 100644
--- a/erts/include/internal/libatomic_ops/ethr_atomic.h
+++ b/erts/include/internal/libatomic_ops/ethr_atomic.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2010-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2014. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -32,22 +32,23 @@
  * These operations need to be defined by libatomic_ops;
  * otherwise, we won't compile:
  * - AO_nop_full()
- * - AO_load()
- * - AO_store()
- * - AO_compare_and_swap()
+ * - AO_load() || AO_load_aquire()
+ * - AO_store() || AO_store_release()
+ * - AO_compare_and_swap() || AO_compare_and_swap_acquire()
+ *   || AO_compare_and_swap_release() || AO_compare_and_swap_full()
  *
  */
 
 #if ETHR_SIZEOF_AO_T == 4
 #define ETHR_HAVE_NATIVE_ATOMIC32 1
-#define ETHR_NATIVE_ATOMIC32_IMPL "libatomic_ops"
+#define ETHR_NATIVE_ATOMIC32_IMPL ETHR_NATIVE_IMPL__
 #define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
 #define ETHR_ATMC_T__ ethr_native_atomic32_t
 #define ETHR_AINT_T__ ethr_sint32_t
 #define ETHR_AINT_SUFFIX__ "l"
 #elif ETHR_SIZEOF_AO_T == 8
 #define ETHR_HAVE_NATIVE_ATOMIC64 1
-#define ETHR_NATIVE_ATOMIC64_IMPL "libatomic_ops"
+#define ETHR_NATIVE_ATOMIC64_IMPL ETHR_NATIVE_IMPL__
 #define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
 #define ETHR_ATMC_T__ ethr_native_atomic64_t
 #define ETHR_AINT_T__ ethr_sint64_t
@@ -74,6 +75,8 @@ ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
     return (ETHR_AINT_T__ *) &var->counter;
 }
 
+#ifdef AO_HAVE_store
+
 #if ETHR_SIZEOF_AO_T == 4
 #  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
 #else
@@ -86,6 +89,24 @@ ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
     AO_store(&var->counter, (AO_t) value);
 }
 
+#endif
+
+#ifdef AO_HAVE_store_write
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB 1
+#endif
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_wb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+{
+    AO_store_write(&var->counter, (AO_t) value);
+}
+
+#endif
+
 #ifdef AO_HAVE_store_release
 
 #if ETHR_SIZEOF_AO_T == 4
@@ -102,6 +123,24 @@ ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
 
 #endif
 
+#ifdef AO_HAVE_store_full
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB 1
+#endif
+
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+{
+    AO_store_full(&var->counter, (AO_t) value);
+}
+
+#endif
+
+#ifdef AO_HAVE_load
+
 #if ETHR_SIZEOF_AO_T == 4
 #  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
 #else
@@ -114,6 +153,24 @@ ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
     return (ETHR_AINT_T__) AO_load(&var->counter);
 }
 
+#endif
+
+#ifdef AO_HAVE_load_read
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read_rb)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__) AO_load_read(&var->counter);
+}
+
+#endif
+
 #ifdef AO_HAVE_load_acquire
 
 #if ETHR_SIZEOF_AO_T == 4
@@ -130,6 +187,22 @@ ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
 
 #endif
 
+#ifdef AO_HAVE_load_full
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read_mb)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__) AO_load_full(&var->counter);
+}
+
+#endif
+
 #ifdef AO_HAVE_fetch_and_add
 
 #if ETHR_SIZEOF_AO_T == 4
@@ -146,6 +219,54 @@ ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
 
 #endif
 
+#ifdef AO_HAVE_fetch_and_add_acquire
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(add_return_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
+{
+    return ((ETHR_AINT_T__) AO_fetch_and_add_acquire(&var->counter, (AO_t) incr)) + incr;
+}
+
+#endif
+
+#ifdef AO_HAVE_fetch_and_add_release
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(add_return_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
+{
+    return ((ETHR_AINT_T__) AO_fetch_and_add_release(&var->counter, (AO_t) incr)) + incr;
+}
+
+#endif
+
+#ifdef AO_HAVE_fetch_and_add_full
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
+{
+    return ((ETHR_AINT_T__) AO_fetch_and_add_full(&var->counter, (AO_t) incr)) + incr;
+}
+
+#endif
+
 #ifdef AO_HAVE_fetch_and_add1
 
 #if ETHR_SIZEOF_AO_T == 4
@@ -178,6 +299,38 @@ ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
 
 #endif
 
+#ifdef AO_HAVE_fetch_and_add1_release
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return_relb)(ETHR_ATMC_T__ *var)
+{
+    return ((ETHR_AINT_T__) AO_fetch_and_add1_release(&var->counter)) + 1;
+}
+
+#endif
+
+#ifdef AO_HAVE_fetch_and_add1_full
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return_mb)(ETHR_ATMC_T__ *var)
+{
+    return ((ETHR_AINT_T__) AO_fetch_and_add1_full(&var->counter)) + 1;
+}
+
+#endif
+
 #ifdef AO_HAVE_fetch_and_sub1
 
 #if ETHR_SIZEOF_AO_T == 4
@@ -194,6 +347,22 @@ ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
 
 #endif
 
+#ifdef AO_HAVE_fetch_and_sub1_acquire
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return_acqb)(ETHR_ATMC_T__ *var)
+{
+    return ((ETHR_AINT_T__) AO_fetch_and_sub1_acquire(&var->counter)) - 1;
+}
+
+#endif
+
 #ifdef AO_HAVE_fetch_and_sub1_release
 
 #if ETHR_SIZEOF_AO_T == 4
@@ -210,7 +379,60 @@ ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
 
 #endif
 
-#ifdef AO_HAVE_compare_and_swap
+#ifdef AO_HAVE_fetch_and_sub1_full
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return_mb)(ETHR_ATMC_T__ *var)
+{
+    return ((ETHR_AINT_T__) AO_fetch_and_sub1_full(&var->counter)) - 1;
+}
+
+#endif
+
+#if defined(AO_HAVE_compare_and_swap_full) || defined(AO_HAVE_fetch_compare_and_swap_full)
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var,
+			      ETHR_AINT_T__ new,
+			      ETHR_AINT_T__ exp)
+{
+#if defined(AO_HAVE_fetch_compare_and_swap_full)
+    return (ETHR_AINT_T__) AO_fetch_compare_and_swap_full(&var->counter,
+							  (AO_t) exp,
+							  (AO_t) new);
+#else
+    ETHR_AINT_T__ act;
+    do {
+	if (AO_compare_and_swap_full(&var->counter, (AO_t) exp, (AO_t) new))
+	    return exp;
+#ifdef AO_HAVE_load_acquire
+	act = (ETHR_AINT_T__) AO_load_acquire(&var->counter);
+#else
+	act = (ETHR_AINT_T__) AO_load(&var->counter);
+#endif
+    } while (act == exp);
+#ifndef AO_HAVE_load_acquire
+    AO_nop_full();
+#endif
+    return act;
+#endif
+}
+
+#endif
+
+#if defined(AO_HAVE_compare_and_swap) || defined(AO_HAVE_fetch_compare_and_swap)
 
 #if ETHR_SIZEOF_AO_T == 4
 #  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1
@@ -223,18 +445,28 @@ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
 			   ETHR_AINT_T__ new,
 			   ETHR_AINT_T__ exp)
 {
+#if defined(AO_HAVE_fetch_compare_and_swap)
+    return (ETHR_AINT_T__) AO_fetch_compare_and_swap(&var->counter,
+						     (AO_t) exp,
+						     (AO_t) new);
+#else
     ETHR_AINT_T__ act;
     do {
 	if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new))
 	    return exp;
+#ifdef AO_HAVE_load
 	act = (ETHR_AINT_T__) AO_load(&var->counter);
+#else
+	act = (ETHR_AINT_T__) AO_load_aquire(&var->counter);
+#endif
     } while (act == exp);
     return act;
+#endif
 }
 
 #endif
 
-#ifdef AO_HAVE_compare_and_swap_acquire
+#if defined(AO_HAVE_compare_and_swap_acquire) || defined(AO_HAVE_fetch_compare_and_swap_acquire)
 
 #if ETHR_SIZEOF_AO_T == 4
 #  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
@@ -247,6 +479,11 @@ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
 				ETHR_AINT_T__ new,
 				ETHR_AINT_T__ exp)
 {
+#if defined(AO_HAVE_fetch_compare_and_swap_acquire)
+    return (ETHR_AINT_T__) AO_fetch_compare_and_swap_acquire(&var->counter,
+							     (AO_t) exp,
+							     (AO_t) new);
+#else
     ETHR_AINT_T__ act;
     do {
 	if (AO_compare_and_swap_acquire(&var->counter, (AO_t) exp, (AO_t) new))
@@ -261,11 +498,55 @@ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
     AO_nop_full();
 #endif
     return act;
+#endif
+}
+
+#endif
+
+#if defined(AO_HAVE_compare_and_swap_read) || defined(AO_HAVE_fetch_compare_and_swap_read)
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_rb)(ETHR_ATMC_T__ *var,
+			      ETHR_AINT_T__ new,
+			      ETHR_AINT_T__ exp)
+{
+#if defined(AO_HAVE_fetch_compare_and_swap_read)
+    return (ETHR_AINT_T__) AO_fetch_compare_and_swap_read(&var->counter,
+							     (AO_t) exp,
+							     (AO_t) new);
+#else
+    ETHR_AINT_T__ act;
+    do {
+	if (AO_compare_and_swap_read(&var->counter, (AO_t) exp, (AO_t) new))
+	    return exp;
+#if defined(AO_HAVE_load_read)
+	act = (ETHR_AINT_T__) AO_load_read(&var->counter);
+#elif defined(AO_HAVE_load)
+	act = (ETHR_AINT_T__) AO_load(&var->counter);
+#else
+	act = (ETHR_AINT_T__) AO_load_acquire(&var->counter);
+#endif
+    } while (act == exp);
+#ifndef AO_HAVE_load_read
+#ifdef AO_HAVE_nop_read
+    AO_nop_read();
+#else
+    AO_nop_full();
+#endif
+#endif
+    return act;
+#endif
 }
 
 #endif
 
-#ifdef AO_HAVE_compare_and_swap_release
+#if defined(AO_HAVE_compare_and_swap_release) || defined(AO_HAVE_fetch_compare_and_swap_release)
 
 #if ETHR_SIZEOF_AO_T == 4
 #  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1
@@ -278,13 +559,57 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
 				ETHR_AINT_T__ new,
 				ETHR_AINT_T__ exp)
 {
+#if defined(AO_HAVE_fetch_compare_and_swap_release)
+    return (ETHR_AINT_T__) AO_fetch_compare_and_swap_release(&var->counter,
+							     (AO_t) exp,
+							     (AO_t) new);
+#else
     ETHR_AINT_T__ act;
     do {
 	if (AO_compare_and_swap_release(&var->counter, (AO_t) exp, (AO_t) new))
 	    return exp;
+#ifdef AO_HAVE_load
 	act = (ETHR_AINT_T__) AO_load(&var->counter);
+#else
+	act = (ETHR_AINT_T__) AO_load_acquire(&var->counter);
+#endif
+    } while (act == exp);
+    return act;
+#endif
+}
+
+#endif
+
+#if defined(AO_HAVE_compare_and_swap_write) || defined(AO_HAVE_fetch_compare_and_swap_write)
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB 1
+#endif
+
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg_wb)(ETHR_ATMC_T__ *var,
+			      ETHR_AINT_T__ new,
+			      ETHR_AINT_T__ exp)
+{
+#if defined(AO_HAVE_fetch_compare_and_swap_write)
+    return (ETHR_AINT_T__) AO_fetch_compare_and_swap_write(&var->counter,
+							   (AO_t) exp,
+							   (AO_t) new);
+#else
+    ETHR_AINT_T__ act;
+    do {
+	if (AO_compare_and_swap_write(&var->counter, (AO_t) exp, (AO_t) new))
+	    return exp;
+#ifdef AO_HAVE_load
+	act = (ETHR_AINT_T__) AO_load(&var->counter);
+#else
+	act = (ETHR_AINT_T__) AO_load_acquire(&var->counter);
+#endif
     } while (act == exp);
     return act;
+#endif
 }
 
 #endif
diff --git a/erts/include/internal/libatomic_ops/ethread.h b/erts/include/internal/libatomic_ops/ethread.h
index e1fdd588bb..7375b73793 100644
--- a/erts/include/internal/libatomic_ops/ethread.h
+++ b/erts/include/internal/libatomic_ops/ethread.h
@@ -33,6 +33,8 @@
 #define AO_USE_PENTIUM4_INSTRS
 #endif
 
+#define ETHR_NATIVE_IMPL__ "libatomic_ops"
+
 #include "atomic_ops.h"
 #include "ethr_membar.h"
 #include "ethr_atomic.h"