Improve ethread atomics

The ethread atomics API now also provide double word size atomics.
Double word size atomics are implemented using native atomic
instructions on x86 (when the cmpxchg8b instruction is available)
and on x86_64 (when the cmpxchg16b instruction is available). On
other hardware where 32-bit atomics or word size atomics are
available, an optimized fallback is used; otherwise, a spinlock,
or a mutex based fallback is used.

The ethread library now performs runtime tests for presence of
hardware features, such as for example SSE2 instructions, instead
of requiring this to be determined at compile time.

There are now functions implementing each atomic operation with the
following implied memory barrier semantics: none, read, write,
acquire, release, and full. Some of the operation-barrier
combinations aren't especially useful. But instead of filtering
useful ones out, and potentially miss a useful one, we implement
them all.

A much smaller set of functionality for native atomics are required
to be implemented than before. More or less only cmpxchg and a
membar macro are required to be implemented for each atomic size.
Other functions will automatically be constructed from these. It is,
of course, often wise to implement more that this if possible from a
performance perspective.

1 files changed, 125 insertions, 173 deletions

diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h
index 93bc06036f..fb1288c330 100644
--- a/erts/include/internal/libatomic_ops/ethr_atomic.h
+++ b/erts/include/internal/libatomic_ops/ethr_atomic.h
@@ -25,16 +25,6 @@
 #ifndef ETHR_LIBATOMIC_OPS_ATOMIC_H__
 #define ETHR_LIBATOMIC_OPS_ATOMIC_H__
 
-#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS)
-#define ETHR_HAVE_NATIVE_ATOMICS 1
-
-#if (defined(__i386__) && !defined(ETHR_PRE_PENTIUM4_COMPAT)) \
-    || defined(__x86_64__)
-#define AO_USE_PENTIUM4_INSTRS
-#endif
-
-#include "atomic_ops.h"
-
 /*
  * libatomic_ops can be downloaded from:
  *   http://www.hpl.hp.com/research/linux/atomic_ops/
@@ -46,21 +36,18 @@
  * - AO_store()
  * - AO_compare_and_swap()
  *
- * The `AO_t' type also have to be at least as large as the `void *' type.
  */
 
-#if ETHR_SIZEOF_AO_T < ETHR_SIZEOF_PTR
-#error The AO_t type is too small
-#endif
-
 #if ETHR_SIZEOF_AO_T == 4
 #define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATIVE_ATOMIC32_IMPL "libatomic_ops"
 #define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
 #define ETHR_ATMC_T__ ethr_native_atomic32_t
 #define ETHR_AINT_T__ ethr_sint32_t
 #define ETHR_AINT_SUFFIX__ "l"
 #elif ETHR_SIZEOF_AO_T == 8
 #define ETHR_HAVE_NATIVE_ATOMIC64 1
+#define ETHR_NATIVE_ATOMIC64_IMPL "libatomic_ops"
 #define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
 #define ETHR_ATMC_T__ ethr_native_atomic64_t
 #define ETHR_AINT_T__ ethr_sint64_t
@@ -69,61 +56,29 @@
 #error "Unsupported integer size"
 #endif
 
-#if ETHR_SIZEOF_AO_T == 8
-typedef union {
-    volatile AO_t counter;
-    ethr_sint32_t sint32[2];
-} ETHR_ATMC_T__;
-#else
 typedef struct {
     volatile AO_t counter;
 } ETHR_ATMC_T__;
-#endif
 
-#define ETHR_MEMORY_BARRIER AO_nop_full()
-#ifdef AO_HAVE_nop_write
-# define ETHR_WRITE_MEMORY_BARRIER AO_nop_write()
-#else
-# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER
-#endif
-#ifdef AO_HAVE_nop_read
-#  define ETHR_READ_MEMORY_BARRIER AO_nop_read()
-#else
-#  define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER
-#endif
-#ifdef AO_NO_DD_ORDERING
-#  define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
 #else
-#  define ETHR_READ_DEPEND_MEMORY_BARRIER AO_compiler_barrier()
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
 #endif
 
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
-
 static ETHR_INLINE ETHR_AINT_T__ *
 ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
 {
     return (ETHR_AINT_T__ *) &var->counter;
 }
 
-#if ETHR_SIZEOF_AO_T == 8
-/*
- * We also need to provide an ethr_native_atomic32_addr(), since
- * this 64-bit implementation will be used implementing 32-bit
- * native atomics.
- */
-
-static ETHR_INLINE ethr_sint32_t *
-ethr_native_atomic32_addr(ETHR_ATMC_T__ *var)
-{
-    ETHR_ASSERT(((void *) &var->sint32[0]) == ((void *) &var->counter));
-#ifdef ETHR_BIGENDIAN
-    return &var->sint32[1];
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
 #else
-    return &var->sint32[0];
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
 #endif
-}
-
-#endif /* ETHR_SIZEOF_AO_T == 8 */
 
 static ETHR_INLINE void
 ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
@@ -131,197 +86,198 @@ ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
     AO_store(&var->counter, (AO_t) value);
 }
 
+#ifdef AO_HAVE_store_release
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1
+#endif
+
 static ETHR_INLINE void
-ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
 {
-    ETHR_NATMC_FUNC__(set)(var, value);
+    AO_store_release(&var->counter, (AO_t) value);
 }
 
+#endif
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
+#endif
+
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
 {
     return (ETHR_AINT_T__) AO_load(&var->counter);
 }
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
-{
-#ifdef AO_HAVE_fetch_and_add_full
-    return ((ETHR_AINT_T__) AO_fetch_and_add_full(&var->counter, (AO_t) incr)) + incr;
+#ifdef AO_HAVE_load_acquire
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB 1
 #else
-    while (1) {
-	AO_t exp = AO_load(&var->counter);
-	AO_t new = exp + (AO_t) incr;
-	if (AO_compare_and_swap_full(&var->counter, exp, new))
-	    return (ETHR_AINT_T__) new;
-    }
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB 1
 #endif
-}
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
 {
-    (void) ETHR_NATMC_FUNC__(add_return)(var, incr);
+    return (ETHR_AINT_T__) AO_load_acquire(&var->counter);
 }
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
-{
-#ifdef AO_HAVE_fetch_and_add1_full
-    return ((ETHR_AINT_T__) AO_fetch_and_add1_full(&var->counter)) + 1;
-#else
-    return ETHR_NATMC_FUNC__(add_return)(var, 1);
 #endif
-}
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
-{
-    (void) ETHR_NATMC_FUNC__(inc_return)(var);
-}
+#ifdef AO_HAVE_fetch_and_add
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
-{
-#ifdef AO_HAVE_fetch_and_sub1_full
-    return ((ETHR_AINT_T__) AO_fetch_and_sub1_full(&var->counter)) - 1;
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1
 #else
-    return ETHR_NATMC_FUNC__(add_return)(var, -1);
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN 1
 #endif
-}
-
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var)
-{
-    (void) ETHR_NATMC_FUNC__(dec_return)(var);
-}
 
 static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
 {
-    while (1) {
-	AO_t exp = AO_load(&var->counter);
-	AO_t new = exp & ((AO_t) mask);
-	if (AO_compare_and_swap_full(&var->counter, exp, new))
-	    return (ETHR_AINT_T__) exp;
-    }
+    return ((ETHR_AINT_T__) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr;
 }
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
-{
-    while (1) {
-	AO_t exp = AO_load(&var->counter);
-	AO_t new = exp | ((AO_t) mask);
-	if (AO_compare_and_swap_full(&var->counter, exp, new))
-	    return (ETHR_AINT_T__) exp;
-    }
-}
+#endif
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
-			   ETHR_AINT_T__ new,
-			   ETHR_AINT_T__ exp)
-{
-    ETHR_AINT_T__ act;
-    do {
-	if (AO_compare_and_swap_full(&var->counter, (AO_t) exp, (AO_t) new))
-	    return exp;
-	act = (ETHR_AINT_T__) AO_load(&var->counter);
-    } while (act == exp);
-    return act;
-}
+#ifdef AO_HAVE_fetch_and_add1
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN 1
+#endif
 
 static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
+ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
 {
-    while (1) {
-	AO_t exp = AO_load(&var->counter);
-	if (AO_compare_and_swap_full(&var->counter, exp, (AO_t) new))
-	    return (ETHR_AINT_T__) exp;
-    }
+    return ((ETHR_AINT_T__) AO_fetch_and_add1(&var->counter)) + 1;
 }
 
-/*
- * Atomic ops with at least specified barriers.
- */
+#endif
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
-{
-#ifdef AO_HAVE_load_acquire
-    return (ETHR_AINT_T__) AO_load_acquire(&var->counter);
+#ifdef AO_HAVE_fetch_and_add1_acquire
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1
 #else
-    ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var);
-    ETHR_MEMORY_BARRIER;
-    return res;
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB 1
 #endif
-}
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
 {
-#ifdef AO_HAVE_fetch_and_add1_acquire
     return ((ETHR_AINT_T__) AO_fetch_and_add1_acquire(&var->counter)) + 1;
+}
+
+#endif
+
+#ifdef AO_HAVE_fetch_and_sub1
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN 1
 #else
-    ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(add_return)(var, 1);
-    return res;
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN 1
 #endif
-}
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
 {
-#ifdef AO_HAVE_store_release
-    AO_store_release(&var->counter, (AO_t) value);
+    return ((ETHR_AINT_T__) AO_fetch_and_sub1(&var->counter)) - 1;
+}
+
+#endif
+
+#ifdef AO_HAVE_fetch_and_sub1_release
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1
 #else
-    ETHR_MEMORY_BARRIER;
-    ETHR_NATMC_FUNC__(set)(var, value);
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB 1
 #endif
-}
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
 {
-#ifdef AO_HAVE_fetch_and_sub1_release
     return ((ETHR_AINT_T__) AO_fetch_and_sub1_release(&var->counter)) - 1;
+}
+
+#endif
+
+#ifdef AO_HAVE_compare_and_swap
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1
 #else
-    return ETHR_NATMC_FUNC__(dec_return)(var);
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG 1
 #endif
-}
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
+			   ETHR_AINT_T__ new,
+			   ETHR_AINT_T__ exp)
 {
-    (void) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+    ETHR_AINT_T__ act;
+    do {
+	if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new))
+	    return exp;
+	act = (ETHR_AINT_T__) AO_load(&var->counter);
+    } while (act == exp);
+    return act;
 }
 
+#endif
+
+#ifdef AO_HAVE_compare_and_swap_acquire
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1
+#endif
+
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
 				ETHR_AINT_T__ new,
 				ETHR_AINT_T__ exp)
 {
-#ifdef AO_HAVE_compare_and_swap_acquire
     ETHR_AINT_T__ act;
     do {
 	if (AO_compare_and_swap_acquire(&var->counter, (AO_t) exp, (AO_t) new))
 	    return exp;
+#ifdef AO_HAVE_load_acquire
+	act = (ETHR_AINT_T__) AO_load_acquire(&var->counter);
+#else
 	act = (ETHR_AINT_T__) AO_load(&var->counter);
+#endif
     } while (act == exp);
+#ifndef AO_HAVE_load_acquire
     AO_nop_full();
+#endif
     return act;
+}
+
+#endif
+
+#ifdef AO_HAVE_compare_and_swap_release
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1
 #else
-    ETHR_AINT_T__ act = ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp);
-    return act;
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB 1
 #endif
-}
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
 				ETHR_AINT_T__ new,
 				ETHR_AINT_T__ exp)
 {
-#ifdef AO_HAVE_compare_and_swap_release
     ETHR_AINT_T__ act;
     do {
 	if (AO_compare_and_swap_release(&var->counter, (AO_t) exp, (AO_t) new))
@@ -329,11 +285,9 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
 	act = (ETHR_AINT_T__) AO_load(&var->counter);
     } while (act == exp);
     return act;
-#else
-    return ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp);
-#endif
 }
 
+#endif
 
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
@@ -341,6 +295,4 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
 #undef ETHR_ATMC_T__
 #undef ETHR_AINT_T__
 
-#endif /* !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS) */
-
 #endif /* ETHR_LIBATOMIC_OPS_ATOMIC_H__ */