Merge branch 'rickard/atomics-api/OTP-9014' and OTP_R14B04

Conflicts:
	erts/aclocal.m4
	erts/emulator/beam/erl_db.c
	erts/emulator/sys/win32/sys.c
	erts/include/internal/ethread_header_config.h.in

1 files changed, 125 insertions, 173 deletions

diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h
index 93bc06036f..fb1288c330 100644
--- a/erts/include/internal/libatomic_ops/ethr_atomic.h
+++ b/erts/include/internal/libatomic_ops/ethr_atomic.h
@@ -25,16 +25,6 @@
 #ifndef ETHR_LIBATOMIC_OPS_ATOMIC_H__
 #define ETHR_LIBATOMIC_OPS_ATOMIC_H__
 
-#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS)
-#define ETHR_HAVE_NATIVE_ATOMICS 1
-
-#if (defined(__i386__) && !defined(ETHR_PRE_PENTIUM4_COMPAT)) \
-    || defined(__x86_64__)
-#define AO_USE_PENTIUM4_INSTRS
-#endif
-
-#include "atomic_ops.h"
-
 /*
  * libatomic_ops can be downloaded from:
  *   http://www.hpl.hp.com/research/linux/atomic_ops/
@@ -46,21 +36,18 @@
  * - AO_store()
  * - AO_compare_and_swap()
  *
- * The `AO_t' type also have to be at least as large as the `void *' type.
  */
 
-#if ETHR_SIZEOF_AO_T < ETHR_SIZEOF_PTR
-#error The AO_t type is too small
-#endif
-
 #if ETHR_SIZEOF_AO_T == 4
 #define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATIVE_ATOMIC32_IMPL "libatomic_ops"
 #define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
 #define ETHR_ATMC_T__ ethr_native_atomic32_t
 #define ETHR_AINT_T__ ethr_sint32_t
 #define ETHR_AINT_SUFFIX__ "l"
 #elif ETHR_SIZEOF_AO_T == 8
 #define ETHR_HAVE_NATIVE_ATOMIC64 1
+#define ETHR_NATIVE_ATOMIC64_IMPL "libatomic_ops"
 #define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
 #define ETHR_ATMC_T__ ethr_native_atomic64_t
 #define ETHR_AINT_T__ ethr_sint64_t
@@ -69,61 +56,29 @@
 #error "Unsupported integer size"
 #endif
 
-#if ETHR_SIZEOF_AO_T == 8
-typedef union {
-    volatile AO_t counter;
-    ethr_sint32_t sint32[2];
-} ETHR_ATMC_T__;
-#else
 typedef struct {
     volatile AO_t counter;
 } ETHR_ATMC_T__;
-#endif
 
-#define ETHR_MEMORY_BARRIER AO_nop_full()
-#ifdef AO_HAVE_nop_write
-# define ETHR_WRITE_MEMORY_BARRIER AO_nop_write()
-#else
-# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER
-#endif
-#ifdef AO_HAVE_nop_read
-#  define ETHR_READ_MEMORY_BARRIER AO_nop_read()
-#else
-#  define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER
-#endif
-#ifdef AO_NO_DD_ORDERING
-#  define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
 #else
-#  define ETHR_READ_DEPEND_MEMORY_BARRIER AO_compiler_barrier()
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
 #endif
 
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
-
 static ETHR_INLINE ETHR_AINT_T__ *
 ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
 {
     return (ETHR_AINT_T__ *) &var->counter;
 }
 
-#if ETHR_SIZEOF_AO_T == 8
-/*
- * We also need to provide an ethr_native_atomic32_addr(), since
- * this 64-bit implementation will be used implementing 32-bit
- * native atomics.
- */
-
-static ETHR_INLINE ethr_sint32_t *
-ethr_native_atomic32_addr(ETHR_ATMC_T__ *var)
-{
-    ETHR_ASSERT(((void *) &var->sint32[0]) == ((void *) &var->counter));
-#ifdef ETHR_BIGENDIAN
-    return &var->sint32[1];
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
 #else
-    return &var->sint32[0];
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
 #endif
-}
-
-#endif /* ETHR_SIZEOF_AO_T == 8 */
 
 static ETHR_INLINE void
 ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
@@ -131,197 +86,198 @@ ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
     AO_store(&var->counter, (AO_t) value);
 }
 
+#ifdef AO_HAVE_store_release
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1
+#endif
+
 static ETHR_INLINE void
-ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
 {
-    ETHR_NATMC_FUNC__(set)(var, value);
+    AO_store_release(&var->counter, (AO_t) value);
 }
 
+#endif
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
+#endif
+
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
 {
     return (ETHR_AINT_T__) AO_load(&var->counter);
 }
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
-{
-#ifdef AO_HAVE_fetch_and_add_full
-    return ((ETHR_AINT_T__) AO_fetch_and_add_full(&var->counter, (AO_t) incr)) + incr;
+#ifdef AO_HAVE_load_acquire
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB 1
 #else
-    while (1) {
-	AO_t exp = AO_load(&var->counter);
-	AO_t new = exp + (AO_t) incr;
-	if (AO_compare_and_swap_full(&var->counter, exp, new))
-	    return (ETHR_AINT_T__) new;
-    }
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB 1
 #endif
-}
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
 {
-    (void) ETHR_NATMC_FUNC__(add_return)(var, incr);
+    return (ETHR_AINT_T__) AO_load_acquire(&var->counter);
 }
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
-{
-#ifdef AO_HAVE_fetch_and_add1_full
-    return ((ETHR_AINT_T__) AO_fetch_and_add1_full(&var->counter)) + 1;
-#else
-    return ETHR_NATMC_FUNC__(add_return)(var, 1);
 #endif
-}
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
-{
-    (void) ETHR_NATMC_FUNC__(inc_return)(var);
-}
+#ifdef AO_HAVE_fetch_and_add
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
-{
-#ifdef AO_HAVE_fetch_and_sub1_full
-    return ((ETHR_AINT_T__) AO_fetch_and_sub1_full(&var->counter)) - 1;
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1
 #else
-    return ETHR_NATMC_FUNC__(add_return)(var, -1);
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN 1
 #endif
-}
-
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var)
-{
-    (void) ETHR_NATMC_FUNC__(dec_return)(var);
-}
 
 static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
 {
-    while (1) {
-	AO_t exp = AO_load(&var->counter);
-	AO_t new = exp & ((AO_t) mask);
-	if (AO_compare_and_swap_full(&var->counter, exp, new))
-	    return (ETHR_AINT_T__) exp;
-    }
+    return ((ETHR_AINT_T__) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr;
 }
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
-{
-    while (1) {
-	AO_t exp = AO_load(&var->counter);
-	AO_t new = exp | ((AO_t) mask);
-	if (AO_compare_and_swap_full(&var->counter, exp, new))
-	    return (ETHR_AINT_T__) exp;
-    }
-}
+#endif
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
-			   ETHR_AINT_T__ new,
-			   ETHR_AINT_T__ exp)
-{
-    ETHR_AINT_T__ act;
-    do {
-	if (AO_compare_and_swap_full(&var->counter, (AO_t) exp, (AO_t) new))
-	    return exp;
-	act = (ETHR_AINT_T__) AO_load(&var->counter);
-    } while (act == exp);
-    return act;
-}
+#ifdef AO_HAVE_fetch_and_add1
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN 1
+#endif
 
 static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
+ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
 {
-    while (1) {
-	AO_t exp = AO_load(&var->counter);
-	if (AO_compare_and_swap_full(&var->counter, exp, (AO_t) new))
-	    return (ETHR_AINT_T__) exp;
-    }
+    return ((ETHR_AINT_T__) AO_fetch_and_add1(&var->counter)) + 1;
 }
 
-/*
- * Atomic ops with at least specified barriers.
- */
+#endif
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
-{
-#ifdef AO_HAVE_load_acquire
-    return (ETHR_AINT_T__) AO_load_acquire(&var->counter);
+#ifdef AO_HAVE_fetch_and_add1_acquire
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1
 #else
-    ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var);
-    ETHR_MEMORY_BARRIER;
-    return res;
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB 1
 #endif
-}
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
 {
-#ifdef AO_HAVE_fetch_and_add1_acquire
     return ((ETHR_AINT_T__) AO_fetch_and_add1_acquire(&var->counter)) + 1;
+}
+
+#endif
+
+#ifdef AO_HAVE_fetch_and_sub1
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN 1
 #else
-    ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(add_return)(var, 1);
-    return res;
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN 1
 #endif
-}
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
 {
-#ifdef AO_HAVE_store_release
-    AO_store_release(&var->counter, (AO_t) value);
+    return ((ETHR_AINT_T__) AO_fetch_and_sub1(&var->counter)) - 1;
+}
+
+#endif
+
+#ifdef AO_HAVE_fetch_and_sub1_release
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1
 #else
-    ETHR_MEMORY_BARRIER;
-    ETHR_NATMC_FUNC__(set)(var, value);
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB 1
 #endif
-}
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
 {
-#ifdef AO_HAVE_fetch_and_sub1_release
     return ((ETHR_AINT_T__) AO_fetch_and_sub1_release(&var->counter)) - 1;
+}
+
+#endif
+
+#ifdef AO_HAVE_compare_and_swap
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1
 #else
-    return ETHR_NATMC_FUNC__(dec_return)(var);
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG 1
 #endif
-}
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
+			   ETHR_AINT_T__ new,
+			   ETHR_AINT_T__ exp)
 {
-    (void) ETHR_NATMC_FUNC__(dec_return_relb)(var);
+    ETHR_AINT_T__ act;
+    do {
+	if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new))
+	    return exp;
+	act = (ETHR_AINT_T__) AO_load(&var->counter);
+    } while (act == exp);
+    return act;
 }
 
+#endif
+
+#ifdef AO_HAVE_compare_and_swap_acquire
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1
+#endif
+
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
 				ETHR_AINT_T__ new,
 				ETHR_AINT_T__ exp)
 {
-#ifdef AO_HAVE_compare_and_swap_acquire
     ETHR_AINT_T__ act;
     do {
 	if (AO_compare_and_swap_acquire(&var->counter, (AO_t) exp, (AO_t) new))
 	    return exp;
+#ifdef AO_HAVE_load_acquire
+	act = (ETHR_AINT_T__) AO_load_acquire(&var->counter);
+#else
 	act = (ETHR_AINT_T__) AO_load(&var->counter);
+#endif
     } while (act == exp);
+#ifndef AO_HAVE_load_acquire
     AO_nop_full();
+#endif
     return act;
+}
+
+#endif
+
+#ifdef AO_HAVE_compare_and_swap_release
+
+#if ETHR_SIZEOF_AO_T == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1
 #else
-    ETHR_AINT_T__ act = ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp);
-    return act;
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB 1
 #endif
-}
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
 				ETHR_AINT_T__ new,
 				ETHR_AINT_T__ exp)
 {
-#ifdef AO_HAVE_compare_and_swap_release
     ETHR_AINT_T__ act;
     do {
 	if (AO_compare_and_swap_release(&var->counter, (AO_t) exp, (AO_t) new))
@@ -329,11 +285,9 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
 	act = (ETHR_AINT_T__) AO_load(&var->counter);
     } while (act == exp);
     return act;
-#else
-    return ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp);
-#endif
 }
 
+#endif
 
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
@@ -341,6 +295,4 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
 #undef ETHR_ATMC_T__
 #undef ETHR_AINT_T__
 
-#endif /* !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS) */
-
 #endif /* ETHR_LIBATOMIC_OPS_ATOMIC_H__ */