Move atomic API into own files

9 files changed, 349 insertions, 314 deletions

diff --git a/erts/include/internal/ethr_atomics.h b/erts/include/internal/ethr_atomics.h
new file mode 100644
index 0000000000..30f14ddee5
--- /dev/null
+++ b/erts/include/internal/ethr_atomics.h
@@ -0,0 +1,341 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: The ethread atomic API
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_ATOMIC_H__
+#define ETHR_ATOMIC_H__
+
+#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+#  define ETHR_NEED_ATOMIC_PROTOTYPES__
+#endif
+
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+/*
+ * Map ethread native atomics to ethread API atomics.
+ */
+typedef ethr_native_atomic_t ethr_atomic_t;
+#else
+typedef ethr_sint_t ethr_atomic_t;
+#endif
+
+#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__
+void ethr_atomic_init(ethr_atomic_t *, ethr_sint_t);
+void ethr_atomic_set(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_read(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *);
+void ethr_atomic_inc(ethr_atomic_t *);
+void ethr_atomic_dec(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *, ethr_sint_t);
+void ethr_atomic_add(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
+ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *);
+void ethr_atomic_set_relb(ethr_atomic_t *, ethr_sint_t);
+void ethr_atomic_dec_relb(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
+ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
+#endif
+
+int ethr_init_atomics(void);
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#ifndef ETHR_HAVE_NATIVE_ATOMICS
+/*
+ * Fallbacks for atomics used in absence of a native implementation.
+ */
+
+#define ETHR_ATOMIC_ADDR_BITS 10
+#define ETHR_ATOMIC_ADDR_SHIFT 6
+
+typedef struct {
+    union {
+	ethr_spinlock_t lck;
+	char buf[ETHR_CACHE_LINE_SIZE];
+    } u;
+} ethr_atomic_protection_t;
+
+extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
+
+#define ETHR_ATOMIC_PTR2LCK__(PTR) \
+(&ethr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \
+			& ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck)
+
+
+#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS)			\
+do {									\
+    ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP));		\
+    ethr_spin_lock(slp__);						\
+    { EXPS; }								\
+    ethr_spin_unlock(slp__);						\
+} while (0)
+
+#endif
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, ethr_sint_t i)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ethr_native_atomic_init(var, i);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, ethr_sint_t i)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ethr_native_atomic_set(var, i);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return ethr_native_atomic_read(var);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+    return res;
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, ethr_sint_t incr)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ethr_native_atomic_add(var, incr);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr);
+#endif
+}   
+    
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, ethr_sint_t i)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return ethr_native_atomic_add_return(var, i);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var);
+    return res;
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ethr_native_atomic_inc(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ethr_native_atomic_dec(var);
+#else
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return ethr_native_atomic_inc_return(var);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return ethr_native_atomic_dec_return(var);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var,
+					      ethr_sint_t mask)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return ethr_native_atomic_and_retold(var, mask);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask);
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var,
+					     ethr_sint_t mask)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return ethr_native_atomic_or_retold(var, mask);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask);
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, ethr_sint_t new)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return ethr_native_atomic_xchg(var, new);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new);
+    return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var,
+					    ethr_sint_t new,
+                                            ethr_sint_t exp)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return ethr_native_atomic_cmpxchg(var, new, exp);
+#else
+    ethr_sint_t res;
+    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var,
+    {
+	res = *var;
+	if (__builtin_expect(res == exp, 1))
+	    *var = new;
+    });
+    return res;
+#endif
+}
+
+/*
+ * Important memory barrier requirements.
+ *
+ * The following atomic operations *must* supply a memory barrier of
+ * at least the type specified by its suffix:
+ *    _acqb = acquire barrier
+ *    _relb = release barrier
+ */
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return ethr_native_atomic_read_acqb(var);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return ethr_native_atomic_inc_return_acqb(var);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var,
+					     ethr_sint_t val)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ethr_native_atomic_set_relb(var, val);
+#else
+    ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    ethr_native_atomic_dec_relb(var);
+#else
+    ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return ethr_native_atomic_dec_return_relb(var);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var,
+						 ethr_sint_t new,
+						 ethr_sint_t exp)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return ethr_native_atomic_cmpxchg_acqb(var, new, exp);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var,
+						 ethr_sint_t new,
+						 ethr_sint_t exp)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+    return ethr_native_atomic_cmpxchg_relb(var, new, exp);
+#else
+    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp);
+#endif
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#endif
diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h
index 84554d2d77..8b99e0a1e6 100644
--- a/erts/include/internal/ethread.h
+++ b/erts/include/internal/ethread.h
@@ -406,7 +406,6 @@ typedef struct {
 #if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
 #  define ETHR_NEED_SPINLOCK_PROTOTYPES__
 #  define ETHR_NEED_RWSPINLOCK_PROTOTYPES__
-#  define ETHR_NEED_ATOMIC_PROTOTYPES__
 #endif
 
 int ethr_init(ethr_init_data *);
@@ -521,312 +520,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock)
 
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-/*
- * Map ethread native atomics to ethread API atomics.
- */
-typedef ethr_native_atomic_t ethr_atomic_t;
-#else
-typedef ethr_sint_t ethr_atomic_t;
-#endif
-
-#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__
-void ethr_atomic_init(ethr_atomic_t *, ethr_sint_t);
-void ethr_atomic_set(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_read(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *);
-void ethr_atomic_inc(ethr_atomic_t *);
-void ethr_atomic_dec(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *, ethr_sint_t);
-void ethr_atomic_add(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
-ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *);
-void ethr_atomic_set_relb(ethr_atomic_t *, ethr_sint_t);
-void ethr_atomic_dec_relb(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
-ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
-#endif
-
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
-
-#ifndef ETHR_HAVE_NATIVE_ATOMICS
-/*
- * Fallbacks for atomics used in absence of a native implementation.
- */
-
-#define ETHR_ATOMIC_ADDR_BITS 10
-#define ETHR_ATOMIC_ADDR_SHIFT 6
-
-typedef struct {
-    union {
-	ethr_spinlock_t lck;
-	char buf[ETHR_CACHE_LINE_SIZE];
-    } u;
-} ethr_atomic_protection_t;
-
-extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
-
-#define ETHR_ATOMIC_PTR2LCK__(PTR) \
-(&ethr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \
-			& ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck)
-
-
-#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS)			\
-do {									\
-    ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP));		\
-    ethr_spin_lock(slp__);						\
-    { EXPS; }								\
-    ethr_spin_unlock(slp__);						\
-} while (0)
-
-#endif
-
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, ethr_sint_t i)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    ethr_native_atomic_init(var, i);
-#else
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
-#endif
-}
-
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, ethr_sint_t i)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    ethr_native_atomic_set(var, i);
-#else
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    return ethr_native_atomic_read(var);
-#else
-    ethr_sint_t res;
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
-    return res;
-#endif
-}
-
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, ethr_sint_t incr)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    ethr_native_atomic_add(var, incr);
-#else
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr);
-#endif
-}   
-    
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, ethr_sint_t i)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    return ethr_native_atomic_add_return(var, i);
-#else
-    ethr_sint_t res;
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var);
-    return res;
-#endif
-}
-
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    ethr_native_atomic_inc(var);
-#else
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
-#endif
-}
-
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    ethr_native_atomic_dec(var);
-#else
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    return ethr_native_atomic_inc_return(var);
-#else
-    ethr_sint_t res;
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
-    return res;
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    return ethr_native_atomic_dec_return(var);
-#else
-    ethr_sint_t res;
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
-    return res;
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var,
-					      ethr_sint_t mask)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    return ethr_native_atomic_and_retold(var, mask);
-#else
-    ethr_sint_t res;
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask);
-    return res;
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var,
-					     ethr_sint_t mask)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    return ethr_native_atomic_or_retold(var, mask);
-#else
-    ethr_sint_t res;
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask);
-    return res;
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, ethr_sint_t new)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    return ethr_native_atomic_xchg(var, new);
-#else
-    ethr_sint_t res;
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new);
-    return res;
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var,
-					    ethr_sint_t new,
-                                            ethr_sint_t exp)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    return ethr_native_atomic_cmpxchg(var, new, exp);
-#else
-    ethr_sint_t res;
-    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var,
-    {
-	res = *var;
-	if (__builtin_expect(res == exp, 1))
-	    *var = new;
-    });
-    return res;
-#endif
-}
-
-/*
- * Important memory barrier requirements.
- *
- * The following atomic operations *must* supply a memory barrier of
- * at least the type specified by its suffix:
- *    _acqb = acquire barrier
- *    _relb = release barrier
- */
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    return ethr_native_atomic_read_acqb(var);
-#else
-    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var);
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    return ethr_native_atomic_inc_return_acqb(var);
-#else
-    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var);
-#endif
-}
-
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var,
-					     ethr_sint_t val)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    ethr_native_atomic_set_relb(var, val);
-#else
-    ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val);
-#endif
-}
-
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    ethr_native_atomic_dec_relb(var);
-#else
-    ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var);
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    return ethr_native_atomic_dec_return_relb(var);
-#else
-    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var);
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var,
-						 ethr_sint_t new,
-						 ethr_sint_t exp)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    return ethr_native_atomic_cmpxchg_acqb(var, new, exp);
-#else
-    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp);
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var,
-						 ethr_sint_t new,
-						 ethr_sint_t exp)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-    return ethr_native_atomic_cmpxchg_relb(var, new, exp);
-#else
-    return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp);
-#endif
-}
-
-#endif /* ETHR_TRY_INLINE_FUNCS */
+#include "ethr_atomics.h"
 
 typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */
 
diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h
index 49a8ceeab1..fd678aa97d 100644
--- a/erts/include/internal/gcc/ethr_atomic.h
+++ b/erts/include/internal/gcc/ethr_atomic.h
@@ -65,7 +65,7 @@ do { \
 #endif
 #define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMORY_BARRIER
 
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
 
 static ETHR_INLINE void
 ethr_native_atomic_set(ethr_native_atomic_t *var, long value)
diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h
index 709a6a17fc..b3f4a1a60d 100644
--- a/erts/include/internal/i386/atomic.h
+++ b/erts/include/internal/i386/atomic.h
@@ -51,7 +51,7 @@ do { \
 
 #define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1
 
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
 
 #ifdef __x86_64__
 #define LONG_SUFFIX "q"
diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h
index 7def51ebd1..f244e9c626 100644
--- a/erts/include/internal/libatomic_ops/ethr_atomic.h
+++ b/erts/include/internal/libatomic_ops/ethr_atomic.h
@@ -74,7 +74,7 @@ typedef struct {
 #  define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("":::"memory")
 #endif
 
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
 
 static ETHR_INLINE void
 ethr_native_atomic_set(ethr_native_atomic_t *var, ethr_sint_t value)
diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h
index daa0de88ab..6a06044f15 100644
--- a/erts/include/internal/ppc32/atomic.h
+++ b/erts/include/internal/ppc32/atomic.h
@@ -38,7 +38,7 @@ typedef struct {
 
 #define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory")
 
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
 
 static ETHR_INLINE void
 ethr_native_atomic_init(ethr_native_atomic_t *var, int i)
diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h
index 8d94fa939f..96c1fe50a2 100644
--- a/erts/include/internal/sparc32/atomic.h
+++ b/erts/include/internal/sparc32/atomic.h
@@ -36,7 +36,7 @@ typedef struct {
   __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore\n" \
                        : : : "memory")
 
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
 
 #if defined(__arch64__)
 #define CASX "casx"
diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h
index 05246a2aa9..b5b905b996 100644
--- a/erts/include/internal/tile/atomic.h
+++ b/erts/include/internal/tile/atomic.h
@@ -38,7 +38,7 @@ typedef struct {
 
 #define ETHR_MEMORY_BARRIER __insn_mf()
 
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
 
 static ETHR_INLINE void
 ethr_native_atomic_init(ethr_native_atomic_t *var, long i)
diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h
index 9e2dd08728..7ae5bb6cd3 100644
--- a/erts/include/internal/win/ethr_atomic.h
+++ b/erts/include/internal/win/ethr_atomic.h
@@ -141,7 +141,7 @@ typedef struct {
     volatile ETHR_AINT_T__ value;
 } ETHR_ATMC_T__;
 
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
 
 static ETHR_INLINE void
 ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)