aboutsummaryrefslogtreecommitdiffstats
path: root/erts/include
diff options
context:
space:
mode:
Diffstat (limited to 'erts/include')
-rw-r--r--erts/include/internal/ethr_atomics.h341
-rw-r--r--erts/include/internal/ethread.h308
-rw-r--r--erts/include/internal/gcc/ethr_atomic.h2
-rw-r--r--erts/include/internal/i386/atomic.h2
-rw-r--r--erts/include/internal/libatomic_ops/ethr_atomic.h2
-rw-r--r--erts/include/internal/ppc32/atomic.h2
-rw-r--r--erts/include/internal/sparc32/atomic.h2
-rw-r--r--erts/include/internal/tile/atomic.h2
-rw-r--r--erts/include/internal/win/ethr_atomic.h2
9 files changed, 349 insertions, 314 deletions
diff --git a/erts/include/internal/ethr_atomics.h b/erts/include/internal/ethr_atomics.h
new file mode 100644
index 0000000000..30f14ddee5
--- /dev/null
+++ b/erts/include/internal/ethr_atomics.h
@@ -0,0 +1,341 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: The ethread atomic API
+ * Author: Rickard Green
+ */
+
+#ifndef ETHR_ATOMIC_H__
+#define ETHR_ATOMIC_H__
+
+#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+# define ETHR_NEED_ATOMIC_PROTOTYPES__
+#endif
+
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+/*
+ * Map ethread native atomics to ethread API atomics.
+ */
+typedef ethr_native_atomic_t ethr_atomic_t;
+#else
+typedef ethr_sint_t ethr_atomic_t;
+#endif
+
+#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__
+void ethr_atomic_init(ethr_atomic_t *, ethr_sint_t);
+void ethr_atomic_set(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_read(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *);
+void ethr_atomic_inc(ethr_atomic_t *);
+void ethr_atomic_dec(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *, ethr_sint_t);
+void ethr_atomic_add(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *, ethr_sint_t);
+ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
+ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *);
+void ethr_atomic_set_relb(ethr_atomic_t *, ethr_sint_t);
+void ethr_atomic_dec_relb(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *);
+ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
+ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
+#endif
+
+int ethr_init_atomics(void);
+
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
+
+#ifndef ETHR_HAVE_NATIVE_ATOMICS
+/*
+ * Fallbacks for atomics used in absence of a native implementation.
+ */
+
+#define ETHR_ATOMIC_ADDR_BITS 10
+#define ETHR_ATOMIC_ADDR_SHIFT 6
+
+typedef struct {
+ union {
+ ethr_spinlock_t lck;
+ char buf[ETHR_CACHE_LINE_SIZE];
+ } u;
+} ethr_atomic_protection_t;
+
+extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
+
+#define ETHR_ATOMIC_PTR2LCK__(PTR) \
+(&ethr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \
+ & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck)
+
+
+#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \
+do { \
+ ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \
+ ethr_spin_lock(slp__); \
+ { EXPS; } \
+ ethr_spin_unlock(slp__); \
+} while (0)
+
+#endif
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, ethr_sint_t i)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ ethr_native_atomic_init(var, i);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, ethr_sint_t i)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ ethr_native_atomic_set(var, i);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ return ethr_native_atomic_read(var);
+#else
+ ethr_sint_t res;
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
+ return res;
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, ethr_sint_t incr)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ ethr_native_atomic_add(var, incr);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, ethr_sint_t i)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ return ethr_native_atomic_add_return(var, i);
+#else
+ ethr_sint_t res;
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var);
+ return res;
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ ethr_native_atomic_inc(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ ethr_native_atomic_dec(var);
+#else
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ return ethr_native_atomic_inc_return(var);
+#else
+ ethr_sint_t res;
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
+ return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ return ethr_native_atomic_dec_return(var);
+#else
+ ethr_sint_t res;
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
+ return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var,
+ ethr_sint_t mask)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ return ethr_native_atomic_and_retold(var, mask);
+#else
+ ethr_sint_t res;
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask);
+ return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var,
+ ethr_sint_t mask)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ return ethr_native_atomic_or_retold(var, mask);
+#else
+ ethr_sint_t res;
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask);
+ return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, ethr_sint_t new)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ return ethr_native_atomic_xchg(var, new);
+#else
+ ethr_sint_t res;
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new);
+ return res;
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var,
+ ethr_sint_t new,
+ ethr_sint_t exp)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ return ethr_native_atomic_cmpxchg(var, new, exp);
+#else
+ ethr_sint_t res;
+ ETHR_ATOMIC_OP_FALLBACK_IMPL__(var,
+ {
+ res = *var;
+ if (__builtin_expect(res == exp, 1))
+ *var = new;
+ });
+ return res;
+#endif
+}
+
+/*
+ * Important memory barrier requirements.
+ *
+ * The following atomic operations *must* supply a memory barrier of
+ * at least the type specified by its suffix:
+ * _acqb = acquire barrier
+ * _relb = release barrier
+ */
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ return ethr_native_atomic_read_acqb(var);
+#else
+ return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ return ethr_native_atomic_inc_return_acqb(var);
+#else
+ return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var,
+ ethr_sint_t val)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ ethr_native_atomic_set_relb(var, val);
+#else
+ ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val);
+#endif
+}
+
+static ETHR_INLINE void
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ ethr_native_atomic_dec_relb(var);
+#else
+ ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ return ethr_native_atomic_dec_return_relb(var);
+#else
+ return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var,
+ ethr_sint_t new,
+ ethr_sint_t exp)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ return ethr_native_atomic_cmpxchg_acqb(var, new, exp);
+#else
+ return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp);
+#endif
+}
+
+static ETHR_INLINE ethr_sint_t
+ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var,
+ ethr_sint_t new,
+ ethr_sint_t exp)
+{
+#ifdef ETHR_HAVE_NATIVE_ATOMICS
+ return ethr_native_atomic_cmpxchg_relb(var, new, exp);
+#else
+ return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp);
+#endif
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#endif
diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h
index 84554d2d77..8b99e0a1e6 100644
--- a/erts/include/internal/ethread.h
+++ b/erts/include/internal/ethread.h
@@ -406,7 +406,6 @@ typedef struct {
#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
# define ETHR_NEED_SPINLOCK_PROTOTYPES__
# define ETHR_NEED_RWSPINLOCK_PROTOTYPES__
-# define ETHR_NEED_ATOMIC_PROTOTYPES__
#endif
int ethr_init(ethr_init_data *);
@@ -521,312 +520,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock)
#endif /* ETHR_TRY_INLINE_FUNCS */
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
-/*
- * Map ethread native atomics to ethread API atomics.
- */
-typedef ethr_native_atomic_t ethr_atomic_t;
-#else
-typedef ethr_sint_t ethr_atomic_t;
-#endif
-
-#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__
-void ethr_atomic_init(ethr_atomic_t *, ethr_sint_t);
-void ethr_atomic_set(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_read(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *);
-void ethr_atomic_inc(ethr_atomic_t *);
-void ethr_atomic_dec(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *, ethr_sint_t);
-void ethr_atomic_add(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *, ethr_sint_t);
-ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
-ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *);
-void ethr_atomic_set_relb(ethr_atomic_t *, ethr_sint_t);
-void ethr_atomic_dec_relb(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *);
-ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
-ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t);
-#endif
-
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
-
-#ifndef ETHR_HAVE_NATIVE_ATOMICS
-/*
- * Fallbacks for atomics used in absence of a native implementation.
- */
-
-#define ETHR_ATOMIC_ADDR_BITS 10
-#define ETHR_ATOMIC_ADDR_SHIFT 6
-
-typedef struct {
- union {
- ethr_spinlock_t lck;
- char buf[ETHR_CACHE_LINE_SIZE];
- } u;
-} ethr_atomic_protection_t;
-
-extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
-
-#define ETHR_ATOMIC_PTR2LCK__(PTR) \
-(&ethr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \
- & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck)
-
-
-#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \
-do { \
- ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \
- ethr_spin_lock(slp__); \
- { EXPS; } \
- ethr_spin_unlock(slp__); \
-} while (0)
-
-#endif
-
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, ethr_sint_t i)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ethr_native_atomic_init(var, i);
-#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
-#endif
-}
-
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, ethr_sint_t i)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ethr_native_atomic_set(var, i);
-#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i);
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return ethr_native_atomic_read(var);
-#else
- ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
- return res;
-#endif
-}
-
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, ethr_sint_t incr)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ethr_native_atomic_add(var, incr);
-#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr);
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, ethr_sint_t i)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return ethr_native_atomic_add_return(var, i);
-#else
- ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var);
- return res;
-#endif
-}
-
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ethr_native_atomic_inc(var);
-#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
-#endif
-}
-
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ethr_native_atomic_dec(var);
-#else
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return ethr_native_atomic_inc_return(var);
-#else
- ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
- return res;
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return ethr_native_atomic_dec_return(var);
-#else
- ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
- return res;
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var,
- ethr_sint_t mask)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return ethr_native_atomic_and_retold(var, mask);
-#else
- ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask);
- return res;
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var,
- ethr_sint_t mask)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return ethr_native_atomic_or_retold(var, mask);
-#else
- ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask);
- return res;
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, ethr_sint_t new)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return ethr_native_atomic_xchg(var, new);
-#else
- ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new);
- return res;
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var,
- ethr_sint_t new,
- ethr_sint_t exp)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return ethr_native_atomic_cmpxchg(var, new, exp);
-#else
- ethr_sint_t res;
- ETHR_ATOMIC_OP_FALLBACK_IMPL__(var,
- {
- res = *var;
- if (__builtin_expect(res == exp, 1))
- *var = new;
- });
- return res;
-#endif
-}
-
-/*
- * Important memory barrier requirements.
- *
- * The following atomic operations *must* supply a memory barrier of
- * at least the type specified by its suffix:
- * _acqb = acquire barrier
- * _relb = release barrier
- */
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return ethr_native_atomic_read_acqb(var);
-#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var);
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return ethr_native_atomic_inc_return_acqb(var);
-#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var);
-#endif
-}
-
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var,
- ethr_sint_t val)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ethr_native_atomic_set_relb(var, val);
-#else
- ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val);
-#endif
-}
-
-static ETHR_INLINE void
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- ethr_native_atomic_dec_relb(var);
-#else
- ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var);
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return ethr_native_atomic_dec_return_relb(var);
-#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var);
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var,
- ethr_sint_t new,
- ethr_sint_t exp)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return ethr_native_atomic_cmpxchg_acqb(var, new, exp);
-#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp);
-#endif
-}
-
-static ETHR_INLINE ethr_sint_t
-ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var,
- ethr_sint_t new,
- ethr_sint_t exp)
-{
-#ifdef ETHR_HAVE_NATIVE_ATOMICS
- return ethr_native_atomic_cmpxchg_relb(var, new, exp);
-#else
- return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp);
-#endif
-}
-
-#endif /* ETHR_TRY_INLINE_FUNCS */
+#include "ethr_atomics.h"
typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */
diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h
index 49a8ceeab1..fd678aa97d 100644
--- a/erts/include/internal/gcc/ethr_atomic.h
+++ b/erts/include/internal/gcc/ethr_atomic.h
@@ -65,7 +65,7 @@ do { \
#endif
#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMORY_BARRIER
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
static ETHR_INLINE void
ethr_native_atomic_set(ethr_native_atomic_t *var, long value)
diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h
index 709a6a17fc..b3f4a1a60d 100644
--- a/erts/include/internal/i386/atomic.h
+++ b/erts/include/internal/i386/atomic.h
@@ -51,7 +51,7 @@ do { \
#define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
#ifdef __x86_64__
#define LONG_SUFFIX "q"
diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h
index 7def51ebd1..f244e9c626 100644
--- a/erts/include/internal/libatomic_ops/ethr_atomic.h
+++ b/erts/include/internal/libatomic_ops/ethr_atomic.h
@@ -74,7 +74,7 @@ typedef struct {
# define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("":::"memory")
#endif
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
static ETHR_INLINE void
ethr_native_atomic_set(ethr_native_atomic_t *var, ethr_sint_t value)
diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h
index daa0de88ab..6a06044f15 100644
--- a/erts/include/internal/ppc32/atomic.h
+++ b/erts/include/internal/ppc32/atomic.h
@@ -38,7 +38,7 @@ typedef struct {
#define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory")
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
static ETHR_INLINE void
ethr_native_atomic_init(ethr_native_atomic_t *var, int i)
diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h
index 8d94fa939f..96c1fe50a2 100644
--- a/erts/include/internal/sparc32/atomic.h
+++ b/erts/include/internal/sparc32/atomic.h
@@ -36,7 +36,7 @@ typedef struct {
__asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore\n" \
: : : "memory")
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
#if defined(__arch64__)
#define CASX "casx"
diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h
index 05246a2aa9..b5b905b996 100644
--- a/erts/include/internal/tile/atomic.h
+++ b/erts/include/internal/tile/atomic.h
@@ -38,7 +38,7 @@ typedef struct {
#define ETHR_MEMORY_BARRIER __insn_mf()
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
static ETHR_INLINE void
ethr_native_atomic_init(ethr_native_atomic_t *var, long i)
diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h
index 9e2dd08728..7ae5bb6cd3 100644
--- a/erts/include/internal/win/ethr_atomic.h
+++ b/erts/include/internal/win/ethr_atomic.h
@@ -141,7 +141,7 @@ typedef struct {
volatile ETHR_AINT_T__ value;
} ETHR_ATMC_T__;
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
static ETHR_INLINE void
ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 07:38:12 +0000