diff options
Diffstat (limited to 'erts/include/internal')
| -rw-r--r-- | erts/include/internal/ethr_atomics.h | 341 | ||||
| -rw-r--r-- | erts/include/internal/ethread.h | 308 | ||||
| -rw-r--r-- | erts/include/internal/gcc/ethr_atomic.h | 2 | ||||
| -rw-r--r-- | erts/include/internal/i386/atomic.h | 2 | ||||
| -rw-r--r-- | erts/include/internal/libatomic_ops/ethr_atomic.h | 2 | ||||
| -rw-r--r-- | erts/include/internal/ppc32/atomic.h | 2 | ||||
| -rw-r--r-- | erts/include/internal/sparc32/atomic.h | 2 | ||||
| -rw-r--r-- | erts/include/internal/tile/atomic.h | 2 | ||||
| -rw-r--r-- | erts/include/internal/win/ethr_atomic.h | 2 |
9 files changed, 349 insertions, 314 deletions
diff --git a/erts/include/internal/ethr_atomics.h b/erts/include/internal/ethr_atomics.h new file mode 100644 index 0000000000..30f14ddee5 --- /dev/null +++ b/erts/include/internal/ethr_atomics.h @@ -0,0 +1,341 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: The ethread atomic API + * Author: Rickard Green + */ + +#ifndef ETHR_ATOMIC_H__ +#define ETHR_ATOMIC_H__ + +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_ATOMIC_PROTOTYPES__ +#endif + +#ifdef ETHR_HAVE_NATIVE_ATOMICS +/* + * Map ethread native atomics to ethread API atomics. + */ +typedef ethr_native_atomic_t ethr_atomic_t; +#else +typedef ethr_sint_t ethr_atomic_t; +#endif + +#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__ +void ethr_atomic_init(ethr_atomic_t *, ethr_sint_t); +void ethr_atomic_set(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_read(ethr_atomic_t *); +ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *); +ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *); +void ethr_atomic_inc(ethr_atomic_t *); +void ethr_atomic_dec(ethr_atomic_t *); +ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *, ethr_sint_t); +void ethr_atomic_add(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); +ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *); +ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *); +void ethr_atomic_set_relb(ethr_atomic_t *, ethr_sint_t); +void ethr_atomic_dec_relb(ethr_atomic_t *); +ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *); +ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); +ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); +#endif + +int ethr_init_atomics(void); + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +#ifndef ETHR_HAVE_NATIVE_ATOMICS +/* + * Fallbacks for atomics used in absence of a native implementation. + */ + +#define ETHR_ATOMIC_ADDR_BITS 10 +#define ETHR_ATOMIC_ADDR_SHIFT 6 + +typedef struct { + union { + ethr_spinlock_t lck; + char buf[ETHR_CACHE_LINE_SIZE]; + } u; +} ethr_atomic_protection_t; + +extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; + +#define ETHR_ATOMIC_PTR2LCK__(PTR) \ +(ðr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ + & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck) + + +#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ +do { \ + ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ + ethr_spin_lock(slp__); \ + { EXPS; } \ + ethr_spin_unlock(slp__); \ +} while (0) + +#endif + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, ethr_sint_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_init(var, i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, ethr_sint_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_set(var, i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_read(var); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + return res; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, ethr_sint_t incr) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_add(var, incr); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, ethr_sint_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_add_return(var, i); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); + return res; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_inc(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_dec(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_inc_return(var); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_dec_return(var); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var, + ethr_sint_t mask) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_and_retold(var, mask); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var, + ethr_sint_t mask) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_or_retold(var, mask); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, ethr_sint_t new) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_xchg(var, new); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, + ethr_sint_t new, + ethr_sint_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_cmpxchg(var, new, exp); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, + { + res = *var; + if (__builtin_expect(res == exp, 1)) + *var = new; + }); + return res; +#endif +} + +/* + * Important memory barrier requirements. + * + * The following atomic operations *must* supply a memory barrier of + * at least the type specified by its suffix: + * _acqb = acquire barrier + * _relb = release barrier + */ + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_read_acqb(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_inc_return_acqb(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var, + ethr_sint_t val) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_set_relb(var, val); +#else + ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_dec_relb(var); +#else + ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_dec_return_relb(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var, + ethr_sint_t new, + ethr_sint_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_cmpxchg_acqb(var, new, exp); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var, + ethr_sint_t new, + ethr_sint_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_cmpxchg_relb(var, new, exp); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); +#endif +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#endif diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h index 84554d2d77..8b99e0a1e6 100644 --- a/erts/include/internal/ethread.h +++ b/erts/include/internal/ethread.h @@ -406,7 +406,6 @@ typedef struct { #if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) # define ETHR_NEED_SPINLOCK_PROTOTYPES__ # define ETHR_NEED_RWSPINLOCK_PROTOTYPES__ -# define ETHR_NEED_ATOMIC_PROTOTYPES__ #endif int ethr_init(ethr_init_data *); @@ -521,312 +520,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) #endif /* ETHR_TRY_INLINE_FUNCS */ -#ifdef ETHR_HAVE_NATIVE_ATOMICS -/* - * Map ethread native atomics to ethread API atomics. - */ -typedef ethr_native_atomic_t ethr_atomic_t; -#else -typedef ethr_sint_t ethr_atomic_t; -#endif - -#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__ -void ethr_atomic_init(ethr_atomic_t *, ethr_sint_t); -void ethr_atomic_set(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_read(ethr_atomic_t *); -ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *); -ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *); -void ethr_atomic_inc(ethr_atomic_t *); -void ethr_atomic_dec(ethr_atomic_t *); -ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *, ethr_sint_t); -void ethr_atomic_add(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); -ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *); -ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *); -void ethr_atomic_set_relb(ethr_atomic_t *, ethr_sint_t); -void ethr_atomic_dec_relb(ethr_atomic_t *); -ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *); -ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); -ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); -#endif - -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) - -#ifndef ETHR_HAVE_NATIVE_ATOMICS -/* - * Fallbacks for atomics used in absence of a native implementation. - */ - -#define ETHR_ATOMIC_ADDR_BITS 10 -#define ETHR_ATOMIC_ADDR_SHIFT 6 - -typedef struct { - union { - ethr_spinlock_t lck; - char buf[ETHR_CACHE_LINE_SIZE]; - } u; -} ethr_atomic_protection_t; - -extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; - -#define ETHR_ATOMIC_PTR2LCK__(PTR) \ -(ðr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ - & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck) - - -#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ -do { \ - ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ - ethr_spin_lock(slp__); \ - { EXPS; } \ - ethr_spin_unlock(slp__); \ -} while (0) - -#endif - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, ethr_sint_t i) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_init(var, i); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, ethr_sint_t i) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_set(var, i); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_read(var); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); - return res; -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, ethr_sint_t incr) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_add(var, incr); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, ethr_sint_t i) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_add_return(var, i); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); - return res; -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_inc(var); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_dec(var); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_inc_return(var); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); - return res; -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_dec_return(var); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); - return res; -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var, - ethr_sint_t mask) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_and_retold(var, mask); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); - return res; -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var, - ethr_sint_t mask) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_or_retold(var, mask); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); - return res; -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, ethr_sint_t new) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_xchg(var, new); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); - return res; -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, - ethr_sint_t new, - ethr_sint_t exp) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_cmpxchg(var, new, exp); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, - { - res = *var; - if (__builtin_expect(res == exp, 1)) - *var = new; - }); - return res; -#endif -} - -/* - * Important memory barrier requirements. - * - * The following atomic operations *must* supply a memory barrier of - * at least the type specified by its suffix: - * _acqb = acquire barrier - * _relb = release barrier - */ - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_read_acqb(var); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var); -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_inc_return_acqb(var); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var); -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var, - ethr_sint_t val) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_set_relb(var, val); -#else - ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val); -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_dec_relb(var); -#else - ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var); -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_dec_return_relb(var); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var); -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var, - ethr_sint_t new, - ethr_sint_t exp) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_cmpxchg_acqb(var, new, exp); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var, - ethr_sint_t new, - ethr_sint_t exp) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_cmpxchg_relb(var, new, exp); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); -#endif -} - -#endif /* ETHR_TRY_INLINE_FUNCS */ +#include "ethr_atomics.h" typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */ diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h index 49a8ceeab1..fd678aa97d 100644 --- a/erts/include/internal/gcc/ethr_atomic.h +++ b/erts/include/internal/gcc/ethr_atomic.h @@ -65,7 +65,7 @@ do { \ #endif #define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMORY_BARRIER -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) static ETHR_INLINE void ethr_native_atomic_set(ethr_native_atomic_t *var, long value) diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h index 709a6a17fc..b3f4a1a60d 100644 --- a/erts/include/internal/i386/atomic.h +++ b/erts/include/internal/i386/atomic.h @@ -51,7 +51,7 @@ do { \ #define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1 -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) #ifdef __x86_64__ #define LONG_SUFFIX "q" diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h index 7def51ebd1..f244e9c626 100644 --- a/erts/include/internal/libatomic_ops/ethr_atomic.h +++ b/erts/include/internal/libatomic_ops/ethr_atomic.h @@ -74,7 +74,7 @@ typedef struct { # define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("":::"memory") #endif -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) static ETHR_INLINE void ethr_native_atomic_set(ethr_native_atomic_t *var, ethr_sint_t value) diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h index daa0de88ab..6a06044f15 100644 --- a/erts/include/internal/ppc32/atomic.h +++ b/erts/include/internal/ppc32/atomic.h @@ -38,7 +38,7 @@ typedef struct { #define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory") -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) static ETHR_INLINE void ethr_native_atomic_init(ethr_native_atomic_t *var, int i) diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h index 8d94fa939f..96c1fe50a2 100644 --- a/erts/include/internal/sparc32/atomic.h +++ b/erts/include/internal/sparc32/atomic.h @@ -36,7 +36,7 @@ typedef struct { __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore\n" \ : : : "memory") -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) #if defined(__arch64__) #define CASX "casx" diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h index 05246a2aa9..b5b905b996 100644 --- a/erts/include/internal/tile/atomic.h +++ b/erts/include/internal/tile/atomic.h @@ -38,7 +38,7 @@ typedef struct { #define ETHR_MEMORY_BARRIER __insn_mf() -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) static ETHR_INLINE void ethr_native_atomic_init(ethr_native_atomic_t *var, long i) diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h index 9e2dd08728..7ae5bb6cd3 100644 --- a/erts/include/internal/win/ethr_atomic.h +++ b/erts/include/internal/win/ethr_atomic.h @@ -141,7 +141,7 @@ typedef struct { volatile ETHR_AINT_T__ value; } ETHR_ATMC_T__; -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) static ETHR_INLINE void ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) |