diff options
| -rw-r--r-- | erts/include/internal/ethr_atomics.h | 341 | ||||
| -rw-r--r-- | erts/include/internal/ethread.h | 308 | ||||
| -rw-r--r-- | erts/include/internal/gcc/ethr_atomic.h | 2 | ||||
| -rw-r--r-- | erts/include/internal/i386/atomic.h | 2 | ||||
| -rw-r--r-- | erts/include/internal/libatomic_ops/ethr_atomic.h | 2 | ||||
| -rw-r--r-- | erts/include/internal/ppc32/atomic.h | 2 | ||||
| -rw-r--r-- | erts/include/internal/sparc32/atomic.h | 2 | ||||
| -rw-r--r-- | erts/include/internal/tile/atomic.h | 2 | ||||
| -rw-r--r-- | erts/include/internal/win/ethr_atomic.h | 2 | ||||
| -rw-r--r-- | erts/lib_src/Makefile.in | 2 | ||||
| -rw-r--r-- | erts/lib_src/common/ethr_atomics.c | 214 | ||||
| -rw-r--r-- | erts/lib_src/common/ethr_aux.c | 181 |
12 files changed, 568 insertions, 492 deletions
diff --git a/erts/include/internal/ethr_atomics.h b/erts/include/internal/ethr_atomics.h new file mode 100644 index 0000000000..30f14ddee5 --- /dev/null +++ b/erts/include/internal/ethr_atomics.h @@ -0,0 +1,341 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: The ethread atomic API + * Author: Rickard Green + */ + +#ifndef ETHR_ATOMIC_H__ +#define ETHR_ATOMIC_H__ + +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_ATOMIC_PROTOTYPES__ +#endif + +#ifdef ETHR_HAVE_NATIVE_ATOMICS +/* + * Map ethread native atomics to ethread API atomics. + */ +typedef ethr_native_atomic_t ethr_atomic_t; +#else +typedef ethr_sint_t ethr_atomic_t; +#endif + +#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__ +void ethr_atomic_init(ethr_atomic_t *, ethr_sint_t); +void ethr_atomic_set(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_read(ethr_atomic_t *); +ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *); +ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *); +void ethr_atomic_inc(ethr_atomic_t *); +void ethr_atomic_dec(ethr_atomic_t *); +ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *, ethr_sint_t); +void ethr_atomic_add(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); +ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *); +ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *); +void ethr_atomic_set_relb(ethr_atomic_t *, ethr_sint_t); +void ethr_atomic_dec_relb(ethr_atomic_t *); +ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *); +ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); +ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); +#endif + +int ethr_init_atomics(void); + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +#ifndef ETHR_HAVE_NATIVE_ATOMICS +/* + * Fallbacks for atomics used in absence of a native implementation. + */ + +#define ETHR_ATOMIC_ADDR_BITS 10 +#define ETHR_ATOMIC_ADDR_SHIFT 6 + +typedef struct { + union { + ethr_spinlock_t lck; + char buf[ETHR_CACHE_LINE_SIZE]; + } u; +} ethr_atomic_protection_t; + +extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; + +#define ETHR_ATOMIC_PTR2LCK__(PTR) \ +(ðr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ + & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck) + + +#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ +do { \ + ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ + ethr_spin_lock(slp__); \ + { EXPS; } \ + ethr_spin_unlock(slp__); \ +} while (0) + +#endif + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, ethr_sint_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_init(var, i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, ethr_sint_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_set(var, i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_read(var); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + return res; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, ethr_sint_t incr) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_add(var, incr); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, ethr_sint_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_add_return(var, i); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); + return res; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_inc(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_dec(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_inc_return(var); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_dec_return(var); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var, + ethr_sint_t mask) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_and_retold(var, mask); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var, + ethr_sint_t mask) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_or_retold(var, mask); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, ethr_sint_t new) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_xchg(var, new); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, + ethr_sint_t new, + ethr_sint_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_cmpxchg(var, new, exp); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, + { + res = *var; + if (__builtin_expect(res == exp, 1)) + *var = new; + }); + return res; +#endif +} + +/* + * Important memory barrier requirements. + * + * The following atomic operations *must* supply a memory barrier of + * at least the type specified by its suffix: + * _acqb = acquire barrier + * _relb = release barrier + */ + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_read_acqb(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_inc_return_acqb(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var, + ethr_sint_t val) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_set_relb(var, val); +#else + ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ethr_native_atomic_dec_relb(var); +#else + ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_dec_return_relb(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var, + ethr_sint_t new, + ethr_sint_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_cmpxchg_acqb(var, new, exp); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var, + ethr_sint_t new, + ethr_sint_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic_cmpxchg_relb(var, new, exp); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); +#endif +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#endif diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h index 84554d2d77..8b99e0a1e6 100644 --- a/erts/include/internal/ethread.h +++ b/erts/include/internal/ethread.h @@ -406,7 +406,6 @@ typedef struct { #if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) # define ETHR_NEED_SPINLOCK_PROTOTYPES__ # define ETHR_NEED_RWSPINLOCK_PROTOTYPES__ -# define ETHR_NEED_ATOMIC_PROTOTYPES__ #endif int ethr_init(ethr_init_data *); @@ -521,312 +520,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) #endif /* ETHR_TRY_INLINE_FUNCS */ -#ifdef ETHR_HAVE_NATIVE_ATOMICS -/* - * Map ethread native atomics to ethread API atomics. - */ -typedef ethr_native_atomic_t ethr_atomic_t; -#else -typedef ethr_sint_t ethr_atomic_t; -#endif - -#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__ -void ethr_atomic_init(ethr_atomic_t *, ethr_sint_t); -void ethr_atomic_set(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_read(ethr_atomic_t *); -ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *); -ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *); -void ethr_atomic_inc(ethr_atomic_t *); -void ethr_atomic_dec(ethr_atomic_t *); -ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *, ethr_sint_t); -void ethr_atomic_add(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *, ethr_sint_t); -ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); -ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *); -ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *); -void ethr_atomic_set_relb(ethr_atomic_t *, ethr_sint_t); -void ethr_atomic_dec_relb(ethr_atomic_t *); -ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *); -ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); -ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); -#endif - -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) - -#ifndef ETHR_HAVE_NATIVE_ATOMICS -/* - * Fallbacks for atomics used in absence of a native implementation. - */ - -#define ETHR_ATOMIC_ADDR_BITS 10 -#define ETHR_ATOMIC_ADDR_SHIFT 6 - -typedef struct { - union { - ethr_spinlock_t lck; - char buf[ETHR_CACHE_LINE_SIZE]; - } u; -} ethr_atomic_protection_t; - -extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; - -#define ETHR_ATOMIC_PTR2LCK__(PTR) \ -(ðr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ - & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck) - - -#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ -do { \ - ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ - ethr_spin_lock(slp__); \ - { EXPS; } \ - ethr_spin_unlock(slp__); \ -} while (0) - -#endif - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, ethr_sint_t i) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_init(var, i); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, ethr_sint_t i) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_set(var, i); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_read(var); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); - return res; -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, ethr_sint_t incr) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_add(var, incr); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, ethr_sint_t i) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_add_return(var, i); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); - return res; -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_inc(var); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_dec(var); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_inc_return(var); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); - return res; -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_dec_return(var); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); - return res; -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var, - ethr_sint_t mask) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_and_retold(var, mask); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); - return res; -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var, - ethr_sint_t mask) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_or_retold(var, mask); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); - return res; -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, ethr_sint_t new) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_xchg(var, new); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); - return res; -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, - ethr_sint_t new, - ethr_sint_t exp) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_cmpxchg(var, new, exp); -#else - ethr_sint_t res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, - { - res = *var; - if (__builtin_expect(res == exp, 1)) - *var = new; - }); - return res; -#endif -} - -/* - * Important memory barrier requirements. - * - * The following atomic operations *must* supply a memory barrier of - * at least the type specified by its suffix: - * _acqb = acquire barrier - * _relb = release barrier - */ - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_read_acqb(var); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var); -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_inc_return_acqb(var); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var); -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var, - ethr_sint_t val) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_set_relb(var, val); -#else - ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val); -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_dec_relb(var); -#else - ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var); -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_dec_return_relb(var); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var); -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var, - ethr_sint_t new, - ethr_sint_t exp) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_cmpxchg_acqb(var, new, exp); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); -#endif -} - -static ETHR_INLINE ethr_sint_t -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var, - ethr_sint_t new, - ethr_sint_t exp) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_cmpxchg_relb(var, new, exp); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); -#endif -} - -#endif /* ETHR_TRY_INLINE_FUNCS */ +#include "ethr_atomics.h" typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */ diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h index 49a8ceeab1..fd678aa97d 100644 --- a/erts/include/internal/gcc/ethr_atomic.h +++ b/erts/include/internal/gcc/ethr_atomic.h @@ -65,7 +65,7 @@ do { \ #endif #define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMORY_BARRIER -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) static ETHR_INLINE void ethr_native_atomic_set(ethr_native_atomic_t *var, long value) diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h index 709a6a17fc..b3f4a1a60d 100644 --- a/erts/include/internal/i386/atomic.h +++ b/erts/include/internal/i386/atomic.h @@ -51,7 +51,7 @@ do { \ #define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1 -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) #ifdef __x86_64__ #define LONG_SUFFIX "q" diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h index 7def51ebd1..f244e9c626 100644 --- a/erts/include/internal/libatomic_ops/ethr_atomic.h +++ b/erts/include/internal/libatomic_ops/ethr_atomic.h @@ -74,7 +74,7 @@ typedef struct { # define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("":::"memory") #endif -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) static ETHR_INLINE void ethr_native_atomic_set(ethr_native_atomic_t *var, ethr_sint_t value) diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h index daa0de88ab..6a06044f15 100644 --- a/erts/include/internal/ppc32/atomic.h +++ b/erts/include/internal/ppc32/atomic.h @@ -38,7 +38,7 @@ typedef struct { #define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory") -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) static ETHR_INLINE void ethr_native_atomic_init(ethr_native_atomic_t *var, int i) diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h index 8d94fa939f..96c1fe50a2 100644 --- a/erts/include/internal/sparc32/atomic.h +++ b/erts/include/internal/sparc32/atomic.h @@ -36,7 +36,7 @@ typedef struct { __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore\n" \ : : : "memory") -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) #if defined(__arch64__) #define CASX "casx" diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h index 05246a2aa9..b5b905b996 100644 --- a/erts/include/internal/tile/atomic.h +++ b/erts/include/internal/tile/atomic.h @@ -38,7 +38,7 @@ typedef struct { #define ETHR_MEMORY_BARRIER __insn_mf() -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) static ETHR_INLINE void ethr_native_atomic_init(ethr_native_atomic_t *var, long i) diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h index 9e2dd08728..7ae5bb6cd3 100644 --- a/erts/include/internal/win/ethr_atomic.h +++ b/erts/include/internal/win/ethr_atomic.h @@ -141,7 +141,7 @@ typedef struct { volatile ETHR_AINT_T__ value; } ETHR_ATMC_T__; -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) static ETHR_INLINE void ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in index 0d3181cace..71fc98a959 100644 --- a/erts/lib_src/Makefile.in +++ b/erts/lib_src/Makefile.in @@ -283,6 +283,7 @@ endif ETHR_THR_LIB_BASE_DIR=@ETHR_THR_LIB_BASE_DIR@ ifneq ($(strip $(ETHR_LIB_NAME)),) ETHREAD_LIB_SRC=common/ethr_aux.c \ + common/ethr_atomics.c \ common/ethr_mutex.c \ common/ethr_cbf.c \ $(ETHR_THR_LIB_BASE_DIR)/ethread.c \ @@ -445,6 +446,7 @@ INTERNAL_RELEASE_INCLUDES= \ $(ERTS_INCL_INT)/ethread.h \ $(ERTS_INCL_INT)/ethr_mutex.h \ $(ERTS_INCL_INT)/ethr_optimized_fallbacks.h \ + $(ERTS_INCL_INT)/ethr_atomics.h \ $(ERTS_INCL_INT)/$(TARGET)/ethread.mk \ $(ERTS_INCL_INT)/$(TARGET)/erts_internal.mk \ $(ERTS_INCL_INT)/$(TARGET)/ethread_header_config.h \ diff --git a/erts/lib_src/common/ethr_atomics.c b/erts/lib_src/common/ethr_atomics.c new file mode 100644 index 0000000000..c41f4a7de7 --- /dev/null +++ b/erts/lib_src/common/ethr_atomics.c @@ -0,0 +1,214 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: The ethread atomic API + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_ATOMIC_IMPL__ + +#include "ethread.h" +#include "ethr_internal.h" + +#ifndef ETHR_HAVE_NATIVE_ATOMICS +ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; +#endif + +int +ethr_init_atomics(void) +{ +#ifndef ETHR_HAVE_NATIVE_ATOMICS + { + int i; + for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) { + res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck); + if (res != 0) + return res; + } + } +#endif + return 0; +} + +void +ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t i) +{ + ETHR_ASSERT(var); + ethr_atomic_init__(var, i); +} + +void +ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t i) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_set__(var, i); +} + +ethr_sint_t +ethr_atomic_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read__(var); +} + + +ethr_sint_t +ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t incr) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_add_read__(var, incr); +} + +ethr_sint_t +ethr_atomic_inc_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_inc_read__(var); +} + +ethr_sint_t +ethr_atomic_dec_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_dec_read__(var); +} + +void +ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t incr) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_add__(var, incr); +} + +void +ethr_atomic_inc(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_inc__(var); +} + +void +ethr_atomic_dec(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_dec__(var); +} + +ethr_sint_t +ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t mask) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_band__(var, mask); +} + +ethr_sint_t +ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t mask) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_bor__(var, mask); +} + +ethr_sint_t +ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t new) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_xchg__(var, new); +} + +ethr_sint_t +ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t expected) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg__(var, new, expected); +} + +ethr_sint_t +ethr_atomic_read_acqb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_acqb__(var); +} + +ethr_sint_t +ethr_atomic_inc_read_acqb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_inc_read_acqb__(var); +} + +void +ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t i) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_set_relb__(var, i); +} + +void +ethr_atomic_dec_relb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_dec_relb__(var); +} + +ethr_sint_t +ethr_atomic_dec_read_relb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_dec_read_relb__(var); +} + +ethr_sint_t +ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg_acqb__(var, new, exp); +} + +ethr_sint_t +ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg_relb__(var, new, exp); +} + diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c index 38cf80c7df..e4569e9f5d 100644 --- a/erts/lib_src/common/ethr_aux.c +++ b/erts/lib_src/common/ethr_aux.c @@ -51,10 +51,6 @@ int ethr_not_inited__ = 1; ethr_memory_allocators ethr_mem__ = ETHR_MEM_ALLOCS_DEF_INITER__; -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; -#endif - void *(*ethr_thr_prepare_func__)(void) = NULL; void (*ethr_thr_parent_func__)(void *) = NULL; void (*ethr_thr_child_func__)(void *) = NULL; @@ -138,16 +134,9 @@ ethr_init_common__(ethr_init_data *id) #endif ethr_max_stack_size__ = ETHR_B2KW(ethr_max_stack_size__); -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS - { - int i; - for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) { - res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck); - if (res != 0) - return res; - } - } -#endif + res = ethr_init_atomics(); + if (res != 0) + return res; res = ethr_mutex_lib_init(erts_get_cpu_configured(ethr_cpu_info__)); if (res != 0) @@ -458,170 +447,6 @@ int ethr_get_main_thr_status(int *on) return 0; } - -/* Atomics */ - -void -ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t i) -{ - ETHR_ASSERT(var); - ethr_atomic_init__(var, i); -} - -void -ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t i) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - ethr_atomic_set__(var, i); -} - -ethr_sint_t -ethr_atomic_read(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_read__(var); -} - - -ethr_sint_t -ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t incr) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_add_read__(var, incr); -} - -ethr_sint_t -ethr_atomic_inc_read(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_inc_read__(var); -} - -ethr_sint_t -ethr_atomic_dec_read(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_dec_read__(var); -} - -void -ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t incr) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - ethr_atomic_add__(var, incr); -} - -void -ethr_atomic_inc(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - ethr_atomic_inc__(var); -} - -void -ethr_atomic_dec(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - ethr_atomic_dec__(var); -} - -ethr_sint_t -ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t mask) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_read_band__(var, mask); -} - -ethr_sint_t -ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t mask) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_read_bor__(var, mask); -} - -ethr_sint_t -ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t new) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_xchg__(var, new); -} - -ethr_sint_t -ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t expected) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_cmpxchg__(var, new, expected); -} - -ethr_sint_t -ethr_atomic_read_acqb(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_read_acqb__(var); -} - -ethr_sint_t -ethr_atomic_inc_read_acqb(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_inc_read_acqb__(var); -} - -void -ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t i) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - ethr_atomic_set_relb__(var, i); -} - -void -ethr_atomic_dec_relb(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - ethr_atomic_dec_relb__(var); -} - -ethr_sint_t -ethr_atomic_dec_read_relb(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_dec_read_relb__(var); -} - -ethr_sint_t -ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_cmpxchg_acqb__(var, new, exp); -} - -ethr_sint_t -ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_cmpxchg_relb__(var, new, exp); -} - - /* Spinlocks and rwspinlocks */ int |