diff options
Diffstat (limited to 'erts/include/internal/win/ethr_atomic.h')
| -rw-r--r-- | erts/include/internal/win/ethr_atomic.h | 190 |
1 files changed, 147 insertions, 43 deletions
diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h index 7ae5bb6cd3..60def01a7e 100644 --- a/erts/include/internal/win/ethr_atomic.h +++ b/erts/include/internal/win/ethr_atomic.h @@ -22,10 +22,27 @@ * Author: Rickard Green */ -#ifndef ETHR_WIN_ATOMIC_H__ -#define ETHR_WIN_ATOMIC_H__ +#undef ETHR_INCLUDE_ATOMIC_IMPL__ +#if !defined(ETHR_WIN_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) +#define ETHR_WIN_ATOMIC32_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +#undef ETHR_ATOMIC_WANT_32BIT_IMPL__ +#elif !defined(ETHR_WIN_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) +#define ETHR_WIN_ATOMIC64_H__ +#ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 +/* _InterlockedCompareExchange64() required... */ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +#endif +#undef ETHR_ATOMIC_WANT_64BIT_IMPL__ +#endif + +#ifdef ETHR_INCLUDE_ATOMIC_IMPL__ #if defined(_MSC_VER) && _MSC_VER >= 1400 + +#ifndef ETHR_WIN_ATOMIC_COMMON__ +#define ETHR_WIN_ATOMIC_COMMON__ + #define ETHR_HAVE_NATIVE_ATOMICS 1 #if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64) @@ -79,7 +96,15 @@ do { \ #endif -#if ETHR_SIZEOF_PTR == 4 +#endif /* ETHR_WIN_ATOMIC_COMMON__ */ + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 + +#define ETHR_HAVE_NATIVE_ATOMIC32 1 + +/* + * All used operations available as 32-bit intrinsics + */ #pragma intrinsic(_InterlockedDecrement) #pragma intrinsic(_InterlockedIncrement) @@ -95,23 +120,94 @@ do { \ #pragma intrinsic(_InterlockedCompareExchange_rel) #endif -#define ETHR_ATMC__(X) X +#define ETHR_ILCKD__(X) _Interlocked ## X #ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS -#define ETHR_ATMC_ACQ__(X) X ## _acq -#define ETHR_ATMC_REL__(X) X ## _rel +#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq +#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel +#else +#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X +#define ETHR_ILCKD_REL__(X) _Interlocked ## X +#endif + +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic32_t +#define ETHR_AINT_T__ ethr_sint32_t + +#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 + +#define ETHR_HAVE_NATIVE_ATOMIC64 1 + +/* + * _InterlockedCompareExchange64() is required. The other may not + * be available, but if so, we can generate them. + */ +#pragma intrinsic(_InterlockedCompareExchange64) + +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ +#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) *(PTR) #else -#define ETHR_ATMC_ACQ__(X) X -#define ETHR_ATMC_REL__(X) X +#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) (__int64) 0 #endif -#elif ETHR_SIZEOF_PTR == 8 +#define ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, PTR, NEW, ACT, EXP, OPS, RET) \ +{ \ + __int64 NEW, ACT, EXP; \ + ACT = ETHR_OWN_ILCKD_INIT_VAL__(PTR); \ + do { \ + EXP = ACT; \ + { OPS; } \ + ACT = _InterlockedCompareExchange64(PTR, NEW, EXP); \ + } while (ACT != EXP); \ + return RET; \ +} + +#define ETHR_OWN_ILCKD_1_IMPL__(FUNC, NEW, ACT, EXP, OPS, RET) \ +static __forceinline __int64 \ +FUNC(__int64 volatile *ptr) \ +ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET) + +#define ETHR_OWN_ILCKD_2_IMPL__(FUNC, NEW, ACT, EXP, OPS, ARG, RET) \ +static __forceinline __int64 \ +FUNC(__int64 volatile *ptr, __int64 ARG) \ +ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET) + +#ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64 #pragma intrinsic(_InterlockedDecrement64) +#else +ETHR_OWN_ILCKD_1_IMPL__(_InterlockedDecrement64, new, act, exp, + new = act - 1, new) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64 #pragma intrinsic(_InterlockedIncrement64) +#else +ETHR_OWN_ILCKD_1_IMPL__(_InterlockedIncrement64, new, act, exp, + new = act + 1, new) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 #pragma intrinsic(_InterlockedExchangeAdd64) +#else +ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchangeAdd64, new, act, exp, + new = act + arg, arg, act) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64 #pragma intrinsic(_InterlockedExchange64) +#else +ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchange64, new, act, exp, + new = arg, arg, act) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDAND64 #pragma intrinsic(_InterlockedAnd64) +#else +ETHR_OWN_ILCKD_2_IMPL__(_InterlockedAnd64, new, act, exp, + new = act & arg, arg, act) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDOR64 #pragma intrinsic(_InterlockedOr64) +#else +ETHR_OWN_ILCKD_2_IMPL__(_InterlockedOr64, new, act, exp, + new = act | arg, arg, act) +#endif #ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS #pragma intrinsic(_InterlockedExchangeAdd64_acq) #pragma intrinsic(_InterlockedIncrement64_acq) @@ -120,36 +216,42 @@ do { \ #pragma intrinsic(_InterlockedCompareExchange64_rel) #endif -#define ETHR_ATMC__(X) X ## 64 +#define ETHR_ILCKD__(X) _Interlocked ## X ## 64 #ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS -#define ETHR_ATMC_ACQ__(X) X ## 64_acq -#define ETHR_ATMC_REL__(X) X ## 64_rel +#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq +#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel #else -#define ETHR_ATMC_ACQ__(X) X ## 64 -#define ETHR_ATMC_REL__(X) X ## 64 +#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64 +#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64 #endif +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic64_t +#define ETHR_AINT_T__ ethr_sint64_t + #else -#error "Unsupported word size" +#error "Unsupported integer size" #endif -#define ETHR_NATMC_FUNC__(X) ethr_native_atomic_ ## X -#define ETHR_ATMC_T__ ethr_native_atomic_t -#define ETHR_AINT_T__ ethr_sint_t - typedef struct { volatile ETHR_AINT_T__ value; } ETHR_ATMC_T__; #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +static ETHR_INLINE ETHR_AINT_T__ * +ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__ *) &var->value; +} + static ETHR_INLINE void ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { #if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ var->value = i; #else - (void) ETHR_ATMC__(_InterlockedExchange)(&var->value, i); + (void) ETHR_ILCKD__(Exchange)(&var->value, i); #endif } @@ -159,7 +261,7 @@ ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) #if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ var->value = i; #else - (void) ETHR_ATMC__(_InterlockedExchange)(&var->value, i); + (void) ETHR_ILCKD__(Exchange)(&var->value, i); #endif } @@ -169,56 +271,56 @@ ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) #if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ return var->value; #else - return ETHR_ATMC__(_InterlockedExchangeAdd)(&var->value, (ETHR_AINT_T__) 0); + return ETHR_ILCKD__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0); #endif } static ETHR_INLINE void ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - (void) ETHR_ATMC__(_InterlockedExchangeAdd)(&var->value, incr); + (void) ETHR_ILCKD__(ExchangeAdd)(&var->value, incr); } static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { - return ETHR_ATMC__(_InterlockedExchangeAdd)(&var->value, i) + i; + return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i; } static ETHR_INLINE void ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) { - (void) ETHR_ATMC__(_InterlockedIncrement)(&var->value); + (void) ETHR_ILCKD__(Increment)(&var->value); } static ETHR_INLINE void ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) { - (void) ETHR_ATMC__(_InterlockedDecrement)(&var->value); + (void) ETHR_ILCKD__(Decrement)(&var->value); } static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) { - return ETHR_ATMC__(_InterlockedIncrement)(&var->value); + return ETHR_ILCKD__(Increment)(&var->value); } static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) { - return ETHR_ATMC__(_InterlockedDecrement)(&var->value); + return ETHR_ILCKD__(Decrement)(&var->value); } static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - return ETHR_ATMC__(_InterlockedAnd)(&var->value, mask); + return ETHR_ILCKD__(And)(&var->value, mask); } static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - return ETHR_ATMC__(_InterlockedOr)(&var->value, mask); + return ETHR_ILCKD__(Or)(&var->value, mask); } static ETHR_INLINE ETHR_AINT_T__ @@ -226,14 +328,14 @@ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { - return ETHR_ATMC__(_InterlockedCompareExchange)(&var->value, new, old); + return ETHR_ILCKD__(CompareExchange)(&var->value, new, old); } static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) { - return ETHR_ATMC__(_InterlockedExchange)(&var->value, new); + return ETHR_ILCKD__(Exchange)(&var->value, new); } /* @@ -248,15 +350,14 @@ ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) ETHR_COMPILER_BARRIER; return val; #else - return ETHR_ATMC_ACQ__(_InterlockedExchangeAdd)(&var->value, - (ETHR_AINT_T__) 0); + return ETHR_ILCKD_ACQ__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0); #endif } static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) { - return ETHR_ATMC_ACQ__(_InterlockedIncrement)(&var->value); + return ETHR_ILCKD_ACQ__(Increment)(&var->value); } static ETHR_INLINE void @@ -266,20 +367,20 @@ ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) ETHR_COMPILER_BARRIER; var->value = i; #else - (void) ETHR_ATMC_REL__(_InterlockedExchange)(&var->value, i); + (void) ETHR_ILCKD_REL__(Exchange)(&var->value, i); #endif } static ETHR_INLINE void ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) { - (void) ETHR_ATMC_REL__(_InterlockedDecrement)(&var->value); + (void) ETHR_ILCKD_REL__(Decrement)(&var->value); } static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) { - return ETHR_ATMC_REL__(_InterlockedDecrement)(&var->value); + return ETHR_ILCKD_REL__(Decrement)(&var->value); } static ETHR_INLINE ETHR_AINT_T__ @@ -287,7 +388,7 @@ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { - return ETHR_ATMC_ACQ__(_InterlockedCompareExchange)(&var->value, new, old); + return ETHR_ILCKD_ACQ__(CompareExchange)(&var->value, new, old); } static ETHR_INLINE ETHR_AINT_T__ @@ -295,17 +396,20 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { - return ETHR_ATMC_REL__(_InterlockedCompareExchange)(&var->value, new, old); + return ETHR_ILCKD_REL__(CompareExchange)(&var->value, new, old); } -#endif +#endif /* ETHR_TRY_INLINE_FUNCS */ +#undef ETHR_ILCKD__ +#undef ETHR_ILCKD_ACQ__ +#undef ETHR_ILCKD_REL__ #undef ETHR_NATMC_FUNC__ #undef ETHR_ATMC_T__ #undef ETHR_AINT_T__ #undef ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ #undef ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ -#endif +#endif /* _MSC_VER */ -#endif +#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */ |