diff options
| -rw-r--r-- | erts/aclocal.m4 | 2 | ||||
| -rw-r--r-- | erts/include/internal/ethread.h | 14 | ||||
| -rw-r--r-- | erts/include/internal/ethread_header_config.h.in | 3 | ||||
| -rw-r--r-- | erts/include/internal/win/ethr_atomic.h | 300 | ||||
| -rw-r--r-- | erts/include/internal/win/ethread.h | 1 |
5 files changed, 199 insertions, 121 deletions
diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index d45b36bfed..799dcc2c7c 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -1093,6 +1093,8 @@ AC_CHECK_SIZEOF(long) AC_DEFINE_UNQUOTED(ETHR_SIZEOF_LONG, $ac_cv_sizeof_long, [Define to the size of long]) AC_CHECK_SIZEOF(long long) AC_DEFINE_UNQUOTED(ETHR_SIZEOF_LONG_LONG, $ac_cv_sizeof_long_long, [Define to the size of long long]) +AC_CHECK_SIZEOF(__int64) +AC_DEFINE_UNQUOTED(ETHR_SIZEOF___INT64, $ac_cv_sizeof___int64, [Define to the size of __int64]) AC_ARG_ENABLE(native-ethr-impls, AS_HELP_STRING([--disable-native-ethr-impls], diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h index c32ea42a92..84554d2d77 100644 --- a/erts/include/internal/ethread.h +++ b/erts/include/internal/ethread.h @@ -167,14 +167,20 @@ typedef pthread_key_t ethr_tsd_key; #endif #if defined(_MSC_VER) -#if ETHR_SIZEOF_PTR == 4 + #define ETHR_HAVE_INT_T 1 -typedef LONG ethr_sint_t; -typedef ULONG ethr_uint_t; +#if ETHR_SIZEOF_LONG == ETHR_SIZEOF_PTR +typedef long ethr_sint_t; +typedef unsigned long ethr_uint_t; +#elif ETHR_SIZEOF___INT64 == ETHR_SIZEOF_PTR +typedef __int64 ethr_sint_t; +typedef unsigned __int64 ethr_uint_t; #else -#error "Only 32-bit windows is supported" +#error "No integer type of the same size as pointers found" #endif + #endif + struct ethr_join_data_; /* Types */ diff --git a/erts/include/internal/ethread_header_config.h.in b/erts/include/internal/ethread_header_config.h.in index 8c6fbc1fdd..3b62196780 100644 --- a/erts/include/internal/ethread_header_config.h.in +++ b/erts/include/internal/ethread_header_config.h.in @@ -29,6 +29,9 @@ /* Define to the size of long long */ #undef ETHR_SIZEOF_LONG_LONG +/* Define to the size of __int64 */ +#undef ETHR_SIZEOF___INT64 + /* Define if you want to disable native ethread implementations */ #undef ETHR_DISABLE_NATIVE_IMPLS diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h index 47068927a6..9e2dd08728 100644 --- a/erts/include/internal/win/ethr_atomic.h +++ b/erts/include/internal/win/ethr_atomic.h @@ -25,218 +25,286 @@ #ifndef ETHR_WIN_ATOMIC_H__ #define ETHR_WIN_ATOMIC_H__ -#ifdef _MSC_VER -# if _MSC_VER < 1300 -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 /* Dont trust really old compilers */ -# else -# if defined(_M_IX86) -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 -# else /* I.e. IA64 */ -# if _MSC_VER >= 1400 -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 -# else -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 -# endif -# endif -# endif -# if _MSC_VER >= 1400 -# include <intrin.h> -# undef ETHR_COMPILER_BARRIER -# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() -# endif -#pragma intrinsic(_ReadWriteBarrier) -#pragma intrinsic(_InterlockedAnd) -#pragma intrinsic(_InterlockedOr) +#if defined(_MSC_VER) && _MSC_VER >= 1400 +#define ETHR_HAVE_NATIVE_ATOMICS 1 + +#if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64) +# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1 #else -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 +# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0 #endif +#if defined(_M_AMD64) || (defined(_M_IX86) \ + && !defined(ETHR_PRE_PENTIUM4_COMPAT)) +# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1 +#else +# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0 +#endif /* - * No configure test checking for _Interlocked*_{acq,rel} and - * Interlocked*{Acquire,Release} have been written yet... + * No configure test checking for interlocked acquire/release + * versions have been written, yet. It should define + * ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS if, and + * only if, all used interlocked operations with barriers + * exists. * * Note, that these are pure optimizations for the itanium * processor. */ -#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_ACQ -#pragma intrinsic(_InterlockedCompareExchange_acq) +#include <intrin.h> +#undef ETHR_COMPILER_BARRIER +#define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +#pragma intrinsic(_ReadWriteBarrier) +#pragma intrinsic(_InterlockedCompareExchange) + +#if defined(_M_AMD64) || (defined(_M_IX86) \ + && !defined(ETHR_PRE_PENTIUM4_COMPAT)) +#include <emmintrin.h> +#include <mmintrin.h> +#pragma intrinsic(_mm_mfence) +#define ETHR_MEMORY_BARRIER _mm_mfence() +#pragma intrinsic(_mm_sfence) +#define ETHR_WRITE_MEMORY_BARRIER _mm_sfence() +#pragma intrinsic(_mm_lfence) +#define ETHR_READ_MEMORY_BARRIER _mm_lfence() +#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER + +#else + +#define ETHR_MEMORY_BARRIER \ +do { \ + volatile long x___ = 0; \ + _InterlockedCompareExchange(&x___, (long) 1, (long) 0); \ +} while (0) + #endif -#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_REL + +#if ETHR_SIZEOF_PTR == 4 + +#pragma intrinsic(_InterlockedDecrement) +#pragma intrinsic(_InterlockedIncrement) +#pragma intrinsic(_InterlockedExchangeAdd) +#pragma intrinsic(_InterlockedExchange) +#pragma intrinsic(_InterlockedAnd) +#pragma intrinsic(_InterlockedOr) +#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS +#pragma intrinsic(_InterlockedExchangeAdd_acq) +#pragma intrinsic(_InterlockedIncrement_acq) +#pragma intrinsic(_InterlockedDecrement_rel) +#pragma intrinsic(_InterlockedCompareExchange_acq) #pragma intrinsic(_InterlockedCompareExchange_rel) #endif -typedef struct { - volatile LONG value; -} ethr_native_atomic_t; +#define ETHR_ATMC__(X) X +#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS +#define ETHR_ATMC_ACQ__(X) X ## _acq +#define ETHR_ATMC_REL__(X) X ## _rel +#else +#define ETHR_ATMC_ACQ__(X) X +#define ETHR_ATMC_REL__(X) X +#endif -#define ETHR_MEMORY_BARRIER \ -do { \ - volatile LONG x___ = 0; \ - _InterlockedCompareExchange(&x___, (LONG) 1, (LONG) 0); \ -} while (0) +#elif ETHR_SIZEOF_PTR == 8 + +#pragma intrinsic(_InterlockedDecrement64) +#pragma intrinsic(_InterlockedIncrement64) +#pragma intrinsic(_InterlockedExchangeAdd64) +#pragma intrinsic(_InterlockedExchange64) +#pragma intrinsic(_InterlockedAnd64) +#pragma intrinsic(_InterlockedOr64) +#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS +#pragma intrinsic(_InterlockedExchangeAdd64_acq) +#pragma intrinsic(_InterlockedIncrement64_acq) +#pragma intrinsic(_InterlockedDecrement64_rel) +#pragma intrinsic(_InterlockedCompareExchange64_acq) +#pragma intrinsic(_InterlockedCompareExchange64_rel) +#endif + +#define ETHR_ATMC__(X) X ## 64 +#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS +#define ETHR_ATMC_ACQ__(X) X ## 64_acq +#define ETHR_ATMC_REL__(X) X ## 64_rel +#else +#define ETHR_ATMC_ACQ__(X) X ## 64 +#define ETHR_ATMC_REL__(X) X ## 64 +#endif + +#else +#error "Unsupported word size" +#endif + +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic_t +#define ETHR_AINT_T__ ethr_sint_t + +typedef struct { + volatile ETHR_AINT_T__ value; +} ETHR_ATMC_T__; #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void -ethr_native_atomic_init(ethr_native_atomic_t *var, long i) +ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { - var->value = (LONG) i; +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ + var->value = i; +#else + (void) ETHR_ATMC__(_InterlockedExchange)(&var->value, i); +#endif } static ETHR_INLINE void -ethr_native_atomic_set(ethr_native_atomic_t *var, long i) +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - var->value = (LONG) i; +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ + var->value = i; #else - (void) InterlockedExchange(&var->value, (LONG) i); + (void) ETHR_ATMC__(_InterlockedExchange)(&var->value, i); #endif } -static ETHR_INLINE long -ethr_native_atomic_read(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ return var->value; #else - return InterlockedExchangeAdd(&var->value, (LONG) 0); + return ETHR_ATMC__(_InterlockedExchangeAdd)(&var->value, (ETHR_AINT_T__) 0); #endif } static ETHR_INLINE void -ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - (void) InterlockedExchangeAdd(&var->value, (LONG) incr); + (void) ETHR_ATMC__(_InterlockedExchangeAdd)(&var->value, incr); } -static ETHR_INLINE long -ethr_native_atomic_add_return(ethr_native_atomic_t *var, long i) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { - LONG tmp = InterlockedExchangeAdd(&var->value, (LONG) i); - return tmp + i; + return ETHR_ATMC__(_InterlockedExchangeAdd)(&var->value, i) + i; } static ETHR_INLINE void -ethr_native_atomic_inc(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) { - (void) InterlockedIncrement(&var->value); + (void) ETHR_ATMC__(_InterlockedIncrement)(&var->value); } static ETHR_INLINE void -ethr_native_atomic_dec(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) { - (void) InterlockedDecrement(&var->value); + (void) ETHR_ATMC__(_InterlockedDecrement)(&var->value); } -static ETHR_INLINE long -ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) { - return (long) InterlockedIncrement(&var->value); + return ETHR_ATMC__(_InterlockedIncrement)(&var->value); } -static ETHR_INLINE long -ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) { - return (long) InterlockedDecrement(&var->value); + return ETHR_ATMC__(_InterlockedDecrement)(&var->value); } -static ETHR_INLINE long -ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - return (long) _InterlockedAnd(&var->value, mask); + return ETHR_ATMC__(_InterlockedAnd)(&var->value, mask); } -static ETHR_INLINE long -ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - return (long) _InterlockedOr(&var->value, mask); + return ETHR_ATMC__(_InterlockedOr)(&var->value, mask); } - -static ETHR_INLINE long -ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { - return (long) _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) old); + return ETHR_ATMC__(_InterlockedCompareExchange)(&var->value, new, old); } -static ETHR_INLINE long -ethr_native_atomic_xchg(ethr_native_atomic_t *var, long new) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) { - return (long) InterlockedExchange(&var->value, (LONG) new); + return ETHR_ATMC__(_InterlockedExchange)(&var->value, new); } /* * Atomic ops with at least specified barriers. */ -static ETHR_INLINE long -ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) { -#ifdef ETHR_HAVE_INTERLOCKEDEXCHANGEADDACQUIRE - return (long) InterlockedExchangeAddAcquire(&var->value, (LONG) 0); +#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ + ETHR_AINT_T__ val = var->value; + ETHR_COMPILER_BARRIER; + return val; #else - return (long) InterlockedExchangeAdd(&var->value, (LONG) 0); + return ETHR_ATMC_ACQ__(_InterlockedExchangeAdd)(&var->value, + (ETHR_AINT_T__) 0); #endif } -static ETHR_INLINE long -ethr_native_atomic_inc_return_acqb(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) { -#ifdef ETHR_HAVE_INTERLOCKEDINCREMENTACQUIRE - return (long) InterlockedIncrementAcquire(&var->value); -#else - return (long) InterlockedIncrement(&var->value); -#endif + return ETHR_ATMC_ACQ__(_InterlockedIncrement)(&var->value); } static ETHR_INLINE void -ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long i) +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { - (void) InterlockedExchange(&var->value, (LONG) i); +#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ + ETHR_COMPILER_BARRIER; + var->value = i; +#else + (void) ETHR_ATMC_REL__(_InterlockedExchange)(&var->value, i); +#endif } static ETHR_INLINE void -ethr_native_atomic_dec_relb(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) { -#ifdef ETHR_HAVE_INTERLOCKEDDECREMENTRELEASE - (void) InterlockedDecrementRelease(&var->value); -#else - (void) InterlockedDecrement(&var->value); -#endif + (void) ETHR_ATMC_REL__(_InterlockedDecrement)(&var->value); } -static ETHR_INLINE long -ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) { -#ifdef ETHR_HAVE_INTERLOCKEDDECREMENTRELEASE - return (long) InterlockedDecrementRelease(&var->value); -#else - return (long) InterlockedDecrement(&var->value); -#endif + return ETHR_ATMC_REL__(_InterlockedDecrement)(&var->value); } -static ETHR_INLINE long -ethr_native_atomic_cmpxchg_acqb(ethr_native_atomic_t *var, long new, long old) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { -#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_ACQ - return (long) _InterlockedCompareExchange_acq(&var->value, (LONG) new, (LONG) old); -#else - return (long) _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) old); -#endif + return ETHR_ATMC_ACQ__(_InterlockedCompareExchange)(&var->value, new, old); } -static ETHR_INLINE long -ethr_native_atomic_cmpxchg_relb(ethr_native_atomic_t *var, long new, long old) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { + return ETHR_ATMC_REL__(_InterlockedCompareExchange)(&var->value, new, old); +} -#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_REL - return (long) _InterlockedCompareExchange_rel(&var->value, (LONG) new, (LONG) old); -#else - return (long) _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) old); #endif -} + +#undef ETHR_NATMC_FUNC__ +#undef ETHR_ATMC_T__ +#undef ETHR_AINT_T__ +#undef ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ +#undef ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ #endif diff --git a/erts/include/internal/win/ethread.h b/erts/include/internal/win/ethread.h index b52710f6a3..3cbd4aeda1 100644 --- a/erts/include/internal/win/ethread.h +++ b/erts/include/internal/win/ethread.h @@ -26,6 +26,5 @@ #define ETHREAD_WIN_H__ #include "ethr_atomic.h" -#define ETHR_HAVE_NATIVE_ATOMICS 1 #endif |