diff options
Diffstat (limited to 'erts/include/internal/win')
| -rw-r--r-- | erts/include/internal/win/ethr_atomic.h | 595 | ||||
| -rw-r--r-- | erts/include/internal/win/ethr_dw_atomic.h | 154 | ||||
| -rw-r--r-- | erts/include/internal/win/ethr_event.h | 16 | ||||
| -rw-r--r-- | erts/include/internal/win/ethr_membar.h | 145 | ||||
| -rw-r--r-- | erts/include/internal/win/ethread.h | 4 |
5 files changed, 640 insertions, 274 deletions
diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h index 60def01a7e..e11f1abf47 100644 --- a/erts/include/internal/win/ethr_atomic.h +++ b/erts/include/internal/win/ethr_atomic.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,364 +24,408 @@ #undef ETHR_INCLUDE_ATOMIC_IMPL__ #if !defined(ETHR_WIN_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) -#define ETHR_WIN_ATOMIC32_H__ -#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 -#undef ETHR_ATOMIC_WANT_32BIT_IMPL__ +# define ETHR_WIN_ATOMIC32_H__ +# if (defined(ETHR_MEMBAR) \ + && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE) \ + && defined(ETHR_HAVE__INTERLOCKEDEXCHANGE)) +# define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +# endif +# undef ETHR_ATOMIC_WANT_32BIT_IMPL__ #elif !defined(ETHR_WIN_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) -#define ETHR_WIN_ATOMIC64_H__ -#ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 -/* _InterlockedCompareExchange64() required... */ -#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +# define ETHR_WIN_ATOMIC64_H__ +# if (defined(ETHR_MEMBAR) \ + && (defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64) \ + || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ) \ + || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL))) +# define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +# endif +# undef ETHR_ATOMIC_WANT_64BIT_IMPL__ #endif -#undef ETHR_ATOMIC_WANT_64BIT_IMPL__ + +#if !defined(_MSC_VER) || _MSC_VER < 1400 +# undef ETHR_INCLUDE_ATOMIC_IMPL__ #endif #ifdef ETHR_INCLUDE_ATOMIC_IMPL__ -#if defined(_MSC_VER) && _MSC_VER >= 1400 +# ifndef ETHR_WIN_ATOMIC_COMMON__ +# define ETHR_WIN_ATOMIC_COMMON__ + +# if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64) +# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1 +# else +# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0 +# endif + +# endif /* ETHR_WIN_ATOMIC_COMMON__ */ + +# if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 + +# define ETHR_HAVE_NATIVE_ATOMIC32 1 +# define ETHR_NATIVE_ATOMIC32_IMPL "windows-interlocked" + +# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT +# define ETHR_WIN_HAVE_DEC +# pragma intrinsic(_InterlockedDecrement) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT_REL +# define ETHR_WIN_HAVE_DEC_REL +# pragma intrinsic(_InterlockedDecrement_rel) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT +# define ETHR_WIN_HAVE_INC +# pragma intrinsic(_InterlockedIncrement) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ +# define ETHR_WIN_HAVE_INC_ACQ +# pragma intrinsic(_InterlockedIncrement_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD +# define ETHR_WIN_HAVE_XCHG_ADD +# pragma intrinsic(_InterlockedExchangeAdd) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ +# define ETHR_WIN_HAVE_XCHG_ADD_ACQ +# pragma intrinsic(_InterlockedExchangeAdd_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE +# define ETHR_WIN_HAVE_XCHG +# pragma intrinsic(_InterlockedExchange) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDAND +# define ETHR_WIN_HAVE_AND +# pragma intrinsic(_InterlockedAnd) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDOR +# define ETHR_WIN_HAVE_OR +# pragma intrinsic(_InterlockedOr) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE +# define ETHR_WIN_HAVE_CMPXCHG +# pragma intrinsic(_InterlockedCompareExchange) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ +# define ETHR_WIN_HAVE_CMPXCHG_ACQ +# pragma intrinsic(_InterlockedCompareExchange_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL +# define ETHR_WIN_HAVE_CMPXCHG_REL +# pragma intrinsic(_InterlockedCompareExchange_rel) +# endif + +# define ETHR_ILCKD__(X) _Interlocked ## X +# define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq +# define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel + +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +# define ETHR_ATMC_T__ ethr_native_atomic32_t +# define ETHR_AINT_T__ ethr_sint32_t + +# elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 + +# define ETHR_HAVE_NATIVE_ATOMIC64 1 +# define ETHR_NATIVE_ATOMIC64_IMPL "windows-interlocked" + +# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64 +# define ETHR_WIN_HAVE_DEC +# pragma intrinsic(_InterlockedDecrement64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64_REL +# define ETHR_WIN_HAVE_DEC_REL +# pragma intrinsic(_InterlockedDecrement64_rel) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ +# define ETHR_WIN_HAVE_INC_ACQ +# pragma intrinsic(_InterlockedIncrement64_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64 +# define ETHR_WIN_HAVE_INC +# pragma intrinsic(_InterlockedIncrement64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 +# define ETHR_WIN_HAVE_XCHG_ADD +# pragma intrinsic(_InterlockedExchangeAdd64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ +# define ETHR_WIN_HAVE_XCHG_ADD_ACQ +# pragma intrinsic(_InterlockedExchangeAdd64_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64 +# define ETHR_WIN_HAVE_XCHG +# pragma intrinsic(_InterlockedExchange64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDAND64 +# define ETHR_WIN_HAVE_AND +# pragma intrinsic(_InterlockedAnd64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDOR64 +# define ETHR_WIN_HAVE_OR +# pragma intrinsic(_InterlockedOr64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 +# define ETHR_WIN_HAVE_CMPXCHG +# pragma intrinsic(_InterlockedCompareExchange64) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ +# define ETHR_WIN_HAVE_CMPXCHG_ACQ +# pragma intrinsic(_InterlockedCompareExchange64_acq) +# endif +# ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL +# define ETHR_WIN_HAVE_CMPXCHG_REL +# pragma intrinsic(_InterlockedCompareExchange64_rel) +# endif + +# define ETHR_ILCKD__(X) _Interlocked ## X ## 64 +# define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq +# define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel + +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# define ETHR_ATMC_T__ ethr_native_atomic64_t +# define ETHR_AINT_T__ ethr_sint64_t + +# else +# error "Unsupported integer size" +# endif -#ifndef ETHR_WIN_ATOMIC_COMMON__ -#define ETHR_WIN_ATOMIC_COMMON__ +typedef struct { + volatile ETHR_AINT_T__ value; +} ETHR_ATMC_T__; -#define ETHR_HAVE_NATIVE_ATOMICS 1 +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) -#if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64) -# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1 +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1 #else -# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1 #endif -#if defined(_M_AMD64) || (defined(_M_IX86) \ - && !defined(ETHR_PRE_PENTIUM4_COMPAT)) -# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1 -#else -# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0 -#endif -/* - * No configure test checking for interlocked acquire/release - * versions have been written, yet. It should define - * ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS if, and - * only if, all used interlocked operations with barriers - * exists. - * - * Note, that these are pure optimizations for the itanium - * processor. - */ +static ETHR_INLINE ETHR_AINT_T__ * +ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__ *) &var->value; +} -#include <intrin.h> -#undef ETHR_COMPILER_BARRIER -#define ETHR_COMPILER_BARRIER _ReadWriteBarrier() -#pragma intrinsic(_ReadWriteBarrier) -#pragma intrinsic(_InterlockedCompareExchange) - -#if defined(_M_AMD64) || (defined(_M_IX86) \ - && !defined(ETHR_PRE_PENTIUM4_COMPAT)) -#include <emmintrin.h> -#include <mmintrin.h> -#pragma intrinsic(_mm_mfence) -#define ETHR_MEMORY_BARRIER _mm_mfence() -#pragma intrinsic(_mm_sfence) -#define ETHR_WRITE_MEMORY_BARRIER _mm_sfence() -#pragma intrinsic(_mm_lfence) -#define ETHR_READ_MEMORY_BARRIER _mm_lfence() -#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1 #else - -#define ETHR_MEMORY_BARRIER \ -do { \ - volatile long x___ = 0; \ - _InterlockedCompareExchange(&x___, (long) 1, (long) 0); \ -} while (0) - +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1 #endif -#endif /* ETHR_WIN_ATOMIC_COMMON__ */ +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + var->value = i; +} #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1 +#endif -#define ETHR_HAVE_NATIVE_ATOMIC32 1 +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ +#if defined(_M_IX86) + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + (void) ETHR_ILCKD__(Exchange)(&var->value, i); + else +#endif /* _M_IX86 */ + { + ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore); + var->value = i; + } +} -/* - * All used operations available as 32-bit intrinsics - */ +#endif /* ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ */ -#pragma intrinsic(_InterlockedDecrement) -#pragma intrinsic(_InterlockedIncrement) -#pragma intrinsic(_InterlockedExchangeAdd) -#pragma intrinsic(_InterlockedExchange) -#pragma intrinsic(_InterlockedAnd) -#pragma intrinsic(_InterlockedOr) -#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS -#pragma intrinsic(_InterlockedExchangeAdd_acq) -#pragma intrinsic(_InterlockedIncrement_acq) -#pragma intrinsic(_InterlockedDecrement_rel) -#pragma intrinsic(_InterlockedCompareExchange_acq) -#pragma intrinsic(_InterlockedCompareExchange_rel) -#endif +#if defined(ETHR_WIN_HAVE_XCHG) -#define ETHR_ILCKD__(X) _Interlocked ## X -#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS -#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq -#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB 1 #else -#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X -#define ETHR_ILCKD_REL__(X) _Interlocked ## X +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB 1 #endif -#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X -#define ETHR_ATMC_T__ ethr_native_atomic32_t -#define ETHR_AINT_T__ ethr_sint32_t - -#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + (void) ETHR_ILCKD__(Exchange)(&var->value, i); +} -#define ETHR_HAVE_NATIVE_ATOMIC64 1 +#endif -/* - * _InterlockedCompareExchange64() is required. The other may not - * be available, but if so, we can generate them. - */ -#pragma intrinsic(_InterlockedCompareExchange64) +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ -#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ -#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) *(PTR) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 #else -#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) (__int64) 0 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1 #endif -#define ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, PTR, NEW, ACT, EXP, OPS, RET) \ -{ \ - __int64 NEW, ACT, EXP; \ - ACT = ETHR_OWN_ILCKD_INIT_VAL__(PTR); \ - do { \ - EXP = ACT; \ - { OPS; } \ - ACT = _InterlockedCompareExchange64(PTR, NEW, EXP); \ - } while (ACT != EXP); \ - return RET; \ +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) +{ + return var->value; } -#define ETHR_OWN_ILCKD_1_IMPL__(FUNC, NEW, ACT, EXP, OPS, RET) \ -static __forceinline __int64 \ -FUNC(__int64 volatile *ptr) \ -ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET) - -#define ETHR_OWN_ILCKD_2_IMPL__(FUNC, NEW, ACT, EXP, OPS, ARG, RET) \ -static __forceinline __int64 \ -FUNC(__int64 volatile *ptr, __int64 ARG) \ -ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET) +#endif /* ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ */ +#if defined(ETHR_WIN_HAVE_XCHG_ADD) -#ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64 -#pragma intrinsic(_InterlockedDecrement64) -#else -ETHR_OWN_ILCKD_1_IMPL__(_InterlockedDecrement64, new, act, exp, - new = act - 1, new) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64 -#pragma intrinsic(_InterlockedIncrement64) -#else -ETHR_OWN_ILCKD_1_IMPL__(_InterlockedIncrement64, new, act, exp, - new = act + 1, new) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 -#pragma intrinsic(_InterlockedExchangeAdd64) -#else -ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchangeAdd64, new, act, exp, - new = act + arg, arg, act) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64 -#pragma intrinsic(_InterlockedExchange64) -#else -ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchange64, new, act, exp, - new = arg, arg, act) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDAND64 -#pragma intrinsic(_InterlockedAnd64) -#else -ETHR_OWN_ILCKD_2_IMPL__(_InterlockedAnd64, new, act, exp, - new = act & arg, arg, act) -#endif -#ifdef ETHR_HAVE__INTERLOCKEDOR64 -#pragma intrinsic(_InterlockedOr64) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1 #else -ETHR_OWN_ILCKD_2_IMPL__(_InterlockedOr64, new, act, exp, - new = act | arg, arg, act) -#endif -#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS -#pragma intrinsic(_InterlockedExchangeAdd64_acq) -#pragma intrinsic(_InterlockedIncrement64_acq) -#pragma intrinsic(_InterlockedDecrement64_rel) -#pragma intrinsic(_InterlockedCompareExchange64_acq) -#pragma intrinsic(_InterlockedCompareExchange64_rel) +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1 #endif -#define ETHR_ILCKD__(X) _Interlocked ## X ## 64 -#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS -#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq -#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel -#else -#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64 -#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64 +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i; +} + #endif -#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X -#define ETHR_ATMC_T__ ethr_native_atomic64_t -#define ETHR_AINT_T__ ethr_sint64_t +#if defined(ETHR_WIN_HAVE_INC) +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB 1 #else -#error "Unsupported integer size" +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB 1 #endif -typedef struct { - volatile ETHR_AINT_T__ value; -} ETHR_ATMC_T__; - -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) - -static ETHR_INLINE ETHR_AINT_T__ * -ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_mb)(ETHR_ATMC_T__ *var) { - return (ETHR_AINT_T__ *) &var->value; + return ETHR_ILCKD__(Increment)(&var->value); } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ -#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ - var->value = i; -#else - (void) ETHR_ILCKD__(Exchange)(&var->value, i); #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ -#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ - var->value = i; +#if defined(ETHR_WIN_HAVE_INC_ACQ) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1 #else - (void) ETHR_ILCKD__(Exchange)(&var->value, i); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB 1 #endif -} static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) { -#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ - return var->value; -#else - return ETHR_ILCKD__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0); -#endif + return ETHR_ILCKD_ACQ__(Increment)(&var->value); } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) -{ - (void) ETHR_ILCKD__(ExchangeAdd)(&var->value, incr); -} +#endif + +#if defined(ETHR_WIN_HAVE_DEC) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +ETHR_NATMC_FUNC__(dec_return_mb)(ETHR_ATMC_T__ *var) { - return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i; + return ETHR_ILCKD__(Decrement)(&var->value); } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) -{ - (void) ETHR_ILCKD__(Increment)(&var->value); -} +#endif -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) -{ - (void) ETHR_ILCKD__(Decrement)(&var->value); -} +#if defined(ETHR_WIN_HAVE_DEC_REL) -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) -{ - return ETHR_ILCKD__(Increment)(&var->value); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) { - return ETHR_ILCKD__(Decrement)(&var->value); + return ETHR_ILCKD_REL__(Decrement)(&var->value); } +#endif + +#if defined(ETHR_WIN_HAVE_AND) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) +ETHR_NATMC_FUNC__(and_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { return ETHR_ILCKD__(And)(&var->value, mask); } +#endif + +#if defined(ETHR_WIN_HAVE_OR) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) +ETHR_NATMC_FUNC__(or_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { return ETHR_ILCKD__(Or)(&var->value, mask); } -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, - ETHR_AINT_T__ new, - ETHR_AINT_T__ old) -{ - return ETHR_ILCKD__(CompareExchange)(&var->value, new, old); -} +#endif +#if defined(ETHR_WIN_HAVE_XCHG) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) +ETHR_NATMC_FUNC__(xchg_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) { return ETHR_ILCKD__(Exchange)(&var->value, new); } -/* - * Atomic ops with at least specified barriers. - */ +#endif -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) -{ -#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ - ETHR_AINT_T__ val = var->value; - ETHR_COMPILER_BARRIER; - return val; +#if defined(ETHR_WIN_HAVE_CMPXCHG) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1 #else - return ETHR_ILCKD_ACQ__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0); +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1 #endif -} static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) +ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { - return ETHR_ILCKD_ACQ__(Increment)(&var->value); + return ETHR_ILCKD__(CompareExchange)(&var->value, new, old); } -static ETHR_INLINE void -ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) -{ -#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ - ETHR_COMPILER_BARRIER; - var->value = i; -#else - (void) ETHR_ILCKD_REL__(Exchange)(&var->value, i); #endif -} -static ETHR_INLINE void -ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) -{ - (void) ETHR_ILCKD_REL__(Decrement)(&var->value); -} +#if defined(ETHR_WIN_HAVE_CMPXCHG_ACQ) -static ETHR_INLINE ETHR_AINT_T__ -ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) -{ - return ETHR_ILCKD_REL__(Decrement)(&var->value); -} +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1 +#endif static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, @@ -391,6 +435,16 @@ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, return ETHR_ILCKD_ACQ__(CompareExchange)(&var->value, new, old); } +#endif + +#if defined(ETHR_WIN_HAVE_CMPXCHG_REL) + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB 1 +#endif + static ETHR_INLINE ETHR_AINT_T__ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, @@ -399,6 +453,8 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, return ETHR_ILCKD_REL__(CompareExchange)(&var->value, new, old); } +#endif + #endif /* ETHR_TRY_INLINE_FUNCS */ #undef ETHR_ILCKD__ @@ -408,8 +464,17 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, #undef ETHR_ATMC_T__ #undef ETHR_AINT_T__ #undef ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ -#undef ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ - -#endif /* _MSC_VER */ +#undef ETHR_WIN_HAVE_CMPXCHG +#undef ETHR_WIN_HAVE_DEC +#undef ETHR_WIN_HAVE_INC +#undef ETHR_WIN_HAVE_XCHG_ADD +#undef ETHR_WIN_HAVE_XCHG +#undef ETHR_WIN_HAVE_AND +#undef ETHR_WIN_HAVE_OR +#undef ETHR_WIN_HAVE_XCHG_ADD_ACQ +#undef ETHR_WIN_HAVE_INC_ACQ +#undef ETHR_WIN_HAVE_DEC_REL +#undef ETHR_WIN_HAVE_CMPXCHG_ACQ +#undef ETHR_WIN_HAVE_CMPXCHG_REL #endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */ diff --git a/erts/include/internal/win/ethr_dw_atomic.h b/erts/include/internal/win/ethr_dw_atomic.h new file mode 100644 index 0000000000..a3e7ffc3aa --- /dev/null +++ b/erts/include/internal/win/ethr_dw_atomic.h @@ -0,0 +1,154 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native double word atomics for windows + * Author: Rickard Green + */ + +#undef ETHR_INCLUDE_DW_ATOMIC_IMPL__ +#ifndef ETHR_X86_DW_ATOMIC_H__ +# define ETHR_X86_DW_ATOMIC_H__ +# if ((ETHR_SIZEOF_PTR == 4 \ + && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64)) \ + || (ETHR_SIZEOF_PTR == 8 \ + && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128))) +# define ETHR_INCLUDE_DW_ATOMIC_IMPL__ +# endif +#endif + +#ifdef ETHR_INCLUDE_DW_ATOMIC_IMPL__ + +# if ETHR_SIZEOF_PTR == 4 +# define ETHR_HAVE_NATIVE_SU_DW_ATOMIC +# else +# define ETHR_HAVE_NATIVE_DW_ATOMIC +# endif +# define ETHR_NATIVE_DW_ATOMIC_IMPL "windows-interlocked" + +# if defined(_M_IX86) || defined(_M_AMD64) +/* + * If ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ is defined, it will be used + * at runtime in order to determine if native or fallback implementation + * should be used. + */ +# define ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__ \ + ETHR_X86_RUNTIME_CONF_HAVE_DW_CMPXCHG__ +# endif + +# include <intrin.h> +# if ETHR_SIZEOF_PTR == 4 +# pragma intrinsic(_InterlockedCompareExchange64) +# define ETHR_DW_NATMC_ALIGN_MASK__ 0x7 +# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t +# else +# pragma intrinsic(_InterlockedCompareExchange128) +# define ETHR_DW_NATMC_ALIGN_MASK__ 0xf +# endif + +typedef volatile __int64 * ethr_native_dw_ptr_t; + +/* + * We need 16 byte aligned memory in 64-bit mode, and 8 byte aligned + * memory in 32-bit mode. 16 byte aligned malloc in 64-bit mode is + * not common, and at least some glibc malloc implementations + * only 4 byte align in 32-bit mode. + * + * This code assumes 8 byte aligned memory in 64-bit mode, and 4 byte + * aligned memory in 32-bit mode. A malloc implementation that does + * not adhere to these alignment requirements is seriously broken, + * and we wont bother trying to work around it. + * + * Since memory alignment may be off by one word we need to align at + * runtime. We, therefore, need an extra word allocated. + */ +#define ETHR_DW_NATMC_MEM__(VAR) \ + (&var->c[(int) ((ethr_uint_t) &(VAR)->c[0]) & ETHR_DW_NATMC_ALIGN_MASK__]) +typedef union { +#ifdef ETHR_NATIVE_SU_DW_SINT_T + volatile ETHR_NATIVE_SU_DW_SINT_T dw_sint; +#endif + volatile ethr_sint_t sint[3]; + volatile char c[ETHR_SIZEOF_PTR*3]; +} ethr_native_dw_atomic_t; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +#ifdef ETHR_DEBUG +# define ETHR_DW_DBG_ALIGNED__(PTR) \ + ETHR_ASSERT((((ethr_uint_t) (PTR)) & ETHR_DW_NATMC_ALIGN_MASK__) == 0); +#else +# define ETHR_DW_DBG_ALIGNED__(PTR) +#endif + +#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR + +static ETHR_INLINE ethr_sint_t * +ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var) +{ + ethr_sint_t *p = (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var); + ETHR_DW_DBG_ALIGNED__(p); + return p; +} + + +#if ETHR_SIZEOF_PTR == 4 + +#define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB + +static ETHR_INLINE ethr_sint64_t +ethr_native_su_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var, + ethr_sint64_t new, + ethr_sint64_t exp) +{ + ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var); + ETHR_DW_DBG_ALIGNED__(p); + return (ethr_sint64_t) _InterlockedCompareExchange64(p, new, exp); +} + +#elif ETHR_SIZEOF_PTR == 8 + +#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB + +#ifdef ETHR_BIGENDIAN +# define ETHR_WIN_LOW_WORD__ 1 +# define ETHR_WIN_HIGH_WORD__ 0 +#else +# define ETHR_WIN_LOW_WORD__ 0 +# define ETHR_WIN_HIGH_WORD__ 1 +#endif + +static ETHR_INLINE int +ethr_native_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var, + ethr_sint_t *new, + ethr_sint_t *xchg) +{ + ethr_native_dw_ptr_t p = (ethr_native_dw_ptr_t) ETHR_DW_NATMC_MEM__(var); + ETHR_DW_DBG_ALIGNED__(p); + return (int) _InterlockedCompareExchange128(p, + new[ETHR_WIN_HIGH_WORD__], + new[ETHR_WIN_LOW_WORD__], + xchg); +} + +#endif + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#endif /* ETHR_INCLUDE_DW_ATOMIC_IMPL__ */ diff --git a/erts/include/internal/win/ethr_event.h b/erts/include/internal/win/ethr_event.h index 598816b2c6..6363174a74 100644 --- a/erts/include/internal/win/ethr_event.h +++ b/erts/include/internal/win/ethr_event.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * Copyright Ericsson AB 2009-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -21,12 +21,12 @@ * Author: Rickard Green */ -#define ETHR_EVENT_OFF_WAITER__ ((long) -1) -#define ETHR_EVENT_OFF__ ((long) 1) -#define ETHR_EVENT_ON__ ((long) 0) +#define ETHR_EVENT_OFF_WAITER__ ((ethr_sint32_t) -1) +#define ETHR_EVENT_OFF__ ((ethr_sint32_t) 1) +#define ETHR_EVENT_ON__ ((ethr_sint32_t) 0) typedef struct { - volatile long state; + ethr_atomic32_t state; HANDLE handle; } ethr_event; @@ -38,7 +38,7 @@ static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) { /* _InterlockedExchange() imply a full memory barrier which is important */ - long state = _InterlockedExchange(&e->state, ETHR_EVENT_ON__); + ethr_sint32_t state = ethr_atomic32_xchg_wb(&e->state, ETHR_EVENT_ON__); if (state == ETHR_EVENT_OFF_WAITER__) { if (!SetEvent(e->handle)) ETHR_FATAL_ERROR__(ethr_win_get_errno__()); @@ -48,8 +48,8 @@ ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) { - /* _InterlockedExchange() imply a full memory barrier which is important */ - InterlockedExchange(&e->state, ETHR_EVENT_OFF__); + ethr_atomic32_set(&e->state, ETHR_EVENT_OFF__); + ETHR_MEMORY_BARRIER; } #endif diff --git a/erts/include/internal/win/ethr_membar.h b/erts/include/internal/win/ethr_membar.h new file mode 100644 index 0000000000..8237660b2c --- /dev/null +++ b/erts/include/internal/win/ethr_membar.h @@ -0,0 +1,145 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Memory barriers for Windows + * Author: Rickard Green + */ + +#if (!defined(ETHR_WIN_MEMBAR_H__) \ + && (defined(_MSC_VER) && _MSC_VER >= 1400) \ + && (defined(_M_AMD64) \ + || defined(_M_IA64) \ + || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE))) +#define ETHR_WIN_MEMBAR_H__ + +#define ETHR_LoadLoad (1 << 0) +#define ETHR_LoadStore (1 << 1) +#define ETHR_StoreLoad (1 << 2) +#define ETHR_StoreStore (1 << 3) + +#include <intrin.h> +#undef ETHR_COMPILER_BARRIER +#define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +#pragma intrinsic(_ReadWriteBarrier) + +#pragma intrinsic(_InterlockedCompareExchange) + +#define ETHR_MB_USING_INTERLOCKED__ \ +do { \ + volatile long x___ = 0; \ + (void) _InterlockedCompareExchange(&x___, 2, 1); \ +} while (0) + +#if defined(_M_IA64) + +#define ETHR_MEMBAR(B) __mf() + +#elif defined(_M_AMD64) || defined(_M_IX86) + +#include <emmintrin.h> +#include <mmintrin.h> + +#if ETHR_SIZEOF_PTR == 4 +# define ETHR_NO_SSE2_MB__ ETHR_MB_USING_INTERLOCKED__ +#endif +#pragma intrinsic(_mm_mfence) +#pragma intrinsic(_mm_sfence) +#pragma intrinsic(_mm_lfence) + +static __forceinline void +ethr_cfence__(void) +{ + _ReadWriteBarrier(); +} + +static __forceinline void +ethr_mfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MB__; + else +#endif + _mm_mfence(); +} + +static __forceinline void +ethr_sfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MB__; + else +#endif + _mm_sfence(); +} + +static __forceinline void +ethr_lfence__(void) +{ +#if ETHR_SIZEOF_PTR == 4 + if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__) + ETHR_NO_SSE2_MB__; + else +#endif + _mm_lfence(); +} + +#define ETHR_X86_OUT_OF_ORDER_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, \ + ethr_sfence__(), \ + ETHR_CHOOSE_EXPR((B) == ETHR_LoadLoad, \ + ethr_lfence__(), \ + ethr_mfence__())) + +#ifdef ETHR_X86_OUT_OF_ORDER + +#define ETHR_MEMBAR(B) \ + ETHR_X86_OUT_OF_ORDER_MEMBAR((B)) + +#else /* !ETHR_X86_OUT_OF_ORDER (the default) */ + +/* + * We assume that only stores before loads may be reordered. That is, + * we assume that *no* instructions like these are used: + * - CLFLUSH, + * - streaming stores executed with non-temporal move, + * - string operations, or + * - other instructions which aren't LoadLoad, LoadStore, and StoreStore + * ordered by themselves + * If such instructions are used, either insert memory barriers + * using ETHR_X86_OUT_OF_ORDER_MEMBAR() at appropriate places, or + * define ETHR_X86_OUT_OF_ORDER. For more info see Intel 64 and IA-32 + * Architectures Software Developer's Manual; Vol 3A; Chapter 8.2.2. + */ + +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) & ETHR_StoreLoad, ethr_mfence__(), ethr_cfence__()) + +#endif + +#else /* No knowledge about platform; use interlocked fallback */ + +#define ETHR_MEMBAR(B) ETHR_MB_USING_INTERLOCKED__ +#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MB_USING_INTERLOCKED__ + +#endif + +#endif /* ETHR_WIN_MEMBAR_H__ */ diff --git a/erts/include/internal/win/ethread.h b/erts/include/internal/win/ethread.h index c01b17cf14..8be35e810e 100644 --- a/erts/include/internal/win/ethread.h +++ b/erts/include/internal/win/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010. All Rights Reserved. + * Copyright Ericsson AB 2010-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,11 +25,13 @@ #ifndef ETHREAD_WIN_H__ #define ETHREAD_WIN_H__ +#include "ethr_membar.h" #define ETHR_ATOMIC_WANT_32BIT_IMPL__ #include "ethr_atomic.h" #if ETHR_SIZEOF_PTR == 8 # define ETHR_ATOMIC_WANT_64BIT_IMPL__ # include "ethr_atomic.h" #endif +#include "ethr_dw_atomic.h" #endif |