Improve ethread atomics

The ethread atomics API now also provide double word size atomics.
Double word size atomics are implemented using native atomic
instructions on x86 (when the cmpxchg8b instruction is available)
and on x86_64 (when the cmpxchg16b instruction is available). On
other hardware where 32-bit atomics or word size atomics are
available, an optimized fallback is used; otherwise, a spinlock,
or a mutex based fallback is used.

The ethread library now performs runtime tests for presence of
hardware features, such as for example SSE2 instructions, instead
of requiring this to be determined at compile time.

There are now functions implementing each atomic operation with the
following implied memory barrier semantics: none, read, write,
acquire, release, and full. Some of the operation-barrier
combinations aren't especially useful. But instead of filtering
useful ones out, and potentially miss a useful one, we implement
them all.

A much smaller set of functionality for native atomics are required
to be implemented than before. More or less only cmpxchg and a
membar macro are required to be implemented for each atomic size.
Other functions will automatically be constructed from these. It is,
of course, often wise to implement more that this if possible from a
performance perspective.

1 files changed, 330 insertions, 265 deletions

diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h
index 60def01a7e..e11f1abf47 100644
--- a/erts/include/internal/win/ethr_atomic.h
+++ b/erts/include/internal/win/ethr_atomic.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2010. All Rights Reserved.
+ * Copyright Ericsson AB 2010-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -24,364 +24,408 @@
 
 #undef ETHR_INCLUDE_ATOMIC_IMPL__
 #if !defined(ETHR_WIN_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
-#define ETHR_WIN_ATOMIC32_H__
-#define ETHR_INCLUDE_ATOMIC_IMPL__ 4
-#undef ETHR_ATOMIC_WANT_32BIT_IMPL__
+#  define ETHR_WIN_ATOMIC32_H__
+#  if (defined(ETHR_MEMBAR) \
+       && defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE) \
+       && defined(ETHR_HAVE__INTERLOCKEDEXCHANGE))
+#    define ETHR_INCLUDE_ATOMIC_IMPL__ 4
+#  endif
+#  undef ETHR_ATOMIC_WANT_32BIT_IMPL__
 #elif !defined(ETHR_WIN_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
-#define ETHR_WIN_ATOMIC64_H__
-#ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64
-/* _InterlockedCompareExchange64() required... */
-#define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+#  define ETHR_WIN_ATOMIC64_H__
+#  if (defined(ETHR_MEMBAR) \
+     && (defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64) \
+	 || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ) \
+	 || defined(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL)))
+#    define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+#  endif
+#  undef ETHR_ATOMIC_WANT_64BIT_IMPL__
 #endif
-#undef ETHR_ATOMIC_WANT_64BIT_IMPL__
+
+#if !defined(_MSC_VER) || _MSC_VER < 1400
+#  undef ETHR_INCLUDE_ATOMIC_IMPL__
 #endif
 
 #ifdef ETHR_INCLUDE_ATOMIC_IMPL__
 
-#if defined(_MSC_VER) && _MSC_VER >= 1400
+#  ifndef ETHR_WIN_ATOMIC_COMMON__
+#    define ETHR_WIN_ATOMIC_COMMON__
+
+#    if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64)
+#      define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1
+#    else
+#      define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0
+#    endif
+
+#  endif /* ETHR_WIN_ATOMIC_COMMON__ */
+
+#  if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+
+#    define ETHR_HAVE_NATIVE_ATOMIC32 1
+#    define ETHR_NATIVE_ATOMIC32_IMPL "windows-interlocked"
+
+#    ifdef ETHR_HAVE__INTERLOCKEDDECREMENT
+#      define ETHR_WIN_HAVE_DEC
+#      pragma intrinsic(_InterlockedDecrement)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDDECREMENT_REL
+#      define ETHR_WIN_HAVE_DEC_REL
+#      pragma intrinsic(_InterlockedDecrement_rel)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDINCREMENT
+#      define ETHR_WIN_HAVE_INC
+#      pragma intrinsic(_InterlockedIncrement)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ
+#      define ETHR_WIN_HAVE_INC_ACQ
+#      pragma intrinsic(_InterlockedIncrement_acq)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD
+#      define ETHR_WIN_HAVE_XCHG_ADD
+#      pragma intrinsic(_InterlockedExchangeAdd)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ
+#      define ETHR_WIN_HAVE_XCHG_ADD_ACQ
+#      pragma intrinsic(_InterlockedExchangeAdd_acq)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE
+#      define ETHR_WIN_HAVE_XCHG
+#      pragma intrinsic(_InterlockedExchange)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDAND
+#      define ETHR_WIN_HAVE_AND
+#      pragma intrinsic(_InterlockedAnd)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDOR
+#      define ETHR_WIN_HAVE_OR
+#      pragma intrinsic(_InterlockedOr)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE
+#      define ETHR_WIN_HAVE_CMPXCHG
+#        pragma intrinsic(_InterlockedCompareExchange)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ
+#      define ETHR_WIN_HAVE_CMPXCHG_ACQ
+#      pragma intrinsic(_InterlockedCompareExchange_acq)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL
+#      define ETHR_WIN_HAVE_CMPXCHG_REL
+#      pragma intrinsic(_InterlockedCompareExchange_rel)
+#    endif
+
+#    define ETHR_ILCKD__(X) _Interlocked ## X
+#    define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq
+#    define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel
+
+#    define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#    define ETHR_ATMC_T__ ethr_native_atomic32_t
+#    define ETHR_AINT_T__ ethr_sint32_t
+
+#  elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
+
+#    define ETHR_HAVE_NATIVE_ATOMIC64 1
+#    define ETHR_NATIVE_ATOMIC64_IMPL "windows-interlocked"
+
+#    ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64
+#      define ETHR_WIN_HAVE_DEC
+#      pragma intrinsic(_InterlockedDecrement64)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64_REL
+#      define ETHR_WIN_HAVE_DEC_REL
+#      pragma intrinsic(_InterlockedDecrement64_rel)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ
+#      define ETHR_WIN_HAVE_INC_ACQ
+#      pragma intrinsic(_InterlockedIncrement64_acq)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64
+#      define ETHR_WIN_HAVE_INC
+#      pragma intrinsic(_InterlockedIncrement64)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64
+#      define ETHR_WIN_HAVE_XCHG_ADD
+#      pragma intrinsic(_InterlockedExchangeAdd64)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ
+#      define ETHR_WIN_HAVE_XCHG_ADD_ACQ
+#      pragma intrinsic(_InterlockedExchangeAdd64_acq)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64
+#      define ETHR_WIN_HAVE_XCHG
+#      pragma intrinsic(_InterlockedExchange64)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDAND64
+#      define ETHR_WIN_HAVE_AND
+#      pragma intrinsic(_InterlockedAnd64)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDOR64
+#      define ETHR_WIN_HAVE_OR
+#      pragma intrinsic(_InterlockedOr64)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64
+#      define ETHR_WIN_HAVE_CMPXCHG
+#      pragma intrinsic(_InterlockedCompareExchange64)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ
+#      define ETHR_WIN_HAVE_CMPXCHG_ACQ
+#      pragma intrinsic(_InterlockedCompareExchange64_acq)
+#    endif
+#    ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL
+#      define ETHR_WIN_HAVE_CMPXCHG_REL
+#      pragma intrinsic(_InterlockedCompareExchange64_rel)
+#    endif
+
+#    define ETHR_ILCKD__(X) _Interlocked ## X ## 64
+#    define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq
+#    define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel
+
+#    define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#    define ETHR_ATMC_T__ ethr_native_atomic64_t
+#    define ETHR_AINT_T__ ethr_sint64_t
+
+#  else
+#    error "Unsupported integer size"
+#  endif
 
-#ifndef ETHR_WIN_ATOMIC_COMMON__
-#define ETHR_WIN_ATOMIC_COMMON__
+typedef struct {
+    volatile ETHR_AINT_T__ value;
+} ETHR_ATMC_T__;
 
-#define ETHR_HAVE_NATIVE_ATOMICS 1
+#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
 
-#if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64)
-#  define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
 #else
-#  define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR 1
 #endif
 
-#if defined(_M_AMD64) || (defined(_M_IX86) \
-			  && !defined(ETHR_PRE_PENTIUM4_COMPAT))
-#  define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1
-#else
-#  define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0
-#endif
-/*
- * No configure test checking for interlocked acquire/release
- * versions have been written, yet. It should define
- * ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS if, and
- * only if, all used interlocked operations with barriers
- * exists.
- *
- * Note, that these are pure optimizations for the itanium
- * processor.
- */
+static ETHR_INLINE ETHR_AINT_T__ *
+ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__ *) &var->value;
+}
 
-#include <intrin.h>
-#undef ETHR_COMPILER_BARRIER
-#define ETHR_COMPILER_BARRIER _ReadWriteBarrier()
-#pragma intrinsic(_ReadWriteBarrier)
-#pragma intrinsic(_InterlockedCompareExchange)
-
-#if defined(_M_AMD64) || (defined(_M_IX86) \
-			  && !defined(ETHR_PRE_PENTIUM4_COMPAT))
-#include <emmintrin.h>
-#include <mmintrin.h>
-#pragma intrinsic(_mm_mfence)
-#define ETHR_MEMORY_BARRIER _mm_mfence()
-#pragma intrinsic(_mm_sfence)
-#define ETHR_WRITE_MEMORY_BARRIER _mm_sfence()
-#pragma intrinsic(_mm_lfence)
-#define ETHR_READ_MEMORY_BARRIER _mm_lfence()
-#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER
+#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
 
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
 #else
-
-#define ETHR_MEMORY_BARRIER					\
-do {								\
-    volatile long x___ = 0;					\
-    _InterlockedCompareExchange(&x___, (long) 1, (long) 0);	\
-} while (0)
-
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET 1
 #endif
 
-#endif /* ETHR_WIN_ATOMIC_COMMON__ */
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+    var->value = i;
+}
 
 #if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB 1
+#endif
 
-#define ETHR_HAVE_NATIVE_ATOMIC32 1
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+#if defined(_M_IX86)
+    if (ETHR_X86_RUNTIME_CONF_HAVE_NO_SSE2__)
+	(void) ETHR_ILCKD__(Exchange)(&var->value, i);
+    else
+#endif /* _M_IX86 */
+    {
+	ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
+	var->value = i;
+    }
+}
 
-/*
- * All used operations available as 32-bit intrinsics
- */
+#endif /* ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ */
 
-#pragma intrinsic(_InterlockedDecrement)
-#pragma intrinsic(_InterlockedIncrement)
-#pragma intrinsic(_InterlockedExchangeAdd)
-#pragma intrinsic(_InterlockedExchange)
-#pragma intrinsic(_InterlockedAnd)
-#pragma intrinsic(_InterlockedOr)
-#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
-#pragma intrinsic(_InterlockedExchangeAdd_acq)
-#pragma intrinsic(_InterlockedIncrement_acq)
-#pragma intrinsic(_InterlockedDecrement_rel)
-#pragma intrinsic(_InterlockedCompareExchange_acq)
-#pragma intrinsic(_InterlockedCompareExchange_rel)
-#endif
+#if defined(ETHR_WIN_HAVE_XCHG)
 
-#define ETHR_ILCKD__(X) _Interlocked ## X
-#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
-#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq
-#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB 1
 #else
-#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X
-#define ETHR_ILCKD_REL__(X) _Interlocked ## X
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB 1
 #endif
 
-#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
-#define ETHR_ATMC_T__ ethr_native_atomic32_t
-#define ETHR_AINT_T__ ethr_sint32_t
-
-#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
+static ETHR_INLINE void
+ETHR_NATMC_FUNC__(set_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+    (void) ETHR_ILCKD__(Exchange)(&var->value, i);
+}
 
-#define ETHR_HAVE_NATIVE_ATOMIC64 1
+#endif
 
-/*
- * _InterlockedCompareExchange64() is required. The other may not
- * be available, but if so, we can generate them.
- */
-#pragma intrinsic(_InterlockedCompareExchange64)
+#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
 
-#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__		
-#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) *(PTR)
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
 #else
-#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) (__int64) 0
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ 1
 #endif
 
-#define ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, PTR, NEW, ACT, EXP, OPS, RET)	\
-{									\
-    __int64 NEW, ACT, EXP;						\
-    ACT = ETHR_OWN_ILCKD_INIT_VAL__(PTR);				\
-    do {								\
-	EXP = ACT;							\
-	{ OPS; }							\
-	ACT = _InterlockedCompareExchange64(PTR, NEW, EXP);		\
-    } while (ACT != EXP);						\
-    return RET;								\
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
+{
+    return var->value;
 }
 
-#define ETHR_OWN_ILCKD_1_IMPL__(FUNC, NEW, ACT, EXP, OPS, RET)		\
-static __forceinline __int64						\
-FUNC(__int64 volatile *ptr)						\
-ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET)
-
-#define ETHR_OWN_ILCKD_2_IMPL__(FUNC, NEW, ACT, EXP, OPS, ARG, RET)	\
-static __forceinline __int64						\
-FUNC(__int64 volatile *ptr, __int64 ARG)				\
-ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET)
+#endif /* ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ */
 
+#if defined(ETHR_WIN_HAVE_XCHG_ADD)
 
-#ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64
-#pragma intrinsic(_InterlockedDecrement64)
-#else
-ETHR_OWN_ILCKD_1_IMPL__(_InterlockedDecrement64, new, act, exp,
-			new = act - 1, new)
-#endif
-#ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64
-#pragma intrinsic(_InterlockedIncrement64)
-#else
-ETHR_OWN_ILCKD_1_IMPL__(_InterlockedIncrement64, new, act, exp,
-			new = act + 1, new)
-#endif
-#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64
-#pragma intrinsic(_InterlockedExchangeAdd64)
-#else
-ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchangeAdd64, new, act, exp,
-			new = act + arg, arg, act)
-#endif
-#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64
-#pragma intrinsic(_InterlockedExchange64)
-#else
-ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchange64, new, act, exp,
-			new = arg, arg, act)
-#endif
-#ifdef ETHR_HAVE__INTERLOCKEDAND64
-#pragma intrinsic(_InterlockedAnd64)
-#else
-ETHR_OWN_ILCKD_2_IMPL__(_InterlockedAnd64, new, act, exp,
-			new = act & arg, arg, act)
-#endif
-#ifdef ETHR_HAVE__INTERLOCKEDOR64
-#pragma intrinsic(_InterlockedOr64)
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1
 #else
-ETHR_OWN_ILCKD_2_IMPL__(_InterlockedOr64, new, act, exp,
-			new = act | arg, arg, act)
-#endif
-#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
-#pragma intrinsic(_InterlockedExchangeAdd64_acq)
-#pragma intrinsic(_InterlockedIncrement64_acq)
-#pragma intrinsic(_InterlockedDecrement64_rel)
-#pragma intrinsic(_InterlockedCompareExchange64_acq)
-#pragma intrinsic(_InterlockedCompareExchange64_rel)
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1
 #endif
 
-#define ETHR_ILCKD__(X) _Interlocked ## X ## 64
-#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
-#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq
-#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel
-#else
-#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64
-#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+{
+    return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i;
+}
+
 #endif
 
-#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
-#define ETHR_ATMC_T__ ethr_native_atomic64_t
-#define ETHR_AINT_T__ ethr_sint64_t
+#if defined(ETHR_WIN_HAVE_INC)
 
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB 1
 #else
-#error "Unsupported integer size"
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB 1
 #endif
 
-typedef struct {
-    volatile ETHR_AINT_T__ value;
-} ETHR_ATMC_T__;
-
-#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
-
-static ETHR_INLINE ETHR_AINT_T__ *
-ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
+static ETHR_INLINE ETHR_AINT_T__
+ETHR_NATMC_FUNC__(inc_return_mb)(ETHR_ATMC_T__ *var)
 {
-    return (ETHR_AINT_T__ *) &var->value;
+    return ETHR_ILCKD__(Increment)(&var->value);
 }
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
-{
-#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
-    var->value = i;
-#else
-    (void) ETHR_ILCKD__(Exchange)(&var->value, i);
 #endif
-}
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
-{
-#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
-    var->value = i;
+#if defined(ETHR_WIN_HAVE_INC_ACQ)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1
 #else
-    (void) ETHR_ILCKD__(Exchange)(&var->value, i);
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB 1
 #endif
-}
 
 static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
+ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
 {
-#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
-    return var->value;
-#else
-    return ETHR_ILCKD__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0);
-#endif
+    return ETHR_ILCKD_ACQ__(Increment)(&var->value);
 }
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
-{
-    (void) ETHR_ILCKD__(ExchangeAdd)(&var->value, incr);
-}
+#endif
+
+#if defined(ETHR_WIN_HAVE_DEC)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB 1
+#endif
 
 static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
+ETHR_NATMC_FUNC__(dec_return_mb)(ETHR_ATMC_T__ *var)
 {
-    return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i;
+    return ETHR_ILCKD__(Decrement)(&var->value);
 }
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
-{
-    (void) ETHR_ILCKD__(Increment)(&var->value);
-}
+#endif
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var)
-{
-    (void) ETHR_ILCKD__(Decrement)(&var->value);
-}
+#if defined(ETHR_WIN_HAVE_DEC_REL)
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
-{
-    return ETHR_ILCKD__(Increment)(&var->value);
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB 1
+#endif
 
 static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
+ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
 {
-    return ETHR_ILCKD__(Decrement)(&var->value);
+    return ETHR_ILCKD_REL__(Decrement)(&var->value);
 }
 
+#endif
+
+#if defined(ETHR_WIN_HAVE_AND)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB 1
+#endif
+
 static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+ETHR_NATMC_FUNC__(and_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
 {
     return ETHR_ILCKD__(And)(&var->value, mask);
 }
 
+#endif
+
+#if defined(ETHR_WIN_HAVE_OR)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB 1
+#endif
+
 static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
+ETHR_NATMC_FUNC__(or_retold_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
 {
     return ETHR_ILCKD__(Or)(&var->value, mask);
 }
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
-			   ETHR_AINT_T__ new,
-			   ETHR_AINT_T__ old)
-{
-    return ETHR_ILCKD__(CompareExchange)(&var->value, new, old);
-}
+#endif
 
+#if defined(ETHR_WIN_HAVE_XCHG)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB 1
+#endif
 
 static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
+ETHR_NATMC_FUNC__(xchg_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
 {
     return ETHR_ILCKD__(Exchange)(&var->value, new);
 }
 
-/*
- * Atomic ops with at least specified barriers.
- */
+#endif
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
-{
-#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__
-    ETHR_AINT_T__ val = var->value;
-    ETHR_COMPILER_BARRIER;
-    return val;
+#if defined(ETHR_WIN_HAVE_CMPXCHG)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1
 #else
-    return ETHR_ILCKD_ACQ__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0);
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1
 #endif
-}
 
 static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
+ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var,
+			      ETHR_AINT_T__ new,
+			      ETHR_AINT_T__ old)
 {
-    return ETHR_ILCKD_ACQ__(Increment)(&var->value);
+    return ETHR_ILCKD__(CompareExchange)(&var->value, new, old);
 }
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
-{
-#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__
-    ETHR_COMPILER_BARRIER;
-    var->value = i;
-#else
-    (void) ETHR_ILCKD_REL__(Exchange)(&var->value, i);
 #endif
-}
 
-static ETHR_INLINE void
-ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
-{
-    (void) ETHR_ILCKD_REL__(Decrement)(&var->value);
-}
+#if defined(ETHR_WIN_HAVE_CMPXCHG_ACQ)
 
-static ETHR_INLINE ETHR_AINT_T__
-ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
-{
-    return ETHR_ILCKD_REL__(Decrement)(&var->value);
-}
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB 1
+#endif
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
@@ -391,6 +435,16 @@ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
     return ETHR_ILCKD_ACQ__(CompareExchange)(&var->value, new, old);
 }
 
+#endif
+
+#if defined(ETHR_WIN_HAVE_CMPXCHG_REL)
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1
+#else
+#  define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB 1
+#endif
+
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
 				ETHR_AINT_T__ new,
@@ -399,6 +453,8 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
     return ETHR_ILCKD_REL__(CompareExchange)(&var->value, new, old);
 }
 
+#endif
+
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
 #undef ETHR_ILCKD__
@@ -408,8 +464,17 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
 #undef ETHR_ATMC_T__
 #undef ETHR_AINT_T__
 #undef ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
-#undef ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__
-
-#endif /* _MSC_VER */
+#undef ETHR_WIN_HAVE_CMPXCHG
+#undef ETHR_WIN_HAVE_DEC
+#undef ETHR_WIN_HAVE_INC
+#undef ETHR_WIN_HAVE_XCHG_ADD
+#undef ETHR_WIN_HAVE_XCHG
+#undef ETHR_WIN_HAVE_AND
+#undef ETHR_WIN_HAVE_OR
+#undef ETHR_WIN_HAVE_XCHG_ADD_ACQ
+#undef ETHR_WIN_HAVE_INC_ACQ
+#undef ETHR_WIN_HAVE_DEC_REL
+#undef ETHR_WIN_HAVE_CMPXCHG_ACQ
+#undef ETHR_WIN_HAVE_CMPXCHG_REL
 
 #endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */