Add support for 32-bit atomics

3 files changed, 161 insertions, 50 deletions

diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h
index 7ae5bb6cd3..60def01a7e 100644
--- a/erts/include/internal/win/ethr_atomic.h
+++ b/erts/include/internal/win/ethr_atomic.h
@@ -22,10 +22,27 @@
  * Author: Rickard Green
  */
 
-#ifndef ETHR_WIN_ATOMIC_H__
-#define ETHR_WIN_ATOMIC_H__
+#undef ETHR_INCLUDE_ATOMIC_IMPL__
+#if !defined(ETHR_WIN_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__)
+#define ETHR_WIN_ATOMIC32_H__
+#define ETHR_INCLUDE_ATOMIC_IMPL__ 4
+#undef ETHR_ATOMIC_WANT_32BIT_IMPL__
+#elif !defined(ETHR_WIN_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__)
+#define ETHR_WIN_ATOMIC64_H__
+#ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64
+/* _InterlockedCompareExchange64() required... */
+#define ETHR_INCLUDE_ATOMIC_IMPL__ 8
+#endif
+#undef ETHR_ATOMIC_WANT_64BIT_IMPL__
+#endif
+
+#ifdef ETHR_INCLUDE_ATOMIC_IMPL__
 
 #if defined(_MSC_VER) && _MSC_VER >= 1400
+
+#ifndef ETHR_WIN_ATOMIC_COMMON__
+#define ETHR_WIN_ATOMIC_COMMON__
+
 #define ETHR_HAVE_NATIVE_ATOMICS 1
 
 #if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64)
@@ -79,7 +96,15 @@ do {								\
 
 #endif
 
-#if ETHR_SIZEOF_PTR == 4
+#endif /* ETHR_WIN_ATOMIC_COMMON__ */
+
+#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4
+
+#define ETHR_HAVE_NATIVE_ATOMIC32 1
+
+/*
+ * All used operations available as 32-bit intrinsics
+ */
 
 #pragma intrinsic(_InterlockedDecrement)
 #pragma intrinsic(_InterlockedIncrement)
@@ -95,23 +120,94 @@ do {								\
 #pragma intrinsic(_InterlockedCompareExchange_rel)
 #endif
 
-#define ETHR_ATMC__(X) X
+#define ETHR_ILCKD__(X) _Interlocked ## X
 #ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
-#define ETHR_ATMC_ACQ__(X) X ## _acq
-#define ETHR_ATMC_REL__(X) X ## _rel
+#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq
+#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel
+#else
+#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X
+#define ETHR_ILCKD_REL__(X) _Interlocked ## X
+#endif
+
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic32_t
+#define ETHR_AINT_T__ ethr_sint32_t
+
+#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8
+
+#define ETHR_HAVE_NATIVE_ATOMIC64 1
+
+/*
+ * _InterlockedCompareExchange64() is required. The other may not
+ * be available, but if so, we can generate them.
+ */
+#pragma intrinsic(_InterlockedCompareExchange64)
+
+#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__		
+#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) *(PTR)
 #else
-#define ETHR_ATMC_ACQ__(X) X
-#define ETHR_ATMC_REL__(X) X
+#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) (__int64) 0
 #endif
 
-#elif ETHR_SIZEOF_PTR == 8
+#define ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, PTR, NEW, ACT, EXP, OPS, RET)	\
+{									\
+    __int64 NEW, ACT, EXP;						\
+    ACT = ETHR_OWN_ILCKD_INIT_VAL__(PTR);				\
+    do {								\
+	EXP = ACT;							\
+	{ OPS; }							\
+	ACT = _InterlockedCompareExchange64(PTR, NEW, EXP);		\
+    } while (ACT != EXP);						\
+    return RET;								\
+}
+
+#define ETHR_OWN_ILCKD_1_IMPL__(FUNC, NEW, ACT, EXP, OPS, RET)		\
+static __forceinline __int64						\
+FUNC(__int64 volatile *ptr)						\
+ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET)
+
+#define ETHR_OWN_ILCKD_2_IMPL__(FUNC, NEW, ACT, EXP, OPS, ARG, RET)	\
+static __forceinline __int64						\
+FUNC(__int64 volatile *ptr, __int64 ARG)				\
+ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET)
 
+
+#ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64
 #pragma intrinsic(_InterlockedDecrement64)
+#else
+ETHR_OWN_ILCKD_1_IMPL__(_InterlockedDecrement64, new, act, exp,
+			new = act - 1, new)
+#endif
+#ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64
 #pragma intrinsic(_InterlockedIncrement64)
+#else
+ETHR_OWN_ILCKD_1_IMPL__(_InterlockedIncrement64, new, act, exp,
+			new = act + 1, new)
+#endif
+#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64
 #pragma intrinsic(_InterlockedExchangeAdd64)
+#else
+ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchangeAdd64, new, act, exp,
+			new = act + arg, arg, act)
+#endif
+#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64
 #pragma intrinsic(_InterlockedExchange64)
+#else
+ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchange64, new, act, exp,
+			new = arg, arg, act)
+#endif
+#ifdef ETHR_HAVE__INTERLOCKEDAND64
 #pragma intrinsic(_InterlockedAnd64)
+#else
+ETHR_OWN_ILCKD_2_IMPL__(_InterlockedAnd64, new, act, exp,
+			new = act & arg, arg, act)
+#endif
+#ifdef ETHR_HAVE__INTERLOCKEDOR64
 #pragma intrinsic(_InterlockedOr64)
+#else
+ETHR_OWN_ILCKD_2_IMPL__(_InterlockedOr64, new, act, exp,
+			new = act | arg, arg, act)
+#endif
 #ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
 #pragma intrinsic(_InterlockedExchangeAdd64_acq)
 #pragma intrinsic(_InterlockedIncrement64_acq)
@@ -120,36 +216,42 @@ do {								\
 #pragma intrinsic(_InterlockedCompareExchange64_rel)
 #endif
 
-#define ETHR_ATMC__(X) X ## 64
+#define ETHR_ILCKD__(X) _Interlocked ## X ## 64
 #ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS
-#define ETHR_ATMC_ACQ__(X) X ## 64_acq
-#define ETHR_ATMC_REL__(X) X ## 64_rel
+#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq
+#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel
 #else
-#define ETHR_ATMC_ACQ__(X) X ## 64
-#define ETHR_ATMC_REL__(X) X ## 64
+#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64
+#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64
 #endif
 
+#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X
+#define ETHR_ATMC_T__ ethr_native_atomic64_t
+#define ETHR_AINT_T__ ethr_sint64_t
+
 #else
-#error "Unsupported word size"
+#error "Unsupported integer size"
 #endif
 
-#define ETHR_NATMC_FUNC__(X) ethr_native_atomic_ ## X
-#define ETHR_ATMC_T__ ethr_native_atomic_t
-#define ETHR_AINT_T__ ethr_sint_t
-
 typedef struct {
     volatile ETHR_AINT_T__ value;
 } ETHR_ATMC_T__;
 
 #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
 
+static ETHR_INLINE ETHR_AINT_T__ *
+ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var)
+{
+    return (ETHR_AINT_T__ *) &var->value;
+}
+
 static ETHR_INLINE void
 ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
 {
 #if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
     var->value = i;
 #else
-    (void) ETHR_ATMC__(_InterlockedExchange)(&var->value, i);
+    (void) ETHR_ILCKD__(Exchange)(&var->value, i);
 #endif
 }
 
@@ -159,7 +261,7 @@ ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
 #if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
     var->value = i;
 #else
-    (void) ETHR_ATMC__(_InterlockedExchange)(&var->value, i);
+    (void) ETHR_ILCKD__(Exchange)(&var->value, i);
 #endif
 }
 
@@ -169,56 +271,56 @@ ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
 #if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
     return var->value;
 #else
-    return ETHR_ATMC__(_InterlockedExchangeAdd)(&var->value, (ETHR_AINT_T__) 0);
+    return ETHR_ILCKD__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0);
 #endif
 }
 
 static ETHR_INLINE void
 ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
 {
-    (void) ETHR_ATMC__(_InterlockedExchangeAdd)(&var->value, incr);
+    (void) ETHR_ILCKD__(ExchangeAdd)(&var->value, incr);
 }
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
 {
-    return ETHR_ATMC__(_InterlockedExchangeAdd)(&var->value, i) + i;
+    return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i;
 }
 
 static ETHR_INLINE void
 ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
 {
-    (void) ETHR_ATMC__(_InterlockedIncrement)(&var->value);
+    (void) ETHR_ILCKD__(Increment)(&var->value);
 }
 
 static ETHR_INLINE void
 ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var)
 {
-    (void) ETHR_ATMC__(_InterlockedDecrement)(&var->value);
+    (void) ETHR_ILCKD__(Decrement)(&var->value);
 }
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
 {
-    return ETHR_ATMC__(_InterlockedIncrement)(&var->value);
+    return ETHR_ILCKD__(Increment)(&var->value);
 }
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
 {
-    return ETHR_ATMC__(_InterlockedDecrement)(&var->value);
+    return ETHR_ILCKD__(Decrement)(&var->value);
 }
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
 {
-    return ETHR_ATMC__(_InterlockedAnd)(&var->value, mask);
+    return ETHR_ILCKD__(And)(&var->value, mask);
 }
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
 {
-    return ETHR_ATMC__(_InterlockedOr)(&var->value, mask);
+    return ETHR_ILCKD__(Or)(&var->value, mask);
 }
 
 static ETHR_INLINE ETHR_AINT_T__
@@ -226,14 +328,14 @@ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
 			   ETHR_AINT_T__ new,
 			   ETHR_AINT_T__ old)
 {
-    return ETHR_ATMC__(_InterlockedCompareExchange)(&var->value, new, old);
+    return ETHR_ILCKD__(CompareExchange)(&var->value, new, old);
 }
 
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
 {
-    return ETHR_ATMC__(_InterlockedExchange)(&var->value, new);
+    return ETHR_ILCKD__(Exchange)(&var->value, new);
 }
 
 /*
@@ -248,15 +350,14 @@ ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
     ETHR_COMPILER_BARRIER;
     return val;
 #else
-    return ETHR_ATMC_ACQ__(_InterlockedExchangeAdd)(&var->value,
-						    (ETHR_AINT_T__) 0);
+    return ETHR_ILCKD_ACQ__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0);
 #endif
 }
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
 {
-    return ETHR_ATMC_ACQ__(_InterlockedIncrement)(&var->value);
+    return ETHR_ILCKD_ACQ__(Increment)(&var->value);
 }
 
 static ETHR_INLINE void
@@ -266,20 +367,20 @@ ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i)
     ETHR_COMPILER_BARRIER;
     var->value = i;
 #else
-    (void) ETHR_ATMC_REL__(_InterlockedExchange)(&var->value, i);
+    (void) ETHR_ILCKD_REL__(Exchange)(&var->value, i);
 #endif
 }
 
 static ETHR_INLINE void
 ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
 {
-    (void) ETHR_ATMC_REL__(_InterlockedDecrement)(&var->value);
+    (void) ETHR_ILCKD_REL__(Decrement)(&var->value);
 }
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
 {
-    return ETHR_ATMC_REL__(_InterlockedDecrement)(&var->value);
+    return ETHR_ILCKD_REL__(Decrement)(&var->value);
 }
 
 static ETHR_INLINE ETHR_AINT_T__
@@ -287,7 +388,7 @@ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
 				ETHR_AINT_T__ new,
 				ETHR_AINT_T__ old)
 {
-    return ETHR_ATMC_ACQ__(_InterlockedCompareExchange)(&var->value, new, old);
+    return ETHR_ILCKD_ACQ__(CompareExchange)(&var->value, new, old);
 }
 
 static ETHR_INLINE ETHR_AINT_T__
@@ -295,17 +396,20 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
 				ETHR_AINT_T__ new,
 				ETHR_AINT_T__ old)
 {
-    return ETHR_ATMC_REL__(_InterlockedCompareExchange)(&var->value, new, old);
+    return ETHR_ILCKD_REL__(CompareExchange)(&var->value, new, old);
 }
 
-#endif
+#endif /* ETHR_TRY_INLINE_FUNCS */
 
+#undef ETHR_ILCKD__
+#undef ETHR_ILCKD_ACQ__
+#undef ETHR_ILCKD_REL__
 #undef ETHR_NATMC_FUNC__
 #undef ETHR_ATMC_T__
 #undef ETHR_AINT_T__
 #undef ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
 #undef ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__
 
-#endif
+#endif /* _MSC_VER */
 
-#endif
+#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */
diff --git a/erts/include/internal/win/ethr_event.h b/erts/include/internal/win/ethr_event.h
index af57c20f91..598816b2c6 100644
--- a/erts/include/internal/win/ethr_event.h
+++ b/erts/include/internal/win/ethr_event.h
@@ -21,22 +21,24 @@
  * Author: Rickard Green
  */
 
-#define ETHR_EVENT_OFF_WAITER__		((LONG) -1)
-#define ETHR_EVENT_OFF__		((LONG) 1)
-#define ETHR_EVENT_ON__ 		((LONG) 0)
+#define ETHR_EVENT_OFF_WAITER__		((long) -1)
+#define ETHR_EVENT_OFF__		((long) 1)
+#define ETHR_EVENT_ON__ 		((long) 0)
 
 typedef struct {
-    volatile LONG state;
+    volatile long state;
     HANDLE handle;
 } ethr_event;
 
 #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__)
 
+#pragma intrinsic(_InterlockedExchange)
+
 static ETHR_INLINE void
 ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e)
 {
-    /* InterlockedExchange() imply a full memory barrier which is important */
-    LONG state = InterlockedExchange(&e->state, ETHR_EVENT_ON__);
+    /* _InterlockedExchange() imply a full memory barrier which is important */
+    long state = _InterlockedExchange(&e->state, ETHR_EVENT_ON__);
     if (state == ETHR_EVENT_OFF_WAITER__) {
 	if (!SetEvent(e->handle))
 	    ETHR_FATAL_ERROR__(ethr_win_get_errno__());
@@ -46,7 +48,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e)
 static ETHR_INLINE void
 ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e)
 {
-    /* InterlockedExchange() imply a full memory barrier which is important */
+    /* _InterlockedExchange() imply a full memory barrier which is important */
     InterlockedExchange(&e->state, ETHR_EVENT_OFF__);
 }
 
diff --git a/erts/include/internal/win/ethread.h b/erts/include/internal/win/ethread.h
index 3cbd4aeda1..c01b17cf14 100644
--- a/erts/include/internal/win/ethread.h
+++ b/erts/include/internal/win/ethread.h
@@ -25,6 +25,11 @@
 #ifndef ETHREAD_WIN_H__
 #define ETHREAD_WIN_H__
 
+#define ETHR_ATOMIC_WANT_32BIT_IMPL__
 #include "ethr_atomic.h"
+#if ETHR_SIZEOF_PTR == 8
+#  define ETHR_ATOMIC_WANT_64BIT_IMPL__
+#  include "ethr_atomic.h"
+#endif
 
 #endif