Miscellaneous rwmutex bug fixes and improvements

The ERTS internal rwlock implementation could get
into an inconsistent state. This bug was very seldom
triggered, but could be during heavy contention. The
bug was introduced in R14B (erts-5.8.1).

The bug was most likely to be triggered when using the
read_concurrency option on an ETS table that
was frequently accessed from multiple processes doing
lots of writes and reads. That is, in a situation where
you typically don't want to use the read_concurrency
option in the first place.

1 files changed, 43 insertions, 6 deletions

diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h
index f28258059f..52d01aab32 100644
--- a/erts/include/internal/i386/atomic.h
+++ b/erts/include/internal/i386/atomic.h
@@ -167,15 +167,52 @@ ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val)
  * Atomic ops with at least specified barriers.
  */
 
-#define ethr_native_atomic_read_acqb ethr_native_atomic_read
-#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return
+static ETHR_INLINE long
+ethr_native_atomic_read_acqb(ethr_native_atomic_t *var)
+{
+    long val;
+#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT)
+    val = var->counter;
+#else
+    val = ethr_native_atomic_add_return(var, 0);
+#endif
+    __asm__ __volatile__("" : : : "memory");
+    return val;
+}
+
+static ETHR_INLINE void
+ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long i)
+{
+    __asm__ __volatile__("" : : : "memory");
 #if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT)
-#define ethr_native_atomic_set_relb ethr_native_atomic_set
+    var->counter = i;
 #else
-#define ethr_native_atomic_set_relb ethr_native_atomic_xchg
+    (void) ethr_native_atomic_xchg(var, i);
 #endif
-#define ethr_native_atomic_dec_relb ethr_native_atomic_dec
-#define ethr_native_atomic_dec_return_relb ethr_native_atomic_dec_return
+}
+
+static ETHR_INLINE long
+ethr_native_atomic_inc_return_acqb(ethr_native_atomic_t *var)
+{
+    long res = ethr_native_atomic_inc_return(var);
+    __asm__ __volatile__("" : : : "memory");
+    return res;
+}
+
+static ETHR_INLINE void
+ethr_native_atomic_dec_relb(ethr_native_atomic_t *var)
+{
+    __asm__ __volatile__("" : : : "memory");
+    ethr_native_atomic_dec(var);
+}
+
+static ETHR_INLINE long
+ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var)
+{
+    __asm__ __volatile__("" : : : "memory");
+    return ethr_native_atomic_dec_return(var);
+}
+
 #define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg
 #define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg