Miscellaneous rwmutex bug fixes and improvements

The ERTS internal rwlock implementation could get
into an inconsistent state. This bug was very seldom
triggered, but could be during heavy contention. The
bug was introduced in R14B (erts-5.8.1).

The bug was most likely to be triggered when using the
read_concurrency option on an ETS table that
was frequently accessed from multiple processes doing
lots of writes and reads. That is, in a situation where
you typically don't want to use the read_concurrency
option in the first place.

4 files changed, 588 insertions, 272 deletions

diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h
index 8d9d5e3d08..636fdc1e2f 100644
--- a/erts/include/internal/ethr_mutex.h
+++ b/erts/include/internal/ethr_mutex.h
@@ -33,6 +33,13 @@
 #  define ETHR_MTX_HARD_DEBUG
 #endif
 
+#if 0
+#  define ETHR_MTX_CHK_EXCL
+#if 1
+#    define ETHR_MTX_CHK_NON_EXCL
+#endif
+#endif
+
 #ifdef ETHR_MTX_HARD_DEBUG
 #  ifdef __GNUC__
 #    warning ETHR_MTX_HARD_DEBUG
@@ -49,6 +56,15 @@
 
 #if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)
 
+#ifdef ETHR_DEBUG
+#  ifndef ETHR_MTX_CHK_EXCL
+#    define ETHR_MTX_CHK_EXCL
+#  endif
+#  ifndef ETHR_MTX_CHK_NON_EXCL
+#    define ETHR_MTX_CHK_NON_EXCL
+#  endif
+#endif
+
 #if 0
 #  define ETHR_MTX_Q_LOCK_SPINLOCK__
 #  define ETHR_MTX_QLOCK_TYPE__ ethr_spinlock_t
@@ -68,8 +84,8 @@
 
 /* frequent read kind */
 #define ETHR_RWMTX_R_FLG__		(((long) 1) << 28)
-#define ETHR_RWMTX_R_PEND_UNLCK_MASK__	(ETHR_RWMTX_R_FLG__ - 1)
-#define ETHR_RWMTX_R_MASK__		(ETHR_RWMTX_R_WAIT_FLG__ - 1)
+#define ETHR_RWMTX_R_ABRT_UNLCK_FLG__	(((long) 1) << 27)
+#define ETHR_RWMTX_R_PEND_UNLCK_MASK__	(ETHR_RWMTX_R_ABRT_UNLCK_FLG__ - 1)
 
 /* normal kind */
 #define ETHR_RWMTX_RS_MASK__		(ETHR_RWMTX_R_WAIT_FLG__ - 1)
@@ -91,6 +107,12 @@ struct ethr_mutex_base_ {
 #ifdef ETHR_MTX_HARD_DEBUG_WSQ
     int ws;
 #endif
+#ifdef ETHR_MTX_CHK_EXCL
+    ethr_atomic_t exclusive;
+#endif
+#ifdef ETHR_MTX_CHK_NON_EXCL
+    ethr_atomic_t non_exclusive;
+#endif
 #ifdef ETHR_MTX_HARD_DEBUG_LFS
     ethr_atomic_t hdbg_lfs;
 #endif
@@ -344,6 +366,116 @@ do { \
 #define ETHR_MTX_HARD_DEBUG_FENCE_INIT(X)
 #endif
 
+#ifdef ETHR_MTX_CHK_EXCL
+
+#if !defined(ETHR_DEBUG) && defined(__GNUC__)
+#warning "check exclusive is enabled"
+#endif
+
+#  define ETHR_MTX_CHK_EXCL_INIT__(MTXB)		\
+    ethr_atomic_init(&(MTXB)->exclusive, 0)
+
+#  define ETHR_MTX_CHK_EXCL_IS_EXCL(MTXB)		\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    if (!ethr_atomic_read(&(MTXB)->exclusive))		\
+	ethr_assert_failed(__FILE__, __LINE__, __func__,\
+			   "is exclusive");		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#  define ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(MTXB)		\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    if (ethr_atomic_read(&(MTXB)->exclusive))		\
+	ethr_assert_failed(__FILE__, __LINE__, __func__,\
+			   "is not exclusive");		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#  define ETHR_MTX_CHK_EXCL_SET_EXCL(MTXB)		\
+do {							\
+    ETHR_MTX_CHK_EXCL_IS_NOT_EXCL((MTXB));		\
+    ethr_atomic_set(&(MTXB)->exclusive, 1);		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#  define ETHR_MTX_CHK_EXCL_UNSET_EXCL(MTXB)		\
+do {							\
+    ETHR_MTX_CHK_EXCL_IS_EXCL((MTXB));			\
+    ethr_atomic_set(&(MTXB)->exclusive, 0);		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+
+#ifdef ETHR_MTX_CHK_NON_EXCL
+
+#if !defined(ETHR_DEBUG) && defined(__GNUC__)
+#warning "check non-exclusive is enabled"
+#endif
+
+#    define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB)		\
+    ethr_atomic_init(&(MTXB)->non_exclusive, 0)
+#    define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB)		\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    if (!ethr_atomic_read(&(MTXB)->non_exclusive))	\
+	ethr_assert_failed(__FILE__, __LINE__, __func__,\
+			   "is non-exclusive");		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#    define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB)	\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    if (ethr_atomic_read(&(MTXB)->non_exclusive))	\
+	ethr_assert_failed(__FILE__, __LINE__, __func__,\
+			   "is not non-exclusive");	\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB)	\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    ethr_atomic_inc(&(MTXB)->non_exclusive);		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO)	\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    ethr_atomic_add(&(MTXB)->non_exclusive, (NO));	\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#    define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB)	\
+do {							\
+    ETHR_COMPILER_BARRIER;				\
+    ethr_atomic_dec(&(MTXB)->non_exclusive);		\
+    ETHR_COMPILER_BARRIER;				\
+} while (0)
+#else
+#    define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB)
+#    define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB)
+#    define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB)
+#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO)
+#    define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB)
+#    define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB)
+#endif
+
+#else
+#  define ETHR_MTX_CHK_EXCL_INIT__(MTXB)
+#  define ETHR_MTX_CHK_EXCL_IS_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_SET_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_UNSET_EXCL(MTXB)
+#  define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB)
+#  define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO)
+#  define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB)
+#  define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB)
+#endif
+
+#  define ETHR_MTX_CHK_EXCL_INIT(MTXB)			\
+do {							\
+    ETHR_MTX_CHK_EXCL_INIT__((MTXB));			\
+    ETHR_MTX_CHK_NON_EXCL_INIT__((MTXB));		\
+} while (0)
+
+
 #ifdef ETHR_USE_OWN_MTX_IMPL__
 
 #define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000
@@ -369,6 +501,11 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx)
     act = ethr_atomic_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0);
     res = (act == 0) ? 0 : EBUSY;
 
+#ifdef ETHR_MTX_CHK_EXCL
+    if (res == 0)
+	ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb);
+#endif
+
     ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&mtx->mtxb, res);
     ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
 
@@ -386,6 +523,8 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx)
     if (act != 0)
 	ethr_mutex_lock_wait__(mtx, act);
 
+    ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb);
+
     ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb);
     ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
 
@@ -400,6 +539,8 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
     ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
     ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&mtx->mtxb);
 
+    ETHR_MTX_CHK_EXCL_UNSET_EXCL(&mtx->mtxb);
+
     act = ethr_atomic_cmpxchg_relb(&mtx->mtxb.flgs, 0, ETHR_RWMTX_W_FLG__);
     if (act != ETHR_RWMTX_W_FLG__)
 	ethr_mutex_unlock_wake__(mtx, act);
diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h
index f28258059f..52d01aab32 100644
--- a/erts/include/internal/i386/atomic.h
+++ b/erts/include/internal/i386/atomic.h
@@ -167,15 +167,52 @@ ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val)
  * Atomic ops with at least specified barriers.
  */
 
-#define ethr_native_atomic_read_acqb ethr_native_atomic_read
-#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return
+static ETHR_INLINE long
+ethr_native_atomic_read_acqb(ethr_native_atomic_t *var)
+{
+    long val;
+#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT)
+    val = var->counter;
+#else
+    val = ethr_native_atomic_add_return(var, 0);
+#endif
+    __asm__ __volatile__("" : : : "memory");
+    return val;
+}
+
+static ETHR_INLINE void
+ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long i)
+{
+    __asm__ __volatile__("" : : : "memory");
 #if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT)
-#define ethr_native_atomic_set_relb ethr_native_atomic_set
+    var->counter = i;
 #else
-#define ethr_native_atomic_set_relb ethr_native_atomic_xchg
+    (void) ethr_native_atomic_xchg(var, i);
 #endif
-#define ethr_native_atomic_dec_relb ethr_native_atomic_dec
-#define ethr_native_atomic_dec_return_relb ethr_native_atomic_dec_return
+}
+
+static ETHR_INLINE long
+ethr_native_atomic_inc_return_acqb(ethr_native_atomic_t *var)
+{
+    long res = ethr_native_atomic_inc_return(var);
+    __asm__ __volatile__("" : : : "memory");
+    return res;
+}
+
+static ETHR_INLINE void
+ethr_native_atomic_dec_relb(ethr_native_atomic_t *var)
+{
+    __asm__ __volatile__("" : : : "memory");
+    ethr_native_atomic_dec(var);
+}
+
+static ETHR_INLINE long
+ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var)
+{
+    __asm__ __volatile__("" : : : "memory");
+    return ethr_native_atomic_dec_return(var);
+}
+
 #define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg
 #define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg
 
diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h
index 2a995d4465..2da6472393 100644
--- a/erts/include/internal/sparc32/atomic.h
+++ b/erts/include/internal/sparc32/atomic.h
@@ -176,38 +176,59 @@ ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old)
  * Atomic ops with at least specified barriers.
  */
 
+/* TODO: relax acquire barriers */
+
 static ETHR_INLINE long
 ethr_native_atomic_read_acqb(ethr_native_atomic_t *var)
 {
     long res = ethr_native_atomic_read(var);
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+    __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore" : : : "memory");
     return res;
 }
 
 static ETHR_INLINE void
 ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long i)
 {
-    __asm__ __volatile__("membar #LoadStore|#StoreStore");
+    __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory");
     ethr_native_atomic_set(var, i);
 }
 
+static ETHR_INLINE long
+ethr_native_atomic_inc_return_acqb(ethr_native_atomic_t *var)
+{
+    long res = ethr_native_atomic_inc_return(var);
+    __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory");
+    return res;
+}
+
 static ETHR_INLINE void
 ethr_native_atomic_dec_relb(ethr_native_atomic_t *var)
 {
-    __asm__ __volatile__("membar #LoadStore|#StoreStore");
+    __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory");
     ethr_native_atomic_dec(var);
 }
 
 static ETHR_INLINE long
 ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var)
 {
-    __asm__ __volatile__("membar #LoadStore|#StoreStore");
+    __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory");
     return ethr_native_atomic_dec_return(var);
 }
 
-#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return
-#define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg
-#define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg
+static ETHR_INLINE long
+ethr_native_atomic_cmpxchg_acqb(ethr_native_atomic_t *var, long new, long old)
+{
+    long res = ethr_native_atomic_cmpxchg(var, new, old);
+    __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory");
+    return res;
+}
+
+static ETHR_INLINE long
+ethr_native_atomic_cmpxchg_relb(ethr_native_atomic_t *var, long new, long old)
+{
+    __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory");
+    return ethr_native_atomic_cmpxchg(var, new, old);
+}
 
 #endif /* ETHR_TRY_INLINE_FUNCS */
 
diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c
index 78323b62a3..c5738c3d77 100644
--- a/erts/lib_src/common/ethr_mutex.c
+++ b/erts/lib_src/common/ethr_mutex.c
@@ -205,9 +205,14 @@ static void hard_debug_chk_q__(struct ethr_mutex_base_ *, int);
 
 #ifdef ETHR_USE_OWN_RWMTX_IMPL__
 static void
+rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx,
+			   long initial,
+			   int q_locked);
+static void
 rwmutex_unlock_wake(ethr_rwmutex *rwmtx,
 		    int have_w,
-		    long initial);
+		    long initial,
+		    int transfer_read_lock);
 static int
 rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
 			     long initial,
@@ -334,7 +339,7 @@ rwmutex_freqread_rdrs_read(ethr_rwmutex *rwmtx, int ix)
 	ETHR_ASSERT(res >= 0);
 	break;
     case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
-	ETHR_ASSERT(res == 0 || res == 1);
+	ETHR_ASSERT(ix == 0 ? res >= 0 : (res == 0 || res == 1));
 	break;
     default:
 	ETHR_ASSERT(0);
@@ -400,6 +405,7 @@ event_wait(struct ethr_mutex_base_ *mtxb,
     int locked = 0;
     long act;
     int need_try_complete_runlock = 0;
+    int transfer_read_lock = 0;
 
     /* Need to enqueue and wait... */
 
@@ -444,8 +450,8 @@ event_wait(struct ethr_mutex_base_ *mtxb,
 
 	while (1) {
 	    long new, exp = act;
-	    int freqread_tryrlock = 0;
 	    need_try_complete_runlock = 0;
+	    transfer_read_lock = 0;
 
 	    if (type == ETHR_RWMTX_W_WAIT_FLG__) {
 		if (is_freq_read && act == ETHR_RWMTX_R_FLG__)
@@ -465,14 +471,11 @@ event_wait(struct ethr_mutex_base_ *mtxb,
 			new = act + 1; /* Try to get it */
 		}
 		else {
-		    if (act & ~ETHR_RWMTX_R_FLG__)
-			new = act | ETHR_RWMTX_R_WAIT_FLG__;
-		    else { /* Try to get it */
-			ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
-			rwmutex_freqread_rdrs_inc(rwmtx, tse);
-			ETHR_MEMORY_BARRIER;
-			new = act | ETHR_RWMTX_R_FLG__;
-			freqread_tryrlock = 1;
+		    new = act | ETHR_RWMTX_R_WAIT_FLG__;
+		    if ((act & (ETHR_RWMTX_W_FLG__
+				| ETHR_RWMTX_W_WAIT_FLG__)) == 0) {
+			/* Transfer read lock to this thread. */
+			transfer_read_lock = 1;
 		    }
 		}
 	    }
@@ -488,24 +491,6 @@ event_wait(struct ethr_mutex_base_ *mtxb,
 		    goto done;
 		}
 	    }
-
-	    if (freqread_tryrlock) {
-		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
-
-		/* We didn't set ETHR_RWMTX_R_FLG__, however someone
-		   else might have */
-		if (act == ETHR_RWMTX_R_FLG__)
-		    goto done; /* Got it by help from someone else */
-
-		ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
-		/*
-		 * We know that no waiter flags have been set, i.e.,
-		 * we cannot get into a situation where we need to wake
-		 * someone up here. Just restore the readers counter
-		 * and do it over again...
-		 */
-		rwmutex_freqread_rdrs_dec(rwmtx, tse);
-	    }
 	}
 
 	/* Enqueue */
@@ -535,20 +520,36 @@ event_wait(struct ethr_mutex_base_ *mtxb,
 
     /* Wait */
     locked = 0;
-    ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
 
-    if (need_try_complete_runlock) {
+    ETHR_ASSERT(!(transfer_read_lock && need_try_complete_runlock));
+
+    if (transfer_read_lock) {
 	ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type
 		    != ETHR_RWMUTEX_TYPE_NORMAL);
 	/*
-	 * We were the only one in queue when we enqueued, and it
-	 * was seemingly read locked. We need to try to complete a
-	 * runlock otherwise we might be hanging forever. If the
-	 * runlock could be completed we will be dequeued and
-	 * woken by ourselves.
+	 * We are the only one in the queue and we are not write
+	 * locked; rwmutex_transfer_read_lock() will:
+	 * - transfer a read lock to us (since we're first in q)
+	 * - unlock the Q-lock
 	 */
-	rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb,
-				     act, tse, 0, 1, 0);
+	rwmutex_transfer_read_lock(((ethr_rwmutex *) mtxb), act, 1);
+    }
+    else {
+	ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+
+	if (need_try_complete_runlock) {
+	    ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type
+			!= ETHR_RWMUTEX_TYPE_NORMAL);
+	    /*
+	     * We were the only one in queue when we enqueued, and it
+	     * was seemingly read locked. We need to try to complete a
+	     * runlock otherwise we might be hanging forever. If the
+	     * runlock could be completed we will be dequeued and
+	     * woken by ourselves.
+	     */
+	    rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb,
+					 act, tse, 0, 1, 0);
+	}
     }
 
     while (1) {
@@ -653,8 +654,6 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb,
     ethr_ts_event *tse = NULL;
     int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
     int res;
-    int freq_read_size = -1;
-    int freq_read_start_ix = -1;
 
     ETHR_ASSERT(!is_freq_read || is_rwmtx);
 
@@ -666,44 +665,23 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb,
      */
 
     while (1) {
-	long exp;
-
 	while (act != 0) {
 
 	    if (is_freq_read && act == ETHR_RWMTX_R_FLG__) {
 		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+		scnt--;
 		if (!tse)
 		    tse = ethr_get_ts_event();
-		if (freq_read_size < 0) {
-		    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
-			freq_read_size = reader_groups_array_size;
-			freq_read_start_ix = tse->rgix;
-		    }
-		    else {
-			freq_read_size = main_threads_array_size;
-			freq_read_start_ix = tse->mtix;
-		    }
-		}
-		res = check_readers_array(rwmtx,
-					  freq_read_start_ix,
-					  freq_read_size);
-		scnt--;
-		if (res == 0) {
-		    act = ethr_atomic_read(&mtxb->flgs);
-		    if (act & ETHR_RWMTX_R_MASK__) {
-			res = rwmutex_try_complete_runlock(rwmtx, act,
-							   tse, 0, 0,
-							   1);
-			if (res != EBUSY)
-			    goto done; /* Got it */
-		    }
-		    if (scnt <= 0)
-			goto chk_spin;
-		    if (--until_yield == 0) {
-			until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
-			ETHR_YIELD();
-		    }
-		    continue;
+		res = rwmutex_try_complete_runlock(rwmtx, act,
+						   tse, 0, 0,
+						   1);
+		if (res != EBUSY)
+		    goto done; /* Got it */
+		if (scnt <= 0)
+		    goto chk_spin;
+		if (--until_yield == 0) {
+		    until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		    ETHR_YIELD();
 		}
 	    }
 
@@ -728,11 +706,9 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb,
 	    scnt--;
 	}
 
-	exp = act;
-
 	act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs,
 				       ETHR_RWMTX_W_FLG__,
-				       exp);
+				       0);
 	if (act == 0)
 	    goto done; /* Got it */
     }
@@ -753,6 +729,7 @@ mtxb_init(struct ethr_mutex_base_ *mtxb,
 #ifdef ETHR_MTX_HARD_DEBUG_WSQ
     mtxb->ws = 0;
 #endif
+    ETHR_MTX_CHK_EXCL_INIT(mtxb);
     if (no_spin) {
 	mtxb->main_scnt = 0;
 	mtxb->aux_scnt = 0;
@@ -1254,7 +1231,7 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
     ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb);
     ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
     ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
-
+    ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb);
     tse->udata = udata;
     ethr_leave_ts_event(tse);
     return 0;
@@ -1499,7 +1476,62 @@ int check_readers_array(ethr_rwmutex *rwmtx,
     return 0;
 }
 
-static ETHR_INLINE void
+static void
+rwmutex_freqread_rdrs_dec_chk_wakeup(ethr_rwmutex *rwmtx,
+				     ethr_ts_event *tse,
+				     long initial)
+{
+    long act = initial;
+
+    if ((act & (ETHR_RWMTX_W_FLG__|
+		ETHR_RWMTX_R_ABRT_UNLCK_FLG__)) == 0) {
+	if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	    if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) {
+		/*
+		 * We *need* to try to complete the runlock.
+		 * A writer that just enqueued (not seen by us
+		 * in flag field) may depend on someone else
+		 * completing the runlock. We just took over
+		 * that responsibilty since we modified reader
+		 * groups.
+		 */
+		rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+	    }
+	}
+	else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_R_WAIT_FLG__)
+	    rwmutex_transfer_read_lock(rwmtx, act, 0);
+	else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_W_WAIT_FLG__)
+	    rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+	else {
+	    /*
+	     * Don't know if we got readers or writers
+	     * first in queue; need to peek
+	     */
+	    ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+	    if (!rwmtx->mtxb.q)
+		ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+	    else if (is_w_waiter(rwmtx->mtxb.q)) {
+		act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+		ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+		if ((act & ETHR_RWMTX_W_FLG__) == 0)
+		    rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+	    }
+	    else {
+		/*
+		 * rwmutex_transfer_read_lock() will
+		 * unlock Q lock.
+		 */
+		act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+		if (act & ETHR_RWMTX_W_FLG__)
+		    ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+		else
+		    rwmutex_transfer_read_lock(rwmtx, act, 1);
+	    }
+	}
+    }
+}
+
+static void
 rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx,
 					 ethr_ts_event *tse)
 {
@@ -1509,24 +1541,11 @@ rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx,
      */
     act = rwmutex_freqread_rdrs_dec_read(rwmtx, tse);
 
-    ETHR_WRITE_MEMORY_BARRIER;
+    ETHR_MEMORY_BARRIER;
 
     if (act == 0) {
-
-#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL
-	ETHR_READ_MEMORY_BARRIER;
-#endif
-
 	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
-
-	if ((act & ETHR_RWMTX_W_FLG__) == 0
-	    && act & (ETHR_RWMTX_WAIT_FLGS__|ETHR_RWMTX_R_PEND_UNLCK_MASK__)) {
-	    /*
-	     * We either got waiters, or someone else trying
-	     * to read unlock which we might have to help.
-	     */
-	    rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 1, 0);
-	}
+	rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act);
     }
 }
 
@@ -1542,12 +1561,16 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
     long act = initial;
     int six, res, length;
 
+    ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
+
+    if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__)
+	return try_write_lock ? EBUSY : 0;
+
     tse_tmp = tse;
     if (!tse_tmp)
 	tse_tmp = ethr_get_ts_event();
 
-    if ((act & ETHR_RWMTX_WAIT_FLGS__)
-	&& (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0)
+    if ((act & ETHR_RWMTX_WAIT_FLGS__) && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0)
 	goto check_waiters;
 
     if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
@@ -1569,14 +1592,21 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
 
     if (check_before_try) {
 	res = check_readers_array(rwmtx, six, length);
+
+	ETHR_MEMORY_BARRIER;
+
 	if (res == EBUSY)
 	    return try_write_lock ? EBUSY : 0;
     }
 
+ restart:
+
     while (1) {
 	long exp = act;
 	long new = act+1;
 
+	ETHR_ASSERT((act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) == 0);
+
 	ETHR_ASSERT((act & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
 		    < ETHR_RWMTX_R_PEND_UNLCK_MASK__);
 
@@ -1585,8 +1615,10 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
 	    act = new;
 	    break;
 	}
+
 	if (!try_write_lock) {
-	    if (act == ETHR_RWMTX_W_FLG__ || act == 0)
+	    if (act == 0 || (act & (ETHR_RWMTX_W_FLG__
+				    | ETHR_RWMTX_R_ABRT_UNLCK_FLG__)))
 		return 0;
 	    if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
 		if ((act & ETHR_RWMTX_R_FLG__) == 0)
@@ -1601,33 +1633,50 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
 	else {
 	    if (act == 0)
 		goto tryrwlock;
-	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__))
+	    if (act & (ETHR_RWMTX_W_FLG__
+		       | ETHR_RWMTX_R_ABRT_UNLCK_FLG__))
 		return EBUSY;
 	}
     }
 
     res = check_readers_array(rwmtx, six, length);
-    if (res == EBUSY) {
-	act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs);
-	if (act & ETHR_RWMTX_R_MASK__)
-	    return try_write_lock ? EBUSY : 0;
-    }
-    else {
-	while (1) {
-	    long exp = act;
-	    long new = act;
-	    new &= ~ETHR_RWMTX_R_FLG__;
-	    new--;
 
-	    ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__);
+    ETHR_MEMORY_BARRIER;
 
-	    act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
-	    if (exp == act) {
-		if (new & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
-		    return try_write_lock ? EBUSY : 0;
-		act = new;
-		break;
+    ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
+
+    while (1) {
+	int finished_abort = 0;
+	long exp = act;
+	long new = act;
+
+	new--;
+	if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) {
+	    if ((new & ETHR_RWMTX_R_PEND_UNLCK_MASK__) == 0) {
+		new &= ~ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+		finished_abort = 1;
 	    }
+	    ETHR_ASSERT(act & ETHR_RWMTX_R_FLG__);
+	}
+	else if ((act & ETHR_RWMTX_R_FLG__) && res != EBUSY) {
+	    new &= ~ETHR_RWMTX_R_FLG__;
+	}
+
+	ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__);
+
+	act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	if (exp == act) {
+	    act = new;
+	    if (act & ETHR_RWMTX_W_FLG__)
+		return try_write_lock ? EBUSY : 0;
+	    if (finished_abort && (act & ETHR_RWMTX_WAIT_FLGS__))
+		goto restart;
+	    if (act & (ETHR_RWMTX_R_FLG__
+		       | ETHR_RWMTX_R_ABRT_UNLCK_FLG__
+		       | ETHR_RWMTX_R_PEND_UNLCK_MASK__))
+		return try_write_lock ? EBUSY : 0;
+	    /* Read unlock completed */
+	    break;
 	}
     }
 
@@ -1637,12 +1686,9 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
      * to write lock it).
      */
 
-    if (act & ETHR_RWMTX_W_FLG__)
-	return try_write_lock ? EBUSY : 0;
-
     if (act & ETHR_RWMTX_WAIT_FLGS__) {
     check_waiters:
-	rwmutex_unlock_wake(rwmtx, 0, act);
+	rwmutex_unlock_wake(rwmtx, 0, act, 0);
 	return try_write_lock ? EBUSY : 0;
     }
 
@@ -1670,7 +1716,7 @@ rwmutex_incdec_restore_failed_tryrlock(ethr_rwmutex *rwmtx)
     act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs);
     if ((act & ETHR_RWMTX_WAIT_FLGS__)
 	&& (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
-	rwmutex_unlock_wake(rwmtx, 0, act);
+	rwmutex_unlock_wake(rwmtx, 0, act, 0);
     }
 }
 
@@ -1700,7 +1746,7 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
 #endif
 
 	while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
-	    if (scnt >= 0) {
+	    if (scnt <= 0) {
 		tse = ethr_get_ts_event();
 		if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
 		    event_wait(&rwmtx->mtxb, tse, scnt,
@@ -1736,10 +1782,83 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
 
 static void
 rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
-			    ethr_ts_event *tse,
-			    long initial)
+			    ethr_ts_event *tse);
+
+static int
+rwmutex_freqread_rlock(ethr_rwmutex *rwmtx, ethr_ts_event *tse, int trylock)
 {
-    long act = initial;
+    int res = 0;
+    long act;
+
+    rwmutex_freqread_rdrs_inc(rwmtx, tse);
+
+    ETHR_MEMORY_BARRIER;
+
+    act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs);
+
+    if (act != ETHR_RWMTX_R_FLG__) {
+	int wake_other_readers;
+
+	while (1) {
+	    long exp, new;
+
+	    wake_other_readers = 0;
+
+	    if (act == 0)
+		new = act | ETHR_RWMTX_R_FLG__;
+	    else if (act == ETHR_RWMTX_R_FLG__)
+		break; /* Got it */
+	    else if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+		rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
+		if (trylock)
+		    res = EBUSY;
+		else
+		    rwmutex_freqread_rlock_wait(rwmtx, tse);
+		break;
+	    }
+	    else if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) {
+		if ((act & ETHR_RWMTX_R_FLG__) == 0)
+		    ETHR_FATAL_ERROR__(EFAULT);
+		/*
+		 * An aborted runlock, not write locked, and no write
+		 * waiters, i.e., we got it...
+		 */
+		if (act & ETHR_RWMTX_R_WAIT_FLG__)
+		    wake_other_readers = 1;
+		break;
+	    }
+	    else {
+		new = act | ETHR_RWMTX_R_FLG__;
+		if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) {
+		    /*
+		     * Someone is doing tryrwlock (no writer and no
+		     * write waiters); we will try to abort that...
+		     */
+		    new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+		}
+
+		if (act & ETHR_RWMTX_R_WAIT_FLG__)
+		    wake_other_readers = 1;
+	    }
+
+	    exp = act;
+	    act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
+	    if (act == exp)
+		break;
+	}
+
+	if (wake_other_readers)
+	    rwmutex_transfer_read_lock(rwmtx, act, 0);
+    }
+
+    return res;
+}
+
+static void
+rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
+			    ethr_ts_event *tse)
+{
+    long act;
     int scnt, start_scnt;
     int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
 
@@ -1752,12 +1871,10 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
 
     while (1) {
 
-	rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
-
 	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
 
-	while (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
-	    if (scnt >= 0) {
+	while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+	    if (scnt <= 0) {
 		if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
 		    event_wait(&rwmtx->mtxb, tse, scnt,
 			       ETHR_RWMTX_R_WAIT_FLG__, 1, 1);
@@ -1773,30 +1890,8 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
 	    scnt--;
 	}
 
-	rwmutex_freqread_rdrs_inc(rwmtx, tse);
-
-	ETHR_MEMORY_BARRIER;
-
-	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
-
-	if (act == ETHR_RWMTX_R_FLG__)
-	    return; /* Got it */
-
-	while (1) {
-	    long exp, new;
-
-	    if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__))
-		break; /* Busy (need to restore inc) */
-
-	    if (act & ETHR_RWMTX_R_FLG__)
-		return; /* Got it */
-
-	    exp = act;
-	    new = act | ETHR_RWMTX_R_FLG__;
-	    act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
-	    if (act == exp)
-		return; /* Got it */
-	}
+	if (rwmutex_freqread_rlock(rwmtx, tse, 1) != EBUSY)
+	    break; /* Got it */
     }
 }
 
@@ -1816,14 +1911,23 @@ static ETHR_INLINE void
 rwlock_wake_set_flags(ethr_rwmutex *rwmtx, long new_initial, int act_initial)
 {
     long act, act_mask;
+    int chk_abrt_flg;
+
+    ETHR_MEMORY_BARRIER;
+
     if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
 	/* r pend unlock mask may vary and must be retained */
 	act_mask = ETHR_RWMTX_R_PEND_UNLCK_MASK__;
+	if (new_initial & ETHR_RWMTX_R_FLG__)
+	    chk_abrt_flg = 1;
+	else
+	    chk_abrt_flg = 0;
     }
     else {
 #ifdef ETHR_RLOCK_WITH_INC_DEC
 	/* rs mask may vary and must be retained */
 	act_mask = ETHR_RWMTX_RS_MASK__;
+	chk_abrt_flg = 0;
 #else
 	/* rs mask always zero */
 	ETHR_ASSERT((act_initial & ETHR_RWMTX_RS_MASK__) == 0);
@@ -1836,6 +1940,8 @@ rwlock_wake_set_flags(ethr_rwmutex *rwmtx, long new_initial, int act_initial)
     while (1) {
 	long exp = act;
 	long new = new_initial + (act & act_mask);
+	if (chk_abrt_flg && (act & act_mask))
+	    new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
 	act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
 	if (act == exp)
 	    break;
@@ -1883,6 +1989,14 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx,
 	exp |= ETHR_RWMTX_R_WAIT_FLG__;
 	if (rwmtx->rq_end->next != rwmtx->mtxb.q)
 	    exp |= ETHR_RWMTX_W_WAIT_FLG__;
+	else if (exp == ETHR_RWMTX_R_WAIT_FLG__) {
+	    if (!have_w) {
+		if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
+		    imask |= ETHR_RWMTX_R_FLG__;
+		else
+		    imask |= ETHR_RWMTX_RS_MASK__;
+	    }
+	}
 	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
 	ETHR_ASSERT((exp & ~imask) == (act & ~imask));
 
@@ -1894,41 +2008,83 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx,
 #endif
 
 static void
-rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
+rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx, long initial, int q_locked)
+{
+    long act = initial;
+
+    if (!q_locked) {
+	ethr_ts_event *tse;
+	ETHR_ASSERT(initial & ETHR_RWMTX_R_WAIT_FLG__);
+	ETHR_ASSERT((initial & ETHR_RWMTX_W_FLG__) == 0);
+	ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	tse = rwmtx->mtxb.q;
+	if ((act & ETHR_RWMTX_W_FLG__) || !tse || is_w_waiter(tse)) {
+	    /* Someone else woke the readers up... */
+	    ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+	    return;
+	}
+    }
+
+    rwmutex_unlock_wake(rwmtx, 0, initial, 1);
+}
+
+static void
+rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial,
+		    int transfer_read_lock)
 {
     long new, act = initial;
     ethr_ts_event *tse;
 
-    if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
-	if (!have_w)
-	    return;
-	else {
-	    while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
-		long exp = act;
-		new = exp & ~ETHR_RWMTX_W_FLG__;
-		act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
-		if (act == exp)
-		    return;
+    if (transfer_read_lock) {
+	/*
+	 * - Q already locked
+	 * - Got R waiters first in Q
+	 * - Not W locked
+	 */
+	tse = rwmtx->mtxb.q;
+
+	ETHR_ASSERT(act & ETHR_RWMTX_R_WAIT_FLG__);
+	ETHR_ASSERT((act & (ETHR_RWMTX_W_FLG__)) == 0);
+	ETHR_ASSERT(tse && !is_w_waiter(tse));
+    }
+    else {
+
+	if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	    if (!have_w)
+		return;
+	    else {
+		while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+		    long exp = act;
+		    new = exp & ~ETHR_RWMTX_W_FLG__;
+		    act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+		    if (act == exp)
+			return;
+		}
 	    }
 	}
-    }
 
-    ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
-    tse = rwmtx->mtxb.q;
+	ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+	tse = rwmtx->mtxb.q;
 
-    if (!have_w) {
-	if (!tse) {
+	if (!have_w) {
+	    if (!tse) {
 #ifdef ETHR_DEBUG
-	    act = ethr_atomic_read(&rwmtx->mtxb.flgs);
-	    ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
+		act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+		ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
 #endif
-	    goto already_served;
-	}
-	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
-	if (act & ~ETHR_RWMTX_WAIT_FLGS__) {
-	already_served:
-	    ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
-	    return;
+		goto already_served;
+	    }
+	    act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	    if (act == (ETHR_RWMTX_R_WAIT_FLG__|ETHR_RWMTX_R_FLG__)) {
+		ETHR_ASSERT(tse && !is_w_waiter(tse));
+	    }
+	    else if (act & ~ETHR_RWMTX_WAIT_FLGS__) {
+	    already_served:
+		ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+		return;
+	    }
 	}
     }
 
@@ -1988,6 +2144,7 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
 		    rwmutex_freqread_rdrs_add(rwmtx, type, ix, wrs);
 		}
 	    }
+
 	    new = ETHR_RWMTX_R_FLG__;
 	}
 
@@ -1995,6 +2152,7 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
 	    new |= ETHR_RWMTX_W_WAIT_FLG__;
 
 	rwlock_wake_set_flags(rwmtx, new, act);
+
 	wake_readers(rwmtx, rs);
     }
 }
@@ -2225,39 +2383,19 @@ ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
     case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
     case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
 	ethr_ts_event *tse = ethr_get_ts_event();
-
-	rwmutex_freqread_rdrs_inc(rwmtx, tse);
-
-	ETHR_MEMORY_BARRIER;
-
-	act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs);
-
-	if (act != ETHR_RWMTX_R_FLG__) {
-	    while (1) {
-		long exp, new;
-
-		if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
-		    rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
-		    res = EBUSY;
-		    break;
-		}
-
-		if (act & ETHR_RWMTX_R_FLG__)
-		    break;
-
-		exp = act;
-		new = act | ETHR_RWMTX_R_FLG__;
-		act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
-		if (act == exp)
-		    break;
-	    }
-	}
-
+	res = rwmutex_freqread_rlock(rwmtx, tse, 1);
 	ethr_leave_ts_event(tse);
 	break;
     }
     }
 
+#ifdef ETHR_MTX_CHK_EXCL
+    if (res == 0) {
+	ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb);
+	ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
+    }
+#endif
+
     ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(&rwmtx->mtxb, res);
     ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
 
@@ -2286,9 +2424,8 @@ ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
 
 	while (1) {
 	    act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
-	    if (act == exp) {
+	    if (act == exp)
 		break;
-	    }
 
 	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
 		rwmutex_normal_rlock_wait(rwmtx, act);
@@ -2303,38 +2440,14 @@ ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
     case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
     case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
 	ethr_ts_event *tse = ethr_get_ts_event();
-
-	rwmutex_freqread_rdrs_inc(rwmtx, tse);
-
-	ETHR_MEMORY_BARRIER;
-
-	act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs);
-
-	if (act != ETHR_RWMTX_R_FLG__) {
-	    while (1) {
-		long exp, new;
-
-		if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
-		    rwmutex_freqread_rlock_wait(rwmtx, tse, act);
-		    break;
-		}
-
-		if (act & ETHR_RWMTX_R_FLG__)
-		    break;
-
-		exp = act;
-		new = act | ETHR_RWMTX_R_FLG__;
-		act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
-		if (act == exp)
-		    break;
-	    }
-	}
-
+	rwmutex_freqread_rlock(rwmtx, tse, 0);
 	ethr_leave_ts_event(tse);
 	break;
     }
     }
 
+    ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
     ETHR_MTX_HARD_DEBUG_LFS_RLOCK(&rwmtx->mtxb);
     ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
 }
@@ -2344,6 +2457,8 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
 {
     long act;
 
+    ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(&rwmtx->mtxb);
     ETHR_ASSERT(!ethr_not_inited__);
     ETHR_ASSERT(rwmtx);
     ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
@@ -2357,7 +2472,7 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
 	if ((act & ETHR_RWMTX_WAIT_FLGS__)
 	    && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
 	    ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
-	    rwmutex_unlock_wake(rwmtx, 0, act);
+	    rwmutex_unlock_wake(rwmtx, 0, act, 0);
 	}
 	break;
 
@@ -2369,21 +2484,12 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
 
 	ETHR_ASSERT(act >= 0);
 
-	ETHR_WRITE_MEMORY_BARRIER;
+	ETHR_MEMORY_BARRIER;
 
 	if (act == 0) {
-
-#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL
-	    ETHR_READ_MEMORY_BARRIER;
-#endif
 	    act = ethr_atomic_read(&rwmtx->mtxb.flgs);
-
-	    if ((act & ETHR_RWMTX_W_FLG__) == 0
-		&& (act & (ETHR_RWMTX_WAIT_FLGS__
-			   | ETHR_RWMTX_R_PEND_UNLCK_MASK__))) {
-		rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
-	    }
-
+	    if (act != ETHR_RWMTX_R_FLG__)
+		rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act);
 	}
 
 	ethr_leave_ts_event(tse);
@@ -2422,25 +2528,31 @@ ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
 
 	do {
 
-	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__)) {
-		res = EBUSY;
-		break;
-	    }
-
-	    if (act & ETHR_RWMTX_R_MASK__) {
+	    if (act == 0)
+		act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+					       ETHR_RWMTX_W_FLG__, 0);
+	    else if (act == ETHR_RWMTX_R_FLG__) {
 		res = rwmutex_try_complete_runlock(rwmtx, act, NULL,
 						   0, 1, 1);
 		break;
 	    }
-
-	    act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
-					   ETHR_RWMTX_W_FLG__, 0);
+	    else {
+		res = EBUSY;
+		break;
+	    }
 
 	} while (act != 0);
 
 	break;
     }
 
+#ifdef ETHR_MTX_CHK_EXCL
+    if (res == 0) {
+	ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb);
+	ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
+    }
+#endif
+
     ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&rwmtx->mtxb, res);
     ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
 
@@ -2485,6 +2597,8 @@ ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
 	break;
     }
 
+    ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
     ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&rwmtx->mtxb);
     ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
 
@@ -2501,12 +2615,15 @@ ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
     ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
     ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&rwmtx->mtxb);
 
+    ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
+    ETHR_MTX_CHK_EXCL_UNSET_EXCL(&rwmtx->mtxb);
+
     switch (rwmtx->type) {
     case ETHR_RWMUTEX_TYPE_NORMAL:
 	act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs,
 				       0, ETHR_RWMTX_W_FLG__);
 	if (act != ETHR_RWMTX_W_FLG__)
-	    rwmutex_unlock_wake(rwmtx, 1, act);
+	    rwmutex_unlock_wake(rwmtx, 1, act, 0);
 	break;
 
     case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
@@ -2514,7 +2631,7 @@ ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
 	act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs, 0,
 				       ETHR_RWMTX_W_FLG__);
 	if (act != ETHR_RWMTX_W_FLG__)
-	    rwmutex_unlock_wake(rwmtx, 1, act);
+	    rwmutex_unlock_wake(rwmtx, 1, act, 0);
 	break;
     }