Merge branch 'rickard/rwmtx-spin/OTP-8819' into dev

* rickard/rwmtx-spin/OTP-8819:
  Fix deadlock in reader optimized rwlock implementation
  Remove unused variables
  Increase spincount with many schedulers
  Re-enable spin wait on ethreads rwlocks

9 files changed, 58 insertions, 48 deletions

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index 6b3c106a97..b97705ed96 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -303,7 +303,7 @@ init_atom_table(void)
     HashFunctions f;
     int i;
     Atom a;
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
 
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index b0369a402b..52d5f86ee0 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -265,7 +265,7 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
 				     char *rwname, char* fixname)
 {
 #ifdef ERTS_SMP
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     if (use_frequent_read_lock)
 	rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
 #endif
@@ -2746,7 +2746,7 @@ void init_db(void)
     size_t size;
 
 #ifdef ERTS_SMP
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 5abd2e50fa..fa707f4eed 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -623,7 +623,7 @@ int db_create_hash(Process *p, DbTable *tbl)
     erts_smp_atomic_init(&tb->is_resizing, 0);
 #ifdef ERTS_SMP
     if (tb->common.type & DB_FINE_LOCKED) {
-	erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+	erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
 	int i;
 	if (tb->common.type & DB_FREQ_READ)
 	    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index 5dce5ad262..84869f12d6 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -55,7 +55,7 @@ void
 erts_init_fun_table(void)
 {
     HashFunctions f;
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index e430b4ad77..d0b08bf72e 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -80,7 +80,7 @@ dist_table_alloc(void *dep_tmpl)
     Eterm chnl_nr;
     Eterm sysname;
     DistEntry *dep;
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
 
     if(((DistEntry *) dep_tmpl) == erts_this_dist_entry)
@@ -710,7 +710,7 @@ erts_set_this_node(Eterm sysname, Uint creation)
 
 void erts_init_node_tables(void)
 {
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     HashFunctions f;
 
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 5e81a2d624..5bc402fe22 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -109,7 +109,7 @@ void
 init_export_table(void)
 {
     HashFunctions f;
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index c9bb7bbe91..26d64887d0 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -145,7 +145,7 @@ static void reg_free(RegProc *obj)
 void init_register_table(void)
 {
     HashFunctions f;
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h
index 4ce3e75c78..8d9d5e3d08 100644
--- a/erts/include/internal/ethr_mutex.h
+++ b/erts/include/internal/ethr_mutex.h
@@ -346,7 +346,9 @@ do { \
 
 #ifdef ETHR_USE_OWN_MTX_IMPL__
 
-#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT 1000
+#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000
+#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE 800
+#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC 50
 #define ETHR_MTX_DEFAULT_AUX_SPINCOUNT 50
 
 #define ETHR_CND_DEFAULT_MAIN_SPINCOUNT 0
@@ -443,7 +445,9 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
 
 #ifdef ETHR_USE_OWN_RWMTX_IMPL__
 
-#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT 1000
+#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000
+#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE 800
+#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC 50
 #define ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT 50
 
 #else /* pthread_rwlock */
diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c
index aac0d44a32..78323b62a3 100644
--- a/erts/lib_src/common/ethr_mutex.c
+++ b/erts/lib_src/common/ethr_mutex.c
@@ -35,7 +35,7 @@
 
 #define ETHR_SPIN_WITH_WAITERS 1
 
-#define ETHR_MTX_MAX_FLGS_SPIN 1000
+#define ETHR_MTX_MAX_FLGS_SPIN 10
 
 #ifdef ETHR_USE_OWN_RWMTX_IMPL__
 static int default_rwmtx_main_spincount;
@@ -90,7 +90,7 @@ ethr_mutex_lib_init(int cpu_conf)
     no_spin = cpu_conf == 1;
 
 #ifdef ETHR_USE_OWN_MTX_IMPL__
-    default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT;
+    default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE;
     default_mtx_aux_spincount = ETHR_MTX_DEFAULT_AUX_SPINCOUNT;
     default_cnd_main_spincount = ETHR_CND_DEFAULT_MAIN_SPINCOUNT;
     default_cnd_aux_spincount = ETHR_CND_DEFAULT_AUX_SPINCOUNT;
@@ -98,7 +98,7 @@ ethr_mutex_lib_init(int cpu_conf)
 
 #ifdef ETHR_USE_OWN_RWMTX_IMPL__
 
-    default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT;
+    default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE;
     default_rwmtx_aux_spincount = ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT;
 
 #else
@@ -146,7 +146,21 @@ static int main_threads_array_size = 0;
 int
 ethr_mutex_lib_late_init(int no_reader_groups, int no_main_threads)
 {
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+    default_mtx_main_spincount += (no_main_threads
+				   * ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC);
+    if (default_mtx_main_spincount > ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX)
+	default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX;
+#endif
+
 #ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+    default_rwmtx_main_spincount += (no_main_threads
+				     * ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC);
+    if (default_rwmtx_main_spincount > ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX)
+	default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX;
+
     reader_groups_array_size = (no_reader_groups <= 1
 				? 1
 				: no_reader_groups + 1);
@@ -593,24 +607,31 @@ initial_spincount(struct ethr_mutex_base_ *mtxb)
 static ETHR_INLINE int
 update_spincount(struct ethr_mutex_base_ *mtxb,
 		 ethr_ts_event *tse,
-		 int *start_scnt,
+		 int *scnt_state,
 		 int *scnt)
 {
-    int sscnt = *start_scnt;
-    if (sscnt < 0) {
-	*scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
-		 ? mtxb->main_scnt
-		 : mtxb->aux_scnt);
-	*scnt -= ETHR_MTX_MAX_FLGS_SPIN;
+    int state = *scnt_state;
+    if (state <= 0) {
+	/* Here state is max spincount to do on event negated */
+	*scnt = -state;
     }
     else {
+	/* Here state is initial spincount made on flags */
 	*scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
 		 ? mtxb->main_scnt
 		 : mtxb->aux_scnt);
-	*scnt -= sscnt;
-	if (*scnt > 0 && sscnt < ETHR_MTX_MAX_FLGS_SPIN) {
-	    *scnt = ETHR_MTX_MAX_FLGS_SPIN - sscnt;
-	    *start_scnt = -1;
+	if (*scnt <= state)
+	    *scnt = 0;
+	else {
+	    if (*scnt <= ETHR_MTX_MAX_FLGS_SPIN)
+		*scnt_state = 0; /* No spin on event */
+	    else {
+		 /* Spin on event after... */
+		*scnt_state = -1*(*scnt - ETHR_MTX_MAX_FLGS_SPIN);
+		/* ... we have spun on flags */
+		*scnt = ETHR_MTX_MAX_FLGS_SPIN;
+	    }
+	    *scnt -= state;
 	    return 0;
 	}
     }
@@ -619,8 +640,7 @@ update_spincount(struct ethr_mutex_base_ *mtxb,
 
 int check_readers_array(ethr_rwmutex *rwmtx,
 			int start_rix,
-			int length,
-			int pre_check);
+			int length);
 
 static ETHR_INLINE void
 write_lock_wait(struct ethr_mutex_base_ *mtxb,
@@ -666,8 +686,7 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb,
 		}
 		res = check_readers_array(rwmtx,
 					  freq_read_start_ix,
-					  freq_read_size,
-					  1);
+					  freq_read_size);
 		scnt--;
 		if (res == 0) {
 		    act = ethr_atomic_read(&mtxb->flgs);
@@ -708,7 +727,6 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb,
 	    act = ethr_atomic_read(&mtxb->flgs);
 	    scnt--;
 	}
-	ETHR_ASSERT(scnt >= 0);
 
 	exp = act;
 
@@ -985,7 +1003,6 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
 int
 ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt)
 {
-    int res;
 #if ETHR_XCHK
     if (!cnd) {
 	ETHR_ASSERT(0);
@@ -1085,7 +1102,6 @@ ethr_cond_signal(ethr_cond *cnd)
 void
 ethr_cond_broadcast(ethr_cond *cnd)
 {
-    int res;
     int got_all;
     ethr_ts_event *tse;
     ETHR_ASSERT(!ethr_not_inited__);
@@ -1153,7 +1169,6 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
 {
     int woken;
     int scnt;
-    int res;
     void *udata = NULL;
     ethr_ts_event *tse;
 
@@ -1466,17 +1481,11 @@ multiple_w_waiters(ethr_rwmutex *rwmtx)
 
 int check_readers_array(ethr_rwmutex *rwmtx,
 			int start_rix,
-			int length,
-			int pre_check)
+			int length)
 {
     int ix = start_rix;
 
-#ifndef ETHR_READ_MEMORY_BARRIER_IS_FULL
-    if (pre_check)
-	ETHR_READ_MEMORY_BARRIER;
-    else
-#endif
-	ETHR_MEMORY_BARRIER;
+    ETHR_MEMORY_BARRIER;
 
     do {
 	long act = rwmutex_freqread_rdrs_read(rwmtx, ix);
@@ -1559,7 +1568,7 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
 	ethr_leave_ts_event(tse_tmp);
 
     if (check_before_try) {
-	res = check_readers_array(rwmtx, six, length, 1);
+	res = check_readers_array(rwmtx, six, length);
 	if (res == EBUSY)
 	    return try_write_lock ? EBUSY : 0;
     }
@@ -1597,7 +1606,7 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
 	}
     }
 
-    res = check_readers_array(rwmtx, six, length, 0);
+    res = check_readers_array(rwmtx, six, length);
     if (res == EBUSY) {
 	act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs);
 	if (act & ETHR_RWMTX_R_MASK__)
@@ -1691,7 +1700,7 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
 #endif
 
 	while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
-	    if (scnt == 0) {
+	    if (scnt >= 0) {
 		tse = ethr_get_ts_event();
 		if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
 		    event_wait(&rwmtx->mtxb, tse, scnt,
@@ -1708,7 +1717,6 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
 	    scnt--;
 	}
 	exp = act;
-	ETHR_ASSERT(scnt >= 0);
 
 #ifdef ETHR_RLOCK_WITH_INC_DEC
 	act = ethr_atomic_inc_read(&rwmtx->mtxb.flgs);
@@ -1749,7 +1757,7 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
 	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
 
 	while (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
-	    if (scnt == 0) {
+	    if (scnt >= 0) {
 		if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
 		    event_wait(&rwmtx->mtxb, tse, scnt,
 			       ETHR_RWMTX_R_WAIT_FLG__, 1, 1);
@@ -1765,8 +1773,6 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
 	    scnt--;
 	}
 
-	ETHR_ASSERT(scnt >= 0);
-
 	rwmutex_freqread_rdrs_inc(rwmtx, tse);
 
 	ETHR_MEMORY_BARRIER;