Re-enable spin wait on ethreads rwlocks

Spin wait on most ethread rwlocks used by the runtime system was
unintentionally disabled during development. Spin wait has now been enabled
again. This bug appeared in commit 59ee2a593090e7d53c97ceba63cbd300d1b9657e,
i.e., it has not been seen in any released versions.

8 files changed, 28 insertions, 21 deletions

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index 6b3c106a97..b97705ed96 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -303,7 +303,7 @@ init_atom_table(void)
     HashFunctions f;
     int i;
     Atom a;
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
 
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index b0369a402b..52d5f86ee0 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -265,7 +265,7 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
 				     char *rwname, char* fixname)
 {
 #ifdef ERTS_SMP
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     if (use_frequent_read_lock)
 	rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
 #endif
@@ -2746,7 +2746,7 @@ void init_db(void)
     size_t size;
 
 #ifdef ERTS_SMP
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 5abd2e50fa..fa707f4eed 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -623,7 +623,7 @@ int db_create_hash(Process *p, DbTable *tbl)
     erts_smp_atomic_init(&tb->is_resizing, 0);
 #ifdef ERTS_SMP
     if (tb->common.type & DB_FINE_LOCKED) {
-	erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+	erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
 	int i;
 	if (tb->common.type & DB_FREQ_READ)
 	    rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index 5dce5ad262..84869f12d6 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -55,7 +55,7 @@ void
 erts_init_fun_table(void)
 {
     HashFunctions f;
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index e430b4ad77..d0b08bf72e 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -80,7 +80,7 @@ dist_table_alloc(void *dep_tmpl)
     Eterm chnl_nr;
     Eterm sysname;
     DistEntry *dep;
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
 
     if(((DistEntry *) dep_tmpl) == erts_this_dist_entry)
@@ -710,7 +710,7 @@ erts_set_this_node(Eterm sysname, Uint creation)
 
 void erts_init_node_tables(void)
 {
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     HashFunctions f;
 
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 5e81a2d624..5bc402fe22 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -109,7 +109,7 @@ void
 init_export_table(void)
 {
     HashFunctions f;
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index c9bb7bbe91..26d64887d0 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -145,7 +145,7 @@ static void reg_free(RegProc *obj)
 void init_register_table(void)
 {
     HashFunctions f;
-    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+    erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
     rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
     rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
 
diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c
index aac0d44a32..ccceb61714 100644
--- a/erts/lib_src/common/ethr_mutex.c
+++ b/erts/lib_src/common/ethr_mutex.c
@@ -35,7 +35,7 @@
 
 #define ETHR_SPIN_WITH_WAITERS 1
 
-#define ETHR_MTX_MAX_FLGS_SPIN 1000
+#define ETHR_MTX_MAX_FLGS_SPIN 10
 
 #ifdef ETHR_USE_OWN_RWMTX_IMPL__
 static int default_rwmtx_main_spincount;
@@ -593,24 +593,31 @@ initial_spincount(struct ethr_mutex_base_ *mtxb)
 static ETHR_INLINE int
 update_spincount(struct ethr_mutex_base_ *mtxb,
 		 ethr_ts_event *tse,
-		 int *start_scnt,
+		 int *scnt_state,
 		 int *scnt)
 {
-    int sscnt = *start_scnt;
-    if (sscnt < 0) {
-	*scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
-		 ? mtxb->main_scnt
-		 : mtxb->aux_scnt);
-	*scnt -= ETHR_MTX_MAX_FLGS_SPIN;
+    int state = *scnt_state;
+    if (state <= 0) {
+	/* Here state is max spincount to do on event negated */
+	*scnt = -state;
     }
     else {
+	/* Here state is initial spincount made on flags */
 	*scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
 		 ? mtxb->main_scnt
 		 : mtxb->aux_scnt);
-	*scnt -= sscnt;
-	if (*scnt > 0 && sscnt < ETHR_MTX_MAX_FLGS_SPIN) {
-	    *scnt = ETHR_MTX_MAX_FLGS_SPIN - sscnt;
-	    *start_scnt = -1;
+	if (*scnt <= state)
+	    *scnt = 0;
+	else {
+	    if (*scnt <= ETHR_MTX_MAX_FLGS_SPIN)
+		*scnt_state = 0; /* No spin on event */
+	    else {
+		 /* Spin on event after... */
+		*scnt_state = -1*(*scnt - ETHR_MTX_MAX_FLGS_SPIN);
+		/* ... we have spun on flags */
+		*scnt = ETHR_MTX_MAX_FLGS_SPIN;
+	    }
+	    *scnt -= state;
 	    return 0;
 	}
     }