From 46959cfb07d78bd4cbece99acfda7de938262611 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Mon, 6 Sep 2010 19:42:43 +0200 Subject: Re-enable spin wait on ethreads rwlocks Spin wait on most ethread rwlocks used by the runtime system was unintentionally disabled during development. Spin wait has now been enabled again. This bug appeared in commit 59ee2a593090e7d53c97ceba63cbd300d1b9657e, i.e., it has not been seen in any released versions. --- erts/emulator/beam/atom.c | 2 +- erts/emulator/beam/erl_db.c | 4 ++-- erts/emulator/beam/erl_db_hash.c | 2 +- erts/emulator/beam/erl_fun.c | 2 +- erts/emulator/beam/erl_node_tables.c | 4 ++-- erts/emulator/beam/export.c | 2 +- erts/emulator/beam/register.c | 2 +- erts/lib_src/common/ethr_mutex.c | 31 +++++++++++++++++++------------ 8 files changed, 28 insertions(+), 21 deletions(-) diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index 6b3c106a97..b97705ed96 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -303,7 +303,7 @@ init_atom_table(void) HashFunctions f; int i; Atom a; - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index b0369a402b..52d5f86ee0 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -265,7 +265,7 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, char *rwname, char* fixname) { #ifdef ERTS_SMP - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; if (use_frequent_read_lock) rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; #endif @@ -2746,7 +2746,7 @@ void init_db(void) size_t size; #ifdef ERTS_SMP - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 5abd2e50fa..fa707f4eed 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -623,7 +623,7 @@ int db_create_hash(Process *p, DbTable *tbl) erts_smp_atomic_init(&tb->is_resizing, 0); #ifdef ERTS_SMP if (tb->common.type & DB_FINE_LOCKED) { - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; int i; if (tb->common.type & DB_FREQ_READ) rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 5dce5ad262..84869f12d6 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -55,7 +55,7 @@ void erts_init_fun_table(void) { HashFunctions f; - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index e430b4ad77..d0b08bf72e 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -80,7 +80,7 @@ dist_table_alloc(void *dep_tmpl) Eterm chnl_nr; Eterm sysname; DistEntry *dep; - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; if(((DistEntry *) dep_tmpl) == erts_this_dist_entry) @@ -710,7 +710,7 @@ erts_set_this_node(Eterm sysname, Uint creation) void erts_init_node_tables(void) { - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; HashFunctions f; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 5e81a2d624..5bc402fe22 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -109,7 +109,7 @@ void init_export_table(void) { HashFunctions f; - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index c9bb7bbe91..26d64887d0 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -145,7 +145,7 @@ static void reg_free(RegProc *obj) void init_register_table(void) { HashFunctions f; - erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c index aac0d44a32..ccceb61714 100644 --- a/erts/lib_src/common/ethr_mutex.c +++ b/erts/lib_src/common/ethr_mutex.c @@ -35,7 +35,7 @@ #define ETHR_SPIN_WITH_WAITERS 1 -#define ETHR_MTX_MAX_FLGS_SPIN 1000 +#define ETHR_MTX_MAX_FLGS_SPIN 10 #ifdef ETHR_USE_OWN_RWMTX_IMPL__ static int default_rwmtx_main_spincount; @@ -593,24 +593,31 @@ initial_spincount(struct ethr_mutex_base_ *mtxb) static ETHR_INLINE int update_spincount(struct ethr_mutex_base_ *mtxb, ethr_ts_event *tse, - int *start_scnt, + int *scnt_state, int *scnt) { - int sscnt = *start_scnt; - if (sscnt < 0) { - *scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR) - ? mtxb->main_scnt - : mtxb->aux_scnt); - *scnt -= ETHR_MTX_MAX_FLGS_SPIN; + int state = *scnt_state; + if (state <= 0) { + /* Here state is max spincount to do on event negated */ + *scnt = -state; } else { + /* Here state is initial spincount made on flags */ *scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR) ? mtxb->main_scnt : mtxb->aux_scnt); - *scnt -= sscnt; - if (*scnt > 0 && sscnt < ETHR_MTX_MAX_FLGS_SPIN) { - *scnt = ETHR_MTX_MAX_FLGS_SPIN - sscnt; - *start_scnt = -1; + if (*scnt <= state) + *scnt = 0; + else { + if (*scnt <= ETHR_MTX_MAX_FLGS_SPIN) + *scnt_state = 0; /* No spin on event */ + else { + /* Spin on event after... */ + *scnt_state = -1*(*scnt - ETHR_MTX_MAX_FLGS_SPIN); + /* ... we have spun on flags */ + *scnt = ETHR_MTX_MAX_FLGS_SPIN; + } + *scnt -= state; return 0; } } -- cgit v1.2.3 From 4e416cc50d9117aaaf46789259489d8cc4440a33 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 7 Sep 2010 15:05:47 +0200 Subject: Increase spincount with many schedulers --- erts/include/internal/ethr_mutex.h | 8 ++++++-- erts/lib_src/common/ethr_mutex.c | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h index 4ce3e75c78..8d9d5e3d08 100644 --- a/erts/include/internal/ethr_mutex.h +++ b/erts/include/internal/ethr_mutex.h @@ -346,7 +346,9 @@ do { \ #ifdef ETHR_USE_OWN_MTX_IMPL__ -#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT 1000 +#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000 +#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE 800 +#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC 50 #define ETHR_MTX_DEFAULT_AUX_SPINCOUNT 50 #define ETHR_CND_DEFAULT_MAIN_SPINCOUNT 0 @@ -443,7 +445,9 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) #ifdef ETHR_USE_OWN_RWMTX_IMPL__ -#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT 1000 +#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000 +#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE 800 +#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC 50 #define ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT 50 #else /* pthread_rwlock */ diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c index ccceb61714..4a588dd705 100644 --- a/erts/lib_src/common/ethr_mutex.c +++ b/erts/lib_src/common/ethr_mutex.c @@ -90,7 +90,7 @@ ethr_mutex_lib_init(int cpu_conf) no_spin = cpu_conf == 1; #ifdef ETHR_USE_OWN_MTX_IMPL__ - default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT; + default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE; default_mtx_aux_spincount = ETHR_MTX_DEFAULT_AUX_SPINCOUNT; default_cnd_main_spincount = ETHR_CND_DEFAULT_MAIN_SPINCOUNT; default_cnd_aux_spincount = ETHR_CND_DEFAULT_AUX_SPINCOUNT; @@ -98,7 +98,7 @@ ethr_mutex_lib_init(int cpu_conf) #ifdef ETHR_USE_OWN_RWMTX_IMPL__ - default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT; + default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE; default_rwmtx_aux_spincount = ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT; #else @@ -146,7 +146,21 @@ static int main_threads_array_size = 0; int ethr_mutex_lib_late_init(int no_reader_groups, int no_main_threads) { + +#ifdef ETHR_USE_OWN_MTX_IMPL__ + default_mtx_main_spincount += (no_main_threads + * ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC); + if (default_mtx_main_spincount > ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX) + default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX; +#endif + #ifdef ETHR_USE_OWN_RWMTX_IMPL__ + + default_rwmtx_main_spincount += (no_main_threads + * ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC); + if (default_rwmtx_main_spincount > ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX) + default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX; + reader_groups_array_size = (no_reader_groups <= 1 ? 1 : no_reader_groups + 1); -- cgit v1.2.3 From e609bf8e5839330f6c52e548541d58ce4da7f48a Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Tue, 7 Sep 2010 15:19:56 +0200 Subject: Remove unused variables --- erts/lib_src/common/ethr_mutex.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c index 4a588dd705..284f85da31 100644 --- a/erts/lib_src/common/ethr_mutex.c +++ b/erts/lib_src/common/ethr_mutex.c @@ -1006,7 +1006,6 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end) int ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt) { - int res; #if ETHR_XCHK if (!cnd) { ETHR_ASSERT(0); @@ -1106,7 +1105,6 @@ ethr_cond_signal(ethr_cond *cnd) void ethr_cond_broadcast(ethr_cond *cnd) { - int res; int got_all; ethr_ts_event *tse; ETHR_ASSERT(!ethr_not_inited__); @@ -1174,7 +1172,6 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) { int woken; int scnt; - int res; void *udata = NULL; ethr_ts_event *tse; -- cgit v1.2.3 From 775365a86f69bd4a9f8db44b7017662854415540 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 8 Sep 2010 22:07:29 +0200 Subject: Fix deadlock in reader optimized rwlock implementation A bug causing a deadlock in the reader optimized rwlock implementation has been fixed. This bug appeared in commit 59ee2a593090e7d53c97ceba63cbd300d1b9657e, i.e., it has not been seen in any released versions. --- erts/lib_src/common/ethr_mutex.c | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c index 284f85da31..78323b62a3 100644 --- a/erts/lib_src/common/ethr_mutex.c +++ b/erts/lib_src/common/ethr_mutex.c @@ -640,8 +640,7 @@ update_spincount(struct ethr_mutex_base_ *mtxb, int check_readers_array(ethr_rwmutex *rwmtx, int start_rix, - int length, - int pre_check); + int length); static ETHR_INLINE void write_lock_wait(struct ethr_mutex_base_ *mtxb, @@ -687,8 +686,7 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb, } res = check_readers_array(rwmtx, freq_read_start_ix, - freq_read_size, - 1); + freq_read_size); scnt--; if (res == 0) { act = ethr_atomic_read(&mtxb->flgs); @@ -729,7 +727,6 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb, act = ethr_atomic_read(&mtxb->flgs); scnt--; } - ETHR_ASSERT(scnt >= 0); exp = act; @@ -1484,17 +1481,11 @@ multiple_w_waiters(ethr_rwmutex *rwmtx) int check_readers_array(ethr_rwmutex *rwmtx, int start_rix, - int length, - int pre_check) + int length) { int ix = start_rix; -#ifndef ETHR_READ_MEMORY_BARRIER_IS_FULL - if (pre_check) - ETHR_READ_MEMORY_BARRIER; - else -#endif - ETHR_MEMORY_BARRIER; + ETHR_MEMORY_BARRIER; do { long act = rwmutex_freqread_rdrs_read(rwmtx, ix); @@ -1577,7 +1568,7 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, ethr_leave_ts_event(tse_tmp); if (check_before_try) { - res = check_readers_array(rwmtx, six, length, 1); + res = check_readers_array(rwmtx, six, length); if (res == EBUSY) return try_write_lock ? EBUSY : 0; } @@ -1615,7 +1606,7 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, } } - res = check_readers_array(rwmtx, six, length, 0); + res = check_readers_array(rwmtx, six, length); if (res == EBUSY) { act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs); if (act & ETHR_RWMTX_R_MASK__) @@ -1709,7 +1700,7 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, #endif while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { - if (scnt == 0) { + if (scnt >= 0) { tse = ethr_get_ts_event(); if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) { event_wait(&rwmtx->mtxb, tse, scnt, @@ -1726,7 +1717,6 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, scnt--; } exp = act; - ETHR_ASSERT(scnt >= 0); #ifdef ETHR_RLOCK_WITH_INC_DEC act = ethr_atomic_inc_read(&rwmtx->mtxb.flgs); @@ -1767,7 +1757,7 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx, act = ethr_atomic_read(&rwmtx->mtxb.flgs); while (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) { - if (scnt == 0) { + if (scnt >= 0) { if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) { event_wait(&rwmtx->mtxb, tse, scnt, ETHR_RWMTX_R_WAIT_FLG__, 1, 1); @@ -1783,8 +1773,6 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx, scnt--; } - ETHR_ASSERT(scnt >= 0); - rwmutex_freqread_rdrs_inc(rwmtx, tse); ETHR_MEMORY_BARRIER; -- cgit v1.2.3