aboutsummaryrefslogtreecommitdiffstats
path: root/erts
diff options
context:
space:
mode:
Diffstat (limited to 'erts')
-rw-r--r--erts/doc/src/erlang.xml2
-rw-r--r--erts/emulator/beam/atom.c2
-rw-r--r--erts/emulator/beam/erl_db.c4
-rw-r--r--erts/emulator/beam/erl_db_hash.c2
-rw-r--r--erts/emulator/beam/erl_fun.c2
-rw-r--r--erts/emulator/beam/erl_node_tables.c4
-rw-r--r--erts/emulator/beam/export.c2
-rw-r--r--erts/emulator/beam/register.c2
-rw-r--r--erts/include/internal/ethr_mutex.h8
-rw-r--r--erts/include/internal/gcc/ethr_atomic.h9
-rw-r--r--erts/lib_src/common/ethr_mutex.c80
11 files changed, 66 insertions, 51 deletions
diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml
index e197791ca9..ba93fe34d3 100644
--- a/erts/doc/src/erlang.xml
+++ b/erts/doc/src/erlang.xml
@@ -147,7 +147,7 @@ iolist() = [char() | binary() | iolist()]
<c>Tuple1</c>, and contains the elements in <c>Tuple1</c>
followed by <c>Term</c> as the last element. Semantically
equivalent to
- <c>list_to_tuple(tuple_to_list(Tuple ++ [Term])</c>, but much
+ <c>list_to_tuple(tuple_to_list(Tuple) ++ [Term])</c>, but much
faster.</p>
<pre>
> <input>erlang:append_element({one, two}, three).</input>
diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index 6b3c106a97..b97705ed96 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -303,7 +303,7 @@ init_atom_table(void)
HashFunctions f;
int i;
Atom a;
- erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+ erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index b0369a402b..52d5f86ee0 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -265,7 +265,7 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock,
char *rwname, char* fixname)
{
#ifdef ERTS_SMP
- erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+ erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
if (use_frequent_read_lock)
rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
#endif
@@ -2746,7 +2746,7 @@ void init_db(void)
size_t size;
#ifdef ERTS_SMP
- erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+ erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 5abd2e50fa..fa707f4eed 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -623,7 +623,7 @@ int db_create_hash(Process *p, DbTable *tbl)
erts_smp_atomic_init(&tb->is_resizing, 0);
#ifdef ERTS_SMP
if (tb->common.type & DB_FINE_LOCKED) {
- erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+ erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
int i;
if (tb->common.type & DB_FREQ_READ)
rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c
index 5dce5ad262..84869f12d6 100644
--- a/erts/emulator/beam/erl_fun.c
+++ b/erts/emulator/beam/erl_fun.c
@@ -55,7 +55,7 @@ void
erts_init_fun_table(void)
{
HashFunctions f;
- erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+ erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c
index e430b4ad77..d0b08bf72e 100644
--- a/erts/emulator/beam/erl_node_tables.c
+++ b/erts/emulator/beam/erl_node_tables.c
@@ -80,7 +80,7 @@ dist_table_alloc(void *dep_tmpl)
Eterm chnl_nr;
Eterm sysname;
DistEntry *dep;
- erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+ erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
if(((DistEntry *) dep_tmpl) == erts_this_dist_entry)
@@ -710,7 +710,7 @@ erts_set_this_node(Eterm sysname, Uint creation)
void erts_init_node_tables(void)
{
- erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+ erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
HashFunctions f;
rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 5e81a2d624..5bc402fe22 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -109,7 +109,7 @@ void
init_export_table(void)
{
HashFunctions f;
- erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+ erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c
index c9bb7bbe91..26d64887d0 100644
--- a/erts/emulator/beam/register.c
+++ b/erts/emulator/beam/register.c
@@ -145,7 +145,7 @@ static void reg_free(RegProc *obj)
void init_register_table(void)
{
HashFunctions f;
- erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_THR_OPTS_DEFAULT_INITER;
+ erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER;
rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ;
rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED;
diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h
index 4ce3e75c78..8d9d5e3d08 100644
--- a/erts/include/internal/ethr_mutex.h
+++ b/erts/include/internal/ethr_mutex.h
@@ -346,7 +346,9 @@ do { \
#ifdef ETHR_USE_OWN_MTX_IMPL__
-#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT 1000
+#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000
+#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE 800
+#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC 50
#define ETHR_MTX_DEFAULT_AUX_SPINCOUNT 50
#define ETHR_CND_DEFAULT_MAIN_SPINCOUNT 0
@@ -443,7 +445,9 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx)
#ifdef ETHR_USE_OWN_RWMTX_IMPL__
-#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT 1000
+#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000
+#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE 800
+#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC 50
#define ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT 50
#else /* pthread_rwlock */
diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h
index 5fe6e23477..e8e529dd48 100644
--- a/erts/include/internal/gcc/ethr_atomic.h
+++ b/erts/include/internal/gcc/ethr_atomic.h
@@ -31,7 +31,7 @@
#define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0
/* Enable immediate read/write on platforms where we know it is safe */
#if defined(__i386__) || defined(__x86_64__) || defined(__sparc__) \
- || defined(__powerpc__) || defined(__ppc__)
+ || defined(__powerpc__) || defined(__ppc__) || defined(__mips__)
# undef ETHR_IMMED_ATOMIC_SET_GET_SAFE__
# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1
#endif
@@ -48,13 +48,18 @@ typedef struct {
* a noop on at least some platforms with some gcc versions.
* This has suposedly been fixed in some gcc version, but we
* don't know from which version. Therefore, we use the
- * workaround implemented below on all gcc versions.
+ * workaround implemented below on all gcc versions except
+ * for gcc 4.2 or above for MIPS, where it's been verified.
*/
+#if defined(__mips__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2))
+#define ETHR_MEMORY_BARRIER __sync_synchronize()
+#else
#define ETHR_MEMORY_BARRIER \
do { \
volatile long x___ = 0; \
(void) __sync_val_compare_and_swap(&x___, (long) 0, (long) 1); \
} while (0)
+#endif
#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMORY_BARRIER
#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__)
diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c
index aac0d44a32..78323b62a3 100644
--- a/erts/lib_src/common/ethr_mutex.c
+++ b/erts/lib_src/common/ethr_mutex.c
@@ -35,7 +35,7 @@
#define ETHR_SPIN_WITH_WAITERS 1
-#define ETHR_MTX_MAX_FLGS_SPIN 1000
+#define ETHR_MTX_MAX_FLGS_SPIN 10
#ifdef ETHR_USE_OWN_RWMTX_IMPL__
static int default_rwmtx_main_spincount;
@@ -90,7 +90,7 @@ ethr_mutex_lib_init(int cpu_conf)
no_spin = cpu_conf == 1;
#ifdef ETHR_USE_OWN_MTX_IMPL__
- default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT;
+ default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE;
default_mtx_aux_spincount = ETHR_MTX_DEFAULT_AUX_SPINCOUNT;
default_cnd_main_spincount = ETHR_CND_DEFAULT_MAIN_SPINCOUNT;
default_cnd_aux_spincount = ETHR_CND_DEFAULT_AUX_SPINCOUNT;
@@ -98,7 +98,7 @@ ethr_mutex_lib_init(int cpu_conf)
#ifdef ETHR_USE_OWN_RWMTX_IMPL__
- default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT;
+ default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE;
default_rwmtx_aux_spincount = ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT;
#else
@@ -146,7 +146,21 @@ static int main_threads_array_size = 0;
int
ethr_mutex_lib_late_init(int no_reader_groups, int no_main_threads)
{
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+ default_mtx_main_spincount += (no_main_threads
+ * ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC);
+ if (default_mtx_main_spincount > ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX)
+ default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX;
+#endif
+
#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+ default_rwmtx_main_spincount += (no_main_threads
+ * ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC);
+ if (default_rwmtx_main_spincount > ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX)
+ default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX;
+
reader_groups_array_size = (no_reader_groups <= 1
? 1
: no_reader_groups + 1);
@@ -593,24 +607,31 @@ initial_spincount(struct ethr_mutex_base_ *mtxb)
static ETHR_INLINE int
update_spincount(struct ethr_mutex_base_ *mtxb,
ethr_ts_event *tse,
- int *start_scnt,
+ int *scnt_state,
int *scnt)
{
- int sscnt = *start_scnt;
- if (sscnt < 0) {
- *scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
- ? mtxb->main_scnt
- : mtxb->aux_scnt);
- *scnt -= ETHR_MTX_MAX_FLGS_SPIN;
+ int state = *scnt_state;
+ if (state <= 0) {
+ /* Here state is max spincount to do on event negated */
+ *scnt = -state;
}
else {
+ /* Here state is initial spincount made on flags */
*scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
? mtxb->main_scnt
: mtxb->aux_scnt);
- *scnt -= sscnt;
- if (*scnt > 0 && sscnt < ETHR_MTX_MAX_FLGS_SPIN) {
- *scnt = ETHR_MTX_MAX_FLGS_SPIN - sscnt;
- *start_scnt = -1;
+ if (*scnt <= state)
+ *scnt = 0;
+ else {
+ if (*scnt <= ETHR_MTX_MAX_FLGS_SPIN)
+ *scnt_state = 0; /* No spin on event */
+ else {
+ /* Spin on event after... */
+ *scnt_state = -1*(*scnt - ETHR_MTX_MAX_FLGS_SPIN);
+ /* ... we have spun on flags */
+ *scnt = ETHR_MTX_MAX_FLGS_SPIN;
+ }
+ *scnt -= state;
return 0;
}
}
@@ -619,8 +640,7 @@ update_spincount(struct ethr_mutex_base_ *mtxb,
int check_readers_array(ethr_rwmutex *rwmtx,
int start_rix,
- int length,
- int pre_check);
+ int length);
static ETHR_INLINE void
write_lock_wait(struct ethr_mutex_base_ *mtxb,
@@ -666,8 +686,7 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb,
}
res = check_readers_array(rwmtx,
freq_read_start_ix,
- freq_read_size,
- 1);
+ freq_read_size);
scnt--;
if (res == 0) {
act = ethr_atomic_read(&mtxb->flgs);
@@ -708,7 +727,6 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb,
act = ethr_atomic_read(&mtxb->flgs);
scnt--;
}
- ETHR_ASSERT(scnt >= 0);
exp = act;
@@ -985,7 +1003,6 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
int
ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt)
{
- int res;
#if ETHR_XCHK
if (!cnd) {
ETHR_ASSERT(0);
@@ -1085,7 +1102,6 @@ ethr_cond_signal(ethr_cond *cnd)
void
ethr_cond_broadcast(ethr_cond *cnd)
{
- int res;
int got_all;
ethr_ts_event *tse;
ETHR_ASSERT(!ethr_not_inited__);
@@ -1153,7 +1169,6 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
{
int woken;
int scnt;
- int res;
void *udata = NULL;
ethr_ts_event *tse;
@@ -1466,17 +1481,11 @@ multiple_w_waiters(ethr_rwmutex *rwmtx)
int check_readers_array(ethr_rwmutex *rwmtx,
int start_rix,
- int length,
- int pre_check)
+ int length)
{
int ix = start_rix;
-#ifndef ETHR_READ_MEMORY_BARRIER_IS_FULL
- if (pre_check)
- ETHR_READ_MEMORY_BARRIER;
- else
-#endif
- ETHR_MEMORY_BARRIER;
+ ETHR_MEMORY_BARRIER;
do {
long act = rwmutex_freqread_rdrs_read(rwmtx, ix);
@@ -1559,7 +1568,7 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
ethr_leave_ts_event(tse_tmp);
if (check_before_try) {
- res = check_readers_array(rwmtx, six, length, 1);
+ res = check_readers_array(rwmtx, six, length);
if (res == EBUSY)
return try_write_lock ? EBUSY : 0;
}
@@ -1597,7 +1606,7 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
}
}
- res = check_readers_array(rwmtx, six, length, 0);
+ res = check_readers_array(rwmtx, six, length);
if (res == EBUSY) {
act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs);
if (act & ETHR_RWMTX_R_MASK__)
@@ -1691,7 +1700,7 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
#endif
while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
- if (scnt == 0) {
+ if (scnt >= 0) {
tse = ethr_get_ts_event();
if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
event_wait(&rwmtx->mtxb, tse, scnt,
@@ -1708,7 +1717,6 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
scnt--;
}
exp = act;
- ETHR_ASSERT(scnt >= 0);
#ifdef ETHR_RLOCK_WITH_INC_DEC
act = ethr_atomic_inc_read(&rwmtx->mtxb.flgs);
@@ -1749,7 +1757,7 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
act = ethr_atomic_read(&rwmtx->mtxb.flgs);
while (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
- if (scnt == 0) {
+ if (scnt >= 0) {
if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
event_wait(&rwmtx->mtxb, tse, scnt,
ETHR_RWMTX_R_WAIT_FLG__, 1, 1);
@@ -1765,8 +1773,6 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
scnt--;
}
- ETHR_ASSERT(scnt >= 0);
-
rwmutex_freqread_rdrs_inc(rwmtx, tse);
ETHR_MEMORY_BARRIER;
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-11 12:16:28 +0000