aboutsummaryrefslogtreecommitdiffstats
path: root/erts/lib_src/common/ethr_mutex.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/lib_src/common/ethr_mutex.c')
-rw-r--r--erts/lib_src/common/ethr_mutex.c1010
1 files changed, 582 insertions, 428 deletions
diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c
index aac0d44a32..2ddef32dfc 100644
--- a/erts/lib_src/common/ethr_mutex.c
+++ b/erts/lib_src/common/ethr_mutex.c
@@ -35,7 +35,7 @@
#define ETHR_SPIN_WITH_WAITERS 1
-#define ETHR_MTX_MAX_FLGS_SPIN 1000
+#define ETHR_MTX_MAX_FLGS_SPIN 10
#ifdef ETHR_USE_OWN_RWMTX_IMPL__
static int default_rwmtx_main_spincount;
@@ -90,7 +90,7 @@ ethr_mutex_lib_init(int cpu_conf)
no_spin = cpu_conf == 1;
#ifdef ETHR_USE_OWN_MTX_IMPL__
- default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT;
+ default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE;
default_mtx_aux_spincount = ETHR_MTX_DEFAULT_AUX_SPINCOUNT;
default_cnd_main_spincount = ETHR_CND_DEFAULT_MAIN_SPINCOUNT;
default_cnd_aux_spincount = ETHR_CND_DEFAULT_AUX_SPINCOUNT;
@@ -98,7 +98,7 @@ ethr_mutex_lib_init(int cpu_conf)
#ifdef ETHR_USE_OWN_RWMTX_IMPL__
- default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT;
+ default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE;
default_rwmtx_aux_spincount = ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT;
#else
@@ -146,7 +146,21 @@ static int main_threads_array_size = 0;
int
ethr_mutex_lib_late_init(int no_reader_groups, int no_main_threads)
{
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+ default_mtx_main_spincount += (no_main_threads
+ * ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC);
+ if (default_mtx_main_spincount > ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX)
+ default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX;
+#endif
+
#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+ default_rwmtx_main_spincount += (no_main_threads
+ * ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC);
+ if (default_rwmtx_main_spincount > ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX)
+ default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX;
+
reader_groups_array_size = (no_reader_groups <= 1
? 1
: no_reader_groups + 1);
@@ -191,12 +205,17 @@ static void hard_debug_chk_q__(struct ethr_mutex_base_ *, int);
#ifdef ETHR_USE_OWN_RWMTX_IMPL__
static void
+rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx,
+ ethr_sint32_t initial,
+ int q_locked);
+static void
rwmutex_unlock_wake(ethr_rwmutex *rwmtx,
int have_w,
- long initial);
+ ethr_sint32_t initial,
+ int transfer_read_lock);
static int
rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
- long initial,
+ ethr_sint32_t initial,
ethr_ts_event *tse,
int start_next_ix,
int check_before_try,
@@ -223,12 +242,12 @@ rwmutex_freqread_rdrs_add(ethr_rwmutex *rwmtx,
int inc)
{
if (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ || ix == 0)
- ethr_atomic_add(&rwmtx->tdata.ra[ix].data.readers, inc);
+ ethr_atomic32_add(&rwmtx->tdata.ra[ix].data.readers, inc);
else {
ETHR_ASSERT(type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
- ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
+ ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
ETHR_ASSERT(inc == 1);
- ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 1);
+ ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 1);
}
}
@@ -239,18 +258,20 @@ rwmutex_freqread_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
ix = tse->rgix;
atomic_inc:
- ethr_atomic_inc(&rwmtx->tdata.ra[ix].data.readers);
+ ethr_atomic32_inc(&rwmtx->tdata.ra[ix].data.readers);
}
else {
ix = tse->mtix;
if (ix == 0)
goto atomic_inc;
ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
- ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
- ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 1);
+ ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
+ ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 1);
}
}
+#if 0 /* Not used */
+
static ETHR_INLINE void
rwmutex_freqread_rdrs_dec(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
{
@@ -258,69 +279,72 @@ rwmutex_freqread_rdrs_dec(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
ix = tse->rgix;
atomic_dec:
- ethr_atomic_dec(&rwmtx->tdata.ra[ix].data.readers);
+ ethr_atomic32_dec(&rwmtx->tdata.ra[ix].data.readers);
}
else {
ix = tse->mtix;
if (ix == 0)
goto atomic_dec;
ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
- ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
- ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 0);
+ ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+ ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 0);
}
}
-static ETHR_INLINE long
+#endif
+
+static ETHR_INLINE ethr_sint32_t
rwmutex_freqread_rdrs_dec_read(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
{
int ix;
if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
ix = tse->rgix;
atomic_dec_read:
- return ethr_atomic_dec_read(&rwmtx->tdata.ra[ix].data.readers);
+ return ethr_atomic32_dec_read(&rwmtx->tdata.ra[ix].data.readers);
}
else {
ix = tse->mtix;
if (ix == 0)
goto atomic_dec_read;
ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
- ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
- ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 0);
- return (long) 0;
+ ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+ ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 0);
+ return (ethr_sint32_t) 0;
}
}
-static ETHR_INLINE long
+static ETHR_INLINE ethr_sint32_t
rwmutex_freqread_rdrs_dec_read_relb(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
{
int ix;
if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
ix = tse->rgix;
atomic_dec_read:
- return ethr_atomic_dec_read_relb(&rwmtx->tdata.ra[ix].data.readers);
+ return ethr_atomic32_dec_read_relb(&rwmtx->tdata.ra[ix].data.readers);
}
else {
ix = tse->mtix;
if (ix == 0)
goto atomic_dec_read;
ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
- ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
- ethr_atomic_set_relb(&rwmtx->tdata.ra[ix].data.readers, (long) 0);
- return (long) 0;
+ ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+ ethr_atomic32_set_relb(&rwmtx->tdata.ra[ix].data.readers,
+ (ethr_sint32_t) 0);
+ return (ethr_sint32_t) 0;
}
}
-static ETHR_INLINE long
+static ETHR_INLINE ethr_sint32_t
rwmutex_freqread_rdrs_read(ethr_rwmutex *rwmtx, int ix)
{
- long res = ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers);
+ ethr_sint32_t res = ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers);
#ifdef ETHR_DEBUG
switch (rwmtx->type) {
case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
ETHR_ASSERT(res >= 0);
break;
case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
- ETHR_ASSERT(res == 0 || res == 1);
+ ETHR_ASSERT(ix == 0 ? res >= 0 : (res == 0 || res == 1));
break;
default:
ETHR_ASSERT(0);
@@ -379,18 +403,19 @@ static void
event_wait(struct ethr_mutex_base_ *mtxb,
ethr_ts_event *tse,
int spincount,
- long type,
+ ethr_sint32_t type,
int is_rwmtx,
int is_freq_read)
{
int locked = 0;
- long act;
+ ethr_sint32_t act;
int need_try_complete_runlock = 0;
+ int transfer_read_lock = 0;
/* Need to enqueue and wait... */
tse->uflgs = type;
- ethr_atomic_set(&tse->uaflgs, type);
+ ethr_atomic32_set(&tse->uaflgs, type);
ETHR_MTX_Q_LOCK(&mtxb->qlck);
locked = 1;
@@ -399,7 +424,7 @@ event_wait(struct ethr_mutex_base_ *mtxb,
hard_debug_chk_q__(mtxb, is_rwmtx);
#endif
- act = ethr_atomic_read(&mtxb->flgs);
+ act = ethr_atomic32_read(&mtxb->flgs);
if (act & type) {
@@ -429,9 +454,9 @@ event_wait(struct ethr_mutex_base_ *mtxb,
/* Set wait bit */
while (1) {
- long new, exp = act;
- int freqread_tryrlock = 0;
+ ethr_sint32_t new, exp = act;
need_try_complete_runlock = 0;
+ transfer_read_lock = 0;
if (type == ETHR_RWMTX_W_WAIT_FLG__) {
if (is_freq_read && act == ETHR_RWMTX_R_FLG__)
@@ -451,19 +476,16 @@ event_wait(struct ethr_mutex_base_ *mtxb,
new = act + 1; /* Try to get it */
}
else {
- if (act & ~ETHR_RWMTX_R_FLG__)
- new = act | ETHR_RWMTX_R_WAIT_FLG__;
- else { /* Try to get it */
- ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
- rwmutex_freqread_rdrs_inc(rwmtx, tse);
- ETHR_MEMORY_BARRIER;
- new = act | ETHR_RWMTX_R_FLG__;
- freqread_tryrlock = 1;
+ new = act | ETHR_RWMTX_R_WAIT_FLG__;
+ if ((act & (ETHR_RWMTX_W_FLG__
+ | ETHR_RWMTX_W_WAIT_FLG__)) == 0) {
+ /* Transfer read lock to this thread. */
+ transfer_read_lock = 1;
}
}
}
- act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs, new, exp);
+ act = ethr_atomic32_cmpxchg_acqb(&mtxb->flgs, new, exp);
if (exp == act) {
if (new & type) {
act = new;
@@ -474,24 +496,6 @@ event_wait(struct ethr_mutex_base_ *mtxb,
goto done;
}
}
-
- if (freqread_tryrlock) {
- ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
-
- /* We didn't set ETHR_RWMTX_R_FLG__, however someone
- else might have */
- if (act == ETHR_RWMTX_R_FLG__)
- goto done; /* Got it by help from someone else */
-
- ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
- /*
- * We know that no waiter flags have been set, i.e.,
- * we cannot get into a situation where we need to wake
- * someone up here. Just restore the readers counter
- * and do it over again...
- */
- rwmutex_freqread_rdrs_dec(rwmtx, tse);
- }
}
/* Enqueue */
@@ -521,26 +525,42 @@ event_wait(struct ethr_mutex_base_ *mtxb,
/* Wait */
locked = 0;
- ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
- if (need_try_complete_runlock) {
+ ETHR_ASSERT(!(transfer_read_lock && need_try_complete_runlock));
+
+ if (transfer_read_lock) {
ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type
!= ETHR_RWMUTEX_TYPE_NORMAL);
/*
- * We were the only one in queue when we enqueued, and it
- * was seemingly read locked. We need to try to complete a
- * runlock otherwise we might be hanging forever. If the
- * runlock could be completed we will be dequeued and
- * woken by ourselves.
+ * We are the only one in the queue and we are not write
+ * locked; rwmutex_transfer_read_lock() will:
+ * - transfer a read lock to us (since we're first in q)
+ * - unlock the Q-lock
*/
- rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb,
- act, tse, 0, 1, 0);
+ rwmutex_transfer_read_lock(((ethr_rwmutex *) mtxb), act, 1);
+ }
+ else {
+ ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+
+ if (need_try_complete_runlock) {
+ ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type
+ != ETHR_RWMUTEX_TYPE_NORMAL);
+ /*
+ * We were the only one in queue when we enqueued, and it
+ * was seemingly read locked. We need to try to complete a
+ * runlock otherwise we might be hanging forever. If the
+ * runlock could be completed we will be dequeued and
+ * woken by ourselves.
+ */
+ rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb,
+ act, tse, 0, 1, 0);
+ }
}
while (1) {
ethr_event_reset(&tse->event);
- act = ethr_atomic_read_acqb(&tse->uaflgs);
+ act = ethr_atomic32_read_acqb(&tse->uaflgs);
if (!act)
goto done; /* Got it */
@@ -548,7 +568,7 @@ event_wait(struct ethr_mutex_base_ *mtxb,
ethr_event_swait(&tse->event, spincount);
/* swait result: 0 || EINTR */
- act = ethr_atomic_read_acqb(&tse->uaflgs);
+ act = ethr_atomic32_read_acqb(&tse->uaflgs);
if (!act)
goto done; /* Got it */
}
@@ -568,7 +588,7 @@ wake_writer(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
dequeue(&mtxb->q, tse, tse);
ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
- ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__);
+ ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__);
#ifdef ETHR_MTX_HARD_DEBUG_WSQ
mtxb->ws--;
#endif
@@ -578,7 +598,7 @@ wake_writer(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
- ethr_atomic_set(&tse->uaflgs, 0);
+ ethr_atomic32_set(&tse->uaflgs, 0);
ethr_event_set(&tse->event);
}
@@ -593,24 +613,31 @@ initial_spincount(struct ethr_mutex_base_ *mtxb)
static ETHR_INLINE int
update_spincount(struct ethr_mutex_base_ *mtxb,
ethr_ts_event *tse,
- int *start_scnt,
+ int *scnt_state,
int *scnt)
{
- int sscnt = *start_scnt;
- if (sscnt < 0) {
- *scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
- ? mtxb->main_scnt
- : mtxb->aux_scnt);
- *scnt -= ETHR_MTX_MAX_FLGS_SPIN;
+ int state = *scnt_state;
+ if (state <= 0) {
+ /* Here state is max spincount to do on event negated */
+ *scnt = -state;
}
else {
+ /* Here state is initial spincount made on flags */
*scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
? mtxb->main_scnt
: mtxb->aux_scnt);
- *scnt -= sscnt;
- if (*scnt > 0 && sscnt < ETHR_MTX_MAX_FLGS_SPIN) {
- *scnt = ETHR_MTX_MAX_FLGS_SPIN - sscnt;
- *start_scnt = -1;
+ if (*scnt <= state)
+ *scnt = 0;
+ else {
+ if (*scnt <= ETHR_MTX_MAX_FLGS_SPIN)
+ *scnt_state = 0; /* No spin on event */
+ else {
+ /* Spin on event after... */
+ *scnt_state = -1*(*scnt - ETHR_MTX_MAX_FLGS_SPIN);
+ /* ... we have spun on flags */
+ *scnt = ETHR_MTX_MAX_FLGS_SPIN;
+ }
+ *scnt -= state;
return 0;
}
}
@@ -619,22 +646,19 @@ update_spincount(struct ethr_mutex_base_ *mtxb,
int check_readers_array(ethr_rwmutex *rwmtx,
int start_rix,
- int length,
- int pre_check);
+ int length);
static ETHR_INLINE void
write_lock_wait(struct ethr_mutex_base_ *mtxb,
- long initial,
+ ethr_sint32_t initial,
int is_rwmtx,
int is_freq_read)
{
- long act = initial;
+ ethr_sint32_t act = initial;
int scnt, start_scnt;
ethr_ts_event *tse = NULL;
int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
int res;
- int freq_read_size = -1;
- int freq_read_start_ix = -1;
ETHR_ASSERT(!is_freq_read || is_rwmtx);
@@ -646,45 +670,23 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb,
*/
while (1) {
- long exp;
-
while (act != 0) {
if (is_freq_read && act == ETHR_RWMTX_R_FLG__) {
ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+ scnt--;
if (!tse)
tse = ethr_get_ts_event();
- if (freq_read_size < 0) {
- if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
- freq_read_size = reader_groups_array_size;
- freq_read_start_ix = tse->rgix;
- }
- else {
- freq_read_size = main_threads_array_size;
- freq_read_start_ix = tse->mtix;
- }
- }
- res = check_readers_array(rwmtx,
- freq_read_start_ix,
- freq_read_size,
- 1);
- scnt--;
- if (res == 0) {
- act = ethr_atomic_read(&mtxb->flgs);
- if (act & ETHR_RWMTX_R_MASK__) {
- res = rwmutex_try_complete_runlock(rwmtx, act,
- tse, 0, 0,
- 1);
- if (res != EBUSY)
- goto done; /* Got it */
- }
- if (scnt <= 0)
- goto chk_spin;
- if (--until_yield == 0) {
- until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
- ETHR_YIELD();
- }
- continue;
+ res = rwmutex_try_complete_runlock(rwmtx, act,
+ tse, 0, 0,
+ 1);
+ if (res != EBUSY)
+ goto done; /* Got it */
+ if (scnt <= 0)
+ goto chk_spin;
+ if (--until_yield == 0) {
+ until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+ ETHR_YIELD();
}
}
@@ -705,16 +707,13 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb,
until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
ETHR_YIELD();
}
- act = ethr_atomic_read(&mtxb->flgs);
+ act = ethr_atomic32_read(&mtxb->flgs);
scnt--;
}
- ETHR_ASSERT(scnt >= 0);
-
- exp = act;
- act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs,
- ETHR_RWMTX_W_FLG__,
- exp);
+ act = ethr_atomic32_cmpxchg_acqb(&mtxb->flgs,
+ ETHR_RWMTX_W_FLG__,
+ 0);
if (act == 0)
goto done; /* Got it */
}
@@ -735,6 +734,7 @@ mtxb_init(struct ethr_mutex_base_ *mtxb,
#ifdef ETHR_MTX_HARD_DEBUG_WSQ
mtxb->ws = 0;
#endif
+ ETHR_MTX_CHK_EXCL_INIT(mtxb);
if (no_spin) {
mtxb->main_scnt = 0;
mtxb->aux_scnt = 0;
@@ -757,16 +757,16 @@ mtxb_init(struct ethr_mutex_base_ *mtxb,
}
mtxb->q = NULL;
- ethr_atomic_init(&mtxb->flgs, 0);
+ ethr_atomic32_init(&mtxb->flgs, 0);
return ETHR_MTX_QLOCK_INIT(&mtxb->qlck);
}
static int
mtxb_destroy(struct ethr_mutex_base_ *mtxb)
{
- long act;
+ ethr_sint32_t act;
ETHR_MTX_Q_LOCK(&mtxb->qlck);
- act = ethr_atomic_read(&mtxb->flgs);
+ act = ethr_atomic32_read(&mtxb->flgs);
ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
if (act != 0)
return EINVAL;
@@ -832,13 +832,13 @@ ethr_mutex_destroy(ethr_mutex *mtx)
}
void
-ethr_mutex_lock_wait__(ethr_mutex *mtx, long initial)
+ethr_mutex_lock_wait__(ethr_mutex *mtx, ethr_sint32_t initial)
{
write_lock_wait(&mtx->mtxb, initial, 0, 0);
}
void
-ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial)
+ethr_mutex_unlock_wake__(ethr_mutex *mtx, ethr_sint32_t initial)
{
ethr_ts_event *tse;
@@ -846,7 +846,7 @@ ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial)
tse = mtx->mtxb.q;
ETHR_ASSERT(tse);
- ETHR_ASSERT(ethr_atomic_read(&mtx->mtxb.flgs)
+ ETHR_ASSERT(ethr_atomic32_read(&mtx->mtxb.flgs)
== (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__));
ETHR_ASSERT(initial & ETHR_RWMTX_W_WAIT_FLG__);
ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
@@ -856,7 +856,7 @@ ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial)
* mtxb->flgs; otherwise, we need to clear the write wait bit...
*/
if (tse->next == mtx->mtxb.q)
- ethr_atomic_set(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__);
+ ethr_atomic32_set(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__);
wake_writer(&mtx->mtxb, 0);
}
@@ -866,7 +866,7 @@ ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial)
static void
enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
{
- long act;
+ ethr_sint32_t act;
/*
* `ethr_cond_signal()' and `ethr_cond_broadcast()' end up here. If `mtx'
@@ -895,7 +895,7 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
}
#endif
- act = ethr_atomic_read(&mtx->mtxb.flgs);
+ act = ethr_atomic32_read(&mtx->mtxb.flgs);
ETHR_ASSERT(act == 0
|| act == ETHR_RWMTX_W_FLG__
|| act == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__));
@@ -903,10 +903,10 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
/* The normal sane case */
if (!(act & ETHR_RWMTX_W_WAIT_FLG__)) {
ETHR_ASSERT(!mtx->mtxb.q);
- act = ethr_atomic_cmpxchg(&mtx->mtxb.flgs,
- (ETHR_RWMTX_W_FLG__
- | ETHR_RWMTX_W_WAIT_FLG__),
- ETHR_RWMTX_W_FLG__);
+ act = ethr_atomic32_cmpxchg(&mtx->mtxb.flgs,
+ (ETHR_RWMTX_W_FLG__
+ | ETHR_RWMTX_W_WAIT_FLG__),
+ ETHR_RWMTX_W_FLG__);
if (act != ETHR_RWMTX_W_FLG__) {
/*
* Sigh... this wasn't so sane after all since, the mutex was
@@ -938,14 +938,14 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
multi = tse_start != tse_end;
while (1) {
- long new, exp = act;
+ ethr_sint32_t new, exp = act;
if (multi || (act & ETHR_RWMTX_W_FLG__))
new = ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__;
else
new = ETHR_RWMTX_W_FLG__;
- act = ethr_atomic_cmpxchg(&mtx->mtxb.flgs, new, exp);
+ act = ethr_atomic32_cmpxchg(&mtx->mtxb.flgs, new, exp);
if (exp == act) {
ETHR_ASSERT(!mtx->mtxb.q);
if (act & ETHR_RWMTX_W_FLG__) {
@@ -973,7 +973,7 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
- ethr_atomic_set(&tse_start->uaflgs, 0);
+ ethr_atomic32_set(&tse_start->uaflgs, 0);
ethr_event_set(&tse_start->event);
}
break;
@@ -985,7 +985,6 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
int
ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt)
{
- int res;
#if ETHR_XCHK
if (!cnd) {
ETHR_ASSERT(0);
@@ -1065,9 +1064,9 @@ ethr_cond_signal(ethr_cond *cnd)
ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
- ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) == ETHR_CND_WAIT_FLG__);
+ ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_CND_WAIT_FLG__);
- ethr_atomic_set(&tse->uaflgs, ETHR_RWMTX_W_WAIT_FLG__);
+ ethr_atomic32_set(&tse->uaflgs, ETHR_RWMTX_W_WAIT_FLG__);
dequeue(&cnd->q, tse, tse);
@@ -1085,7 +1084,6 @@ ethr_cond_signal(ethr_cond *cnd)
void
ethr_cond_broadcast(ethr_cond *cnd)
{
- int res;
int got_all;
ethr_ts_event *tse;
ETHR_ASSERT(!ethr_not_inited__);
@@ -1119,10 +1117,11 @@ ethr_cond_broadcast(ethr_cond *cnd)
/* The normal case */
ETHR_ASSERT(tse_tmp->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
- ETHR_ASSERT(ethr_atomic_read(&tse_tmp->uaflgs)
+ ETHR_ASSERT(ethr_atomic32_read(&tse_tmp->uaflgs)
== ETHR_CND_WAIT_FLG__);
- ethr_atomic_set(&tse_tmp->uaflgs, ETHR_RWMTX_W_WAIT_FLG__);
+ ethr_atomic32_set(&tse_tmp->uaflgs,
+ ETHR_RWMTX_W_WAIT_FLG__);
}
else {
/* Should be very unusual */
@@ -1153,7 +1152,6 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
{
int woken;
int scnt;
- int res;
void *udata = NULL;
ethr_ts_event *tse;
@@ -1176,7 +1174,7 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
tse->udata = (void *) mtx;
tse->uflgs = ETHR_RWMTX_W_WAIT_FLG__; /* Prep for mutex lock op */
- ethr_atomic_set(&tse->uaflgs, ETHR_CND_WAIT_FLG__);
+ ethr_atomic32_set(&tse->uaflgs, ETHR_CND_WAIT_FLG__);
ETHR_MTX_Q_LOCK(&cnd->qlck);
@@ -1189,11 +1187,11 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
/* Wait */
woken = 0;
while (1) {
- long act;
+ ethr_sint32_t act;
ethr_event_reset(&tse->event);
- act = ethr_atomic_read_acqb(&tse->uaflgs);
+ act = ethr_atomic32_read_acqb(&tse->uaflgs);
if (!act)
break; /* Mtx locked */
@@ -1209,7 +1207,7 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
*/
if (act == ETHR_CND_WAIT_FLG__) {
ETHR_MTX_Q_LOCK(&cnd->qlck);
- act = ethr_atomic_read(&tse->uaflgs);
+ act = ethr_atomic32_read(&tse->uaflgs);
ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__
|| act == ETHR_RWMTX_W_WAIT_FLG__);
/*
@@ -1239,7 +1237,7 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
-
+ ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb);
tse->udata = udata;
ethr_leave_ts_event(tse);
return 0;
@@ -1411,7 +1409,7 @@ wake_readers(ethr_rwmutex *rwmtx, int rs)
rwmtx->rq_end = NULL;
ETHR_ASSERT(!rwmtx->mtxb.q
- || (ethr_atomic_read(&rwmtx->mtxb.q->uaflgs)
+ || (ethr_atomic32_read(&rwmtx->mtxb.q->uaflgs)
== ETHR_RWMTX_W_WAIT_FLG__));
ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
@@ -1422,7 +1420,7 @@ wake_readers(ethr_rwmutex *rwmtx, int rs)
#ifdef ETHR_DEBUG
ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
- ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs)
+ ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs)
== ETHR_RWMTX_R_WAIT_FLG__);
drs++;
#endif
@@ -1430,7 +1428,7 @@ wake_readers(ethr_rwmutex *rwmtx, int rs)
tse_next = tse->next; /* we aren't allowed to read tse->next
after we have reset uaflgs */
- ethr_atomic_set(&tse->uaflgs, 0);
+ ethr_atomic32_set(&tse->uaflgs, 0);
ethr_event_set(&tse->event);
tse = tse_next;
}
@@ -1466,20 +1464,14 @@ multiple_w_waiters(ethr_rwmutex *rwmtx)
int check_readers_array(ethr_rwmutex *rwmtx,
int start_rix,
- int length,
- int pre_check)
+ int length)
{
int ix = start_rix;
-#ifndef ETHR_READ_MEMORY_BARRIER_IS_FULL
- if (pre_check)
- ETHR_READ_MEMORY_BARRIER;
- else
-#endif
- ETHR_MEMORY_BARRIER;
+ ETHR_MEMORY_BARRIER;
do {
- long act = rwmutex_freqread_rdrs_read(rwmtx, ix);
+ ethr_sint32_t act = rwmutex_freqread_rdrs_read(rwmtx, ix);
if (act != 0)
return EBUSY;
ix++;
@@ -1490,55 +1482,101 @@ int check_readers_array(ethr_rwmutex *rwmtx,
return 0;
}
-static ETHR_INLINE void
+static void
+rwmutex_freqread_rdrs_dec_chk_wakeup(ethr_rwmutex *rwmtx,
+ ethr_ts_event *tse,
+ ethr_sint32_t initial)
+{
+ ethr_sint32_t act = initial;
+
+ if ((act & (ETHR_RWMTX_W_FLG__|
+ ETHR_RWMTX_R_ABRT_UNLCK_FLG__)) == 0) {
+ if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+ if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) {
+ /*
+ * We *need* to try to complete the runlock.
+ * A writer that just enqueued (not seen by us
+ * in flag field) may depend on someone else
+ * completing the runlock. We just took over
+ * that responsibilty since we modified reader
+ * groups.
+ */
+ rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+ }
+ }
+ else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_R_WAIT_FLG__)
+ rwmutex_transfer_read_lock(rwmtx, act, 0);
+ else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_W_WAIT_FLG__)
+ rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+ else {
+ /*
+ * Don't know if we got readers or writers
+ * first in queue; need to peek
+ */
+ ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+ if (!rwmtx->mtxb.q)
+ ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+ else if (is_w_waiter(rwmtx->mtxb.q)) {
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+ ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+ if ((act & ETHR_RWMTX_W_FLG__) == 0)
+ rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+ }
+ else {
+ /*
+ * rwmutex_transfer_read_lock() will
+ * unlock Q lock.
+ */
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+ if (act & ETHR_RWMTX_W_FLG__)
+ ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+ else
+ rwmutex_transfer_read_lock(rwmtx, act, 1);
+ }
+ }
+ }
+}
+
+static void
rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx,
ethr_ts_event *tse)
{
- long act;
+ ethr_sint32_t act;
/*
* Restore failed increment
*/
act = rwmutex_freqread_rdrs_dec_read(rwmtx, tse);
- ETHR_WRITE_MEMORY_BARRIER;
+ ETHR_MEMORY_BARRIER;
if (act == 0) {
-
-#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL
- ETHR_READ_MEMORY_BARRIER;
-#endif
-
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
-
- if ((act & ETHR_RWMTX_W_FLG__) == 0
- && act & (ETHR_RWMTX_WAIT_FLGS__|ETHR_RWMTX_R_PEND_UNLCK_MASK__)) {
- /*
- * We either got waiters, or someone else trying
- * to read unlock which we might have to help.
- */
- rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 1, 0);
- }
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+ rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act);
}
}
static int
rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
- long initial,
+ ethr_sint32_t initial,
ethr_ts_event *tse,
int start_next_ix,
int check_before_try,
int try_write_lock)
{
ethr_ts_event *tse_tmp;
- long act = initial;
+ ethr_sint32_t act = initial;
int six, res, length;
+ ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
+
+ if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__)
+ return try_write_lock ? EBUSY : 0;
+
tse_tmp = tse;
if (!tse_tmp)
tse_tmp = ethr_get_ts_event();
- if ((act & ETHR_RWMTX_WAIT_FLGS__)
- && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0)
+ if ((act & ETHR_RWMTX_WAIT_FLGS__) && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0)
goto check_waiters;
if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
@@ -1559,25 +1597,34 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
ethr_leave_ts_event(tse_tmp);
if (check_before_try) {
- res = check_readers_array(rwmtx, six, length, 1);
+ res = check_readers_array(rwmtx, six, length);
+
+ ETHR_MEMORY_BARRIER;
+
if (res == EBUSY)
return try_write_lock ? EBUSY : 0;
}
+ restart:
+
while (1) {
- long exp = act;
- long new = act+1;
+ ethr_sint32_t exp = act;
+ ethr_sint32_t new = act+1;
+
+ ETHR_ASSERT((act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) == 0);
ETHR_ASSERT((act & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
< ETHR_RWMTX_R_PEND_UNLCK_MASK__);
- act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+ act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
if (exp == act) {
act = new;
break;
}
+
if (!try_write_lock) {
- if (act == ETHR_RWMTX_W_FLG__ || act == 0)
+ if (act == 0 || (act & (ETHR_RWMTX_W_FLG__
+ | ETHR_RWMTX_R_ABRT_UNLCK_FLG__)))
return 0;
if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
if ((act & ETHR_RWMTX_R_FLG__) == 0)
@@ -1592,33 +1639,50 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
else {
if (act == 0)
goto tryrwlock;
- if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__))
+ if (act & (ETHR_RWMTX_W_FLG__
+ | ETHR_RWMTX_R_ABRT_UNLCK_FLG__))
return EBUSY;
}
}
- res = check_readers_array(rwmtx, six, length, 0);
- if (res == EBUSY) {
- act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs);
- if (act & ETHR_RWMTX_R_MASK__)
- return try_write_lock ? EBUSY : 0;
- }
- else {
- while (1) {
- long exp = act;
- long new = act;
- new &= ~ETHR_RWMTX_R_FLG__;
- new--;
+ res = check_readers_array(rwmtx, six, length);
- ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__);
+ ETHR_MEMORY_BARRIER;
- act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
- if (exp == act) {
- if (new & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
- return try_write_lock ? EBUSY : 0;
- act = new;
- break;
+ ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
+
+ while (1) {
+ int finished_abort = 0;
+ ethr_sint32_t exp = act;
+ ethr_sint32_t new = act;
+
+ new--;
+ if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) {
+ if ((new & ETHR_RWMTX_R_PEND_UNLCK_MASK__) == 0) {
+ new &= ~ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+ finished_abort = 1;
}
+ ETHR_ASSERT(act & ETHR_RWMTX_R_FLG__);
+ }
+ else if ((act & ETHR_RWMTX_R_FLG__) && res != EBUSY) {
+ new &= ~ETHR_RWMTX_R_FLG__;
+ }
+
+ ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__);
+
+ act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+ if (exp == act) {
+ act = new;
+ if (act & ETHR_RWMTX_W_FLG__)
+ return try_write_lock ? EBUSY : 0;
+ if (finished_abort && (act & ETHR_RWMTX_WAIT_FLGS__))
+ goto restart;
+ if (act & (ETHR_RWMTX_R_FLG__
+ | ETHR_RWMTX_R_ABRT_UNLCK_FLG__
+ | ETHR_RWMTX_R_PEND_UNLCK_MASK__))
+ return try_write_lock ? EBUSY : 0;
+ /* Read unlock completed */
+ break;
}
}
@@ -1628,12 +1692,9 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
* to write lock it).
*/
- if (act & ETHR_RWMTX_W_FLG__)
- return try_write_lock ? EBUSY : 0;
-
if (act & ETHR_RWMTX_WAIT_FLGS__) {
check_waiters:
- rwmutex_unlock_wake(rwmtx, 0, act);
+ rwmutex_unlock_wake(rwmtx, 0, act, 0);
return try_write_lock ? EBUSY : 0;
}
@@ -1643,9 +1704,9 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
tryrwlock:
/* Try to write lock it */
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
- ETHR_RWMTX_W_FLG__,
- 0);
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+ ETHR_RWMTX_W_FLG__,
+ 0);
return act == 0 ? 0 : EBUSY;
}
@@ -1654,24 +1715,23 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
static ETHR_INLINE void
rwmutex_incdec_restore_failed_tryrlock(ethr_rwmutex *rwmtx)
{
- long act;
+ ethr_sint32_t act;
/*
* Restore failed increment
*/
- act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_dec_read(&rwmtx->mtxb.flgs);
if ((act & ETHR_RWMTX_WAIT_FLGS__)
&& (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
- rwmutex_unlock_wake(rwmtx, 0, act);
+ rwmutex_unlock_wake(rwmtx, 0, act, 0);
}
}
#endif
static void
-rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
- long initial)
+rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
{
- long act = initial, exp;
+ ethr_sint32_t act = initial, exp;
int scnt, start_scnt;
ethr_ts_event *tse = NULL;
int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
@@ -1687,11 +1747,11 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
#ifdef ETHR_RLOCK_WITH_INC_DEC
rwmutex_incdec_restore_failed_tryrlock(rwmtx);
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
#endif
while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
- if (scnt == 0) {
+ if (scnt <= 0) {
tse = ethr_get_ts_event();
if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
event_wait(&rwmtx->mtxb, tse, scnt,
@@ -1704,18 +1764,17 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
ETHR_YIELD();
}
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
scnt--;
}
exp = act;
- ETHR_ASSERT(scnt >= 0);
#ifdef ETHR_RLOCK_WITH_INC_DEC
- act = ethr_atomic_inc_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_inc_read(&rwmtx->mtxb.flgs);
if ((act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) == 0)
goto done; /* Got it */
#else
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
if (act == exp)
goto done; /* Got it */
#endif
@@ -1728,10 +1787,83 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
static void
rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
- ethr_ts_event *tse,
- long initial)
+ ethr_ts_event *tse);
+
+static int
+rwmutex_freqread_rlock(ethr_rwmutex *rwmtx, ethr_ts_event *tse, int trylock)
{
- long act = initial;
+ int res = 0;
+ ethr_sint32_t act;
+
+ rwmutex_freqread_rdrs_inc(rwmtx, tse);
+
+ ETHR_MEMORY_BARRIER;
+
+ act = ethr_atomic32_read_acqb(&rwmtx->mtxb.flgs);
+
+ if (act != ETHR_RWMTX_R_FLG__) {
+ int wake_other_readers;
+
+ while (1) {
+ ethr_sint32_t exp, new;
+
+ wake_other_readers = 0;
+
+ if (act == 0)
+ new = act | ETHR_RWMTX_R_FLG__;
+ else if (act == ETHR_RWMTX_R_FLG__)
+ break; /* Got it */
+ else if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+ rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
+ if (trylock)
+ res = EBUSY;
+ else
+ rwmutex_freqread_rlock_wait(rwmtx, tse);
+ break;
+ }
+ else if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) {
+ if ((act & ETHR_RWMTX_R_FLG__) == 0)
+ ETHR_FATAL_ERROR__(EFAULT);
+ /*
+ * An aborted runlock, not write locked, and no write
+ * waiters, i.e., we got it...
+ */
+ if (act & ETHR_RWMTX_R_WAIT_FLG__)
+ wake_other_readers = 1;
+ break;
+ }
+ else {
+ new = act | ETHR_RWMTX_R_FLG__;
+ if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) {
+ /*
+ * Someone is doing tryrwlock (no writer and no
+ * write waiters); we will try to abort that...
+ */
+ new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+ }
+
+ if (act & ETHR_RWMTX_R_WAIT_FLG__)
+ wake_other_readers = 1;
+ }
+
+ exp = act;
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
+ if (act == exp)
+ break;
+ }
+
+ if (wake_other_readers)
+ rwmutex_transfer_read_lock(rwmtx, act, 0);
+ }
+
+ return res;
+}
+
+static void
+rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
+ ethr_ts_event *tse)
+{
+ ethr_sint32_t act;
int scnt, start_scnt;
int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
@@ -1744,12 +1876,10 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
while (1) {
- rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
-
- while (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
- if (scnt == 0) {
+ while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+ if (scnt <= 0) {
if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
event_wait(&rwmtx->mtxb, tse, scnt,
ETHR_RWMTX_R_WAIT_FLG__, 1, 1);
@@ -1761,76 +1891,65 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
ETHR_YIELD();
}
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
scnt--;
}
- ETHR_ASSERT(scnt >= 0);
-
- rwmutex_freqread_rdrs_inc(rwmtx, tse);
-
- ETHR_MEMORY_BARRIER;
-
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
-
- if (act == ETHR_RWMTX_R_FLG__)
- return; /* Got it */
-
- while (1) {
- long exp, new;
-
- if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__))
- break; /* Busy (need to restore inc) */
-
- if (act & ETHR_RWMTX_R_FLG__)
- return; /* Got it */
-
- exp = act;
- new = act | ETHR_RWMTX_R_FLG__;
- act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
- if (act == exp)
- return; /* Got it */
- }
+ if (rwmutex_freqread_rlock(rwmtx, tse, 1) != EBUSY)
+ break; /* Got it */
}
}
static void
-rwmutex_normal_rwlock_wait(ethr_rwmutex *rwmtx, long initial)
+rwmutex_normal_rwlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
{
write_lock_wait(&rwmtx->mtxb, initial, 1, 0);
}
static void
-rwmutex_freqread_rwlock_wait(ethr_rwmutex *rwmtx, long initial)
+rwmutex_freqread_rwlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
{
write_lock_wait(&rwmtx->mtxb, initial, 1, 1);
}
static ETHR_INLINE void
-rwlock_wake_set_flags(ethr_rwmutex *rwmtx, long new_initial, int act_initial)
+rwlock_wake_set_flags(ethr_rwmutex *rwmtx,
+ ethr_sint32_t new_initial,
+ ethr_sint32_t act_initial)
{
- long act, act_mask;
+ ethr_sint32_t act, act_mask;
+ int chk_abrt_flg;
+
+ ETHR_MEMORY_BARRIER;
+
if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
/* r pend unlock mask may vary and must be retained */
act_mask = ETHR_RWMTX_R_PEND_UNLCK_MASK__;
+ if (new_initial & ETHR_RWMTX_R_FLG__)
+ chk_abrt_flg = 1;
+ else
+ chk_abrt_flg = 0;
}
else {
#ifdef ETHR_RLOCK_WITH_INC_DEC
/* rs mask may vary and must be retained */
act_mask = ETHR_RWMTX_RS_MASK__;
+ chk_abrt_flg = 0;
#else
/* rs mask always zero */
ETHR_ASSERT((act_initial & ETHR_RWMTX_RS_MASK__) == 0);
- ethr_atomic_set(&rwmtx->mtxb.flgs, new_initial);
+ ethr_atomic32_set(&rwmtx->mtxb.flgs, new_initial);
return;
#endif
}
act = act_initial;
while (1) {
- long exp = act;
- long new = new_initial + (act & act_mask);
- act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+ ethr_sint32_t exp = act;
+ ethr_sint32_t new = new_initial + (act & act_mask);
+ if (chk_abrt_flg && (act & act_mask))
+ new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+ act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
if (act == exp)
break;
exp = act;
@@ -1844,7 +1963,7 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx,
int have_w,
ethr_ts_event *tse)
{
- long exp, act, imask;
+ ethr_sint32_t exp, act, imask;
exp = have_w ? ETHR_RWMTX_W_FLG__ : 0;
@@ -1866,7 +1985,7 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx,
if (rwmtx->rq_end) {
exp |= ETHR_RWMTX_R_WAIT_FLG__;
}
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
ETHR_ASSERT((exp & ~imask) == (act & ~imask));
ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
@@ -1877,7 +1996,15 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx,
exp |= ETHR_RWMTX_R_WAIT_FLG__;
if (rwmtx->rq_end->next != rwmtx->mtxb.q)
exp |= ETHR_RWMTX_W_WAIT_FLG__;
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ else if (exp == ETHR_RWMTX_R_WAIT_FLG__) {
+ if (!have_w) {
+ if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
+ imask |= ETHR_RWMTX_R_FLG__;
+ else
+ imask |= ETHR_RWMTX_RS_MASK__;
+ }
+ }
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
ETHR_ASSERT((exp & ~imask) == (act & ~imask));
ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
@@ -1888,41 +2015,85 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx,
#endif
static void
-rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
+rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx,
+ ethr_sint32_t initial,
+ int q_locked)
{
- long new, act = initial;
- ethr_ts_event *tse;
+ ethr_sint32_t act = initial;
- if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
- if (!have_w)
+ if (!q_locked) {
+ ethr_ts_event *tse;
+ ETHR_ASSERT(initial & ETHR_RWMTX_R_WAIT_FLG__);
+ ETHR_ASSERT((initial & ETHR_RWMTX_W_FLG__) == 0);
+ ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+ tse = rwmtx->mtxb.q;
+ if ((act & ETHR_RWMTX_W_FLG__) || !tse || is_w_waiter(tse)) {
+ /* Someone else woke the readers up... */
+ ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
return;
- else {
- while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
- long exp = act;
- new = exp & ~ETHR_RWMTX_W_FLG__;
- act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
- if (act == exp)
- return;
- }
}
}
- ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
- tse = rwmtx->mtxb.q;
+ rwmutex_unlock_wake(rwmtx, 0, initial, 1);
+}
- if (!have_w) {
- if (!tse) {
+static void
+rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, ethr_sint32_t initial,
+ int transfer_read_lock)
+{
+ ethr_sint32_t new, act = initial;
+ ethr_ts_event *tse;
+
+ if (transfer_read_lock) {
+ /*
+ * - Q already locked
+ * - Got R waiters first in Q
+ * - Not W locked
+ */
+ tse = rwmtx->mtxb.q;
+
+ ETHR_ASSERT(act & ETHR_RWMTX_R_WAIT_FLG__);
+ ETHR_ASSERT((act & (ETHR_RWMTX_W_FLG__)) == 0);
+ ETHR_ASSERT(tse && !is_w_waiter(tse));
+ }
+ else {
+
+ if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+ if (!have_w)
+ return;
+ else {
+ while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+ ethr_sint32_t exp = act;
+ new = exp & ~ETHR_RWMTX_W_FLG__;
+ act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+ if (act == exp)
+ return;
+ }
+ }
+ }
+
+ ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+ tse = rwmtx->mtxb.q;
+
+ if (!have_w) {
+ if (!tse) {
#ifdef ETHR_DEBUG
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
- ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+ ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
#endif
- goto already_served;
- }
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
- if (act & ~ETHR_RWMTX_WAIT_FLGS__) {
- already_served:
- ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
- return;
+ goto already_served;
+ }
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+ if (act == (ETHR_RWMTX_R_WAIT_FLG__|ETHR_RWMTX_R_FLG__)) {
+ ETHR_ASSERT(tse && !is_w_waiter(tse));
+ }
+ else if (act & ~ETHR_RWMTX_WAIT_FLGS__) {
+ already_served:
+ ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+ return;
+ }
}
}
@@ -1933,9 +2104,12 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
if (is_w_waiter(tse)) {
if (!have_w) {
- act = ethr_atomic_read_bor(&rwmtx->mtxb.flgs,
+ act = ethr_atomic32_read_bor(&rwmtx->mtxb.flgs,
ETHR_RWMTX_W_FLG__);
- ETHR_ASSERT((act & ~ETHR_RWMTX_WAIT_FLGS__) == 0);
+ ETHR_ASSERT((act & ~(ETHR_RWMTX_WAIT_FLGS__
+ | (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL
+ ? 0
+ : ETHR_RWMTX_R_PEND_UNLCK_MASK__))) == 0);
ETHR_ASSERT(act & ETHR_RWMTX_W_WAIT_FLG__);
act |= ETHR_RWMTX_W_FLG__;
}
@@ -1962,7 +2136,7 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
if (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL) {
rs = rwmtx->tdata.rs;
- new = (long) rs;
+ new = (ethr_sint32_t) rs;
rwmtx->tdata.rs = 0;
}
else {
@@ -1982,6 +2156,7 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
rwmutex_freqread_rdrs_add(rwmtx, type, ix, wrs);
}
}
+
new = ETHR_RWMTX_R_FLG__;
}
@@ -1989,6 +2164,7 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
new |= ETHR_RWMTX_W_WAIT_FLG__;
rwlock_wake_set_flags(rwmtx, new, act);
+
wake_readers(rwmtx, rs);
}
}
@@ -2016,16 +2192,16 @@ alloc_readers_array(int length, ethr_rwmutex_lived lived)
if (!mem)
return NULL;
- if ((((unsigned long) mem) & ETHR_CACHE_LINE_MASK) == 0) {
+ if ((((ethr_uint_t) mem) & ETHR_CACHE_LINE_MASK) == 0) {
ra = (ethr_rwmtx_readers_array__ *) mem;
ra->data.byte_offset = 0;
}
else {
ra = ((ethr_rwmtx_readers_array__ *)
- ((((unsigned long) mem) & ~ETHR_CACHE_LINE_MASK)
+ ((((ethr_uint_t) mem) & ~ETHR_CACHE_LINE_MASK)
+ ETHR_CACHE_LINE_SIZE));
- ra->data.byte_offset = (int) ((unsigned long) ra
- - (unsigned long) mem);
+ ra->data.byte_offset = (int) ((ethr_uint_t) ra
+ - (ethr_uint_t) mem);
}
ra->data.lived = lived;
return ra;
@@ -2099,7 +2275,7 @@ ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt)
rwmtx->tdata.ra = ra;
for (ix = 0; ix < length; ix++) {
- ethr_atomic_init(&rwmtx->tdata.ra[ix].data.readers, 0);
+ ethr_atomic32_init(&rwmtx->tdata.ra[ix].data.readers, 0);
rwmtx->tdata.ra[ix].data.waiting_readers = 0;
}
break;
@@ -2151,8 +2327,9 @@ ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
return EINVAL;
}
#endif
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
- long act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ ethr_sint32_t act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
if (act == ETHR_RWMTX_R_FLG__)
rwmutex_try_complete_runlock(rwmtx, act, NULL, 0, 0, 0);
}
@@ -2173,7 +2350,7 @@ int
ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
{
int res = 0;
- long act;
+ ethr_sint32_t act;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(rwmtx);
@@ -2181,25 +2358,27 @@ ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
switch (rwmtx->type) {
case ETHR_RWMUTEX_TYPE_NORMAL: {
#ifdef ETHR_RLOCK_WITH_INC_DEC
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))
res = EBUSY;
else {
- act = ethr_atomic_inc_read_acqb(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_inc_read_acqb(&rwmtx->mtxb.flgs);
if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
rwmutex_incdec_restore_failed_tryrlock(rwmtx);
res = EBUSY;
}
}
#else
- long exp = 0;
+ ethr_sint32_t exp = 0;
int tries = 0;
while (1) {
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
if (act == exp) {
res = 0;
break;
@@ -2219,49 +2398,30 @@ ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
ethr_ts_event *tse = ethr_get_ts_event();
-
- rwmutex_freqread_rdrs_inc(rwmtx, tse);
-
- ETHR_MEMORY_BARRIER;
-
- act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs);
-
- if (act != ETHR_RWMTX_R_FLG__) {
- while (1) {
- long exp, new;
-
- if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
- rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
- res = EBUSY;
- break;
- }
-
- if (act & ETHR_RWMTX_R_FLG__)
- break;
-
- exp = act;
- new = act | ETHR_RWMTX_R_FLG__;
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
- if (act == exp)
- break;
- }
- }
-
+ res = rwmutex_freqread_rlock(rwmtx, tse, 1);
ethr_leave_ts_event(tse);
break;
}
}
+#ifdef ETHR_MTX_CHK_EXCL
+ if (res == 0) {
+ ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb);
+ ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
+ }
+#endif
+
ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(&rwmtx->mtxb, res);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
return res;
}
void
ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
{
- long act;
+ ethr_sint32_t act;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(rwmtx);
@@ -2269,20 +2429,21 @@ ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
switch (rwmtx->type) {
case ETHR_RWMUTEX_TYPE_NORMAL: {
#ifdef ETHR_RLOCK_WITH_INC_DEC
- act = ethr_atomic_inc_read_acqb(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_inc_read_acqb(&rwmtx->mtxb.flgs);
if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))
rwmutex_normal_rlock_wait(rwmtx, act);
#else
- long exp = 0;
+ ethr_sint32_t exp = 0;
while (1) {
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
- if (act == exp) {
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+ if (act == exp)
break;
- }
if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
rwmutex_normal_rlock_wait(rwmtx, act);
@@ -2297,38 +2458,15 @@ ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
ethr_ts_event *tse = ethr_get_ts_event();
-
- rwmutex_freqread_rdrs_inc(rwmtx, tse);
-
- ETHR_MEMORY_BARRIER;
-
- act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs);
-
- if (act != ETHR_RWMTX_R_FLG__) {
- while (1) {
- long exp, new;
-
- if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
- rwmutex_freqread_rlock_wait(rwmtx, tse, act);
- break;
- }
-
- if (act & ETHR_RWMTX_R_FLG__)
- break;
-
- exp = act;
- new = act | ETHR_RWMTX_R_FLG__;
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
- if (act == exp)
- break;
- }
- }
-
+ rwmutex_freqread_rlock(rwmtx, tse, 0);
ethr_leave_ts_event(tse);
break;
}
}
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+ ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb);
+ ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
ETHR_MTX_HARD_DEBUG_LFS_RLOCK(&rwmtx->mtxb);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
}
@@ -2336,8 +2474,10 @@ ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
void
ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
{
- long act;
+ ethr_sint32_t act;
+ ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
+ ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(&rwmtx->mtxb);
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(rwmtx);
ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
@@ -2345,13 +2485,15 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(&rwmtx->mtxb);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
switch (rwmtx->type) {
case ETHR_RWMUTEX_TYPE_NORMAL:
- act = ethr_atomic_dec_read_relb(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_dec_read_relb(&rwmtx->mtxb.flgs);
if ((act & ETHR_RWMTX_WAIT_FLGS__)
&& (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
- rwmutex_unlock_wake(rwmtx, 0, act);
+ rwmutex_unlock_wake(rwmtx, 0, act, 0);
}
break;
@@ -2363,21 +2505,12 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
ETHR_ASSERT(act >= 0);
- ETHR_WRITE_MEMORY_BARRIER;
+ ETHR_MEMORY_BARRIER;
if (act == 0) {
-
-#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL
- ETHR_READ_MEMORY_BARRIER;
-#endif
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
-
- if ((act & ETHR_RWMTX_W_FLG__) == 0
- && (act & (ETHR_RWMTX_WAIT_FLGS__
- | ETHR_RWMTX_R_PEND_UNLCK_MASK__))) {
- rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
- }
-
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+ if (act != ETHR_RWMTX_R_FLG__)
+ rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act);
}
ethr_leave_ts_event(tse);
@@ -2385,6 +2518,7 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
}
}
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
}
@@ -2392,7 +2526,7 @@ int
ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
{
int res = 0;
- long act;
+ ethr_sint32_t act;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(rwmtx);
@@ -2400,10 +2534,12 @@ ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
switch (rwmtx->type) {
case ETHR_RWMUTEX_TYPE_NORMAL:
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
- ETHR_RWMTX_W_FLG__, 0);
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+ ETHR_RWMTX_W_FLG__, 0);
if (act != 0)
res = EBUSY;
break;
@@ -2412,29 +2548,36 @@ ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
res = 0;
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
do {
- if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__)) {
- res = EBUSY;
- break;
- }
-
- if (act & ETHR_RWMTX_R_MASK__) {
+ if (act == 0)
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+ ETHR_RWMTX_W_FLG__, 0);
+ else if (act == ETHR_RWMTX_R_FLG__) {
res = rwmutex_try_complete_runlock(rwmtx, act, NULL,
0, 1, 1);
break;
}
-
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
- ETHR_RWMTX_W_FLG__, 0);
+ else {
+ res = EBUSY;
+ break;
+ }
} while (act != 0);
break;
}
+#ifdef ETHR_MTX_CHK_EXCL
+ if (res == 0) {
+ ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb);
+ ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
+ }
+#endif
+
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&rwmtx->mtxb, res);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
@@ -2444,17 +2587,19 @@ ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
void
ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
{
- long act;
+ ethr_sint32_t act;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(rwmtx);
ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
switch (rwmtx->type) {
case ETHR_RWMUTEX_TYPE_NORMAL:
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
- ETHR_RWMTX_W_FLG__, 0);
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+ ETHR_RWMTX_W_FLG__, 0);
if (act != 0)
rwmutex_normal_rwlock_wait(rwmtx, act);
break;
@@ -2462,7 +2607,7 @@ ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
do {
@@ -2471,23 +2616,26 @@ ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
break;
}
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
- ETHR_RWMTX_W_FLG__, 0);
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+ ETHR_RWMTX_W_FLG__, 0);
} while (act != 0);
break;
}
+ ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb);
+ ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&rwmtx->mtxb);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
}
void
ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
{
- long act;
+ ethr_sint32_t act;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(rwmtx);
ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
@@ -2495,24 +2643,30 @@ ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&rwmtx->mtxb);
+ ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
+ ETHR_MTX_CHK_EXCL_UNSET_EXCL(&rwmtx->mtxb);
+
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
switch (rwmtx->type) {
case ETHR_RWMUTEX_TYPE_NORMAL:
- act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs,
- 0, ETHR_RWMTX_W_FLG__);
+ act = ethr_atomic32_cmpxchg_relb(&rwmtx->mtxb.flgs,
+ 0, ETHR_RWMTX_W_FLG__);
if (act != ETHR_RWMTX_W_FLG__)
- rwmutex_unlock_wake(rwmtx, 1, act);
+ rwmutex_unlock_wake(rwmtx, 1, act, 0);
break;
case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
- act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs, 0,
- ETHR_RWMTX_W_FLG__);
+ act = ethr_atomic32_cmpxchg_relb(&rwmtx->mtxb.flgs, 0,
+ ETHR_RWMTX_W_FLG__);
if (act != ETHR_RWMTX_W_FLG__)
- rwmutex_unlock_wake(rwmtx, 1, act);
+ rwmutex_unlock_wake(rwmtx, 1, act, 0);
break;
}
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
}
#else
@@ -2630,7 +2784,7 @@ static void
hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
{
int res;
- long flgs = ethr_atomic_read(&mtxb->flgs);
+ ethr_sint32_t flgs = ethr_atomic32_read(&mtxb->flgs);
ETHR_MTX_HARD_ASSERT(res == 0);
@@ -2653,12 +2807,12 @@ hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
tse = mtxb->q;
do {
- long type;
+ ethr_sint32_t type;
ETHR_MTX_HARD_ASSERT(tse->next->prev == tse);
ETHR_MTX_HARD_ASSERT(tse->prev->next == tse);
- type = ethr_atomic_read(&tse->uaflgs);
+ type = ethr_atomic32_read(&tse->uaflgs);
ETHR_MTX_HARD_ASSERT(type == tse->uflgs);
switch (type) {
case ETHR_RWMTX_W_WAIT_FLG__:
generated by cgit v1.2.3 (git 2.25.1) at 2025-06-28 18:47:56 +0000