diff options
Diffstat (limited to 'erts/lib_src/common')
| -rw-r--r-- | erts/lib_src/common/erl_misc_utils.c | 272 | ||||
| -rw-r--r-- | erts/lib_src/common/ethr_mutex.c | 631 |
2 files changed, 641 insertions, 262 deletions
diff --git a/erts/lib_src/common/erl_misc_utils.c b/erts/lib_src/common/erl_misc_utils.c index 116c9886d8..4c881993a5 100644 --- a/erts/lib_src/common/erl_misc_utils.c +++ b/erts/lib_src/common/erl_misc_utils.c @@ -71,6 +71,19 @@ (CPUSET)) != 0 ? -errno : 0) #define ERTS_MU_SET_THR_AFFINITY__(SETP) \ (sched_setaffinity(0, sizeof(cpu_set_t), (SETP)) != 0 ? -errno : 0) +#elif defined(HAVE_CPUSET_xETAFFINITY) +# include <sys/param.h> +# include <sys/cpuset.h> +# define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__ +#define ERTS_MU_GET_PROC_AFFINITY__(CPUINFOP, CPUSET) \ + (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, \ + sizeof(cpuset_t), \ + (CPUSET)) != 0 ? -errno : 0) +#define ERTS_MU_SET_THR_AFFINITY__(CPUSETP) \ + (cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, \ + sizeof(cpuset_t), \ + (CPUSETP)) != 0 ? -errno : 0) +# define cpu_set_t cpuset_t #elif defined(__WIN32__) # define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__ # define cpu_set_t DWORD @@ -100,6 +113,11 @@ # define ERTS_SYS_CPU_PATH "/sys/devices/system/cpu" #endif +#ifdef __FreeBSD__ +#include <sys/types.h> +#include <sys/sysctl.h> +#endif + static int read_topology(erts_cpu_info_t *cpuinfo); #if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) @@ -1228,7 +1246,10 @@ read_topology(erts_cpu_info_t *cpuinfo) nodes++; } - core_id = malloc(sizeof(int)*(packages ? packages : 1)); + if (!packages) { + packages = 1; + } + core_id = malloc(sizeof(int)*packages); if (!core_id) { res = -ENOMEM; goto error; @@ -1286,11 +1307,13 @@ read_topology(erts_cpu_info_t *cpuinfo) * Nodes and packages may not be supported; pretend * that there are one if this is the case... */ - if (!nodes) - cpuinfo->topology[l].node = 0; - if (!packages) - cpuinfo->topology[l].processor = 0; if (slpip[rix].ProcessorMask & (((ULONG_PTR) 1) << l)) { + if (!nodes) { + cpuinfo->topology[l].node = 0; + } + if (!packages) { + cpuinfo->topology[l].processor = 0; + } if (processor < 0) { processor = cpuinfo->topology[l].processor; if (processor < 0) { @@ -1375,6 +1398,245 @@ read_topology(erts_cpu_info_t *cpuinfo) return res; } +#elif defined(__FreeBSD__) + +/** + * FreeBSD topology detection is based on kern.sched.topology_spec XML as + * exposed by the ULE scheduler and described in SMP(4). It is available in + * 8.0 and higher. + * + * Threads are identified in this XML chunk with a THREAD flag. The function + * (simplistically) distinguishes cores and processors by the amount of cache + * they share (0 => processor, otherwise => core). Nodes are not identified + * (ULE doesn't handle NUMA yet, I believe). + */ + +/** + * Recursively parse a topology_spec <group> tag. + */ +static +const char* parse_topology_spec_group(erts_cpu_info_t *cpuinfo, const char* xml, int parentCacheLevel, int* processor_p, int* core_p, int* index_procs_p) { + int error = 0; + int cacheLevel = parentCacheLevel; + const char* next_group_start = strstr(xml + 1, "<group"); + int is_thread_group = 0; + const char* next_cache_level; + const char* next_thread_flag; + const char* next_group_end; + const char* next_children; + const char* next_children_end; + + /* parse the cache level */ + next_cache_level = strstr(xml, "cache-level=\""); + if (next_cache_level && (next_group_start == NULL || next_cache_level < next_group_start)) { + sscanf(next_cache_level, "cache-level=\"%i\"", &cacheLevel); + } + + /* parse the threads flag */ + next_thread_flag = strstr(xml, "THREAD"); + if (next_thread_flag && (next_group_start == NULL || next_thread_flag < next_group_start)) + is_thread_group = 1; + + /* Determine if it's a leaf with the position of the next children tag */ + next_group_end = strstr(xml, "</group>"); + next_children = strstr(xml, "<children>"); + next_children_end = strstr(xml, "</children>"); + if (next_children == NULL || next_group_end < next_children) { + do { + const char* next_cpu_start; + const char* next_cpu_cdata; + const char* next_cpu_end; + int cpu_str_size; + char* cpu_str; + char* cpu_crsr; + char* brkb; + int thread = 0; + int index_procs = *index_procs_p; + + next_cpu_start = strstr(xml, "<cpu"); + if (!next_cpu_start) { + error = 1; + break; + } + next_cpu_cdata = strstr(next_cpu_start, ">") + 1; + if (!next_cpu_cdata) { + error = 1; + break; + } + next_cpu_end = strstr(next_cpu_cdata, "</cpu>"); + if (!next_cpu_end) { + error = 1; + break; + } + cpu_str_size = next_cpu_end - next_cpu_cdata; + cpu_str = (char*) malloc(cpu_str_size + 1); + memcpy(cpu_str, (const char*) next_cpu_cdata, cpu_str_size); + cpu_str[cpu_str_size] = 0; + for (cpu_crsr = strtok_r(cpu_str, " \t,", &brkb); cpu_crsr; cpu_crsr = strtok_r(NULL, " \t,", &brkb)) { + int cpu_id; + if (index_procs >= cpuinfo->configured) { + void* t = realloc(cpuinfo->topology, (sizeof(erts_cpu_topology_t) * (index_procs + 1))); + if (t) { + cpuinfo->topology = t; + } else { + error = 1; + break; + } + } + cpu_id = atoi(cpu_crsr); + cpuinfo->topology[index_procs].node = -1; + cpuinfo->topology[index_procs].processor = *processor_p; + cpuinfo->topology[index_procs].processor_node = -1; + cpuinfo->topology[index_procs].core = *core_p; + cpuinfo->topology[index_procs].thread = thread; + cpuinfo->topology[index_procs].logical = cpu_id; + if (is_thread_group) { + thread++; + } else { + *core_p = (*core_p)++; + } + index_procs++; + } + *index_procs_p = index_procs; + free(cpu_str); + } while (0); + xml = next_group_end; + } else { + while (next_group_start != NULL && next_group_start < next_children_end) { + xml = parse_topology_spec_group(cpuinfo, next_group_start, cacheLevel, processor_p, core_p, index_procs_p); + if (!xml) + break; + next_group_start = strstr(xml, "<group"); + next_children_end = strstr(xml, "</children>"); + } + } + + if (cacheLevel == 0) { + *core_p = 0; + *processor_p = (*processor_p)++; + } else { + *core_p = (*core_p)++; + } + + if (error) + xml = NULL; + + return xml; +} + +/** + * Parse the topology_spec. Return the number of CPUs or 0 if parsing failed. + */ +static +int parse_topology_spec(erts_cpu_info_t *cpuinfo, const char* xml) { + int res = 1; + int index_procs = 0; + int core = 0; + int processor = 0; + xml = strstr(xml, "<groups"); + if (!xml) + return -1; + + xml += 7; + xml = strstr(xml, "<group"); + while (xml) { + xml = parse_topology_spec_group(cpuinfo, xml, 0, &processor, &core, &index_procs); + if (!xml) { + res = 0; + break; + } + xml = strstr(xml, "<group"); + } + + if (res) + res = index_procs; + + return res; +} + +static int +read_topology(erts_cpu_info_t *cpuinfo) +{ + int ix; + int res = 0; + size_t topology_spec_size = 0; + void* topology_spec = NULL; + + errno = 0; + + if (cpuinfo->configured < 1) + goto error; + + cpuinfo->topology_size = cpuinfo->configured; + cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t) + * cpuinfo->configured); + if (!cpuinfo->topology) { + res = -ENOMEM; + goto error; + } + + for (ix = 0; ix < cpuinfo->configured; ix++) { + cpuinfo->topology[ix].node = -1; + cpuinfo->topology[ix].processor = -1; + cpuinfo->topology[ix].processor_node = -1; + cpuinfo->topology[ix].core = -1; + cpuinfo->topology[ix].thread = -1; + cpuinfo->topology[ix].logical = -1; + } + + if (!sysctlbyname("kern.sched.topology_spec", NULL, &topology_spec_size, NULL, 0)) { + topology_spec = malloc(topology_spec_size); + if (!topology_spec) { + res = -ENOMEM; + goto error; + } + + if (sysctlbyname("kern.sched.topology_spec", topology_spec, &topology_spec_size, NULL, 0)) { + goto error; + } + + res = parse_topology_spec(cpuinfo, topology_spec); + if (!res || res < cpuinfo->online) + res = 0; + else { + cpuinfo->topology_size = res; + + if (cpuinfo->topology_size != cpuinfo->configured) { + void *t = realloc(cpuinfo->topology, (sizeof(erts_cpu_topology_t) + * cpuinfo->topology_size)); + if (t) + cpuinfo->topology = t; + } + + adjust_processor_nodes(cpuinfo, 1); + + qsort(cpuinfo->topology, + cpuinfo->topology_size, + sizeof(erts_cpu_topology_t), + cpu_cmp); + } + } + +error: + + if (res == 0) { + cpuinfo->topology_size = 0; + if (cpuinfo->topology) { + free(cpuinfo->topology); + cpuinfo->topology = NULL; + } + if (errno) + res = -errno; + else + res = -EINVAL; + } + + if (topology_spec) + free(topology_spec); + + return res; +} + #else static int diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c index 78323b62a3..c5738c3d77 100644 --- a/erts/lib_src/common/ethr_mutex.c +++ b/erts/lib_src/common/ethr_mutex.c @@ -205,9 +205,14 @@ static void hard_debug_chk_q__(struct ethr_mutex_base_ *, int); #ifdef ETHR_USE_OWN_RWMTX_IMPL__ static void +rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx, + long initial, + int q_locked); +static void rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, - long initial); + long initial, + int transfer_read_lock); static int rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, long initial, @@ -334,7 +339,7 @@ rwmutex_freqread_rdrs_read(ethr_rwmutex *rwmtx, int ix) ETHR_ASSERT(res >= 0); break; case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: - ETHR_ASSERT(res == 0 || res == 1); + ETHR_ASSERT(ix == 0 ? res >= 0 : (res == 0 || res == 1)); break; default: ETHR_ASSERT(0); @@ -400,6 +405,7 @@ event_wait(struct ethr_mutex_base_ *mtxb, int locked = 0; long act; int need_try_complete_runlock = 0; + int transfer_read_lock = 0; /* Need to enqueue and wait... */ @@ -444,8 +450,8 @@ event_wait(struct ethr_mutex_base_ *mtxb, while (1) { long new, exp = act; - int freqread_tryrlock = 0; need_try_complete_runlock = 0; + transfer_read_lock = 0; if (type == ETHR_RWMTX_W_WAIT_FLG__) { if (is_freq_read && act == ETHR_RWMTX_R_FLG__) @@ -465,14 +471,11 @@ event_wait(struct ethr_mutex_base_ *mtxb, new = act + 1; /* Try to get it */ } else { - if (act & ~ETHR_RWMTX_R_FLG__) - new = act | ETHR_RWMTX_R_WAIT_FLG__; - else { /* Try to get it */ - ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; - rwmutex_freqread_rdrs_inc(rwmtx, tse); - ETHR_MEMORY_BARRIER; - new = act | ETHR_RWMTX_R_FLG__; - freqread_tryrlock = 1; + new = act | ETHR_RWMTX_R_WAIT_FLG__; + if ((act & (ETHR_RWMTX_W_FLG__ + | ETHR_RWMTX_W_WAIT_FLG__)) == 0) { + /* Transfer read lock to this thread. */ + transfer_read_lock = 1; } } } @@ -488,24 +491,6 @@ event_wait(struct ethr_mutex_base_ *mtxb, goto done; } } - - if (freqread_tryrlock) { - ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; - - /* We didn't set ETHR_RWMTX_R_FLG__, however someone - else might have */ - if (act == ETHR_RWMTX_R_FLG__) - goto done; /* Got it by help from someone else */ - - ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0); - /* - * We know that no waiter flags have been set, i.e., - * we cannot get into a situation where we need to wake - * someone up here. Just restore the readers counter - * and do it over again... - */ - rwmutex_freqread_rdrs_dec(rwmtx, tse); - } } /* Enqueue */ @@ -535,20 +520,36 @@ event_wait(struct ethr_mutex_base_ *mtxb, /* Wait */ locked = 0; - ETHR_MTX_Q_UNLOCK(&mtxb->qlck); - if (need_try_complete_runlock) { + ETHR_ASSERT(!(transfer_read_lock && need_try_complete_runlock)); + + if (transfer_read_lock) { ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type != ETHR_RWMUTEX_TYPE_NORMAL); /* - * We were the only one in queue when we enqueued, and it - * was seemingly read locked. We need to try to complete a - * runlock otherwise we might be hanging forever. If the - * runlock could be completed we will be dequeued and - * woken by ourselves. + * We are the only one in the queue and we are not write + * locked; rwmutex_transfer_read_lock() will: + * - transfer a read lock to us (since we're first in q) + * - unlock the Q-lock */ - rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb, - act, tse, 0, 1, 0); + rwmutex_transfer_read_lock(((ethr_rwmutex *) mtxb), act, 1); + } + else { + ETHR_MTX_Q_UNLOCK(&mtxb->qlck); + + if (need_try_complete_runlock) { + ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type + != ETHR_RWMUTEX_TYPE_NORMAL); + /* + * We were the only one in queue when we enqueued, and it + * was seemingly read locked. We need to try to complete a + * runlock otherwise we might be hanging forever. If the + * runlock could be completed we will be dequeued and + * woken by ourselves. + */ + rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb, + act, tse, 0, 1, 0); + } } while (1) { @@ -653,8 +654,6 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb, ethr_ts_event *tse = NULL; int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; int res; - int freq_read_size = -1; - int freq_read_start_ix = -1; ETHR_ASSERT(!is_freq_read || is_rwmtx); @@ -666,44 +665,23 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb, */ while (1) { - long exp; - while (act != 0) { if (is_freq_read && act == ETHR_RWMTX_R_FLG__) { ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + scnt--; if (!tse) tse = ethr_get_ts_event(); - if (freq_read_size < 0) { - if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { - freq_read_size = reader_groups_array_size; - freq_read_start_ix = tse->rgix; - } - else { - freq_read_size = main_threads_array_size; - freq_read_start_ix = tse->mtix; - } - } - res = check_readers_array(rwmtx, - freq_read_start_ix, - freq_read_size); - scnt--; - if (res == 0) { - act = ethr_atomic_read(&mtxb->flgs); - if (act & ETHR_RWMTX_R_MASK__) { - res = rwmutex_try_complete_runlock(rwmtx, act, - tse, 0, 0, - 1); - if (res != EBUSY) - goto done; /* Got it */ - } - if (scnt <= 0) - goto chk_spin; - if (--until_yield == 0) { - until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; - ETHR_YIELD(); - } - continue; + res = rwmutex_try_complete_runlock(rwmtx, act, + tse, 0, 0, + 1); + if (res != EBUSY) + goto done; /* Got it */ + if (scnt <= 0) + goto chk_spin; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + ETHR_YIELD(); } } @@ -728,11 +706,9 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb, scnt--; } - exp = act; - act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs, ETHR_RWMTX_W_FLG__, - exp); + 0); if (act == 0) goto done; /* Got it */ } @@ -753,6 +729,7 @@ mtxb_init(struct ethr_mutex_base_ *mtxb, #ifdef ETHR_MTX_HARD_DEBUG_WSQ mtxb->ws = 0; #endif + ETHR_MTX_CHK_EXCL_INIT(mtxb); if (no_spin) { mtxb->main_scnt = 0; mtxb->aux_scnt = 0; @@ -1254,7 +1231,7 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb); ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); - + ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb); tse->udata = udata; ethr_leave_ts_event(tse); return 0; @@ -1499,7 +1476,62 @@ int check_readers_array(ethr_rwmutex *rwmtx, return 0; } -static ETHR_INLINE void +static void +rwmutex_freqread_rdrs_dec_chk_wakeup(ethr_rwmutex *rwmtx, + ethr_ts_event *tse, + long initial) +{ + long act = initial; + + if ((act & (ETHR_RWMTX_W_FLG__| + ETHR_RWMTX_R_ABRT_UNLCK_FLG__)) == 0) { + if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) { + /* + * We *need* to try to complete the runlock. + * A writer that just enqueued (not seen by us + * in flag field) may depend on someone else + * completing the runlock. We just took over + * that responsibilty since we modified reader + * groups. + */ + rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0); + } + } + else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_R_WAIT_FLG__) + rwmutex_transfer_read_lock(rwmtx, act, 0); + else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_W_WAIT_FLG__) + rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0); + else { + /* + * Don't know if we got readers or writers + * first in queue; need to peek + */ + ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck); + if (!rwmtx->mtxb.q) + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + else if (is_w_waiter(rwmtx->mtxb.q)) { + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + if ((act & ETHR_RWMTX_W_FLG__) == 0) + rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0); + } + else { + /* + * rwmutex_transfer_read_lock() will + * unlock Q lock. + */ + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + if (act & ETHR_RWMTX_W_FLG__) + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + else + rwmutex_transfer_read_lock(rwmtx, act, 1); + } + } + } +} + +static void rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx, ethr_ts_event *tse) { @@ -1509,24 +1541,11 @@ rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx, */ act = rwmutex_freqread_rdrs_dec_read(rwmtx, tse); - ETHR_WRITE_MEMORY_BARRIER; + ETHR_MEMORY_BARRIER; if (act == 0) { - -#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL - ETHR_READ_MEMORY_BARRIER; -#endif - act = ethr_atomic_read(&rwmtx->mtxb.flgs); - - if ((act & ETHR_RWMTX_W_FLG__) == 0 - && act & (ETHR_RWMTX_WAIT_FLGS__|ETHR_RWMTX_R_PEND_UNLCK_MASK__)) { - /* - * We either got waiters, or someone else trying - * to read unlock which we might have to help. - */ - rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 1, 0); - } + rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act); } } @@ -1542,12 +1561,16 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, long act = initial; int six, res, length; + ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0); + + if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) + return try_write_lock ? EBUSY : 0; + tse_tmp = tse; if (!tse_tmp) tse_tmp = ethr_get_ts_event(); - if ((act & ETHR_RWMTX_WAIT_FLGS__) - && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) + if ((act & ETHR_RWMTX_WAIT_FLGS__) && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) goto check_waiters; if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { @@ -1569,14 +1592,21 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, if (check_before_try) { res = check_readers_array(rwmtx, six, length); + + ETHR_MEMORY_BARRIER; + if (res == EBUSY) return try_write_lock ? EBUSY : 0; } + restart: + while (1) { long exp = act; long new = act+1; + ETHR_ASSERT((act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) == 0); + ETHR_ASSERT((act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) < ETHR_RWMTX_R_PEND_UNLCK_MASK__); @@ -1585,8 +1615,10 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, act = new; break; } + if (!try_write_lock) { - if (act == ETHR_RWMTX_W_FLG__ || act == 0) + if (act == 0 || (act & (ETHR_RWMTX_W_FLG__ + | ETHR_RWMTX_R_ABRT_UNLCK_FLG__))) return 0; if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { if ((act & ETHR_RWMTX_R_FLG__) == 0) @@ -1601,33 +1633,50 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, else { if (act == 0) goto tryrwlock; - if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__)) + if (act & (ETHR_RWMTX_W_FLG__ + | ETHR_RWMTX_R_ABRT_UNLCK_FLG__)) return EBUSY; } } res = check_readers_array(rwmtx, six, length); - if (res == EBUSY) { - act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs); - if (act & ETHR_RWMTX_R_MASK__) - return try_write_lock ? EBUSY : 0; - } - else { - while (1) { - long exp = act; - long new = act; - new &= ~ETHR_RWMTX_R_FLG__; - new--; - ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__); + ETHR_MEMORY_BARRIER; - act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); - if (exp == act) { - if (new & ETHR_RWMTX_R_PEND_UNLCK_MASK__) - return try_write_lock ? EBUSY : 0; - act = new; - break; + ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0); + + while (1) { + int finished_abort = 0; + long exp = act; + long new = act; + + new--; + if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) { + if ((new & ETHR_RWMTX_R_PEND_UNLCK_MASK__) == 0) { + new &= ~ETHR_RWMTX_R_ABRT_UNLCK_FLG__; + finished_abort = 1; } + ETHR_ASSERT(act & ETHR_RWMTX_R_FLG__); + } + else if ((act & ETHR_RWMTX_R_FLG__) && res != EBUSY) { + new &= ~ETHR_RWMTX_R_FLG__; + } + + ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__); + + act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (exp == act) { + act = new; + if (act & ETHR_RWMTX_W_FLG__) + return try_write_lock ? EBUSY : 0; + if (finished_abort && (act & ETHR_RWMTX_WAIT_FLGS__)) + goto restart; + if (act & (ETHR_RWMTX_R_FLG__ + | ETHR_RWMTX_R_ABRT_UNLCK_FLG__ + | ETHR_RWMTX_R_PEND_UNLCK_MASK__)) + return try_write_lock ? EBUSY : 0; + /* Read unlock completed */ + break; } } @@ -1637,12 +1686,9 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, * to write lock it). */ - if (act & ETHR_RWMTX_W_FLG__) - return try_write_lock ? EBUSY : 0; - if (act & ETHR_RWMTX_WAIT_FLGS__) { check_waiters: - rwmutex_unlock_wake(rwmtx, 0, act); + rwmutex_unlock_wake(rwmtx, 0, act, 0); return try_write_lock ? EBUSY : 0; } @@ -1670,7 +1716,7 @@ rwmutex_incdec_restore_failed_tryrlock(ethr_rwmutex *rwmtx) act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs); if ((act & ETHR_RWMTX_WAIT_FLGS__) && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) { - rwmutex_unlock_wake(rwmtx, 0, act); + rwmutex_unlock_wake(rwmtx, 0, act, 0); } } @@ -1700,7 +1746,7 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, #endif while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { - if (scnt >= 0) { + if (scnt <= 0) { tse = ethr_get_ts_event(); if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) { event_wait(&rwmtx->mtxb, tse, scnt, @@ -1736,10 +1782,83 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, static void rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx, - ethr_ts_event *tse, - long initial) + ethr_ts_event *tse); + +static int +rwmutex_freqread_rlock(ethr_rwmutex *rwmtx, ethr_ts_event *tse, int trylock) { - long act = initial; + int res = 0; + long act; + + rwmutex_freqread_rdrs_inc(rwmtx, tse); + + ETHR_MEMORY_BARRIER; + + act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs); + + if (act != ETHR_RWMTX_R_FLG__) { + int wake_other_readers; + + while (1) { + long exp, new; + + wake_other_readers = 0; + + if (act == 0) + new = act | ETHR_RWMTX_R_FLG__; + else if (act == ETHR_RWMTX_R_FLG__) + break; /* Got it */ + else if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse); + if (trylock) + res = EBUSY; + else + rwmutex_freqread_rlock_wait(rwmtx, tse); + break; + } + else if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) { + if ((act & ETHR_RWMTX_R_FLG__) == 0) + ETHR_FATAL_ERROR__(EFAULT); + /* + * An aborted runlock, not write locked, and no write + * waiters, i.e., we got it... + */ + if (act & ETHR_RWMTX_R_WAIT_FLG__) + wake_other_readers = 1; + break; + } + else { + new = act | ETHR_RWMTX_R_FLG__; + if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) { + /* + * Someone is doing tryrwlock (no writer and no + * write waiters); we will try to abort that... + */ + new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__; + } + + if (act & ETHR_RWMTX_R_WAIT_FLG__) + wake_other_readers = 1; + } + + exp = act; + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + break; + } + + if (wake_other_readers) + rwmutex_transfer_read_lock(rwmtx, act, 0); + } + + return res; +} + +static void +rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx, + ethr_ts_event *tse) +{ + long act; int scnt, start_scnt; int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; @@ -1752,12 +1871,10 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx, while (1) { - rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse); - act = ethr_atomic_read(&rwmtx->mtxb.flgs); - while (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) { - if (scnt >= 0) { + while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + if (scnt <= 0) { if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) { event_wait(&rwmtx->mtxb, tse, scnt, ETHR_RWMTX_R_WAIT_FLG__, 1, 1); @@ -1773,30 +1890,8 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx, scnt--; } - rwmutex_freqread_rdrs_inc(rwmtx, tse); - - ETHR_MEMORY_BARRIER; - - act = ethr_atomic_read(&rwmtx->mtxb.flgs); - - if (act == ETHR_RWMTX_R_FLG__) - return; /* Got it */ - - while (1) { - long exp, new; - - if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) - break; /* Busy (need to restore inc) */ - - if (act & ETHR_RWMTX_R_FLG__) - return; /* Got it */ - - exp = act; - new = act | ETHR_RWMTX_R_FLG__; - act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); - if (act == exp) - return; /* Got it */ - } + if (rwmutex_freqread_rlock(rwmtx, tse, 1) != EBUSY) + break; /* Got it */ } } @@ -1816,14 +1911,23 @@ static ETHR_INLINE void rwlock_wake_set_flags(ethr_rwmutex *rwmtx, long new_initial, int act_initial) { long act, act_mask; + int chk_abrt_flg; + + ETHR_MEMORY_BARRIER; + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) { /* r pend unlock mask may vary and must be retained */ act_mask = ETHR_RWMTX_R_PEND_UNLCK_MASK__; + if (new_initial & ETHR_RWMTX_R_FLG__) + chk_abrt_flg = 1; + else + chk_abrt_flg = 0; } else { #ifdef ETHR_RLOCK_WITH_INC_DEC /* rs mask may vary and must be retained */ act_mask = ETHR_RWMTX_RS_MASK__; + chk_abrt_flg = 0; #else /* rs mask always zero */ ETHR_ASSERT((act_initial & ETHR_RWMTX_RS_MASK__) == 0); @@ -1836,6 +1940,8 @@ rwlock_wake_set_flags(ethr_rwmutex *rwmtx, long new_initial, int act_initial) while (1) { long exp = act; long new = new_initial + (act & act_mask); + if (chk_abrt_flg && (act & act_mask)) + new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__; act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); if (act == exp) break; @@ -1883,6 +1989,14 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx, exp |= ETHR_RWMTX_R_WAIT_FLG__; if (rwmtx->rq_end->next != rwmtx->mtxb.q) exp |= ETHR_RWMTX_W_WAIT_FLG__; + else if (exp == ETHR_RWMTX_R_WAIT_FLG__) { + if (!have_w) { + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) + imask |= ETHR_RWMTX_R_FLG__; + else + imask |= ETHR_RWMTX_RS_MASK__; + } + } act = ethr_atomic_read(&rwmtx->mtxb.flgs); ETHR_ASSERT((exp & ~imask) == (act & ~imask)); @@ -1894,41 +2008,83 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx, #endif static void -rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial) +rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx, long initial, int q_locked) +{ + long act = initial; + + if (!q_locked) { + ethr_ts_event *tse; + ETHR_ASSERT(initial & ETHR_RWMTX_R_WAIT_FLG__); + ETHR_ASSERT((initial & ETHR_RWMTX_W_FLG__) == 0); + ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck); + + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + tse = rwmtx->mtxb.q; + if ((act & ETHR_RWMTX_W_FLG__) || !tse || is_w_waiter(tse)) { + /* Someone else woke the readers up... */ + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + return; + } + } + + rwmutex_unlock_wake(rwmtx, 0, initial, 1); +} + +static void +rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial, + int transfer_read_lock) { long new, act = initial; ethr_ts_event *tse; - if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { - if (!have_w) - return; - else { - while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { - long exp = act; - new = exp & ~ETHR_RWMTX_W_FLG__; - act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); - if (act == exp) - return; + if (transfer_read_lock) { + /* + * - Q already locked + * - Got R waiters first in Q + * - Not W locked + */ + tse = rwmtx->mtxb.q; + + ETHR_ASSERT(act & ETHR_RWMTX_R_WAIT_FLG__); + ETHR_ASSERT((act & (ETHR_RWMTX_W_FLG__)) == 0); + ETHR_ASSERT(tse && !is_w_waiter(tse)); + } + else { + + if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + if (!have_w) + return; + else { + while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + long exp = act; + new = exp & ~ETHR_RWMTX_W_FLG__; + act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + return; + } } } - } - ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck); - tse = rwmtx->mtxb.q; + ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck); + tse = rwmtx->mtxb.q; - if (!have_w) { - if (!tse) { + if (!have_w) { + if (!tse) { #ifdef ETHR_DEBUG - act = ethr_atomic_read(&rwmtx->mtxb.flgs); - ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0); + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0); #endif - goto already_served; - } - act = ethr_atomic_read(&rwmtx->mtxb.flgs); - if (act & ~ETHR_RWMTX_WAIT_FLGS__) { - already_served: - ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); - return; + goto already_served; + } + act = ethr_atomic_read(&rwmtx->mtxb.flgs); + if (act == (ETHR_RWMTX_R_WAIT_FLG__|ETHR_RWMTX_R_FLG__)) { + ETHR_ASSERT(tse && !is_w_waiter(tse)); + } + else if (act & ~ETHR_RWMTX_WAIT_FLGS__) { + already_served: + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + return; + } } } @@ -1988,6 +2144,7 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial) rwmutex_freqread_rdrs_add(rwmtx, type, ix, wrs); } } + new = ETHR_RWMTX_R_FLG__; } @@ -1995,6 +2152,7 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial) new |= ETHR_RWMTX_W_WAIT_FLG__; rwlock_wake_set_flags(rwmtx, new, act); + wake_readers(rwmtx, rs); } } @@ -2225,39 +2383,19 @@ ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) case ETHR_RWMUTEX_TYPE_FREQUENT_READ: case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { ethr_ts_event *tse = ethr_get_ts_event(); - - rwmutex_freqread_rdrs_inc(rwmtx, tse); - - ETHR_MEMORY_BARRIER; - - act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs); - - if (act != ETHR_RWMTX_R_FLG__) { - while (1) { - long exp, new; - - if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) { - rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse); - res = EBUSY; - break; - } - - if (act & ETHR_RWMTX_R_FLG__) - break; - - exp = act; - new = act | ETHR_RWMTX_R_FLG__; - act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp); - if (act == exp) - break; - } - } - + res = rwmutex_freqread_rlock(rwmtx, tse, 1); ethr_leave_ts_event(tse); break; } } +#ifdef ETHR_MTX_CHK_EXCL + if (res == 0) { + ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb); + } +#endif + ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(&rwmtx->mtxb, res); ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); @@ -2286,9 +2424,8 @@ ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) while (1) { act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); - if (act == exp) { + if (act == exp) break; - } if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { rwmutex_normal_rlock_wait(rwmtx, act); @@ -2303,38 +2440,14 @@ ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) case ETHR_RWMUTEX_TYPE_FREQUENT_READ: case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { ethr_ts_event *tse = ethr_get_ts_event(); - - rwmutex_freqread_rdrs_inc(rwmtx, tse); - - ETHR_MEMORY_BARRIER; - - act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs); - - if (act != ETHR_RWMTX_R_FLG__) { - while (1) { - long exp, new; - - if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) { - rwmutex_freqread_rlock_wait(rwmtx, tse, act); - break; - } - - if (act & ETHR_RWMTX_R_FLG__) - break; - - exp = act; - new = act | ETHR_RWMTX_R_FLG__; - act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp); - if (act == exp) - break; - } - } - + rwmutex_freqread_rlock(rwmtx, tse, 0); ethr_leave_ts_event(tse); break; } } + ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb); ETHR_MTX_HARD_DEBUG_LFS_RLOCK(&rwmtx->mtxb); ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); } @@ -2344,6 +2457,8 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) { long act; + ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(&rwmtx->mtxb); ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(rwmtx); ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); @@ -2357,7 +2472,7 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) if ((act & ETHR_RWMTX_WAIT_FLGS__) && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) { ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0); - rwmutex_unlock_wake(rwmtx, 0, act); + rwmutex_unlock_wake(rwmtx, 0, act, 0); } break; @@ -2369,21 +2484,12 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) ETHR_ASSERT(act >= 0); - ETHR_WRITE_MEMORY_BARRIER; + ETHR_MEMORY_BARRIER; if (act == 0) { - -#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL - ETHR_READ_MEMORY_BARRIER; -#endif act = ethr_atomic_read(&rwmtx->mtxb.flgs); - - if ((act & ETHR_RWMTX_W_FLG__) == 0 - && (act & (ETHR_RWMTX_WAIT_FLGS__ - | ETHR_RWMTX_R_PEND_UNLCK_MASK__))) { - rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0); - } - + if (act != ETHR_RWMTX_R_FLG__) + rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act); } ethr_leave_ts_event(tse); @@ -2422,25 +2528,31 @@ ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) do { - if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__)) { - res = EBUSY; - break; - } - - if (act & ETHR_RWMTX_R_MASK__) { + if (act == 0) + act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); + else if (act == ETHR_RWMTX_R_FLG__) { res = rwmutex_try_complete_runlock(rwmtx, act, NULL, 0, 1, 1); break; } - - act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, - ETHR_RWMTX_W_FLG__, 0); + else { + res = EBUSY; + break; + } } while (act != 0); break; } +#ifdef ETHR_MTX_CHK_EXCL + if (res == 0) { + ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb); + } +#endif + ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&rwmtx->mtxb, res); ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); @@ -2485,6 +2597,8 @@ ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) break; } + ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb); ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&rwmtx->mtxb); ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); @@ -2501,12 +2615,15 @@ ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_UNSET_EXCL(&rwmtx->mtxb); + switch (rwmtx->type) { case ETHR_RWMUTEX_TYPE_NORMAL: act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs, 0, ETHR_RWMTX_W_FLG__); if (act != ETHR_RWMTX_W_FLG__) - rwmutex_unlock_wake(rwmtx, 1, act); + rwmutex_unlock_wake(rwmtx, 1, act, 0); break; case ETHR_RWMUTEX_TYPE_FREQUENT_READ: @@ -2514,7 +2631,7 @@ ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs, 0, ETHR_RWMTX_W_FLG__); if (act != ETHR_RWMTX_W_FLG__) - rwmutex_unlock_wake(rwmtx, 1, act); + rwmutex_unlock_wake(rwmtx, 1, act, 0); break; } |