aboutsummaryrefslogtreecommitdiffstats
path: root/erts/lib_src
diff options
context:
space:
mode:
Diffstat (limited to 'erts/lib_src')
-rw-r--r--erts/lib_src/Makefile.in7
-rw-r--r--erts/lib_src/common/erl_misc_utils.c276
-rw-r--r--erts/lib_src/common/erl_printf_format.c27
-rw-r--r--erts/lib_src/common/ethr_atomics.c402
-rw-r--r--erts/lib_src/common/ethr_aux.c202
-rw-r--r--erts/lib_src/common/ethr_mutex.c1010
-rw-r--r--erts/lib_src/pthread/ethr_event.c34
-rw-r--r--erts/lib_src/pthread/ethread.c40
-rw-r--r--erts/lib_src/win/ethr_event.c7
-rw-r--r--erts/lib_src/win/ethread.c55
10 files changed, 1336 insertions, 724 deletions
diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in
index 0d3181cace..757b3b24e2 100644
--- a/erts/lib_src/Makefile.in
+++ b/erts/lib_src/Makefile.in
@@ -283,6 +283,7 @@ endif
ETHR_THR_LIB_BASE_DIR=@ETHR_THR_LIB_BASE_DIR@
ifneq ($(strip $(ETHR_LIB_NAME)),)
ETHREAD_LIB_SRC=common/ethr_aux.c \
+ common/ethr_atomics.c \
common/ethr_mutex.c \
common/ethr_cbf.c \
$(ETHR_THR_LIB_BASE_DIR)/ethread.c \
@@ -381,6 +382,11 @@ $(ERTS_LIB): $(ERTS_LIB_OBJS)
# Object files
#
+ifeq ($(TYPE)-@GCC@,debug-yes)
+$(r_OBJ_DIR)/ethr_aux.o: common/ethr_aux.c
+ $(CC) $(THR_DEFS) $(CFLAGS) -Wno-unused-function $(INCLUDES) -c $< -o $@
+endif
+
$(r_OBJ_DIR)/%.o: common/%.c
$(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@
@@ -445,6 +451,7 @@ INTERNAL_RELEASE_INCLUDES= \
$(ERTS_INCL_INT)/ethread.h \
$(ERTS_INCL_INT)/ethr_mutex.h \
$(ERTS_INCL_INT)/ethr_optimized_fallbacks.h \
+ $(ERTS_INCL_INT)/ethr_atomics.h \
$(ERTS_INCL_INT)/$(TARGET)/ethread.mk \
$(ERTS_INCL_INT)/$(TARGET)/erts_internal.mk \
$(ERTS_INCL_INT)/$(TARGET)/ethread_header_config.h \
diff --git a/erts/lib_src/common/erl_misc_utils.c b/erts/lib_src/common/erl_misc_utils.c
index 116c9886d8..ec729407bb 100644
--- a/erts/lib_src/common/erl_misc_utils.c
+++ b/erts/lib_src/common/erl_misc_utils.c
@@ -71,6 +71,19 @@
(CPUSET)) != 0 ? -errno : 0)
#define ERTS_MU_SET_THR_AFFINITY__(SETP) \
(sched_setaffinity(0, sizeof(cpu_set_t), (SETP)) != 0 ? -errno : 0)
+#elif defined(HAVE_CPUSET_xETAFFINITY)
+# include <sys/param.h>
+# include <sys/cpuset.h>
+# define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__
+#define ERTS_MU_GET_PROC_AFFINITY__(CPUINFOP, CPUSET) \
+ (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, \
+ sizeof(cpuset_t), \
+ (CPUSET)) != 0 ? -errno : 0)
+#define ERTS_MU_SET_THR_AFFINITY__(CPUSETP) \
+ (cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, \
+ sizeof(cpuset_t), \
+ (CPUSETP)) != 0 ? -errno : 0)
+# define cpu_set_t cpuset_t
#elif defined(__WIN32__)
# define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__
# define cpu_set_t DWORD
@@ -100,6 +113,11 @@
# define ERTS_SYS_CPU_PATH "/sys/devices/system/cpu"
#endif
+#ifdef __FreeBSD__
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#endif
+
static int read_topology(erts_cpu_info_t *cpuinfo);
#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
@@ -795,7 +813,7 @@ read_topology(erts_cpu_info_t *cpuinfo)
cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t)
* cpuinfo->configured);
- if (!cpuinfo)
+ if (!cpuinfo->topology)
goto error;
for (ix = 0; ix < cpuinfo->configured; ix++) {
@@ -1005,7 +1023,7 @@ read_topology(erts_cpu_info_t *cpuinfo)
cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t)
* cpuinfo->configured);
- if (!cpuinfo)
+ if (!cpuinfo->topology)
goto error;
for (ix = 0; ix < cpuinfo->configured; ix++) {
@@ -1228,7 +1246,10 @@ read_topology(erts_cpu_info_t *cpuinfo)
nodes++;
}
- core_id = malloc(sizeof(int)*(packages ? packages : 1));
+ if (!packages) {
+ packages = 1;
+ }
+ core_id = malloc(sizeof(int)*packages);
if (!core_id) {
res = -ENOMEM;
goto error;
@@ -1286,11 +1307,13 @@ read_topology(erts_cpu_info_t *cpuinfo)
* Nodes and packages may not be supported; pretend
* that there are one if this is the case...
*/
- if (!nodes)
- cpuinfo->topology[l].node = 0;
- if (!packages)
- cpuinfo->topology[l].processor = 0;
if (slpip[rix].ProcessorMask & (((ULONG_PTR) 1) << l)) {
+ if (!nodes) {
+ cpuinfo->topology[l].node = 0;
+ }
+ if (!packages) {
+ cpuinfo->topology[l].processor = 0;
+ }
if (processor < 0) {
processor = cpuinfo->topology[l].processor;
if (processor < 0) {
@@ -1375,6 +1398,245 @@ read_topology(erts_cpu_info_t *cpuinfo)
return res;
}
+#elif defined(__FreeBSD__)
+
+/**
+ * FreeBSD topology detection is based on kern.sched.topology_spec XML as
+ * exposed by the ULE scheduler and described in SMP(4). It is available in
+ * 8.0 and higher.
+ *
+ * Threads are identified in this XML chunk with a THREAD flag. The function
+ * (simplistically) distinguishes cores and processors by the amount of cache
+ * they share (0 => processor, otherwise => core). Nodes are not identified
+ * (ULE doesn't handle NUMA yet, I believe).
+ */
+
+/**
+ * Recursively parse a topology_spec <group> tag.
+ */
+static
+const char* parse_topology_spec_group(erts_cpu_info_t *cpuinfo, const char* xml, int parentCacheLevel, int* processor_p, int* core_p, int* index_procs_p) {
+ int error = 0;
+ int cacheLevel = parentCacheLevel;
+ const char* next_group_start = strstr(xml + 1, "<group");
+ int is_thread_group = 0;
+ const char* next_cache_level;
+ const char* next_thread_flag;
+ const char* next_group_end;
+ const char* next_children;
+ const char* next_children_end;
+
+ /* parse the cache level */
+ next_cache_level = strstr(xml, "cache-level=\"");
+ if (next_cache_level && (next_group_start == NULL || next_cache_level < next_group_start)) {
+ sscanf(next_cache_level, "cache-level=\"%i\"", &cacheLevel);
+ }
+
+ /* parse the threads flag */
+ next_thread_flag = strstr(xml, "THREAD");
+ if (next_thread_flag && (next_group_start == NULL || next_thread_flag < next_group_start))
+ is_thread_group = 1;
+
+ /* Determine if it's a leaf with the position of the next children tag */
+ next_group_end = strstr(xml, "</group>");
+ next_children = strstr(xml, "<children>");
+ next_children_end = strstr(xml, "</children>");
+ if (next_children == NULL || next_group_end < next_children) {
+ do {
+ const char* next_cpu_start;
+ const char* next_cpu_cdata;
+ const char* next_cpu_end;
+ int cpu_str_size;
+ char* cpu_str;
+ char* cpu_crsr;
+ char* brkb;
+ int thread = 0;
+ int index_procs = *index_procs_p;
+
+ next_cpu_start = strstr(xml, "<cpu");
+ if (!next_cpu_start) {
+ error = 1;
+ break;
+ }
+ next_cpu_cdata = strstr(next_cpu_start, ">") + 1;
+ if (!next_cpu_cdata) {
+ error = 1;
+ break;
+ }
+ next_cpu_end = strstr(next_cpu_cdata, "</cpu>");
+ if (!next_cpu_end) {
+ error = 1;
+ break;
+ }
+ cpu_str_size = next_cpu_end - next_cpu_cdata;
+ cpu_str = (char*) malloc(cpu_str_size + 1);
+ memcpy(cpu_str, (const char*) next_cpu_cdata, cpu_str_size);
+ cpu_str[cpu_str_size] = 0;
+ for (cpu_crsr = strtok_r(cpu_str, " \t,", &brkb); cpu_crsr; cpu_crsr = strtok_r(NULL, " \t,", &brkb)) {
+ int cpu_id;
+ if (index_procs >= cpuinfo->configured) {
+ void* t = realloc(cpuinfo->topology, (sizeof(erts_cpu_topology_t) * (index_procs + 1)));
+ if (t) {
+ cpuinfo->topology = t;
+ } else {
+ error = 1;
+ break;
+ }
+ }
+ cpu_id = atoi(cpu_crsr);
+ cpuinfo->topology[index_procs].node = -1;
+ cpuinfo->topology[index_procs].processor = *processor_p;
+ cpuinfo->topology[index_procs].processor_node = -1;
+ cpuinfo->topology[index_procs].core = *core_p;
+ cpuinfo->topology[index_procs].thread = thread;
+ cpuinfo->topology[index_procs].logical = cpu_id;
+ if (is_thread_group) {
+ thread++;
+ } else {
+ *core_p = (*core_p)++;
+ }
+ index_procs++;
+ }
+ *index_procs_p = index_procs;
+ free(cpu_str);
+ } while (0);
+ xml = next_group_end;
+ } else {
+ while (next_group_start != NULL && next_group_start < next_children_end) {
+ xml = parse_topology_spec_group(cpuinfo, next_group_start, cacheLevel, processor_p, core_p, index_procs_p);
+ if (!xml)
+ break;
+ next_group_start = strstr(xml, "<group");
+ next_children_end = strstr(xml, "</children>");
+ }
+ }
+
+ if (cacheLevel == 0) {
+ *core_p = 0;
+ *processor_p = (*processor_p)++;
+ } else {
+ *core_p = (*core_p)++;
+ }
+
+ if (error)
+ xml = NULL;
+
+ return xml;
+}
+
+/**
+ * Parse the topology_spec. Return the number of CPUs or 0 if parsing failed.
+ */
+static
+int parse_topology_spec(erts_cpu_info_t *cpuinfo, const char* xml) {
+ int res = 1;
+ int index_procs = 0;
+ int core = 0;
+ int processor = 0;
+ xml = strstr(xml, "<groups");
+ if (!xml)
+ return -1;
+
+ xml += 7;
+ xml = strstr(xml, "<group");
+ while (xml) {
+ xml = parse_topology_spec_group(cpuinfo, xml, 0, &processor, &core, &index_procs);
+ if (!xml) {
+ res = 0;
+ break;
+ }
+ xml = strstr(xml, "<group");
+ }
+
+ if (res)
+ res = index_procs;
+
+ return res;
+}
+
+static int
+read_topology(erts_cpu_info_t *cpuinfo)
+{
+ int ix;
+ int res = 0;
+ size_t topology_spec_size = 0;
+ void* topology_spec = NULL;
+
+ errno = 0;
+
+ if (cpuinfo->configured < 1)
+ goto error;
+
+ cpuinfo->topology_size = cpuinfo->configured;
+ cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t)
+ * cpuinfo->configured);
+ if (!cpuinfo->topology) {
+ res = -ENOMEM;
+ goto error;
+ }
+
+ for (ix = 0; ix < cpuinfo->configured; ix++) {
+ cpuinfo->topology[ix].node = -1;
+ cpuinfo->topology[ix].processor = -1;
+ cpuinfo->topology[ix].processor_node = -1;
+ cpuinfo->topology[ix].core = -1;
+ cpuinfo->topology[ix].thread = -1;
+ cpuinfo->topology[ix].logical = -1;
+ }
+
+ if (!sysctlbyname("kern.sched.topology_spec", NULL, &topology_spec_size, NULL, 0)) {
+ topology_spec = malloc(topology_spec_size);
+ if (!topology_spec) {
+ res = -ENOMEM;
+ goto error;
+ }
+
+ if (sysctlbyname("kern.sched.topology_spec", topology_spec, &topology_spec_size, NULL, 0)) {
+ goto error;
+ }
+
+ res = parse_topology_spec(cpuinfo, topology_spec);
+ if (!res || res < cpuinfo->online)
+ res = 0;
+ else {
+ cpuinfo->topology_size = res;
+
+ if (cpuinfo->topology_size != cpuinfo->configured) {
+ void *t = realloc(cpuinfo->topology, (sizeof(erts_cpu_topology_t)
+ * cpuinfo->topology_size));
+ if (t)
+ cpuinfo->topology = t;
+ }
+
+ adjust_processor_nodes(cpuinfo, 1);
+
+ qsort(cpuinfo->topology,
+ cpuinfo->topology_size,
+ sizeof(erts_cpu_topology_t),
+ cpu_cmp);
+ }
+ }
+
+error:
+
+ if (res == 0) {
+ cpuinfo->topology_size = 0;
+ if (cpuinfo->topology) {
+ free(cpuinfo->topology);
+ cpuinfo->topology = NULL;
+ }
+ if (errno)
+ res = -errno;
+ else
+ res = -EINVAL;
+ }
+
+ if (topology_spec)
+ free(topology_spec);
+
+ return res;
+}
+
#else
static int
diff --git a/erts/lib_src/common/erl_printf_format.c b/erts/lib_src/common/erl_printf_format.c
index bd3d38e649..968d563325 100644
--- a/erts/lib_src/common/erl_printf_format.c
+++ b/erts/lib_src/common/erl_printf_format.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -27,7 +27,7 @@
* length: hh | h | l | ll | L | j | t | b<sz>
* conversion: d,i | o,u,x,X | e,E | f,F | g,G | a,A | c | s | T |
* p | n | %
- * sz: 8 | 16 | 32 | 64 | p
+ * sz: 8 | 16 | 32 | 64 | p | e
*/
/* Without this, variable argument lists break on VxWorks */
@@ -76,6 +76,18 @@
#endif
#endif
+#ifndef ERTS_SIZEOF_ETERM
+# ifdef HALFWORD_HEAP_EMULATOR
+# if SIZEOF_VOID_P == 8
+# define ERTS_SIZEOF_ETERM 4
+# else
+# error "HALFWORD_HEAP_EMULATOR only allowed on 64-bit architecture"
+# endif
+# else
+# define ERTS_SIZEOF_ETERM SIZEOF_VOID_P
+# endif
+#endif
+
#if defined(__GNUC__)
# undef inline
# define inline __inline__
@@ -520,6 +532,17 @@ int erts_printf_format(fmtfn_t fn, void* arg, char* fmt, va_list ap)
#error No integer datatype with the same size as 'void *' found
#endif
}
+ else if (*ptr == 'e') {
+ ptr++;
+#if SIZEOF_INT == ERTS_SIZEOF_ETERM
+#elif SIZEOF_LONG == ERTS_SIZEOF_ETERM
+ fmt |= FMTL_l;
+#elif SIZEOF_LONG_LONG == ERTS_SIZEOF_ETERM
+ fmt |= FMTL_ll;
+#else
+#error No integer datatype with the same size as Eterm found
+#endif
+ }
else {
int bits = 0;
while(isdigit((int) *ptr))
diff --git a/erts/lib_src/common/ethr_atomics.c b/erts/lib_src/common/ethr_atomics.c
new file mode 100644
index 0000000000..94557d904a
--- /dev/null
+++ b/erts/lib_src/common/ethr_atomics.c
@@ -0,0 +1,402 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: The ethread atomic API
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_ATOMIC_IMPL__
+
+#include "ethread.h"
+#include "ethr_internal.h"
+
+#ifndef ETHR_HAVE_NATIVE_ATOMICS
+ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
+#endif
+
+int
+ethr_init_atomics(void)
+{
+#ifndef ETHR_HAVE_NATIVE_ATOMICS
+ {
+ int i;
+ for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) {
+ int res = ethr_spinlock_init(&ethr_atomic_protection__[i].u.lck);
+ if (res != 0)
+ return res;
+ }
+ }
+#endif
+ return 0;
+}
+
+/*
+ * --- Pointer size atomics ---------------------------------------------------
+ */
+
+ethr_sint_t *
+ethr_atomic_addr(ethr_atomic_t *var)
+{
+ ETHR_ASSERT(var);
+ return ethr_atomic_addr__(var);
+}
+
+void
+ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t i)
+{
+ ETHR_ASSERT(var);
+ ethr_atomic_init__(var, i);
+}
+
+void
+ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t i)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_atomic_set__(var, i);
+}
+
+ethr_sint_t
+ethr_atomic_read(ethr_atomic_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic_read__(var);
+}
+
+ethr_sint_t
+ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t incr)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic_add_read__(var, incr);
+}
+
+ethr_sint_t
+ethr_atomic_inc_read(ethr_atomic_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic_inc_read__(var);
+}
+
+ethr_sint_t
+ethr_atomic_dec_read(ethr_atomic_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic_dec_read__(var);
+}
+
+void
+ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t incr)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_atomic_add__(var, incr);
+}
+
+void
+ethr_atomic_inc(ethr_atomic_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_atomic_inc__(var);
+}
+
+void
+ethr_atomic_dec(ethr_atomic_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_atomic_dec__(var);
+}
+
+ethr_sint_t
+ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t mask)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic_read_band__(var, mask);
+}
+
+ethr_sint_t
+ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t mask)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic_read_bor__(var, mask);
+}
+
+ethr_sint_t
+ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t new)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic_xchg__(var, new);
+}
+
+ethr_sint_t
+ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t expected)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic_cmpxchg__(var, new, expected);
+}
+
+ethr_sint_t
+ethr_atomic_read_acqb(ethr_atomic_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic_read_acqb__(var);
+}
+
+ethr_sint_t
+ethr_atomic_inc_read_acqb(ethr_atomic_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic_inc_read_acqb__(var);
+}
+
+void
+ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t i)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_atomic_set_relb__(var, i);
+}
+
+void
+ethr_atomic_dec_relb(ethr_atomic_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_atomic_dec_relb__(var);
+}
+
+ethr_sint_t
+ethr_atomic_dec_read_relb(ethr_atomic_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic_dec_read_relb__(var);
+}
+
+ethr_sint_t
+ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic_cmpxchg_acqb__(var, new, exp);
+}
+
+ethr_sint_t
+ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic_cmpxchg_relb__(var, new, exp);
+}
+
+
+/*
+ * --- 32-bit atomics ---------------------------------------------------------
+ */
+
+ethr_sint32_t *
+ethr_atomic32_addr(ethr_atomic32_t *var)
+{
+ ETHR_ASSERT(var);
+ return ethr_atomic32_addr__(var);
+}
+
+void
+ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t i)
+{
+ ETHR_ASSERT(var);
+ ethr_atomic32_init__(var, i);
+}
+
+void
+ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t i)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_atomic32_set__(var, i);
+}
+
+ethr_sint32_t
+ethr_atomic32_read(ethr_atomic32_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic32_read__(var);
+}
+
+
+ethr_sint32_t
+ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t incr)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic32_add_read__(var, incr);
+}
+
+ethr_sint32_t
+ethr_atomic32_inc_read(ethr_atomic32_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic32_inc_read__(var);
+}
+
+ethr_sint32_t
+ethr_atomic32_dec_read(ethr_atomic32_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic32_dec_read__(var);
+}
+
+void
+ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t incr)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_atomic32_add__(var, incr);
+}
+
+void
+ethr_atomic32_inc(ethr_atomic32_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_atomic32_inc__(var);
+}
+
+void
+ethr_atomic32_dec(ethr_atomic32_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_atomic32_dec__(var);
+}
+
+ethr_sint32_t
+ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t mask)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic32_read_band__(var, mask);
+}
+
+ethr_sint32_t
+ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t mask)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic32_read_bor__(var, mask);
+}
+
+ethr_sint32_t
+ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t new)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic32_xchg__(var, new);
+}
+
+ethr_sint32_t
+ethr_atomic32_cmpxchg(ethr_atomic32_t *var,
+ ethr_sint32_t new,
+ ethr_sint32_t expected)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic32_cmpxchg__(var, new, expected);
+}
+
+ethr_sint32_t
+ethr_atomic32_read_acqb(ethr_atomic32_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic32_read_acqb__(var);
+}
+
+ethr_sint32_t
+ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic32_inc_read_acqb__(var);
+}
+
+void
+ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t i)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_atomic32_set_relb__(var, i);
+}
+
+void
+ethr_atomic32_dec_relb(ethr_atomic32_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ ethr_atomic32_dec_relb__(var);
+}
+
+ethr_sint32_t
+ethr_atomic32_dec_read_relb(ethr_atomic32_t *var)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic32_dec_read_relb__(var);
+}
+
+ethr_sint32_t
+ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var,
+ ethr_sint32_t new,
+ ethr_sint32_t exp)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic32_cmpxchg_acqb__(var, new, exp);
+}
+
+ethr_sint32_t
+ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var,
+ ethr_sint32_t new,
+ ethr_sint32_t exp)
+{
+ ETHR_ASSERT(!ethr_not_inited__);
+ ETHR_ASSERT(var);
+ return ethr_atomic32_cmpxchg_relb__(var, new, exp);
+}
+
diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c
index 4db4cffd3a..2c3e25a805 100644
--- a/erts/lib_src/common/ethr_aux.c
+++ b/erts/lib_src/common/ethr_aux.c
@@ -31,7 +31,10 @@
#define ETHR_INLINE_FUNC_NAME_(X) X ## __
#define ETHR_AUX_IMPL__
-
+#define ETHR_ATOMIC_IMPL__ /* Needed in order to pull in
+ native atomic implementations
+ for optimized fallbacks of
+ spinlocks and rwspinlocks */
#include "ethread.h"
#include "ethr_internal.h"
#include <string.h>
@@ -51,10 +54,6 @@ int ethr_not_inited__ = 1;
ethr_memory_allocators ethr_mem__ = ETHR_MEM_ALLOCS_DEF_INITER__;
-#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
-#endif
-
void *(*ethr_thr_prepare_func__)(void) = NULL;
void (*ethr_thr_parent_func__)(void *) = NULL;
void (*ethr_thr_child_func__)(void *) = NULL;
@@ -138,16 +137,9 @@ ethr_init_common__(ethr_init_data *id)
#endif
ethr_max_stack_size__ = ETHR_B2KW(ethr_max_stack_size__);
-#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
- {
- int i;
- for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) {
- res = ethr_spinlock_init(&ethr_atomic_protection__[i].u.lck);
- if (res != 0)
- return res;
- }
- }
-#endif
+ res = ethr_init_atomics();
+ if (res != 0)
+ return res;
res = ethr_mutex_lib_init(erts_get_cpu_configured(ethr_cpu_info__));
if (res != 0)
@@ -279,14 +271,6 @@ typedef union {
static ethr_spinlock_t ts_ev_alloc_lock;
static ethr_ts_event *free_ts_ev;
-#if SIZEOF_VOID_P == SIZEOF_INT
-typedef unsigned int EthrPtrSzUInt;
-#elif SIZEOF_VOID_P == SIZEOF_LONG
-typedef unsigned long EthrPtrSzUInt;
-#else
-#error No pointer sized integer type
-#endif
-
static ethr_ts_event *ts_event_pool(int size, ethr_ts_event **endpp)
{
int i;
@@ -295,16 +279,16 @@ static ethr_ts_event *ts_event_pool(int size, ethr_ts_event **endpp)
+ ETHR_CACHE_LINE_SIZE);
if (!atsev)
return NULL;
- if ((((EthrPtrSzUInt) atsev) & ETHR_CACHE_LINE_MASK) == 0)
+ if ((((ethr_uint_t) atsev) & ETHR_CACHE_LINE_MASK) == 0)
atsev = ((ethr_aligned_ts_event *)
- ((((EthrPtrSzUInt) atsev) & ~ETHR_CACHE_LINE_MASK)
+ ((((ethr_uint_t) atsev) & ~ETHR_CACHE_LINE_MASK)
+ ETHR_CACHE_LINE_SIZE));
for (i = 1; i < size; i++) {
atsev[i-1].ts_ev.next = &atsev[i].ts_ev;
- ethr_atomic_init(&atsev[i-1].ts_ev.uaflgs, 0);
+ ethr_atomic32_init(&atsev[i-1].ts_ev.uaflgs, 0);
atsev[i-1].ts_ev.iflgs = 0;
}
- ethr_atomic_init(&atsev[size-1].ts_ev.uaflgs, 0);
+ ethr_atomic32_init(&atsev[size-1].ts_ev.uaflgs, 0);
atsev[size-1].ts_ev.iflgs = 0;
atsev[size-1].ts_ev.next = NULL;
if (endpp)
@@ -466,170 +450,6 @@ int ethr_get_main_thr_status(int *on)
return 0;
}
-
-/* Atomics */
-
-void
-ethr_atomic_init(ethr_atomic_t *var, long i)
-{
- ETHR_ASSERT(var);
- ethr_atomic_init__(var, i);
-}
-
-void
-ethr_atomic_set(ethr_atomic_t *var, long i)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- ethr_atomic_set__(var, i);
-}
-
-long
-ethr_atomic_read(ethr_atomic_t *var)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- return ethr_atomic_read__(var);
-}
-
-
-long
-ethr_atomic_add_read(ethr_atomic_t *var, long incr)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- return ethr_atomic_add_read__(var, incr);
-}
-
-long
-ethr_atomic_inc_read(ethr_atomic_t *var)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- return ethr_atomic_inc_read__(var);
-}
-
-long
-ethr_atomic_dec_read(ethr_atomic_t *var)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- return ethr_atomic_dec_read__(var);
-}
-
-void
-ethr_atomic_add(ethr_atomic_t *var, long incr)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- ethr_atomic_add__(var, incr);
-}
-
-void
-ethr_atomic_inc(ethr_atomic_t *var)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- ethr_atomic_inc__(var);
-}
-
-void
-ethr_atomic_dec(ethr_atomic_t *var)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- ethr_atomic_dec__(var);
-}
-
-long
-ethr_atomic_read_band(ethr_atomic_t *var, long mask)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- return ethr_atomic_read_band__(var, mask);
-}
-
-long
-ethr_atomic_read_bor(ethr_atomic_t *var, long mask)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- return ethr_atomic_read_bor__(var, mask);
-}
-
-long
-ethr_atomic_xchg(ethr_atomic_t *var, long new)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- return ethr_atomic_xchg__(var, new);
-}
-
-long
-ethr_atomic_cmpxchg(ethr_atomic_t *var, long new, long expected)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- return ethr_atomic_cmpxchg__(var, new, expected);
-}
-
-long
-ethr_atomic_read_acqb(ethr_atomic_t *var)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- return ethr_atomic_read_acqb__(var);
-}
-
-long
-ethr_atomic_inc_read_acqb(ethr_atomic_t *var)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- return ethr_atomic_inc_read_acqb__(var);
-}
-
-void
-ethr_atomic_set_relb(ethr_atomic_t *var, long i)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- ethr_atomic_set_relb__(var, i);
-}
-
-void
-ethr_atomic_dec_relb(ethr_atomic_t *var)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- ethr_atomic_dec_relb__(var);
-}
-
-long
-ethr_atomic_dec_read_relb(ethr_atomic_t *var)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- return ethr_atomic_dec_read_relb__(var);
-}
-
-long
-ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, long new, long exp)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- return ethr_atomic_cmpxchg_acqb__(var, new, exp);
-}
-
-long
-ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, long new, long exp)
-{
- ETHR_ASSERT(!ethr_not_inited__);
- ETHR_ASSERT(var);
- return ethr_atomic_cmpxchg_relb__(var, new, exp);
-}
-
-
/* Spinlocks and rwspinlocks */
int
diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c
index aac0d44a32..2ddef32dfc 100644
--- a/erts/lib_src/common/ethr_mutex.c
+++ b/erts/lib_src/common/ethr_mutex.c
@@ -35,7 +35,7 @@
#define ETHR_SPIN_WITH_WAITERS 1
-#define ETHR_MTX_MAX_FLGS_SPIN 1000
+#define ETHR_MTX_MAX_FLGS_SPIN 10
#ifdef ETHR_USE_OWN_RWMTX_IMPL__
static int default_rwmtx_main_spincount;
@@ -90,7 +90,7 @@ ethr_mutex_lib_init(int cpu_conf)
no_spin = cpu_conf == 1;
#ifdef ETHR_USE_OWN_MTX_IMPL__
- default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT;
+ default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE;
default_mtx_aux_spincount = ETHR_MTX_DEFAULT_AUX_SPINCOUNT;
default_cnd_main_spincount = ETHR_CND_DEFAULT_MAIN_SPINCOUNT;
default_cnd_aux_spincount = ETHR_CND_DEFAULT_AUX_SPINCOUNT;
@@ -98,7 +98,7 @@ ethr_mutex_lib_init(int cpu_conf)
#ifdef ETHR_USE_OWN_RWMTX_IMPL__
- default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT;
+ default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE;
default_rwmtx_aux_spincount = ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT;
#else
@@ -146,7 +146,21 @@ static int main_threads_array_size = 0;
int
ethr_mutex_lib_late_init(int no_reader_groups, int no_main_threads)
{
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+ default_mtx_main_spincount += (no_main_threads
+ * ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC);
+ if (default_mtx_main_spincount > ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX)
+ default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX;
+#endif
+
#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+ default_rwmtx_main_spincount += (no_main_threads
+ * ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC);
+ if (default_rwmtx_main_spincount > ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX)
+ default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX;
+
reader_groups_array_size = (no_reader_groups <= 1
? 1
: no_reader_groups + 1);
@@ -191,12 +205,17 @@ static void hard_debug_chk_q__(struct ethr_mutex_base_ *, int);
#ifdef ETHR_USE_OWN_RWMTX_IMPL__
static void
+rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx,
+ ethr_sint32_t initial,
+ int q_locked);
+static void
rwmutex_unlock_wake(ethr_rwmutex *rwmtx,
int have_w,
- long initial);
+ ethr_sint32_t initial,
+ int transfer_read_lock);
static int
rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
- long initial,
+ ethr_sint32_t initial,
ethr_ts_event *tse,
int start_next_ix,
int check_before_try,
@@ -223,12 +242,12 @@ rwmutex_freqread_rdrs_add(ethr_rwmutex *rwmtx,
int inc)
{
if (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ || ix == 0)
- ethr_atomic_add(&rwmtx->tdata.ra[ix].data.readers, inc);
+ ethr_atomic32_add(&rwmtx->tdata.ra[ix].data.readers, inc);
else {
ETHR_ASSERT(type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
- ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
+ ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
ETHR_ASSERT(inc == 1);
- ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 1);
+ ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 1);
}
}
@@ -239,18 +258,20 @@ rwmutex_freqread_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
ix = tse->rgix;
atomic_inc:
- ethr_atomic_inc(&rwmtx->tdata.ra[ix].data.readers);
+ ethr_atomic32_inc(&rwmtx->tdata.ra[ix].data.readers);
}
else {
ix = tse->mtix;
if (ix == 0)
goto atomic_inc;
ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
- ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
- ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 1);
+ ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
+ ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 1);
}
}
+#if 0 /* Not used */
+
static ETHR_INLINE void
rwmutex_freqread_rdrs_dec(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
{
@@ -258,69 +279,72 @@ rwmutex_freqread_rdrs_dec(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
ix = tse->rgix;
atomic_dec:
- ethr_atomic_dec(&rwmtx->tdata.ra[ix].data.readers);
+ ethr_atomic32_dec(&rwmtx->tdata.ra[ix].data.readers);
}
else {
ix = tse->mtix;
if (ix == 0)
goto atomic_dec;
ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
- ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
- ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 0);
+ ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+ ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 0);
}
}
-static ETHR_INLINE long
+#endif
+
+static ETHR_INLINE ethr_sint32_t
rwmutex_freqread_rdrs_dec_read(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
{
int ix;
if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
ix = tse->rgix;
atomic_dec_read:
- return ethr_atomic_dec_read(&rwmtx->tdata.ra[ix].data.readers);
+ return ethr_atomic32_dec_read(&rwmtx->tdata.ra[ix].data.readers);
}
else {
ix = tse->mtix;
if (ix == 0)
goto atomic_dec_read;
ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
- ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
- ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 0);
- return (long) 0;
+ ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+ ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 0);
+ return (ethr_sint32_t) 0;
}
}
-static ETHR_INLINE long
+static ETHR_INLINE ethr_sint32_t
rwmutex_freqread_rdrs_dec_read_relb(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
{
int ix;
if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
ix = tse->rgix;
atomic_dec_read:
- return ethr_atomic_dec_read_relb(&rwmtx->tdata.ra[ix].data.readers);
+ return ethr_atomic32_dec_read_relb(&rwmtx->tdata.ra[ix].data.readers);
}
else {
ix = tse->mtix;
if (ix == 0)
goto atomic_dec_read;
ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
- ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
- ethr_atomic_set_relb(&rwmtx->tdata.ra[ix].data.readers, (long) 0);
- return (long) 0;
+ ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+ ethr_atomic32_set_relb(&rwmtx->tdata.ra[ix].data.readers,
+ (ethr_sint32_t) 0);
+ return (ethr_sint32_t) 0;
}
}
-static ETHR_INLINE long
+static ETHR_INLINE ethr_sint32_t
rwmutex_freqread_rdrs_read(ethr_rwmutex *rwmtx, int ix)
{
- long res = ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers);
+ ethr_sint32_t res = ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers);
#ifdef ETHR_DEBUG
switch (rwmtx->type) {
case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
ETHR_ASSERT(res >= 0);
break;
case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
- ETHR_ASSERT(res == 0 || res == 1);
+ ETHR_ASSERT(ix == 0 ? res >= 0 : (res == 0 || res == 1));
break;
default:
ETHR_ASSERT(0);
@@ -379,18 +403,19 @@ static void
event_wait(struct ethr_mutex_base_ *mtxb,
ethr_ts_event *tse,
int spincount,
- long type,
+ ethr_sint32_t type,
int is_rwmtx,
int is_freq_read)
{
int locked = 0;
- long act;
+ ethr_sint32_t act;
int need_try_complete_runlock = 0;
+ int transfer_read_lock = 0;
/* Need to enqueue and wait... */
tse->uflgs = type;
- ethr_atomic_set(&tse->uaflgs, type);
+ ethr_atomic32_set(&tse->uaflgs, type);
ETHR_MTX_Q_LOCK(&mtxb->qlck);
locked = 1;
@@ -399,7 +424,7 @@ event_wait(struct ethr_mutex_base_ *mtxb,
hard_debug_chk_q__(mtxb, is_rwmtx);
#endif
- act = ethr_atomic_read(&mtxb->flgs);
+ act = ethr_atomic32_read(&mtxb->flgs);
if (act & type) {
@@ -429,9 +454,9 @@ event_wait(struct ethr_mutex_base_ *mtxb,
/* Set wait bit */
while (1) {
- long new, exp = act;
- int freqread_tryrlock = 0;
+ ethr_sint32_t new, exp = act;
need_try_complete_runlock = 0;
+ transfer_read_lock = 0;
if (type == ETHR_RWMTX_W_WAIT_FLG__) {
if (is_freq_read && act == ETHR_RWMTX_R_FLG__)
@@ -451,19 +476,16 @@ event_wait(struct ethr_mutex_base_ *mtxb,
new = act + 1; /* Try to get it */
}
else {
- if (act & ~ETHR_RWMTX_R_FLG__)
- new = act | ETHR_RWMTX_R_WAIT_FLG__;
- else { /* Try to get it */
- ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
- rwmutex_freqread_rdrs_inc(rwmtx, tse);
- ETHR_MEMORY_BARRIER;
- new = act | ETHR_RWMTX_R_FLG__;
- freqread_tryrlock = 1;
+ new = act | ETHR_RWMTX_R_WAIT_FLG__;
+ if ((act & (ETHR_RWMTX_W_FLG__
+ | ETHR_RWMTX_W_WAIT_FLG__)) == 0) {
+ /* Transfer read lock to this thread. */
+ transfer_read_lock = 1;
}
}
}
- act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs, new, exp);
+ act = ethr_atomic32_cmpxchg_acqb(&mtxb->flgs, new, exp);
if (exp == act) {
if (new & type) {
act = new;
@@ -474,24 +496,6 @@ event_wait(struct ethr_mutex_base_ *mtxb,
goto done;
}
}
-
- if (freqread_tryrlock) {
- ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
-
- /* We didn't set ETHR_RWMTX_R_FLG__, however someone
- else might have */
- if (act == ETHR_RWMTX_R_FLG__)
- goto done; /* Got it by help from someone else */
-
- ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
- /*
- * We know that no waiter flags have been set, i.e.,
- * we cannot get into a situation where we need to wake
- * someone up here. Just restore the readers counter
- * and do it over again...
- */
- rwmutex_freqread_rdrs_dec(rwmtx, tse);
- }
}
/* Enqueue */
@@ -521,26 +525,42 @@ event_wait(struct ethr_mutex_base_ *mtxb,
/* Wait */
locked = 0;
- ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
- if (need_try_complete_runlock) {
+ ETHR_ASSERT(!(transfer_read_lock && need_try_complete_runlock));
+
+ if (transfer_read_lock) {
ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type
!= ETHR_RWMUTEX_TYPE_NORMAL);
/*
- * We were the only one in queue when we enqueued, and it
- * was seemingly read locked. We need to try to complete a
- * runlock otherwise we might be hanging forever. If the
- * runlock could be completed we will be dequeued and
- * woken by ourselves.
+ * We are the only one in the queue and we are not write
+ * locked; rwmutex_transfer_read_lock() will:
+ * - transfer a read lock to us (since we're first in q)
+ * - unlock the Q-lock
*/
- rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb,
- act, tse, 0, 1, 0);
+ rwmutex_transfer_read_lock(((ethr_rwmutex *) mtxb), act, 1);
+ }
+ else {
+ ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+
+ if (need_try_complete_runlock) {
+ ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type
+ != ETHR_RWMUTEX_TYPE_NORMAL);
+ /*
+ * We were the only one in queue when we enqueued, and it
+ * was seemingly read locked. We need to try to complete a
+ * runlock otherwise we might be hanging forever. If the
+ * runlock could be completed we will be dequeued and
+ * woken by ourselves.
+ */
+ rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb,
+ act, tse, 0, 1, 0);
+ }
}
while (1) {
ethr_event_reset(&tse->event);
- act = ethr_atomic_read_acqb(&tse->uaflgs);
+ act = ethr_atomic32_read_acqb(&tse->uaflgs);
if (!act)
goto done; /* Got it */
@@ -548,7 +568,7 @@ event_wait(struct ethr_mutex_base_ *mtxb,
ethr_event_swait(&tse->event, spincount);
/* swait result: 0 || EINTR */
- act = ethr_atomic_read_acqb(&tse->uaflgs);
+ act = ethr_atomic32_read_acqb(&tse->uaflgs);
if (!act)
goto done; /* Got it */
}
@@ -568,7 +588,7 @@ wake_writer(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
dequeue(&mtxb->q, tse, tse);
ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
- ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__);
+ ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__);
#ifdef ETHR_MTX_HARD_DEBUG_WSQ
mtxb->ws--;
#endif
@@ -578,7 +598,7 @@ wake_writer(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
- ethr_atomic_set(&tse->uaflgs, 0);
+ ethr_atomic32_set(&tse->uaflgs, 0);
ethr_event_set(&tse->event);
}
@@ -593,24 +613,31 @@ initial_spincount(struct ethr_mutex_base_ *mtxb)
static ETHR_INLINE int
update_spincount(struct ethr_mutex_base_ *mtxb,
ethr_ts_event *tse,
- int *start_scnt,
+ int *scnt_state,
int *scnt)
{
- int sscnt = *start_scnt;
- if (sscnt < 0) {
- *scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
- ? mtxb->main_scnt
- : mtxb->aux_scnt);
- *scnt -= ETHR_MTX_MAX_FLGS_SPIN;
+ int state = *scnt_state;
+ if (state <= 0) {
+ /* Here state is max spincount to do on event negated */
+ *scnt = -state;
}
else {
+ /* Here state is initial spincount made on flags */
*scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
? mtxb->main_scnt
: mtxb->aux_scnt);
- *scnt -= sscnt;
- if (*scnt > 0 && sscnt < ETHR_MTX_MAX_FLGS_SPIN) {
- *scnt = ETHR_MTX_MAX_FLGS_SPIN - sscnt;
- *start_scnt = -1;
+ if (*scnt <= state)
+ *scnt = 0;
+ else {
+ if (*scnt <= ETHR_MTX_MAX_FLGS_SPIN)
+ *scnt_state = 0; /* No spin on event */
+ else {
+ /* Spin on event after... */
+ *scnt_state = -1*(*scnt - ETHR_MTX_MAX_FLGS_SPIN);
+ /* ... we have spun on flags */
+ *scnt = ETHR_MTX_MAX_FLGS_SPIN;
+ }
+ *scnt -= state;
return 0;
}
}
@@ -619,22 +646,19 @@ update_spincount(struct ethr_mutex_base_ *mtxb,
int check_readers_array(ethr_rwmutex *rwmtx,
int start_rix,
- int length,
- int pre_check);
+ int length);
static ETHR_INLINE void
write_lock_wait(struct ethr_mutex_base_ *mtxb,
- long initial,
+ ethr_sint32_t initial,
int is_rwmtx,
int is_freq_read)
{
- long act = initial;
+ ethr_sint32_t act = initial;
int scnt, start_scnt;
ethr_ts_event *tse = NULL;
int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
int res;
- int freq_read_size = -1;
- int freq_read_start_ix = -1;
ETHR_ASSERT(!is_freq_read || is_rwmtx);
@@ -646,45 +670,23 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb,
*/
while (1) {
- long exp;
-
while (act != 0) {
if (is_freq_read && act == ETHR_RWMTX_R_FLG__) {
ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+ scnt--;
if (!tse)
tse = ethr_get_ts_event();
- if (freq_read_size < 0) {
- if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
- freq_read_size = reader_groups_array_size;
- freq_read_start_ix = tse->rgix;
- }
- else {
- freq_read_size = main_threads_array_size;
- freq_read_start_ix = tse->mtix;
- }
- }
- res = check_readers_array(rwmtx,
- freq_read_start_ix,
- freq_read_size,
- 1);
- scnt--;
- if (res == 0) {
- act = ethr_atomic_read(&mtxb->flgs);
- if (act & ETHR_RWMTX_R_MASK__) {
- res = rwmutex_try_complete_runlock(rwmtx, act,
- tse, 0, 0,
- 1);
- if (res != EBUSY)
- goto done; /* Got it */
- }
- if (scnt <= 0)
- goto chk_spin;
- if (--until_yield == 0) {
- until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
- ETHR_YIELD();
- }
- continue;
+ res = rwmutex_try_complete_runlock(rwmtx, act,
+ tse, 0, 0,
+ 1);
+ if (res != EBUSY)
+ goto done; /* Got it */
+ if (scnt <= 0)
+ goto chk_spin;
+ if (--until_yield == 0) {
+ until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+ ETHR_YIELD();
}
}
@@ -705,16 +707,13 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb,
until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
ETHR_YIELD();
}
- act = ethr_atomic_read(&mtxb->flgs);
+ act = ethr_atomic32_read(&mtxb->flgs);
scnt--;
}
- ETHR_ASSERT(scnt >= 0);
-
- exp = act;
- act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs,
- ETHR_RWMTX_W_FLG__,
- exp);
+ act = ethr_atomic32_cmpxchg_acqb(&mtxb->flgs,
+ ETHR_RWMTX_W_FLG__,
+ 0);
if (act == 0)
goto done; /* Got it */
}
@@ -735,6 +734,7 @@ mtxb_init(struct ethr_mutex_base_ *mtxb,
#ifdef ETHR_MTX_HARD_DEBUG_WSQ
mtxb->ws = 0;
#endif
+ ETHR_MTX_CHK_EXCL_INIT(mtxb);
if (no_spin) {
mtxb->main_scnt = 0;
mtxb->aux_scnt = 0;
@@ -757,16 +757,16 @@ mtxb_init(struct ethr_mutex_base_ *mtxb,
}
mtxb->q = NULL;
- ethr_atomic_init(&mtxb->flgs, 0);
+ ethr_atomic32_init(&mtxb->flgs, 0);
return ETHR_MTX_QLOCK_INIT(&mtxb->qlck);
}
static int
mtxb_destroy(struct ethr_mutex_base_ *mtxb)
{
- long act;
+ ethr_sint32_t act;
ETHR_MTX_Q_LOCK(&mtxb->qlck);
- act = ethr_atomic_read(&mtxb->flgs);
+ act = ethr_atomic32_read(&mtxb->flgs);
ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
if (act != 0)
return EINVAL;
@@ -832,13 +832,13 @@ ethr_mutex_destroy(ethr_mutex *mtx)
}
void
-ethr_mutex_lock_wait__(ethr_mutex *mtx, long initial)
+ethr_mutex_lock_wait__(ethr_mutex *mtx, ethr_sint32_t initial)
{
write_lock_wait(&mtx->mtxb, initial, 0, 0);
}
void
-ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial)
+ethr_mutex_unlock_wake__(ethr_mutex *mtx, ethr_sint32_t initial)
{
ethr_ts_event *tse;
@@ -846,7 +846,7 @@ ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial)
tse = mtx->mtxb.q;
ETHR_ASSERT(tse);
- ETHR_ASSERT(ethr_atomic_read(&mtx->mtxb.flgs)
+ ETHR_ASSERT(ethr_atomic32_read(&mtx->mtxb.flgs)
== (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__));
ETHR_ASSERT(initial & ETHR_RWMTX_W_WAIT_FLG__);
ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
@@ -856,7 +856,7 @@ ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial)
* mtxb->flgs; otherwise, we need to clear the write wait bit...
*/
if (tse->next == mtx->mtxb.q)
- ethr_atomic_set(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__);
+ ethr_atomic32_set(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__);
wake_writer(&mtx->mtxb, 0);
}
@@ -866,7 +866,7 @@ ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial)
static void
enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
{
- long act;
+ ethr_sint32_t act;
/*
* `ethr_cond_signal()' and `ethr_cond_broadcast()' end up here. If `mtx'
@@ -895,7 +895,7 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
}
#endif
- act = ethr_atomic_read(&mtx->mtxb.flgs);
+ act = ethr_atomic32_read(&mtx->mtxb.flgs);
ETHR_ASSERT(act == 0
|| act == ETHR_RWMTX_W_FLG__
|| act == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__));
@@ -903,10 +903,10 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
/* The normal sane case */
if (!(act & ETHR_RWMTX_W_WAIT_FLG__)) {
ETHR_ASSERT(!mtx->mtxb.q);
- act = ethr_atomic_cmpxchg(&mtx->mtxb.flgs,
- (ETHR_RWMTX_W_FLG__
- | ETHR_RWMTX_W_WAIT_FLG__),
- ETHR_RWMTX_W_FLG__);
+ act = ethr_atomic32_cmpxchg(&mtx->mtxb.flgs,
+ (ETHR_RWMTX_W_FLG__
+ | ETHR_RWMTX_W_WAIT_FLG__),
+ ETHR_RWMTX_W_FLG__);
if (act != ETHR_RWMTX_W_FLG__) {
/*
* Sigh... this wasn't so sane after all since, the mutex was
@@ -938,14 +938,14 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
multi = tse_start != tse_end;
while (1) {
- long new, exp = act;
+ ethr_sint32_t new, exp = act;
if (multi || (act & ETHR_RWMTX_W_FLG__))
new = ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__;
else
new = ETHR_RWMTX_W_FLG__;
- act = ethr_atomic_cmpxchg(&mtx->mtxb.flgs, new, exp);
+ act = ethr_atomic32_cmpxchg(&mtx->mtxb.flgs, new, exp);
if (exp == act) {
ETHR_ASSERT(!mtx->mtxb.q);
if (act & ETHR_RWMTX_W_FLG__) {
@@ -973,7 +973,7 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
- ethr_atomic_set(&tse_start->uaflgs, 0);
+ ethr_atomic32_set(&tse_start->uaflgs, 0);
ethr_event_set(&tse_start->event);
}
break;
@@ -985,7 +985,6 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
int
ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt)
{
- int res;
#if ETHR_XCHK
if (!cnd) {
ETHR_ASSERT(0);
@@ -1065,9 +1064,9 @@ ethr_cond_signal(ethr_cond *cnd)
ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
- ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) == ETHR_CND_WAIT_FLG__);
+ ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_CND_WAIT_FLG__);
- ethr_atomic_set(&tse->uaflgs, ETHR_RWMTX_W_WAIT_FLG__);
+ ethr_atomic32_set(&tse->uaflgs, ETHR_RWMTX_W_WAIT_FLG__);
dequeue(&cnd->q, tse, tse);
@@ -1085,7 +1084,6 @@ ethr_cond_signal(ethr_cond *cnd)
void
ethr_cond_broadcast(ethr_cond *cnd)
{
- int res;
int got_all;
ethr_ts_event *tse;
ETHR_ASSERT(!ethr_not_inited__);
@@ -1119,10 +1117,11 @@ ethr_cond_broadcast(ethr_cond *cnd)
/* The normal case */
ETHR_ASSERT(tse_tmp->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
- ETHR_ASSERT(ethr_atomic_read(&tse_tmp->uaflgs)
+ ETHR_ASSERT(ethr_atomic32_read(&tse_tmp->uaflgs)
== ETHR_CND_WAIT_FLG__);
- ethr_atomic_set(&tse_tmp->uaflgs, ETHR_RWMTX_W_WAIT_FLG__);
+ ethr_atomic32_set(&tse_tmp->uaflgs,
+ ETHR_RWMTX_W_WAIT_FLG__);
}
else {
/* Should be very unusual */
@@ -1153,7 +1152,6 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
{
int woken;
int scnt;
- int res;
void *udata = NULL;
ethr_ts_event *tse;
@@ -1176,7 +1174,7 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
tse->udata = (void *) mtx;
tse->uflgs = ETHR_RWMTX_W_WAIT_FLG__; /* Prep for mutex lock op */
- ethr_atomic_set(&tse->uaflgs, ETHR_CND_WAIT_FLG__);
+ ethr_atomic32_set(&tse->uaflgs, ETHR_CND_WAIT_FLG__);
ETHR_MTX_Q_LOCK(&cnd->qlck);
@@ -1189,11 +1187,11 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
/* Wait */
woken = 0;
while (1) {
- long act;
+ ethr_sint32_t act;
ethr_event_reset(&tse->event);
- act = ethr_atomic_read_acqb(&tse->uaflgs);
+ act = ethr_atomic32_read_acqb(&tse->uaflgs);
if (!act)
break; /* Mtx locked */
@@ -1209,7 +1207,7 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
*/
if (act == ETHR_CND_WAIT_FLG__) {
ETHR_MTX_Q_LOCK(&cnd->qlck);
- act = ethr_atomic_read(&tse->uaflgs);
+ act = ethr_atomic32_read(&tse->uaflgs);
ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__
|| act == ETHR_RWMTX_W_WAIT_FLG__);
/*
@@ -1239,7 +1237,7 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
-
+ ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb);
tse->udata = udata;
ethr_leave_ts_event(tse);
return 0;
@@ -1411,7 +1409,7 @@ wake_readers(ethr_rwmutex *rwmtx, int rs)
rwmtx->rq_end = NULL;
ETHR_ASSERT(!rwmtx->mtxb.q
- || (ethr_atomic_read(&rwmtx->mtxb.q->uaflgs)
+ || (ethr_atomic32_read(&rwmtx->mtxb.q->uaflgs)
== ETHR_RWMTX_W_WAIT_FLG__));
ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
@@ -1422,7 +1420,7 @@ wake_readers(ethr_rwmutex *rwmtx, int rs)
#ifdef ETHR_DEBUG
ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
- ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs)
+ ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs)
== ETHR_RWMTX_R_WAIT_FLG__);
drs++;
#endif
@@ -1430,7 +1428,7 @@ wake_readers(ethr_rwmutex *rwmtx, int rs)
tse_next = tse->next; /* we aren't allowed to read tse->next
after we have reset uaflgs */
- ethr_atomic_set(&tse->uaflgs, 0);
+ ethr_atomic32_set(&tse->uaflgs, 0);
ethr_event_set(&tse->event);
tse = tse_next;
}
@@ -1466,20 +1464,14 @@ multiple_w_waiters(ethr_rwmutex *rwmtx)
int check_readers_array(ethr_rwmutex *rwmtx,
int start_rix,
- int length,
- int pre_check)
+ int length)
{
int ix = start_rix;
-#ifndef ETHR_READ_MEMORY_BARRIER_IS_FULL
- if (pre_check)
- ETHR_READ_MEMORY_BARRIER;
- else
-#endif
- ETHR_MEMORY_BARRIER;
+ ETHR_MEMORY_BARRIER;
do {
- long act = rwmutex_freqread_rdrs_read(rwmtx, ix);
+ ethr_sint32_t act = rwmutex_freqread_rdrs_read(rwmtx, ix);
if (act != 0)
return EBUSY;
ix++;
@@ -1490,55 +1482,101 @@ int check_readers_array(ethr_rwmutex *rwmtx,
return 0;
}
-static ETHR_INLINE void
+static void
+rwmutex_freqread_rdrs_dec_chk_wakeup(ethr_rwmutex *rwmtx,
+ ethr_ts_event *tse,
+ ethr_sint32_t initial)
+{
+ ethr_sint32_t act = initial;
+
+ if ((act & (ETHR_RWMTX_W_FLG__|
+ ETHR_RWMTX_R_ABRT_UNLCK_FLG__)) == 0) {
+ if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+ if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) {
+ /*
+ * We *need* to try to complete the runlock.
+ * A writer that just enqueued (not seen by us
+ * in flag field) may depend on someone else
+ * completing the runlock. We just took over
+ * that responsibilty since we modified reader
+ * groups.
+ */
+ rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+ }
+ }
+ else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_R_WAIT_FLG__)
+ rwmutex_transfer_read_lock(rwmtx, act, 0);
+ else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_W_WAIT_FLG__)
+ rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+ else {
+ /*
+ * Don't know if we got readers or writers
+ * first in queue; need to peek
+ */
+ ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+ if (!rwmtx->mtxb.q)
+ ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+ else if (is_w_waiter(rwmtx->mtxb.q)) {
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+ ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+ if ((act & ETHR_RWMTX_W_FLG__) == 0)
+ rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+ }
+ else {
+ /*
+ * rwmutex_transfer_read_lock() will
+ * unlock Q lock.
+ */
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+ if (act & ETHR_RWMTX_W_FLG__)
+ ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+ else
+ rwmutex_transfer_read_lock(rwmtx, act, 1);
+ }
+ }
+ }
+}
+
+static void
rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx,
ethr_ts_event *tse)
{
- long act;
+ ethr_sint32_t act;
/*
* Restore failed increment
*/
act = rwmutex_freqread_rdrs_dec_read(rwmtx, tse);
- ETHR_WRITE_MEMORY_BARRIER;
+ ETHR_MEMORY_BARRIER;
if (act == 0) {
-
-#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL
- ETHR_READ_MEMORY_BARRIER;
-#endif
-
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
-
- if ((act & ETHR_RWMTX_W_FLG__) == 0
- && act & (ETHR_RWMTX_WAIT_FLGS__|ETHR_RWMTX_R_PEND_UNLCK_MASK__)) {
- /*
- * We either got waiters, or someone else trying
- * to read unlock which we might have to help.
- */
- rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 1, 0);
- }
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+ rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act);
}
}
static int
rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
- long initial,
+ ethr_sint32_t initial,
ethr_ts_event *tse,
int start_next_ix,
int check_before_try,
int try_write_lock)
{
ethr_ts_event *tse_tmp;
- long act = initial;
+ ethr_sint32_t act = initial;
int six, res, length;
+ ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
+
+ if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__)
+ return try_write_lock ? EBUSY : 0;
+
tse_tmp = tse;
if (!tse_tmp)
tse_tmp = ethr_get_ts_event();
- if ((act & ETHR_RWMTX_WAIT_FLGS__)
- && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0)
+ if ((act & ETHR_RWMTX_WAIT_FLGS__) && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0)
goto check_waiters;
if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
@@ -1559,25 +1597,34 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
ethr_leave_ts_event(tse_tmp);
if (check_before_try) {
- res = check_readers_array(rwmtx, six, length, 1);
+ res = check_readers_array(rwmtx, six, length);
+
+ ETHR_MEMORY_BARRIER;
+
if (res == EBUSY)
return try_write_lock ? EBUSY : 0;
}
+ restart:
+
while (1) {
- long exp = act;
- long new = act+1;
+ ethr_sint32_t exp = act;
+ ethr_sint32_t new = act+1;
+
+ ETHR_ASSERT((act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) == 0);
ETHR_ASSERT((act & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
< ETHR_RWMTX_R_PEND_UNLCK_MASK__);
- act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+ act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
if (exp == act) {
act = new;
break;
}
+
if (!try_write_lock) {
- if (act == ETHR_RWMTX_W_FLG__ || act == 0)
+ if (act == 0 || (act & (ETHR_RWMTX_W_FLG__
+ | ETHR_RWMTX_R_ABRT_UNLCK_FLG__)))
return 0;
if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
if ((act & ETHR_RWMTX_R_FLG__) == 0)
@@ -1592,33 +1639,50 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
else {
if (act == 0)
goto tryrwlock;
- if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__))
+ if (act & (ETHR_RWMTX_W_FLG__
+ | ETHR_RWMTX_R_ABRT_UNLCK_FLG__))
return EBUSY;
}
}
- res = check_readers_array(rwmtx, six, length, 0);
- if (res == EBUSY) {
- act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs);
- if (act & ETHR_RWMTX_R_MASK__)
- return try_write_lock ? EBUSY : 0;
- }
- else {
- while (1) {
- long exp = act;
- long new = act;
- new &= ~ETHR_RWMTX_R_FLG__;
- new--;
+ res = check_readers_array(rwmtx, six, length);
- ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__);
+ ETHR_MEMORY_BARRIER;
- act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
- if (exp == act) {
- if (new & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
- return try_write_lock ? EBUSY : 0;
- act = new;
- break;
+ ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
+
+ while (1) {
+ int finished_abort = 0;
+ ethr_sint32_t exp = act;
+ ethr_sint32_t new = act;
+
+ new--;
+ if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) {
+ if ((new & ETHR_RWMTX_R_PEND_UNLCK_MASK__) == 0) {
+ new &= ~ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+ finished_abort = 1;
}
+ ETHR_ASSERT(act & ETHR_RWMTX_R_FLG__);
+ }
+ else if ((act & ETHR_RWMTX_R_FLG__) && res != EBUSY) {
+ new &= ~ETHR_RWMTX_R_FLG__;
+ }
+
+ ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__);
+
+ act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+ if (exp == act) {
+ act = new;
+ if (act & ETHR_RWMTX_W_FLG__)
+ return try_write_lock ? EBUSY : 0;
+ if (finished_abort && (act & ETHR_RWMTX_WAIT_FLGS__))
+ goto restart;
+ if (act & (ETHR_RWMTX_R_FLG__
+ | ETHR_RWMTX_R_ABRT_UNLCK_FLG__
+ | ETHR_RWMTX_R_PEND_UNLCK_MASK__))
+ return try_write_lock ? EBUSY : 0;
+ /* Read unlock completed */
+ break;
}
}
@@ -1628,12 +1692,9 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
* to write lock it).
*/
- if (act & ETHR_RWMTX_W_FLG__)
- return try_write_lock ? EBUSY : 0;
-
if (act & ETHR_RWMTX_WAIT_FLGS__) {
check_waiters:
- rwmutex_unlock_wake(rwmtx, 0, act);
+ rwmutex_unlock_wake(rwmtx, 0, act, 0);
return try_write_lock ? EBUSY : 0;
}
@@ -1643,9 +1704,9 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
tryrwlock:
/* Try to write lock it */
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
- ETHR_RWMTX_W_FLG__,
- 0);
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+ ETHR_RWMTX_W_FLG__,
+ 0);
return act == 0 ? 0 : EBUSY;
}
@@ -1654,24 +1715,23 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
static ETHR_INLINE void
rwmutex_incdec_restore_failed_tryrlock(ethr_rwmutex *rwmtx)
{
- long act;
+ ethr_sint32_t act;
/*
* Restore failed increment
*/
- act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_dec_read(&rwmtx->mtxb.flgs);
if ((act & ETHR_RWMTX_WAIT_FLGS__)
&& (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
- rwmutex_unlock_wake(rwmtx, 0, act);
+ rwmutex_unlock_wake(rwmtx, 0, act, 0);
}
}
#endif
static void
-rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
- long initial)
+rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
{
- long act = initial, exp;
+ ethr_sint32_t act = initial, exp;
int scnt, start_scnt;
ethr_ts_event *tse = NULL;
int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
@@ -1687,11 +1747,11 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
#ifdef ETHR_RLOCK_WITH_INC_DEC
rwmutex_incdec_restore_failed_tryrlock(rwmtx);
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
#endif
while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
- if (scnt == 0) {
+ if (scnt <= 0) {
tse = ethr_get_ts_event();
if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
event_wait(&rwmtx->mtxb, tse, scnt,
@@ -1704,18 +1764,17 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
ETHR_YIELD();
}
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
scnt--;
}
exp = act;
- ETHR_ASSERT(scnt >= 0);
#ifdef ETHR_RLOCK_WITH_INC_DEC
- act = ethr_atomic_inc_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_inc_read(&rwmtx->mtxb.flgs);
if ((act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) == 0)
goto done; /* Got it */
#else
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
if (act == exp)
goto done; /* Got it */
#endif
@@ -1728,10 +1787,83 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
static void
rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
- ethr_ts_event *tse,
- long initial)
+ ethr_ts_event *tse);
+
+static int
+rwmutex_freqread_rlock(ethr_rwmutex *rwmtx, ethr_ts_event *tse, int trylock)
{
- long act = initial;
+ int res = 0;
+ ethr_sint32_t act;
+
+ rwmutex_freqread_rdrs_inc(rwmtx, tse);
+
+ ETHR_MEMORY_BARRIER;
+
+ act = ethr_atomic32_read_acqb(&rwmtx->mtxb.flgs);
+
+ if (act != ETHR_RWMTX_R_FLG__) {
+ int wake_other_readers;
+
+ while (1) {
+ ethr_sint32_t exp, new;
+
+ wake_other_readers = 0;
+
+ if (act == 0)
+ new = act | ETHR_RWMTX_R_FLG__;
+ else if (act == ETHR_RWMTX_R_FLG__)
+ break; /* Got it */
+ else if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+ rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
+ if (trylock)
+ res = EBUSY;
+ else
+ rwmutex_freqread_rlock_wait(rwmtx, tse);
+ break;
+ }
+ else if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) {
+ if ((act & ETHR_RWMTX_R_FLG__) == 0)
+ ETHR_FATAL_ERROR__(EFAULT);
+ /*
+ * An aborted runlock, not write locked, and no write
+ * waiters, i.e., we got it...
+ */
+ if (act & ETHR_RWMTX_R_WAIT_FLG__)
+ wake_other_readers = 1;
+ break;
+ }
+ else {
+ new = act | ETHR_RWMTX_R_FLG__;
+ if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) {
+ /*
+ * Someone is doing tryrwlock (no writer and no
+ * write waiters); we will try to abort that...
+ */
+ new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+ }
+
+ if (act & ETHR_RWMTX_R_WAIT_FLG__)
+ wake_other_readers = 1;
+ }
+
+ exp = act;
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
+ if (act == exp)
+ break;
+ }
+
+ if (wake_other_readers)
+ rwmutex_transfer_read_lock(rwmtx, act, 0);
+ }
+
+ return res;
+}
+
+static void
+rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
+ ethr_ts_event *tse)
+{
+ ethr_sint32_t act;
int scnt, start_scnt;
int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
@@ -1744,12 +1876,10 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
while (1) {
- rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
-
- while (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
- if (scnt == 0) {
+ while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+ if (scnt <= 0) {
if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
event_wait(&rwmtx->mtxb, tse, scnt,
ETHR_RWMTX_R_WAIT_FLG__, 1, 1);
@@ -1761,76 +1891,65 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
ETHR_YIELD();
}
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
scnt--;
}
- ETHR_ASSERT(scnt >= 0);
-
- rwmutex_freqread_rdrs_inc(rwmtx, tse);
-
- ETHR_MEMORY_BARRIER;
-
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
-
- if (act == ETHR_RWMTX_R_FLG__)
- return; /* Got it */
-
- while (1) {
- long exp, new;
-
- if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__))
- break; /* Busy (need to restore inc) */
-
- if (act & ETHR_RWMTX_R_FLG__)
- return; /* Got it */
-
- exp = act;
- new = act | ETHR_RWMTX_R_FLG__;
- act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
- if (act == exp)
- return; /* Got it */
- }
+ if (rwmutex_freqread_rlock(rwmtx, tse, 1) != EBUSY)
+ break; /* Got it */
}
}
static void
-rwmutex_normal_rwlock_wait(ethr_rwmutex *rwmtx, long initial)
+rwmutex_normal_rwlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
{
write_lock_wait(&rwmtx->mtxb, initial, 1, 0);
}
static void
-rwmutex_freqread_rwlock_wait(ethr_rwmutex *rwmtx, long initial)
+rwmutex_freqread_rwlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial)
{
write_lock_wait(&rwmtx->mtxb, initial, 1, 1);
}
static ETHR_INLINE void
-rwlock_wake_set_flags(ethr_rwmutex *rwmtx, long new_initial, int act_initial)
+rwlock_wake_set_flags(ethr_rwmutex *rwmtx,
+ ethr_sint32_t new_initial,
+ ethr_sint32_t act_initial)
{
- long act, act_mask;
+ ethr_sint32_t act, act_mask;
+ int chk_abrt_flg;
+
+ ETHR_MEMORY_BARRIER;
+
if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
/* r pend unlock mask may vary and must be retained */
act_mask = ETHR_RWMTX_R_PEND_UNLCK_MASK__;
+ if (new_initial & ETHR_RWMTX_R_FLG__)
+ chk_abrt_flg = 1;
+ else
+ chk_abrt_flg = 0;
}
else {
#ifdef ETHR_RLOCK_WITH_INC_DEC
/* rs mask may vary and must be retained */
act_mask = ETHR_RWMTX_RS_MASK__;
+ chk_abrt_flg = 0;
#else
/* rs mask always zero */
ETHR_ASSERT((act_initial & ETHR_RWMTX_RS_MASK__) == 0);
- ethr_atomic_set(&rwmtx->mtxb.flgs, new_initial);
+ ethr_atomic32_set(&rwmtx->mtxb.flgs, new_initial);
return;
#endif
}
act = act_initial;
while (1) {
- long exp = act;
- long new = new_initial + (act & act_mask);
- act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+ ethr_sint32_t exp = act;
+ ethr_sint32_t new = new_initial + (act & act_mask);
+ if (chk_abrt_flg && (act & act_mask))
+ new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__;
+ act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
if (act == exp)
break;
exp = act;
@@ -1844,7 +1963,7 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx,
int have_w,
ethr_ts_event *tse)
{
- long exp, act, imask;
+ ethr_sint32_t exp, act, imask;
exp = have_w ? ETHR_RWMTX_W_FLG__ : 0;
@@ -1866,7 +1985,7 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx,
if (rwmtx->rq_end) {
exp |= ETHR_RWMTX_R_WAIT_FLG__;
}
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
ETHR_ASSERT((exp & ~imask) == (act & ~imask));
ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
@@ -1877,7 +1996,15 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx,
exp |= ETHR_RWMTX_R_WAIT_FLG__;
if (rwmtx->rq_end->next != rwmtx->mtxb.q)
exp |= ETHR_RWMTX_W_WAIT_FLG__;
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ else if (exp == ETHR_RWMTX_R_WAIT_FLG__) {
+ if (!have_w) {
+ if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
+ imask |= ETHR_RWMTX_R_FLG__;
+ else
+ imask |= ETHR_RWMTX_RS_MASK__;
+ }
+ }
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
ETHR_ASSERT((exp & ~imask) == (act & ~imask));
ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
@@ -1888,41 +2015,85 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx,
#endif
static void
-rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
+rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx,
+ ethr_sint32_t initial,
+ int q_locked)
{
- long new, act = initial;
- ethr_ts_event *tse;
+ ethr_sint32_t act = initial;
- if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
- if (!have_w)
+ if (!q_locked) {
+ ethr_ts_event *tse;
+ ETHR_ASSERT(initial & ETHR_RWMTX_R_WAIT_FLG__);
+ ETHR_ASSERT((initial & ETHR_RWMTX_W_FLG__) == 0);
+ ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+ tse = rwmtx->mtxb.q;
+ if ((act & ETHR_RWMTX_W_FLG__) || !tse || is_w_waiter(tse)) {
+ /* Someone else woke the readers up... */
+ ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
return;
- else {
- while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
- long exp = act;
- new = exp & ~ETHR_RWMTX_W_FLG__;
- act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
- if (act == exp)
- return;
- }
}
}
- ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
- tse = rwmtx->mtxb.q;
+ rwmutex_unlock_wake(rwmtx, 0, initial, 1);
+}
- if (!have_w) {
- if (!tse) {
+static void
+rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, ethr_sint32_t initial,
+ int transfer_read_lock)
+{
+ ethr_sint32_t new, act = initial;
+ ethr_ts_event *tse;
+
+ if (transfer_read_lock) {
+ /*
+ * - Q already locked
+ * - Got R waiters first in Q
+ * - Not W locked
+ */
+ tse = rwmtx->mtxb.q;
+
+ ETHR_ASSERT(act & ETHR_RWMTX_R_WAIT_FLG__);
+ ETHR_ASSERT((act & (ETHR_RWMTX_W_FLG__)) == 0);
+ ETHR_ASSERT(tse && !is_w_waiter(tse));
+ }
+ else {
+
+ if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+ if (!have_w)
+ return;
+ else {
+ while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+ ethr_sint32_t exp = act;
+ new = exp & ~ETHR_RWMTX_W_FLG__;
+ act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+ if (act == exp)
+ return;
+ }
+ }
+ }
+
+ ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+ tse = rwmtx->mtxb.q;
+
+ if (!have_w) {
+ if (!tse) {
#ifdef ETHR_DEBUG
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
- ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+ ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
#endif
- goto already_served;
- }
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
- if (act & ~ETHR_RWMTX_WAIT_FLGS__) {
- already_served:
- ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
- return;
+ goto already_served;
+ }
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+ if (act == (ETHR_RWMTX_R_WAIT_FLG__|ETHR_RWMTX_R_FLG__)) {
+ ETHR_ASSERT(tse && !is_w_waiter(tse));
+ }
+ else if (act & ~ETHR_RWMTX_WAIT_FLGS__) {
+ already_served:
+ ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+ return;
+ }
}
}
@@ -1933,9 +2104,12 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
if (is_w_waiter(tse)) {
if (!have_w) {
- act = ethr_atomic_read_bor(&rwmtx->mtxb.flgs,
+ act = ethr_atomic32_read_bor(&rwmtx->mtxb.flgs,
ETHR_RWMTX_W_FLG__);
- ETHR_ASSERT((act & ~ETHR_RWMTX_WAIT_FLGS__) == 0);
+ ETHR_ASSERT((act & ~(ETHR_RWMTX_WAIT_FLGS__
+ | (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL
+ ? 0
+ : ETHR_RWMTX_R_PEND_UNLCK_MASK__))) == 0);
ETHR_ASSERT(act & ETHR_RWMTX_W_WAIT_FLG__);
act |= ETHR_RWMTX_W_FLG__;
}
@@ -1962,7 +2136,7 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
if (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL) {
rs = rwmtx->tdata.rs;
- new = (long) rs;
+ new = (ethr_sint32_t) rs;
rwmtx->tdata.rs = 0;
}
else {
@@ -1982,6 +2156,7 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
rwmutex_freqread_rdrs_add(rwmtx, type, ix, wrs);
}
}
+
new = ETHR_RWMTX_R_FLG__;
}
@@ -1989,6 +2164,7 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
new |= ETHR_RWMTX_W_WAIT_FLG__;
rwlock_wake_set_flags(rwmtx, new, act);
+
wake_readers(rwmtx, rs);
}
}
@@ -2016,16 +2192,16 @@ alloc_readers_array(int length, ethr_rwmutex_lived lived)
if (!mem)
return NULL;
- if ((((unsigned long) mem) & ETHR_CACHE_LINE_MASK) == 0) {
+ if ((((ethr_uint_t) mem) & ETHR_CACHE_LINE_MASK) == 0) {
ra = (ethr_rwmtx_readers_array__ *) mem;
ra->data.byte_offset = 0;
}
else {
ra = ((ethr_rwmtx_readers_array__ *)
- ((((unsigned long) mem) & ~ETHR_CACHE_LINE_MASK)
+ ((((ethr_uint_t) mem) & ~ETHR_CACHE_LINE_MASK)
+ ETHR_CACHE_LINE_SIZE));
- ra->data.byte_offset = (int) ((unsigned long) ra
- - (unsigned long) mem);
+ ra->data.byte_offset = (int) ((ethr_uint_t) ra
+ - (ethr_uint_t) mem);
}
ra->data.lived = lived;
return ra;
@@ -2099,7 +2275,7 @@ ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt)
rwmtx->tdata.ra = ra;
for (ix = 0; ix < length; ix++) {
- ethr_atomic_init(&rwmtx->tdata.ra[ix].data.readers, 0);
+ ethr_atomic32_init(&rwmtx->tdata.ra[ix].data.readers, 0);
rwmtx->tdata.ra[ix].data.waiting_readers = 0;
}
break;
@@ -2151,8 +2327,9 @@ ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
return EINVAL;
}
#endif
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
- long act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ ethr_sint32_t act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
if (act == ETHR_RWMTX_R_FLG__)
rwmutex_try_complete_runlock(rwmtx, act, NULL, 0, 0, 0);
}
@@ -2173,7 +2350,7 @@ int
ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
{
int res = 0;
- long act;
+ ethr_sint32_t act;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(rwmtx);
@@ -2181,25 +2358,27 @@ ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
switch (rwmtx->type) {
case ETHR_RWMUTEX_TYPE_NORMAL: {
#ifdef ETHR_RLOCK_WITH_INC_DEC
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))
res = EBUSY;
else {
- act = ethr_atomic_inc_read_acqb(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_inc_read_acqb(&rwmtx->mtxb.flgs);
if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
rwmutex_incdec_restore_failed_tryrlock(rwmtx);
res = EBUSY;
}
}
#else
- long exp = 0;
+ ethr_sint32_t exp = 0;
int tries = 0;
while (1) {
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
if (act == exp) {
res = 0;
break;
@@ -2219,49 +2398,30 @@ ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
ethr_ts_event *tse = ethr_get_ts_event();
-
- rwmutex_freqread_rdrs_inc(rwmtx, tse);
-
- ETHR_MEMORY_BARRIER;
-
- act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs);
-
- if (act != ETHR_RWMTX_R_FLG__) {
- while (1) {
- long exp, new;
-
- if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
- rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
- res = EBUSY;
- break;
- }
-
- if (act & ETHR_RWMTX_R_FLG__)
- break;
-
- exp = act;
- new = act | ETHR_RWMTX_R_FLG__;
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
- if (act == exp)
- break;
- }
- }
-
+ res = rwmutex_freqread_rlock(rwmtx, tse, 1);
ethr_leave_ts_event(tse);
break;
}
}
+#ifdef ETHR_MTX_CHK_EXCL
+ if (res == 0) {
+ ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb);
+ ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
+ }
+#endif
+
ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(&rwmtx->mtxb, res);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
return res;
}
void
ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
{
- long act;
+ ethr_sint32_t act;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(rwmtx);
@@ -2269,20 +2429,21 @@ ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
switch (rwmtx->type) {
case ETHR_RWMUTEX_TYPE_NORMAL: {
#ifdef ETHR_RLOCK_WITH_INC_DEC
- act = ethr_atomic_inc_read_acqb(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_inc_read_acqb(&rwmtx->mtxb.flgs);
if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))
rwmutex_normal_rlock_wait(rwmtx, act);
#else
- long exp = 0;
+ ethr_sint32_t exp = 0;
while (1) {
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
- if (act == exp) {
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+ if (act == exp)
break;
- }
if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
rwmutex_normal_rlock_wait(rwmtx, act);
@@ -2297,38 +2458,15 @@ ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
ethr_ts_event *tse = ethr_get_ts_event();
-
- rwmutex_freqread_rdrs_inc(rwmtx, tse);
-
- ETHR_MEMORY_BARRIER;
-
- act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs);
-
- if (act != ETHR_RWMTX_R_FLG__) {
- while (1) {
- long exp, new;
-
- if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
- rwmutex_freqread_rlock_wait(rwmtx, tse, act);
- break;
- }
-
- if (act & ETHR_RWMTX_R_FLG__)
- break;
-
- exp = act;
- new = act | ETHR_RWMTX_R_FLG__;
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
- if (act == exp)
- break;
- }
- }
-
+ rwmutex_freqread_rlock(rwmtx, tse, 0);
ethr_leave_ts_event(tse);
break;
}
}
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+ ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb);
+ ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
ETHR_MTX_HARD_DEBUG_LFS_RLOCK(&rwmtx->mtxb);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
}
@@ -2336,8 +2474,10 @@ ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
void
ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
{
- long act;
+ ethr_sint32_t act;
+ ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb);
+ ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(&rwmtx->mtxb);
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(rwmtx);
ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
@@ -2345,13 +2485,15 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(&rwmtx->mtxb);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
switch (rwmtx->type) {
case ETHR_RWMUTEX_TYPE_NORMAL:
- act = ethr_atomic_dec_read_relb(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_dec_read_relb(&rwmtx->mtxb.flgs);
if ((act & ETHR_RWMTX_WAIT_FLGS__)
&& (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
- rwmutex_unlock_wake(rwmtx, 0, act);
+ rwmutex_unlock_wake(rwmtx, 0, act, 0);
}
break;
@@ -2363,21 +2505,12 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
ETHR_ASSERT(act >= 0);
- ETHR_WRITE_MEMORY_BARRIER;
+ ETHR_MEMORY_BARRIER;
if (act == 0) {
-
-#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL
- ETHR_READ_MEMORY_BARRIER;
-#endif
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
-
- if ((act & ETHR_RWMTX_W_FLG__) == 0
- && (act & (ETHR_RWMTX_WAIT_FLGS__
- | ETHR_RWMTX_R_PEND_UNLCK_MASK__))) {
- rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
- }
-
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
+ if (act != ETHR_RWMTX_R_FLG__)
+ rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act);
}
ethr_leave_ts_event(tse);
@@ -2385,6 +2518,7 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
}
}
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
}
@@ -2392,7 +2526,7 @@ int
ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
{
int res = 0;
- long act;
+ ethr_sint32_t act;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(rwmtx);
@@ -2400,10 +2534,12 @@ ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
switch (rwmtx->type) {
case ETHR_RWMUTEX_TYPE_NORMAL:
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
- ETHR_RWMTX_W_FLG__, 0);
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+ ETHR_RWMTX_W_FLG__, 0);
if (act != 0)
res = EBUSY;
break;
@@ -2412,29 +2548,36 @@ ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
res = 0;
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
do {
- if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__)) {
- res = EBUSY;
- break;
- }
-
- if (act & ETHR_RWMTX_R_MASK__) {
+ if (act == 0)
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+ ETHR_RWMTX_W_FLG__, 0);
+ else if (act == ETHR_RWMTX_R_FLG__) {
res = rwmutex_try_complete_runlock(rwmtx, act, NULL,
0, 1, 1);
break;
}
-
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
- ETHR_RWMTX_W_FLG__, 0);
+ else {
+ res = EBUSY;
+ break;
+ }
} while (act != 0);
break;
}
+#ifdef ETHR_MTX_CHK_EXCL
+ if (res == 0) {
+ ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb);
+ ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
+ }
+#endif
+
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&rwmtx->mtxb, res);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
@@ -2444,17 +2587,19 @@ ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
void
ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
{
- long act;
+ ethr_sint32_t act;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(rwmtx);
ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
switch (rwmtx->type) {
case ETHR_RWMUTEX_TYPE_NORMAL:
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
- ETHR_RWMTX_W_FLG__, 0);
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+ ETHR_RWMTX_W_FLG__, 0);
if (act != 0)
rwmutex_normal_rwlock_wait(rwmtx, act);
break;
@@ -2462,7 +2607,7 @@ ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
- act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+ act = ethr_atomic32_read(&rwmtx->mtxb.flgs);
do {
@@ -2471,23 +2616,26 @@ ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
break;
}
- act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
- ETHR_RWMTX_W_FLG__, 0);
+ act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+ ETHR_RWMTX_W_FLG__, 0);
} while (act != 0);
break;
}
+ ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb);
+ ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&rwmtx->mtxb);
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
}
void
ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
{
- long act;
+ ethr_sint32_t act;
ETHR_ASSERT(!ethr_not_inited__);
ETHR_ASSERT(rwmtx);
ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
@@ -2495,24 +2643,30 @@ ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&rwmtx->mtxb);
+ ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb);
+ ETHR_MTX_CHK_EXCL_UNSET_EXCL(&rwmtx->mtxb);
+
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
+
switch (rwmtx->type) {
case ETHR_RWMUTEX_TYPE_NORMAL:
- act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs,
- 0, ETHR_RWMTX_W_FLG__);
+ act = ethr_atomic32_cmpxchg_relb(&rwmtx->mtxb.flgs,
+ 0, ETHR_RWMTX_W_FLG__);
if (act != ETHR_RWMTX_W_FLG__)
- rwmutex_unlock_wake(rwmtx, 1, act);
+ rwmutex_unlock_wake(rwmtx, 1, act, 0);
break;
case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
- act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs, 0,
- ETHR_RWMTX_W_FLG__);
+ act = ethr_atomic32_cmpxchg_relb(&rwmtx->mtxb.flgs, 0,
+ ETHR_RWMTX_W_FLG__);
if (act != ETHR_RWMTX_W_FLG__)
- rwmutex_unlock_wake(rwmtx, 1, act);
+ rwmutex_unlock_wake(rwmtx, 1, act, 0);
break;
}
ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+ ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx);
}
#else
@@ -2630,7 +2784,7 @@ static void
hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
{
int res;
- long flgs = ethr_atomic_read(&mtxb->flgs);
+ ethr_sint32_t flgs = ethr_atomic32_read(&mtxb->flgs);
ETHR_MTX_HARD_ASSERT(res == 0);
@@ -2653,12 +2807,12 @@ hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
tse = mtxb->q;
do {
- long type;
+ ethr_sint32_t type;
ETHR_MTX_HARD_ASSERT(tse->next->prev == tse);
ETHR_MTX_HARD_ASSERT(tse->prev->next == tse);
- type = ethr_atomic_read(&tse->uaflgs);
+ type = ethr_atomic32_read(&tse->uaflgs);
ETHR_MTX_HARD_ASSERT(type == tse->uflgs);
switch (type) {
case ETHR_RWMTX_W_WAIT_FLG__:
diff --git a/erts/lib_src/pthread/ethr_event.c b/erts/lib_src/pthread/ethr_event.c
index 6731c0eb46..9434d60d0a 100644
--- a/erts/lib_src/pthread/ethr_event.c
+++ b/erts/lib_src/pthread/ethr_event.c
@@ -24,6 +24,10 @@
#define ETHR_INLINE_FUNC_NAME_(X) X ## __
#define ETHR_EVENT_IMPL__
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
#include "ethread.h"
#if defined(ETHR_LINUX_FUTEX_IMPL__)
@@ -37,7 +41,7 @@
int
ethr_event_init(ethr_event *e)
{
- ethr_atomic_init(&e->futex, ETHR_EVENT_OFF__);
+ ethr_atomic32_init(&e->futex, ETHR_EVENT_OFF__);
return 0;
}
@@ -52,7 +56,7 @@ wait__(ethr_event *e, int spincount)
{
unsigned sc = spincount;
int res;
- long val;
+ ethr_sint32_t val;
int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
if (spincount < 0)
@@ -60,7 +64,7 @@ wait__(ethr_event *e, int spincount)
while (1) {
while (1) {
- val = ethr_atomic_read(&e->futex);
+ val = ethr_atomic32_read(&e->futex);
if (val == ETHR_EVENT_ON__)
return 0;
if (sc == 0)
@@ -76,16 +80,18 @@ wait__(ethr_event *e, int spincount)
}
if (val != ETHR_EVENT_OFF_WAITER__) {
- val = ethr_atomic_cmpxchg(&e->futex,
- ETHR_EVENT_OFF_WAITER__,
- ETHR_EVENT_OFF__);
+ val = ethr_atomic32_cmpxchg(&e->futex,
+ ETHR_EVENT_OFF_WAITER__,
+ ETHR_EVENT_OFF__);
if (val == ETHR_EVENT_ON__)
return 0;
ETHR_ASSERT(val == ETHR_EVENT_OFF__);
}
- res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAIT__, ETHR_EVENT_OFF_WAITER__);
+ res = ETHR_FUTEX__(&e->futex,
+ ETHR_FUTEX_WAIT__,
+ ETHR_EVENT_OFF_WAITER__);
if (res == EINTR)
break;
if (res != 0 && res != EWOULDBLOCK)
@@ -102,7 +108,7 @@ int
ethr_event_init(ethr_event *e)
{
int res;
- ethr_atomic_init(&e->state, ETHR_EVENT_OFF__);
+ ethr_atomic32_init(&e->state, ETHR_EVENT_OFF__);
res = pthread_mutex_init(&e->mtx, NULL);
if (res != 0)
return res;
@@ -131,7 +137,7 @@ static ETHR_INLINE int
wait__(ethr_event *e, int spincount)
{
int sc = spincount;
- long val;
+ ethr_sint32_t val;
int res, ulres;
int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
@@ -139,7 +145,7 @@ wait__(ethr_event *e, int spincount)
ETHR_FATAL_ERROR__(EINVAL);
while (1) {
- val = ethr_atomic_read(&e->state);
+ val = ethr_atomic32_read(&e->state);
if (val == ETHR_EVENT_ON__)
return 0;
if (sc == 0)
@@ -155,9 +161,9 @@ wait__(ethr_event *e, int spincount)
}
if (val != ETHR_EVENT_OFF_WAITER__) {
- val = ethr_atomic_cmpxchg(&e->state,
- ETHR_EVENT_OFF_WAITER__,
- ETHR_EVENT_OFF__);
+ val = ethr_atomic32_cmpxchg(&e->state,
+ ETHR_EVENT_OFF_WAITER__,
+ ETHR_EVENT_OFF__);
if (val == ETHR_EVENT_ON__)
return 0;
ETHR_ASSERT(val == ETHR_EVENT_OFF__);
@@ -172,7 +178,7 @@ wait__(ethr_event *e, int spincount)
while (1) {
- val = ethr_atomic_read(&e->state);
+ val = ethr_atomic32_read(&e->state);
if (val == ETHR_EVENT_ON__)
break;
diff --git a/erts/lib_src/pthread/ethread.c b/erts/lib_src/pthread/ethread.c
index ea1d9d43f0..f047104103 100644
--- a/erts/lib_src/pthread/ethread.c
+++ b/erts/lib_src/pthread/ethread.c
@@ -72,7 +72,7 @@ static void thr_exit_cleanup(void)
/* Argument passed to thr_wrapper() */
typedef struct {
- ethr_atomic_t result;
+ ethr_atomic32_t result;
ethr_ts_event *tse;
void *(*thr_func)(void *);
void *arg;
@@ -81,14 +81,14 @@ typedef struct {
static void *thr_wrapper(void *vtwd)
{
- long result;
+ ethr_sint32_t result;
void *res;
ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd;
void *(*thr_func)(void *) = twd->thr_func;
void *arg = twd->arg;
ethr_ts_event *tsep = NULL;
- result = (long) ethr_make_ts_event__(&tsep);
+ result = (ethr_sint32_t) ethr_make_ts_event__(&tsep);
if (result == 0) {
tsep->iflgs |= ETHR_TS_EV_ETHREAD;
@@ -99,7 +99,7 @@ static void *thr_wrapper(void *vtwd)
tsep = twd->tse; /* We aren't allowed to follow twd after
result has been set! */
- ethr_atomic_set(&twd->result, result);
+ ethr_atomic32_set(&twd->result, result);
ethr_event_set(&tsep->event);
@@ -191,7 +191,7 @@ ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
}
#endif
- ethr_atomic_init(&twd.result, -1);
+ ethr_atomic32_init(&twd.result, (ethr_sint32_t) -1);
twd.tse = ethr_get_ts_event();
twd.thr_func = func;
twd.arg = arg;
@@ -252,10 +252,10 @@ ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
/* Wait for child to initialize... */
while (1) {
- long result;
+ ethr_sint32_t result;
ethr_event_reset(&twd.tse->event);
- result = ethr_atomic_read(&twd.result);
+ result = ethr_atomic32_read(&twd.result);
if (result == 0)
break;
@@ -349,32 +349,6 @@ ethr_leave_ts_event(ethr_ts_event *tsep)
}
/*
- * Current time
- */
-
-int
-ethr_time_now(ethr_timeval *time)
-{
- int res;
- struct timeval tv;
-#if ETHR_XCHK
- if (ethr_not_inited__) {
- ETHR_ASSERT(0);
- return EACCES;
- }
- if (!time) {
- ETHR_ASSERT(0);
- return EINVAL;
- }
-#endif
-
- res = gettimeofday(&tv, NULL);
- time->tv_sec = (long) tv.tv_sec;
- time->tv_nsec = ((long) tv.tv_usec)*1000;
- return res;
-}
-
-/*
* Thread specific data
*/
diff --git a/erts/lib_src/win/ethr_event.c b/erts/lib_src/win/ethr_event.c
index ddb4780ff1..68f093f49c 100644
--- a/erts/lib_src/win/ethr_event.c
+++ b/erts/lib_src/win/ethr_event.c
@@ -28,6 +28,9 @@
/* --- Windows implementation of thread events ------------------------------ */
+#pragma intrinsic(_InterlockedExchangeAdd)
+#pragma intrinsic(_InterlockedCompareExchange)
+
int
ethr_event_init(ethr_event *e)
{
@@ -72,10 +75,10 @@ wait(ethr_event *e, int spincount)
while (1) {
long on;
while (1) {
-#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__
+#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__
state = e->state;
#else
- state = InterlockedExchangeAdd(&e->state, (LONG) 0);
+ state = _InterlockedExchangeAdd(&e->state, (LONG) 0);
#endif
if (state == ETHR_EVENT_ON__)
return 0;
diff --git a/erts/lib_src/win/ethread.c b/erts/lib_src/win/ethread.c
index 69523edf94..789a360b11 100644
--- a/erts/lib_src/win/ethread.c
+++ b/erts/lib_src/win/ethread.c
@@ -49,7 +49,7 @@
/* Argument passed to thr_wrapper() */
typedef struct {
ethr_tid *tid;
- ethr_atomic_t result;
+ ethr_atomic32_t result;
ethr_ts_event *tse;
void *(*thr_func)(void *);
void *arg;
@@ -93,20 +93,20 @@ static void thr_exit_cleanup(ethr_tid *tid, void *res)
static unsigned __stdcall thr_wrapper(LPVOID vtwd)
{
ethr_tid my_tid;
- long result;
+ ethr_sint32_t result;
void *res;
ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd;
void *(*thr_func)(void *) = twd->thr_func;
void *arg = twd->arg;
ethr_ts_event *tsep = NULL;
- result = (long) ethr_make_ts_event__(&tsep);
+ result = (ethr_sint32_t) ethr_make_ts_event__(&tsep);
if (result == 0) {
tsep->iflgs |= ETHR_TS_EV_ETHREAD;
my_tid = *twd->tid;
if (!TlsSetValue(own_tid_key, (LPVOID) &my_tid)) {
- result = (long) ethr_win_get_errno__();
+ result = (ethr_sint32_t) ethr_win_get_errno__();
ethr_free_ts_event__(tsep);
}
else {
@@ -118,7 +118,7 @@ static unsigned __stdcall thr_wrapper(LPVOID vtwd)
tsep = twd->tse; /* We aren't allowed to follow twd after
result has been set! */
- ethr_atomic_set(&twd->result, result);
+ ethr_atomic32_set(&twd->result, result);
ethr_event_set(&tsep->event);
@@ -128,28 +128,6 @@ static unsigned __stdcall thr_wrapper(LPVOID vtwd)
return 0;
}
-#ifdef __GNUC__
-#define LL_LITERAL(X) X##LL
-#else
-#define LL_LITERAL(X) X##i64
-#endif
-
-#define EPOCH_JULIAN_DIFF LL_LITERAL(11644473600)
-
-static ETHR_INLINE void
-get_curr_time(long *sec, long *nsec)
-{
- SYSTEMTIME t;
- FILETIME ft;
- LONGLONG lft;
-
- GetSystemTime(&t);
- SystemTimeToFileTime(&t, &ft);
- memcpy(&lft, &ft, sizeof(lft));
- *nsec = ((long) (lft % LL_LITERAL(10000000)))*100;
- *sec = (long) ((lft / LL_LITERAL(10000000)) - EPOCH_JULIAN_DIFF);
-}
-
/* internal exports */
int
@@ -320,7 +298,7 @@ ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size));
}
- ethr_atomic_init(&twd.result, -1);
+ ethr_atomic32_init(&twd.result, -1);
twd.tid = tid;
twd.thr_func = func;
@@ -352,11 +330,11 @@ ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
/* Wait for child to initialize... */
while (1) {
- long result;
+ ethr_sint32_t result;
int err;
ethr_event_reset(&twd.tse->event);
- result = ethr_atomic_read(&twd.result);
+ result = ethr_atomic32_read(&twd.result);
if (result == 0)
break;
@@ -517,23 +495,6 @@ ethr_equal_tids(ethr_tid tid1, ethr_tid tid2)
return tid1.id == tid2.id && tid1.id != ETHR_INVALID_TID_ID;
}
-int
-ethr_time_now(ethr_timeval *time)
-{
-#if ETHR_XCHK
- if (ethr_not_inited__) {
- ETHR_ASSERT(0);
- return EACCES;
- }
- if (!time) {
- ETHR_ASSERT(0);
- return EINVAL;
- }
-#endif
- get_curr_time(&time->tv_sec, &time->tv_nsec);
- return 0;
-}
-
/*
* Thread specific data
*/
generated by cgit v1.2.3 (git 2.25.1) at 2025-06-29 01:06:37 +0000