Rewrite ethread library

Large parts of the ethread library have been rewritten. The
ethread library is an Erlang runtime system internal, portable
thread library used by the runtime system itself.

Most notable improvement is a reader optimized rwlock
implementation which dramatically improve the performance of
read-lock/read-unlock operations on multi processor systems by
avoiding ping-ponging of the rwlock cache lines. The reader
optimized rwlock implementation is used by miscellaneous
rwlocks in the runtime system that are known to be read-locked
frequently, and can be enabled on ETS tables by passing the
`{read_concurrency, true}' option upon table creation. See the
documentation of `ets:new/2' for more information.

The ethread library can now also use the libatomic_ops library
for atomic memory accesses. This makes it possible for the
Erlang runtime system to utilize optimized atomic operations
on more platforms than before. Use the
`--with-libatomic_ops=PATH' configure command line argument
when specifying where the libatomic_ops installation is
located. The libatomic_ops library can be downloaded from:
http://www.hpl.hp.com/research/linux/atomic_ops/

The changed API of the ethread library has also caused
modifications in the Erlang runtime system. Preparations for
the to come "delayed deallocation" feature has also been done
since it depends on the ethread library.

Note: When building for x86, the ethread library will now use
instructions that first appeared on the pentium 4 processor. If
you want the runtime system to be compatible with older
processors (back to 486) you need to pass the
`--enable-ethread-pre-pentium4-compatibility' configure command
line argument when configuring the system.

5 files changed, 3713 insertions, 3391 deletions

diff --git a/erts/lib_src/common/erl_misc_utils.c b/erts/lib_src/common/erl_misc_utils.c
index f70db86960..d2ef7140a5 100644
--- a/erts/lib_src/common/erl_misc_utils.c
+++ b/erts/lib_src/common/erl_misc_utils.c
@@ -59,8 +59,25 @@
 #  endif
 #endif
 
-#ifdef HAVE_SCHED_xETAFFINITY
+#if defined(HAVE_SCHED_xETAFFINITY)
 #  include <sched.h>
+#  define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__
+#define ERTS_MU_GET_PROC_AFFINITY__(CPUINFOP)				\
+     (sched_getaffinity((CPUINFOP)->pid,				\
+			sizeof(cpu_set_t),				\
+			&(CPUINFOP)->cpuset) != 0 ? -errno : 0)
+#define ERTS_MU_SET_THR_AFFINITY__(SETP)				\
+     (sched_setaffinity(0, sizeof(cpu_set_t), (SETP)) != 0 ? -errno : 0)
+#elif defined(__WIN32__)
+#  define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__
+#  define cpu_set_t DWORD
+#  define CPU_SETSIZE (sizeof(DWORD)*8)
+#  define CPU_ZERO(SETP) (*(SETP) = (DWORD) 0)
+#  define CPU_SET(CPU, SETP) (*(SETP) |= (((DWORD) 1) << (CPU)))
+#  define CPU_CLR(CPU, SETP) (*(SETP) &= ~(((DWORD) 1) << (CPU)))
+#  define CPU_ISSET(CPU, SETP) ((*(SETP) & (((DWORD) 1) << (CPU))) != (DWORD) 0)
+#define ERTS_MU_GET_PROC_AFFINITY__ get_proc_affinity
+#define ERTS_MU_SET_THR_AFFINITY__ set_thr_affinity
 #endif
 #ifdef HAVE_PSET_INFO
 #  include <sys/pset.h>
@@ -105,25 +122,58 @@ struct erts_cpu_info_t_ {
     int available;
     int topology_size;
     erts_cpu_topology_t *topology;
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     char *affinity_str;
     char affinity_str_buf[CPU_SETSIZE/4+2];
     cpu_set_t cpuset;
+#if defined(HAVE_SCHED_xETAFFINITY)
     pid_t pid;
+#endif
 #elif defined(HAVE_PSET_INFO)
     processorid_t *cpuids;
 #endif
 };
 
+#if defined(__WIN32__)
+
+static __forceinline int
+get_proc_affinity(erts_cpu_info_t *cpuinfo)
+{
+    DWORD pamask, samask;
+    if (GetProcessAffinityMask(GetCurrentProcess(), &pamask, &samask)) {
+	cpuinfo->cpuset = (cpu_set_t) pamask;
+	return 0;
+    }
+    else {
+	cpuinfo->cpuset = (cpu_set_t) 0;
+	return -erts_get_last_win_errno();
+    }
+}
+
+static __forceinline int
+set_thr_affinity(cpu_set_t *set)
+{
+    if (*set == (cpu_set_t) 0)
+	return -ENOTSUP;
+    if (SetThreadAffinityMask(GetCurrentThread(), *set) == 0)
+	return -erts_get_last_win_errno();
+    else
+	return 0;
+}
+
+#endif
+
 erts_cpu_info_t *
 erts_cpu_info_create(void)
 {
     erts_cpu_info_t *cpuinfo = malloc(sizeof(erts_cpu_info_t));
     if (!cpuinfo)
 	return NULL;
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     cpuinfo->affinity_str = NULL;
+#if defined(HAVE_SCHED_xETAFFINITY)
     cpuinfo->pid = getpid();
+#endif
 #elif defined(HAVE_PSET_INFO)
     cpuinfo->cpuids = NULL;
 #endif
@@ -162,10 +212,13 @@ erts_cpu_info_update(erts_cpu_info_t *cpuinfo)
 
 #ifdef __WIN32__
     {
+	int i;
 	SYSTEM_INFO sys_info;
 	GetSystemInfo(&sys_info);
 	cpuinfo->configured = (int) sys_info.dwNumberOfProcessors;
-	
+	for (i = 0; i < sizeof(DWORD)*8; i++)
+	    if (sys_info.dwActiveProcessorMask & (((DWORD) 1) << i))
+		cpuinfo->online++;
     }
 #elif !defined(NO_SYSCONF) && (defined(_SC_NPROCESSORS_CONF) \
 			       || defined(_SC_NPROCESSORS_ONLN))
@@ -205,8 +258,8 @@ erts_cpu_info_update(erts_cpu_info_t *cpuinfo)
     if (cpuinfo->online > cpuinfo->configured)
 	cpuinfo->online = cpuinfo->configured;
 
-#ifdef HAVE_SCHED_xETAFFINITY
-    if (sched_getaffinity(cpuinfo->pid, sizeof(cpu_set_t), &cpuinfo->cpuset) == 0) {
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+    if (ERTS_MU_GET_PROC_AFFINITY__(cpuinfo) == 0) {
 	int i, c, cn, si;
 	c = cn = 0;
 	si = sizeof(cpuinfo->affinity_str_buf) - 1;
@@ -289,7 +342,7 @@ erts_get_cpu_available(erts_cpu_info_t *cpuinfo)
 char *
 erts_get_unbind_from_cpu_str(erts_cpu_info_t *cpuinfo)
 {
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     if (!cpuinfo)
 	return "false";
     return cpuinfo->affinity_str;
@@ -303,7 +356,7 @@ erts_get_available_cpu(erts_cpu_info_t *cpuinfo, int no)
 {
     if (!cpuinfo || no < 1 || cpuinfo->available < no)
 	return -EINVAL;
-#ifdef HAVE_SCHED_xETAFFINITY
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     {
 	cpu_set_t *allowed = &cpuinfo->cpuset;
 	int ix, n;
@@ -335,8 +388,8 @@ int
 erts_is_cpu_available(erts_cpu_info_t *cpuinfo, int id)
 {
     if (cpuinfo && 0 <= id) {
-#ifdef HAVE_SCHED_xETAFFINITY
-	if (id <= CPU_SETSIZE)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+	if (id < CPU_SETSIZE)
 	    return CPU_ISSET(id, &cpuinfo->cpuset);
 #elif defined(HAVE_PROCESSOR_BIND)
 	int no;
@@ -388,7 +441,7 @@ erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu)
      */
     if (!cpuinfo)
 	return -EINVAL;
-#ifdef HAVE_SCHED_xETAFFINITY
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     {
 	cpu_set_t bind_set;
 	if (cpu < 0)
@@ -398,9 +451,7 @@ erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu)
 
 	CPU_ZERO(&bind_set);
 	CPU_SET(cpu, &bind_set);
-	if (sched_setaffinity(0, sizeof(cpu_set_t), &bind_set) != 0)
-	    return -errno;
-	return 0;
+	return ERTS_MU_SET_THR_AFFINITY__(&bind_set);
     }
 #elif defined(HAVE_PROCESSOR_BIND)
     if (cpu < 0)
@@ -418,10 +469,8 @@ erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo)
 {
     if (!cpuinfo)
 	return -EINVAL;
-#if defined(HAVE_SCHED_xETAFFINITY)
-    if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuinfo->cpuset) != 0)
-	return -errno;
-    return 0;
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+    return ERTS_MU_SET_THR_AFFINITY__(&cpuinfo->cpuset);
 #elif defined(HAVE_PROCESSOR_BIND)
     if (processor_bind(P_LWPID, P_MYID, PBIND_NONE, NULL) != 0)
 	return -errno;
@@ -434,7 +483,7 @@ erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo)
 int
 erts_unbind_from_cpu_str(char *str)
 {
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     char *c = str;
     int cpus = 0;
     int shft = 0;
@@ -486,9 +535,7 @@ erts_unbind_from_cpu_str(char *str)
     if (!cpus)
 	return -EINVAL;
 
-    if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuset) != 0)
-	return -errno;
-    return 0;
+    return ERTS_MU_SET_THR_AFFINITY__(&cpuset);
 #elif defined(HAVE_PROCESSOR_BIND)
     if (processor_bind(P_LWPID, P_MYID, PBIND_NONE, NULL) != 0)
 	return -errno;
@@ -965,3 +1012,91 @@ read_topology(erts_cpu_info_t *cpuinfo)
 }
 
 #endif
+
+#if defined(__WIN32__)
+
+int
+erts_get_last_win_errno(void)
+{
+    switch (GetLastError()) {
+    case ERROR_INVALID_FUNCTION:		return EINVAL;	/* 1	*/
+    case ERROR_FILE_NOT_FOUND:			return ENOENT;	/* 2	*/
+    case ERROR_PATH_NOT_FOUND:			return ENOENT;	/* 3	*/
+    case ERROR_TOO_MANY_OPEN_FILES:		return EMFILE;	/* 4	*/
+    case ERROR_ACCESS_DENIED:			return EACCES;	/* 5	*/
+    case ERROR_INVALID_HANDLE:			return EBADF;	/* 6	*/
+    case ERROR_ARENA_TRASHED:			return ENOMEM;	/* 7	*/
+    case ERROR_NOT_ENOUGH_MEMORY:		return ENOMEM;	/* 8	*/
+    case ERROR_INVALID_BLOCK:			return ENOMEM;	/* 9	*/
+    case ERROR_BAD_ENVIRONMENT:			return E2BIG;	/* 10	*/
+    case ERROR_BAD_FORMAT:			return ENOEXEC;	/* 11	*/
+    case ERROR_INVALID_ACCESS:			return EINVAL;	/* 12	*/
+    case ERROR_INVALID_DATA:			return EINVAL;	/* 13	*/
+    case ERROR_OUTOFMEMORY:			return ENOMEM;	/* 14	*/
+    case ERROR_INVALID_DRIVE:			return ENOENT;	/* 15	*/
+    case ERROR_CURRENT_DIRECTORY:		return EACCES;	/* 16	*/
+    case ERROR_NOT_SAME_DEVICE:			return EXDEV;	/* 17	*/
+    case ERROR_NO_MORE_FILES:			return ENOENT;	/* 18	*/
+    case ERROR_WRITE_PROTECT:			return EACCES;	/* 19	*/
+    case ERROR_BAD_UNIT:			return EACCES;	/* 20	*/
+    case ERROR_NOT_READY:			return EACCES;	/* 21	*/
+    case ERROR_BAD_COMMAND:			return EACCES;	/* 22	*/
+    case ERROR_CRC:				return EACCES;	/* 23	*/
+    case ERROR_BAD_LENGTH:			return EACCES;	/* 24	*/
+    case ERROR_SEEK:				return EACCES;	/* 25	*/
+    case ERROR_NOT_DOS_DISK:			return EACCES;	/* 26	*/
+    case ERROR_SECTOR_NOT_FOUND:		return EACCES;	/* 27	*/
+    case ERROR_OUT_OF_PAPER:			return EACCES;	/* 28	*/
+    case ERROR_WRITE_FAULT:			return EACCES;	/* 29	*/
+    case ERROR_READ_FAULT:			return EACCES;	/* 30	*/
+    case ERROR_GEN_FAILURE:			return EACCES;	/* 31	*/
+    case ERROR_SHARING_VIOLATION:		return EACCES;	/* 32	*/
+    case ERROR_LOCK_VIOLATION:			return EACCES;	/* 33	*/
+    case ERROR_WRONG_DISK:			return EACCES;	/* 34	*/
+    case ERROR_SHARING_BUFFER_EXCEEDED:		return EACCES;	/* 36	*/
+    case ERROR_BAD_NETPATH:			return ENOENT;	/* 53	*/
+    case ERROR_NETWORK_ACCESS_DENIED:		return EACCES;	/* 65	*/
+    case ERROR_BAD_NET_NAME:			return ENOENT;	/* 67	*/
+    case ERROR_FILE_EXISTS:			return EEXIST;	/* 80	*/
+    case ERROR_CANNOT_MAKE:			return EACCES;	/* 82	*/
+    case ERROR_FAIL_I24:			return EACCES;	/* 83	*/
+    case ERROR_INVALID_PARAMETER:		return EINVAL;	/* 87	*/
+    case ERROR_NO_PROC_SLOTS:			return EAGAIN;	/* 89	*/
+    case ERROR_DRIVE_LOCKED:			return EACCES;	/* 108	*/
+    case ERROR_BROKEN_PIPE:			return EPIPE;	/* 109	*/
+    case ERROR_DISK_FULL:			return ENOSPC;	/* 112	*/
+    case ERROR_INVALID_TARGET_HANDLE:		return EBADF;	/* 114	*/
+    case ERROR_WAIT_NO_CHILDREN:		return ECHILD;	/* 128	*/
+    case ERROR_CHILD_NOT_COMPLETE:		return ECHILD;	/* 129	*/
+    case ERROR_DIRECT_ACCESS_HANDLE:		return EBADF;	/* 130	*/
+    case ERROR_NEGATIVE_SEEK:			return EINVAL;	/* 131	*/
+    case ERROR_SEEK_ON_DEVICE:			return EACCES;	/* 132	*/
+    case ERROR_DIR_NOT_EMPTY:			return ENOTEMPTY;/* 145	*/
+    case ERROR_NOT_LOCKED:			return EACCES;	/* 158	*/
+    case ERROR_BAD_PATHNAME:			return ENOENT;	/* 161	*/
+    case ERROR_MAX_THRDS_REACHED:		return EAGAIN;	/* 164	*/
+    case ERROR_LOCK_FAILED:			return EACCES;	/* 167	*/
+    case ERROR_ALREADY_EXISTS:			return EEXIST;	/* 183	*/
+    case ERROR_INVALID_STARTING_CODESEG:	return ENOEXEC;	/* 188	*/
+    case ERROR_INVALID_STACKSEG:		return ENOEXEC;	/* 189	*/
+    case ERROR_INVALID_MODULETYPE:		return ENOEXEC;	/* 190	*/
+    case ERROR_INVALID_EXE_SIGNATURE:		return ENOEXEC;	/* 191	*/
+    case ERROR_EXE_MARKED_INVALID:		return ENOEXEC;	/* 192	*/
+    case ERROR_BAD_EXE_FORMAT:			return ENOEXEC;	/* 193	*/
+    case ERROR_ITERATED_DATA_EXCEEDS_64k:	return ENOEXEC;	/* 194	*/
+    case ERROR_INVALID_MINALLOCSIZE:		return ENOEXEC;	/* 195	*/
+    case ERROR_DYNLINK_FROM_INVALID_RING:	return ENOEXEC;	/* 196	*/
+    case ERROR_IOPL_NOT_ENABLED:		return ENOEXEC;	/* 197	*/
+    case ERROR_INVALID_SEGDPL:			return ENOEXEC;	/* 198	*/
+    case ERROR_AUTODATASEG_EXCEEDS_64k:		return ENOEXEC;	/* 199	*/
+    case ERROR_RING2SEG_MUST_BE_MOVABLE:	return ENOEXEC;	/* 200	*/
+    case ERROR_RELOC_CHAIN_XEEDS_SEGLIM:	return ENOEXEC;	/* 201	*/
+    case ERROR_INFLOOP_IN_RELOC_CHAIN:		return ENOEXEC;	/* 202	*/
+    case ERROR_FILENAME_EXCED_RANGE:		return ENOENT;	/* 206	*/
+    case ERROR_NESTING_NOT_ALLOWED:		return EAGAIN;	/* 215	*/
+    case ERROR_NOT_ENOUGH_QUOTA:		return ENOMEM;	/* 1816	*/
+    default:					return EINVAL;
+    }
+}
+
+#endif
diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c
new file mode 100644
index 0000000000..4db4cffd3a
--- /dev/null
+++ b/erts/lib_src/common/ethr_aux.c
@@ -0,0 +1,762 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: A Thread library for use in the ERTS and other OTP
+ *              applications.
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_AUX_IMPL__
+
+#include "ethread.h"
+#include "ethr_internal.h"
+#include <string.h>
+#include <limits.h>
+
+#ifndef __WIN32__
+#include <unistd.h>
+#endif
+
+#define ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE 100
+#define ERTS_TS_EV_ALLOC_POOL_SIZE 25
+
+erts_cpu_info_t *ethr_cpu_info__;
+
+int ethr_not_completely_inited__ = 1;
+int ethr_not_inited__ = 1;
+
+ethr_memory_allocators ethr_mem__ = ETHR_MEM_ALLOCS_DEF_INITER__;
+
+#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
+ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
+#endif
+
+void *(*ethr_thr_prepare_func__)(void) = NULL;
+void (*ethr_thr_parent_func__)(void *) = NULL;
+void (*ethr_thr_child_func__)(void *) = NULL;
+
+typedef struct ethr_xhndl_list_ ethr_xhndl_list;
+struct ethr_xhndl_list_ {
+    ethr_xhndl_list *next;
+    void (*funcp)(void);
+};
+
+size_t ethr_pagesize__;
+size_t ethr_min_stack_size__; /* kilo words */
+size_t ethr_max_stack_size__; /* kilo words */
+
+ethr_rwmutex xhndl_rwmtx;
+ethr_xhndl_list *xhndl_list;
+
+static int main_threads;
+
+static int init_ts_event_alloc(void);
+
+int
+ethr_init_common__(ethr_init_data *id)
+{
+    int res;
+    if (id) {
+	ethr_thr_prepare_func__	= id->thread_create_prepare_func;
+	ethr_thr_parent_func__	= id->thread_create_parent_func;
+	ethr_thr_child_func__	= id->thread_create_child_func;
+    }
+
+    ethr_cpu_info__ = erts_cpu_info_create();
+    if (!ethr_cpu_info__)
+	return ENOMEM;
+
+#ifdef _SC_PAGESIZE
+    ethr_pagesize__ = (size_t) sysconf(_SC_PAGESIZE);
+#elif defined(HAVE_GETPAGESIZE)
+    ethr_pagesize__ = (size_t) getpagesize();
+#else
+    ethr_pagesize__ = (size_t) 4*1024; /* Guess 4 KB */
+#endif
+
+    /* User needs at least 4 KB */
+    ethr_min_stack_size__ = 4*1024;
+#if SIZEOF_VOID_P == 8
+    /* Double that on 64-bit archs */
+    ethr_min_stack_size__ *= 2;
+#endif
+    /* On some systems as much as about 4 KB is used by the system */
+    ethr_min_stack_size__ += 4*1024;
+    /* There should be room for signal handlers */
+#ifdef SIGSTKSZ
+    ethr_min_stack_size__ += SIGSTKSZ;
+#else
+    ethr_min_stack_size__ += ethr_pagesize__;
+#endif
+    /* The system may think that we need more stack */
+#if defined(PTHREAD_STACK_MIN)
+    if (ethr_min_stack_size__ < PTHREAD_STACK_MIN)
+	ethr_min_stack_size__ = PTHREAD_STACK_MIN;
+#elif defined(_SC_THREAD_STACK_MIN)
+    {
+	size_t thr_min_stk_sz = (size_t) sysconf(_SC_THREAD_STACK_MIN);
+	if (ethr_min_stack_size__ < thr_min_stk_sz)
+	    ethr_min_stack_size__ = thr_min_stk_sz;
+    }
+#endif
+    /* The guard is at least on some platforms included in the stack size
+       passed when creating threads */
+#ifdef ETHR_STACK_GUARD_SIZE
+    ethr_min_stack_size__ += ETHR_STACK_GUARD_SIZE;
+#endif
+    ethr_min_stack_size__ = ETHR_PAGE_ALIGN(ethr_min_stack_size__);
+
+    ethr_min_stack_size__ = ETHR_B2KW(ethr_min_stack_size__);
+
+    ethr_max_stack_size__ = 32*1024*1024;
+#if SIZEOF_VOID_P == 8
+    ethr_max_stack_size__ *= 2;
+#endif
+    ethr_max_stack_size__ = ETHR_B2KW(ethr_max_stack_size__);
+
+#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
+    {
+	int i;
+	for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) {
+	    res = ethr_spinlock_init(&ethr_atomic_protection__[i].u.lck);
+	    if (res != 0)
+		return res;
+	}
+    }
+#endif
+
+    res = ethr_mutex_lib_init(erts_get_cpu_configured(ethr_cpu_info__));
+    if (res != 0)
+	return res;
+
+    xhndl_list = NULL;
+
+    return 0;
+}
+
+int
+ethr_late_init_common__(ethr_late_init_data *lid)
+{
+    ethr_ts_event *tsep = NULL;
+    int reader_groups;
+    int res;
+    int i;
+    ethr_memory_allocator *m[] = {&ethr_mem__.std,
+				  &ethr_mem__.sl,
+				  &ethr_mem__.ll};
+    if (lid)
+	ethr_mem__ = lid->mem;
+    if (!ethr_mem__.std.alloc
+	|| !ethr_mem__.std.realloc
+	|| !ethr_mem__.std.free) {
+	ethr_mem__.std.alloc = malloc;
+	ethr_mem__.std.realloc = realloc;
+	ethr_mem__.std.free = free;
+    }
+    for (i = 0; i < sizeof(m)/sizeof(m[0]); i++) {
+	if (!m[i]->alloc || !m[i]->realloc || !m[i]->free) {
+	    m[i]->alloc = ethr_mem__.std.alloc;
+	    m[i]->realloc = ethr_mem__.std.realloc;
+	    m[i]->free = ethr_mem__.std.free;
+	}
+
+    }
+    res = init_ts_event_alloc();
+    if (res != 0)
+	return res;
+    res = ethr_make_ts_event__(&tsep);
+    if (res == 0)
+	tsep->iflgs |= ETHR_TS_EV_ETHREAD;
+    if (!lid) {
+	main_threads = 0;
+	reader_groups = 0;
+    }
+    else {
+	if (lid->main_threads < 0 || USHRT_MAX < lid->main_threads)
+	    return res;
+	main_threads = lid->main_threads;
+	reader_groups = lid->reader_groups;
+    }
+    res = ethr_mutex_lib_late_init(reader_groups, main_threads);
+    if (res != 0)
+	return res;
+    ethr_not_completely_inited__ = 0; /* Need it for
+					 rwmutex_init */
+    res = ethr_rwmutex_init(&xhndl_rwmtx);
+    ethr_not_completely_inited__ = 1;
+    if (res != 0)
+	return res;
+    return 0;
+}
+
+int
+ethr_install_exit_handler(void (*funcp)(void))
+{
+    ethr_xhndl_list *xhp;
+
+#if ETHR_XCHK
+    if (ethr_not_completely_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+
+    if (!funcp)
+	return EINVAL;
+
+    xhp = (ethr_xhndl_list *) ethr_mem__.std.alloc(sizeof(ethr_xhndl_list));
+    if (!xhp)
+	return ENOMEM;
+
+    ethr_rwmutex_rwlock(&xhndl_rwmtx);
+
+    xhp->funcp = funcp;
+    xhp->next = xhndl_list;
+    xhndl_list = xhp;
+
+    ethr_rwmutex_rwunlock(&xhndl_rwmtx);
+
+    return 0;
+}
+
+void
+ethr_run_exit_handlers__(void)
+{
+    ethr_xhndl_list *xhp;
+
+    ethr_rwmutex_rlock(&xhndl_rwmtx);
+
+    xhp = xhndl_list;
+
+    ethr_rwmutex_runlock(&xhndl_rwmtx);
+
+    for (; xhp; xhp = xhp->next)
+	(*xhp->funcp)();
+}
+
+/*
+ * Thread specific event alloc, etc.
+ *
+ * Note that we don't know when it is safe to destroy an event, but
+ * we know when it is safe to reuse it. ts_event_free() therefore
+ * never destroys an event (but makes freed events available for
+ * reuse).
+ *
+ * We could easily keep track of the usage of events, and by this
+ * make it possible to destroy events. We would however suffer a
+ * performance penalty for this and save very little memory.
+ */
+
+typedef union {
+    ethr_ts_event ts_ev;
+    char align[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_ts_event))];
+} ethr_aligned_ts_event;
+
+static ethr_spinlock_t ts_ev_alloc_lock;
+static ethr_ts_event *free_ts_ev;
+
+#if SIZEOF_VOID_P == SIZEOF_INT
+typedef unsigned int EthrPtrSzUInt;
+#elif  SIZEOF_VOID_P == SIZEOF_LONG
+typedef unsigned long EthrPtrSzUInt;
+#else
+#error No pointer sized integer type
+#endif
+
+static ethr_ts_event *ts_event_pool(int size, ethr_ts_event **endpp)
+{
+    int i;
+    ethr_aligned_ts_event *atsev;
+    atsev = ethr_mem__.std.alloc(sizeof(ethr_aligned_ts_event) * size
+				 + ETHR_CACHE_LINE_SIZE);
+    if (!atsev)
+	return NULL;
+    if ((((EthrPtrSzUInt) atsev) & ETHR_CACHE_LINE_MASK) == 0)
+	atsev = ((ethr_aligned_ts_event *)
+		 ((((EthrPtrSzUInt) atsev) & ~ETHR_CACHE_LINE_MASK)
+		  + ETHR_CACHE_LINE_SIZE));
+    for (i = 1; i < size; i++) {
+	atsev[i-1].ts_ev.next = &atsev[i].ts_ev;
+	ethr_atomic_init(&atsev[i-1].ts_ev.uaflgs, 0);
+	atsev[i-1].ts_ev.iflgs = 0;
+    }
+    ethr_atomic_init(&atsev[size-1].ts_ev.uaflgs, 0);
+    atsev[size-1].ts_ev.iflgs = 0;
+    atsev[size-1].ts_ev.next = NULL;
+    if (endpp)
+	*endpp = &atsev[size-1].ts_ev;
+    return &atsev[0].ts_ev;
+}
+
+static int init_ts_event_alloc(void)
+{
+    free_ts_ev = ts_event_pool(ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE,
+			       NULL);
+    if (!free_ts_ev)
+	return ENOMEM;
+    return ethr_spinlock_init(&ts_ev_alloc_lock);
+}
+
+static ethr_ts_event *ts_event_alloc(void)
+{
+    ethr_ts_event *ts_ev;
+    ethr_spin_lock(&ts_ev_alloc_lock);
+    if (free_ts_ev) {
+	ts_ev = free_ts_ev;
+	free_ts_ev = ts_ev->next;
+	ethr_spin_unlock(&ts_ev_alloc_lock);
+    }
+    else {
+	ethr_ts_event *ts_ev_pool_end;
+	ethr_spin_unlock(&ts_ev_alloc_lock);
+
+	ts_ev = ts_event_pool(ERTS_TS_EV_ALLOC_POOL_SIZE, &ts_ev_pool_end);
+	if (!ts_ev)
+	    return NULL;
+
+	ethr_spin_lock(&ts_ev_alloc_lock);
+	ts_ev_pool_end->next = free_ts_ev;
+	free_ts_ev = ts_ev->next;
+	ethr_spin_unlock(&ts_ev_alloc_lock);
+    }
+    return ts_ev;
+}
+
+static void ts_event_free(ethr_ts_event *ts_ev)
+{
+    ETHR_ASSERT(!ts_ev->udata);
+    ethr_spin_lock(&ts_ev_alloc_lock);
+    ts_ev->next = free_ts_ev;
+    free_ts_ev = ts_ev;
+    ethr_spin_unlock(&ts_ev_alloc_lock);
+}
+
+int ethr_make_ts_event__(ethr_ts_event **tsepp)
+{
+    int res;
+    ethr_ts_event *tsep = *tsepp;
+
+    if (!tsep) {
+	tsep = ts_event_alloc();
+	if (!tsep)
+	    return ENOMEM;
+    }
+
+    if ((tsep->iflgs & ETHR_TS_EV_INITED) == 0) {
+	res = ethr_event_init(&tsep->event);
+	if (res != 0) {
+	    ts_event_free(tsep);
+	    return res;
+	}
+    }
+
+    tsep->iflgs = ETHR_TS_EV_INITED;
+    tsep->udata = NULL;
+    tsep->rgix = 0;
+    tsep->mtix = 0;
+
+    res = ethr_set_tse__(tsep);
+    if (res != 0 && tsepp && *tsepp) {
+	ts_event_free(tsep);
+	return res;
+    }
+
+    if (tsepp)
+	*tsepp = tsep;
+
+    return 0;
+}
+
+int ethr_get_tmp_ts_event__(ethr_ts_event **tsepp)
+{
+    int res;
+    ethr_ts_event *tsep = *tsepp;
+
+    if (!tsep) {
+	tsep = ts_event_alloc();
+	if (!tsep)
+	    return ENOMEM;
+    }
+
+    if ((tsep->iflgs & ETHR_TS_EV_INITED) == 0) {
+	res = ethr_event_init(&tsep->event);
+	if (res != 0) {
+	    ts_event_free(tsep);
+	    return res;
+	}
+    }
+
+    tsep->iflgs = ETHR_TS_EV_INITED|ETHR_TS_EV_TMP;
+    tsep->udata = NULL;
+
+    if (tsepp)
+	*tsepp = tsep;
+
+    return 0;
+}
+
+int ethr_free_ts_event__(ethr_ts_event *tsep)
+{
+    ts_event_free(tsep);
+    return 0;
+}
+
+void ethr_ts_event_destructor__(void *vtsep)
+{
+    if (vtsep) {
+	ethr_ts_event *tsep = (ethr_ts_event *) vtsep;
+	ts_event_free(tsep);
+	ethr_set_tse__(NULL);
+    }
+}
+
+int ethr_set_main_thr_status(int on, int no)
+{
+    ethr_ts_event *tsep = ethr_get_tse__();
+    if (!tsep)
+	return EINVAL;
+    if (on) {
+	if (no < 1 || main_threads < no)
+	    return EINVAL;
+	tsep->mtix = (unsigned short) no;
+	tsep->iflgs |= ETHR_TS_EV_MAIN_THR;
+    }
+    else {
+	tsep->iflgs &= ~ETHR_TS_EV_MAIN_THR;
+	tsep->mtix = (unsigned short) 0;
+    }
+    return 0;
+}
+
+int ethr_get_main_thr_status(int *on)
+{
+    ethr_ts_event *tsep = ethr_get_tse__();
+    if (!tsep)
+	*on = 0;
+    else {
+	if (tsep->iflgs & ETHR_TS_EV_MAIN_THR)
+	    *on = 1;
+	else
+	    *on = 0;
+    }
+    return 0;
+}
+
+
+/* Atomics */
+
+void
+ethr_atomic_init(ethr_atomic_t *var, long i)
+{
+    ETHR_ASSERT(var);
+    ethr_atomic_init__(var, i);
+}
+
+void
+ethr_atomic_set(ethr_atomic_t *var, long i)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_set__(var, i);
+}
+
+long
+ethr_atomic_read(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_read__(var);
+}
+
+
+long
+ethr_atomic_add_read(ethr_atomic_t *var, long incr)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_add_read__(var, incr);
+}   
+
+long
+ethr_atomic_inc_read(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_inc_read__(var);
+}
+
+long
+ethr_atomic_dec_read(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_dec_read__(var);
+}
+
+void
+ethr_atomic_add(ethr_atomic_t *var, long incr)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_add__(var, incr);
+}   
+    
+void
+ethr_atomic_inc(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_inc__(var);
+}
+
+void
+ethr_atomic_dec(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_dec__(var);
+}
+
+long
+ethr_atomic_read_band(ethr_atomic_t *var, long mask)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_read_band__(var, mask);
+}
+
+long
+ethr_atomic_read_bor(ethr_atomic_t *var, long mask)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_read_bor__(var, mask);
+}
+
+long
+ethr_atomic_xchg(ethr_atomic_t *var, long new)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_xchg__(var, new);
+}   
+
+long
+ethr_atomic_cmpxchg(ethr_atomic_t *var, long new, long expected)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_cmpxchg__(var, new, expected);
+}
+
+long
+ethr_atomic_read_acqb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_read_acqb__(var);
+}
+
+long
+ethr_atomic_inc_read_acqb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_inc_read_acqb__(var);
+}
+
+void
+ethr_atomic_set_relb(ethr_atomic_t *var, long i)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_set_relb__(var, i);
+}
+
+void
+ethr_atomic_dec_relb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_dec_relb__(var);
+}
+
+long
+ethr_atomic_dec_read_relb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_dec_read_relb__(var);
+}
+
+long
+ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, long new, long exp)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_cmpxchg_acqb__(var, new, exp);
+}
+
+long
+ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, long new, long exp)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_cmpxchg_relb__(var, new, exp);
+}
+
+
+/* Spinlocks and rwspinlocks */
+
+int
+ethr_spinlock_init(ethr_spinlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_spinlock_init__(lock);
+}
+
+int
+ethr_spinlock_destroy(ethr_spinlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_spinlock_destroy__(lock);
+}
+
+void
+ethr_spin_unlock(ethr_spinlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_spin_unlock__(lock);
+}
+
+void
+ethr_spin_lock(ethr_spinlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_spin_lock__(lock);
+}
+
+int
+ethr_rwlock_init(ethr_rwlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_rwlock_init__(lock);
+}
+
+int
+ethr_rwlock_destroy(ethr_rwlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_rwlock_destroy__(lock);
+}
+
+void
+ethr_read_unlock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_read_unlock__(lock);
+}
+
+void
+ethr_read_lock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_read_lock__(lock);
+}
+
+void
+ethr_write_unlock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_write_unlock__(lock);
+}
+
+void
+ethr_write_lock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_write_lock__(lock);
+}
+
+ETHR_IMPL_NORETURN__ ethr_fatal_error__(const char *file,
+					int line,
+					const char *func,
+					int err)
+{
+    char *errstr;
+    if (err == ENOTSUP)
+	errstr = "Operation not supported";
+    else {
+	errstr = strerror(err);
+	if (!errstr)
+	    errstr = "Unknown error";
+    }
+    fprintf(stderr, "%s:%d: Fatal error in %s(): %s (%d)\n",
+	    file, line, func, errstr, err);
+    ethr_abort__();
+}
+
+int ethr_assert_failed(const char *file, int line, const char *func, char *a)
+{
+    fprintf(stderr, "%s:%d: %s(): Assertion failed: %s\n", file, line, func, a);
+    ethr_abort__();
+    return 0;
+}
diff --git a/erts/lib_src/common/ethr_cbf.c b/erts/lib_src/common/ethr_cbf.c
new file mode 100644
index 0000000000..04feceec89
--- /dev/null
+++ b/erts/lib_src/common/ethr_cbf.c
@@ -0,0 +1,36 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+
+/*
+ * We keep this function alone in a separate file so the
+ * compiler wont optimize it away.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "ethread.h"
+
+void
+ethr_compiler_barrier_fallback(void)
+{
+
+}
diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c
new file mode 100644
index 0000000000..f918bba81d
--- /dev/null
+++ b/erts/lib_src/common/ethr_mutex.c
@@ -0,0 +1,2758 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Mutex, rwmutex and condition variable implementation
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_MUTEX_IMPL__
+
+#include <limits.h>
+#include "ethread.h"
+#include "ethr_internal.h"
+
+#define ETHR_SPIN_WITH_WAITERS 1
+
+#define ETHR_MTX_MAX_FLGS_SPIN 1000
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+static int default_rwmtx_main_spincount;
+static int default_rwmtx_aux_spincount;
+#endif
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+static int default_mtx_main_spincount;
+static int default_mtx_aux_spincount;
+static int default_cnd_main_spincount;
+static int default_cnd_aux_spincount;
+#endif
+
+static int no_spin;
+
+#ifndef ETHR_USE_OWN_RWMTX_IMPL__
+static pthread_rwlockattr_t write_pref_attr_data;
+static pthread_rwlockattr_t *write_pref_attr;
+#endif
+
+#if defined(ETHR_MTX_Q_LOCK_SPINLOCK__)
+#  define ETHR_MTX_QLOCK_INIT ethr_spinlock_init
+#  define ETHR_MTX_QLOCK_DESTROY ethr_spinlock_destroy
+#  define ETHR_MTX_Q_LOCK ethr_spin_lock
+#  define ETHR_MTX_Q_UNLOCK ethr_spin_unlock
+#elif defined(ETHR_MTX_Q_LOCK_PTHREAD_MUTEX__)
+#  define ETHR_MTX_QLOCK_INIT(QL) pthread_mutex_init((QL), NULL)
+#  define ETHR_MTX_QLOCK_DESTROY pthread_mutex_destroy
+#  define ETHR_MTX_Q_LOCK(L)					\
+do {								\
+    int res__ = pthread_mutex_lock(L);				\
+    if (res__ != 0)						\
+	ETHR_FATAL_ERROR__(res__);				\
+} while (0)
+#  define ETHR_MTX_Q_UNLOCK(L)					\
+do {								\
+    int res__ = pthread_mutex_unlock(L);			\
+    if (res__ != 0)						\
+	ETHR_FATAL_ERROR__(res__);				\
+} while (0)
+#elif defined(ETHR_MTX_Q_LOCK_CRITICAL_SECTION__)
+#  define ETHR_MTX_QLOCK_INIT(QL) (InitializeCriticalSection((QL)), 0)
+#  define ETHR_MTX_QLOCK_DESTROY(QL) (DeleteCriticalSection((QL)), 0)
+#  define ETHR_MTX_Q_LOCK(QL) EnterCriticalSection((QL))
+#  define ETHR_MTX_Q_UNLOCK(QL) LeaveCriticalSection((QL))
+#endif
+
+int
+ethr_mutex_lib_init(int cpu_conf)
+{
+    int res = 0;
+
+    no_spin = cpu_conf == 1;
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+    default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT;
+    default_mtx_aux_spincount = ETHR_MTX_DEFAULT_AUX_SPINCOUNT;
+    default_cnd_main_spincount = ETHR_CND_DEFAULT_MAIN_SPINCOUNT;
+    default_cnd_aux_spincount = ETHR_CND_DEFAULT_AUX_SPINCOUNT;
+#endif
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+    default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT;
+    default_rwmtx_aux_spincount = ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT;
+
+#else
+
+#if defined(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP) \
+    && defined(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)
+    res = pthread_rwlockattr_init(&write_pref_attr_data);
+    if (res != 0)
+	return res;
+    res = pthread_rwlockattr_setkind_np(
+	&write_pref_attr_data,
+	PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
+    write_pref_attr = &write_pref_attr_data;
+#else
+    write_pref_attr = NULL;
+#endif
+
+#endif
+
+    return res;
+}
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+#ifdef ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS
+#if 0 /*
+       * When inc and dec are real atomic instructions as on x86, the
+       * ETHR_RLOCK_WITH_INC_DEC implementations performs better with
+       * lots of read locks compared to the cmpxchg based implementation.
+       * It, however, performs worse with lots of mixed reads and writes.
+       * It could be used for rwlocks that are known to be read locked
+       * much, but the readers array based implementation outperforms it
+       * by far. Therefore, it has been disabled, and will probably be
+       * removed some time in the future.
+       */
+#  define ETHR_RLOCK_WITH_INC_DEC
+#endif
+#endif
+
+static int reader_groups_array_size = 0;
+static int main_threads_array_size = 0;
+
+#endif
+
+int
+ethr_mutex_lib_late_init(int no_reader_groups, int no_main_threads)
+{
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+    reader_groups_array_size = (no_reader_groups <= 1
+				? 1
+				: no_reader_groups + 1);
+    main_threads_array_size = (no_main_threads <= 1
+			       ? 1
+			       : no_main_threads + 1);
+#endif
+    return 0;
+}
+
+int
+ethr_rwmutex_set_reader_group(int ix)
+{
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+    ethr_ts_event *tse;
+
+    if (ix < 0 || reader_groups_array_size <= ix)
+	return EINVAL;
+
+    tse = ethr_get_ts_event();
+
+    if ((tse->iflgs & ETHR_TS_EV_ETHREAD) == 0) {
+	ethr_leave_ts_event(tse);
+	return EINVAL;
+    }
+
+    tse->rgix = ix;
+    
+    ethr_leave_ts_event(tse);
+#endif
+    return 0;
+}
+
+#if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ)
+static void hard_debug_chk_q__(struct ethr_mutex_base_ *, int);
+#define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX) hard_debug_chk_q__(&(RWMTX)->mtxb,1)
+#define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX) hard_debug_chk_q__(&(MTX)->mtxb, 0)
+#else
+#define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX)
+#define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX)
+#endif
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+static void
+rwmutex_unlock_wake(ethr_rwmutex *rwmtx,
+		    int have_w,
+		    long initial);
+static int
+rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
+			     long initial,
+			     ethr_ts_event *tse,
+			     int start_next_ix,
+			     int check_before_try,
+			     int try_write_lock);
+#endif
+
+#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)
+
+/* -- Utilities operating both on ordinary mutexes and read write mutexes -- */
+
+static ETHR_INLINE void
+rwmutex_freqread_wtng_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix = (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ
+	      ? tse->rgix
+	      : tse->mtix);
+    rwmtx->tdata.ra[ix].data.waiting_readers++;
+}
+
+static ETHR_INLINE void
+rwmutex_freqread_rdrs_add(ethr_rwmutex *rwmtx,
+			  ethr_rwmutex_type type,
+			  int ix,
+			  int inc)
+{
+    if (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ || ix == 0)
+	ethr_atomic_add(&rwmtx->tdata.ra[ix].data.readers, inc);
+    else {
+	ETHR_ASSERT(type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
+	ETHR_ASSERT(inc == 1);
+	ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 1);
+    }
+}
+
+static ETHR_INLINE void
+rwmutex_freqread_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_inc:
+	ethr_atomic_inc(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_inc;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
+	ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 1);
+    }
+}
+
+static ETHR_INLINE void
+rwmutex_freqread_rdrs_dec(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_dec:
+	ethr_atomic_dec(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_dec;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+	ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 0);
+    }
+}
+
+static ETHR_INLINE long
+rwmutex_freqread_rdrs_dec_read(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_dec_read:
+	return ethr_atomic_dec_read(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_dec_read;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+	ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 0);
+	return (long) 0;
+    }
+}
+
+static ETHR_INLINE long
+rwmutex_freqread_rdrs_dec_read_relb(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_dec_read:
+	return ethr_atomic_dec_read_relb(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_dec_read;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+	ethr_atomic_set_relb(&rwmtx->tdata.ra[ix].data.readers, (long) 0);
+	return (long) 0;
+    }
+}
+
+static ETHR_INLINE long
+rwmutex_freqread_rdrs_read(ethr_rwmutex *rwmtx, int ix)
+{
+    long res = ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers);
+#ifdef ETHR_DEBUG
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+	ETHR_ASSERT(res >= 0);
+	break;
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+	ETHR_ASSERT(res == 0 || res == 1);
+	break;
+    default:
+	ETHR_ASSERT(0);
+	break;
+    }
+#endif
+    return res;
+}
+
+
+static ETHR_INLINE void
+enqueue(ethr_ts_event **queue,
+	ethr_ts_event *tse_start,
+	ethr_ts_event *tse_end)
+{
+    if (!*queue) {
+	*queue = tse_start;
+	tse_start->prev = tse_end;
+	tse_end->next = tse_start;
+    }
+    else {
+	tse_end->next = *queue;
+	tse_start->prev = (*queue)->prev;
+	(*queue)->prev->next = tse_start;
+	(*queue)->prev = tse_end;
+    }
+}
+
+static ETHR_INLINE void
+insert(ethr_ts_event *tse_pred, ethr_ts_event *tse)
+{
+    tse->next = tse_pred->next;
+    tse->prev = tse_pred;
+    tse_pred->next->prev = tse;
+    tse_pred->next = tse;
+}
+
+static ETHR_INLINE void
+dequeue(ethr_ts_event **queue,
+	ethr_ts_event *tse_start,
+	ethr_ts_event *tse_end)
+{
+    if (tse_start->prev == tse_end) {
+	ETHR_ASSERT(*queue == tse_start && tse_end->next == tse_start);
+	*queue = NULL;
+    }
+    else {
+	if (*queue == tse_start)
+	    *queue = tse_end->next;
+	tse_end->next->prev = tse_start->prev;
+	tse_start->prev->next = tse_end->next;
+    }
+}
+
+static void
+event_wait(struct ethr_mutex_base_ *mtxb,
+	   ethr_ts_event *tse,
+	   int spincount,
+	   long type,
+	   int is_rwmtx,
+	   int is_freq_read)
+{
+    int locked = 0;
+    long act;
+    int need_try_complete_runlock = 0;
+
+    /* Need to enqueue and wait... */
+
+    tse->uflgs = type;
+    ethr_atomic_set(&tse->uaflgs, type);
+
+    ETHR_MTX_Q_LOCK(&mtxb->qlck);
+    locked = 1;
+
+#ifdef ETHR_MTX_HARD_DEBUG_Q
+    hard_debug_chk_q__(mtxb, is_rwmtx);
+#endif
+
+    act = ethr_atomic_read(&mtxb->flgs);
+
+    if (act & type) {
+
+	/* Wait bit already there; enqueue... */
+
+	ETHR_ASSERT(mtxb->q);
+	if (type == ETHR_RWMTX_W_WAIT_FLG__) {
+	    enqueue(&mtxb->q, tse, tse);
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+	    mtxb->ws++;
+#endif
+	}
+	else {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_ASSERT(is_rwmtx);
+	    ETHR_ASSERT(rwmtx->rq_end);
+	    insert(rwmtx->rq_end, tse);
+	    rwmtx->rq_end = tse;
+	    if (is_freq_read)
+		rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse);
+	    else
+		rwmtx->tdata.rs++;
+	}
+    }
+    else {
+
+	/* Set wait bit */
+
+	while (1) {
+	    long new, exp = act;
+	    int freqread_tryrlock = 0;
+	    need_try_complete_runlock = 0;
+
+	    if (type == ETHR_RWMTX_W_WAIT_FLG__) {
+		if (is_freq_read && act == ETHR_RWMTX_R_FLG__)
+		    need_try_complete_runlock = 1;
+		if (act != 0)
+		    new = act | ETHR_RWMTX_W_WAIT_FLG__;
+		else
+		    new = ETHR_RWMTX_W_FLG__; /* Try to get it */
+	    }
+	    else {
+		ETHR_ASSERT(is_rwmtx);
+
+		if (!is_freq_read) {
+		    if (act & (ETHR_RWMTX_W_FLG__| ETHR_RWMTX_W_WAIT_FLG__))
+			new = act | ETHR_RWMTX_R_WAIT_FLG__;
+		    else
+			new = act + 1; /* Try to get it */
+		}
+		else {
+		    if (act & ~ETHR_RWMTX_R_FLG__)
+			new = act | ETHR_RWMTX_R_WAIT_FLG__;
+		    else { /* Try to get it */
+			ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+			rwmutex_freqread_rdrs_inc(rwmtx, tse);
+			ETHR_MEMORY_BARRIER;
+			new = act | ETHR_RWMTX_R_FLG__;
+			freqread_tryrlock = 1;
+		    }
+		}
+	    }
+
+	    act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs, new, exp);
+	    if (exp == act) {
+		if (new & type) {
+		    act = new;
+		    break;
+		}
+		else {
+		    /* Got it */
+		    goto done;
+		}
+	    }
+
+	    if (freqread_tryrlock) {
+		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+
+		/* We didn't set ETHR_RWMTX_R_FLG__, however someone
+		   else might have */
+		if (act == ETHR_RWMTX_R_FLG__)
+		    goto done; /* Got it by help from someone else */
+
+		ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
+		/*
+		 * We know that no waiter flags have been set, i.e.,
+		 * we cannot get into a situation where we need to wake
+		 * someone up here. Just restore the readers counter
+		 * and do it over again...
+		 */
+		rwmutex_freqread_rdrs_dec(rwmtx, tse);
+	    }
+	}
+
+	/* Enqueue */
+
+	if (type == ETHR_RWMTX_R_WAIT_FLG__) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_ASSERT(is_rwmtx);
+	    ETHR_ASSERT(!rwmtx->rq_end);
+	    rwmtx->rq_end =  tse;
+	    if (is_freq_read)
+		rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse);
+	    else
+		rwmtx->tdata.rs++;
+	}
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+	else {
+	    mtxb->ws++;
+	}
+#endif
+
+	enqueue(&mtxb->q, tse, tse);
+    }
+
+#ifdef ETHR_MTX_HARD_DEBUG_Q
+    hard_debug_chk_q__(mtxb, is_rwmtx);
+#endif
+
+    /* Wait */
+    locked = 0;
+    ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+
+    if (need_try_complete_runlock) {
+	ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type
+		    != ETHR_RWMUTEX_TYPE_NORMAL);
+	/*
+	 * We were the only one in queue when we enqueued, and it
+	 * was seemingly read locked. We need to try to complete a
+	 * runlock otherwise we might be hanging forever. If the
+	 * runlock could be completed we will be dequeued and
+	 * woken by ourselves.
+	 */
+	rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb,
+				     act, tse, 0, 1, 0);
+    }
+
+    while (1) {
+	ethr_event_reset(&tse->event);
+
+	act = ethr_atomic_read_acqb(&tse->uaflgs);
+	if (!act)
+	    goto done; /* Got it */
+
+	ETHR_ASSERT(act == type);
+	ethr_event_swait(&tse->event, spincount);
+	/* swait result: 0 || EINTR */
+
+	act = ethr_atomic_read_acqb(&tse->uaflgs);
+	if (!act)
+	    goto done; /* Got it */
+    }
+
+ done:
+    if (locked)
+	ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+}
+
+static void
+wake_writer(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
+{
+    ethr_ts_event *tse;
+
+    tse = mtxb->q;
+    ETHR_ASSERT(tse);
+    dequeue(&mtxb->q, tse, tse);
+
+    ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+    ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__);
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+    mtxb->ws--;
+#endif
+#if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ)
+    hard_debug_chk_q__(mtxb, is_rwmtx);
+#endif
+
+    ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+
+    ethr_atomic_set(&tse->uaflgs, 0);
+    ethr_event_set(&tse->event);
+}
+
+static ETHR_INLINE int
+initial_spincount(struct ethr_mutex_base_ *mtxb)
+{
+    return (mtxb->aux_scnt < ETHR_MTX_MAX_FLGS_SPIN
+	    ? mtxb->aux_scnt
+	    : ETHR_MTX_MAX_FLGS_SPIN);
+}
+
+static ETHR_INLINE int
+update_spincount(struct ethr_mutex_base_ *mtxb,
+		 ethr_ts_event *tse,
+		 int *start_scnt,
+		 int *scnt)
+{
+    int sscnt = *start_scnt;
+    if (sscnt < 0) {
+	*scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
+		 ? mtxb->main_scnt
+		 : mtxb->aux_scnt);
+	*scnt -= ETHR_MTX_MAX_FLGS_SPIN;
+    }
+    else {
+	*scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
+		 ? mtxb->main_scnt
+		 : mtxb->aux_scnt);
+	*scnt -= sscnt;
+	if (*scnt > 0 && sscnt < ETHR_MTX_MAX_FLGS_SPIN) {
+	    *scnt = ETHR_MTX_MAX_FLGS_SPIN - sscnt;
+	    *start_scnt = -1;
+	    return 0;
+	}
+    }
+    return 1;
+}
+
+int check_readers_array(ethr_rwmutex *rwmtx,
+			int start_rix,
+			int length,
+			int pre_check);
+
+static ETHR_INLINE void
+write_lock_wait(struct ethr_mutex_base_ *mtxb,
+		long initial,
+		int is_rwmtx,
+		int is_freq_read)
+{
+    long act = initial;
+    int scnt, start_scnt;
+    ethr_ts_event *tse = NULL;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+    int res;
+    int freq_read_size = -1;
+    int freq_read_start_ix = -1;
+
+    ETHR_ASSERT(!is_freq_read || is_rwmtx);
+
+    start_scnt = scnt = initial_spincount(mtxb);
+
+    /*
+     * Spin trying to write lock for a while. If unsuccessful,
+     * wait on event.
+     */
+
+    while (1) {
+	long exp;
+
+	while (act != 0) {
+
+	    if (is_freq_read && act == ETHR_RWMTX_R_FLG__) {
+		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+		if (!tse)
+		    tse = ethr_get_ts_event();
+		if (freq_read_size < 0) {
+		    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+			freq_read_size = reader_groups_array_size;
+			freq_read_start_ix = tse->rgix;
+		    }
+		    else {
+			freq_read_size = main_threads_array_size;
+			freq_read_start_ix = tse->mtix;
+		    }
+		}
+		res = check_readers_array(rwmtx,
+					  freq_read_start_ix,
+					  freq_read_size,
+					  1);
+		scnt--;
+		if (res == 0) {
+		    act = ethr_atomic_read(&mtxb->flgs);
+		    if (act & ETHR_RWMTX_R_MASK__) {
+			res = rwmutex_try_complete_runlock(rwmtx, act,
+							   tse, 0, 0,
+							   1);
+			if (res != EBUSY)
+			    goto done; /* Got it */
+		    }
+		    if (scnt <= 0)
+			goto chk_spin;
+		    if (--until_yield == 0) {
+			until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+			ETHR_YIELD();
+		    }
+		    continue;
+		}
+	    }
+
+	    if (scnt <= 0) {
+	    chk_spin:
+		scnt = 0;
+
+		if (!tse)
+		    tse = ethr_get_ts_event();
+		if (update_spincount(mtxb, tse, &start_scnt, &scnt)) {
+		    event_wait(mtxb, tse, scnt, ETHR_RWMTX_W_WAIT_FLG__,
+			       is_rwmtx, is_freq_read);
+		    goto done;
+		}
+	    }
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		ETHR_YIELD();
+	    }
+	    act = ethr_atomic_read(&mtxb->flgs);
+	    scnt--;
+	}
+	ETHR_ASSERT(scnt >= 0);
+
+	exp = act;
+
+	act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs,
+				       ETHR_RWMTX_W_FLG__,
+				       exp);
+	if (act == 0)
+	    goto done; /* Got it */
+    }
+
+ done:
+    if (tse)
+	ethr_leave_ts_event(tse);
+}
+
+static int
+mtxb_init(struct ethr_mutex_base_ *mtxb,
+	  int def_main_scnt,
+	  int main_scnt,
+	  int def_aux_scnt,
+	  int aux_scnt)
+{
+    ETHR_MTX_HARD_DEBUG_LFS_INIT(mtxb);
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+    mtxb->ws = 0;
+#endif
+    if (no_spin) {
+	mtxb->main_scnt = 0;
+	mtxb->aux_scnt = 0;
+    }
+    else {
+	if (main_scnt > SHRT_MAX)
+	    mtxb->main_scnt = SHRT_MAX;
+	else if (main_scnt < 0)
+	    mtxb->main_scnt = def_main_scnt;
+	else
+	    mtxb->main_scnt = (short) main_scnt;
+	if (aux_scnt > SHRT_MAX)
+	    mtxb->aux_scnt = SHRT_MAX;
+	else if (aux_scnt < 0)
+	    mtxb->aux_scnt = def_aux_scnt;
+	else
+	    mtxb->aux_scnt = (short) aux_scnt;
+	if (mtxb->main_scnt < mtxb->aux_scnt)
+	    mtxb->main_scnt = mtxb->aux_scnt;
+
+    }
+    mtxb->q = NULL;
+    ethr_atomic_init(&mtxb->flgs, 0);
+    return ETHR_MTX_QLOCK_INIT(&mtxb->qlck);
+}
+
+static int
+mtxb_destroy(struct ethr_mutex_base_ *mtxb)
+{
+    long act;
+    ETHR_MTX_Q_LOCK(&mtxb->qlck);
+    act = ethr_atomic_read(&mtxb->flgs);
+    ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+    if (act != 0)
+	return EINVAL;
+    return ETHR_MTX_QLOCK_DESTROY(&mtxb->qlck);
+}
+
+
+#endif /* ETHR_USE_OWN_RWMTX_IMPL__ || ETHR_USE_OWN_MTX_IMPL__ */
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Mutex and condition variable implementation                               *
+\*                                                                           */
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+
+/* -- Mutex ---------------------------------------------------------------- */
+
+int
+ethr_mutex_init_opt(ethr_mutex *mtx, ethr_mutex_opt *opt)
+{
+    int res;
+#if ETHR_XCHK
+    if (!mtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    mtx->initialized = ETHR_MUTEX_INITIALIZED;
+#endif
+    ETHR_MTX_HARD_DEBUG_FENCE_INIT(mtx);
+    res = mtxb_init(&mtx->mtxb,
+		    default_mtx_main_spincount,
+		    opt ? opt->main_spincount : -1,
+		    default_mtx_aux_spincount,
+		    opt ? opt->aux_spincount : -1);
+#if ETHR_XCHK
+    if (res != 0)
+	mtx->initialized = 0;
+#endif
+    return res;
+}
+
+int
+ethr_mutex_init(ethr_mutex *mtx)
+{
+    return ethr_mutex_init_opt(mtx, NULL);
+}
+
+int
+ethr_mutex_destroy(ethr_mutex *mtx)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!mtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    mtx->initialized = 0;
+#endif
+    return mtxb_destroy(&mtx->mtxb);
+}
+
+void
+ethr_mutex_lock_wait__(ethr_mutex *mtx, long initial)
+{
+    write_lock_wait(&mtx->mtxb, initial, 0, 0);
+}
+
+void
+ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial)
+{
+    ethr_ts_event *tse;
+
+    ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck);
+    tse = mtx->mtxb.q;
+
+    ETHR_ASSERT(tse);
+    ETHR_ASSERT(ethr_atomic_read(&mtx->mtxb.flgs)
+		== (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__));
+    ETHR_ASSERT(initial & ETHR_RWMTX_W_WAIT_FLG__);
+    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+
+    /*
+     * If we have multiple waiters, there is no need to modify
+     * mtxb->flgs; otherwise, we need to clear the write wait bit...
+     */
+    if (tse->next == mtx->mtxb.q)
+	ethr_atomic_set(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__);
+
+    wake_writer(&mtx->mtxb, 0);
+}
+
+/* -- Condition variables -------------------------------------------------- */
+
+static void
+enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
+{
+    long act;
+
+    /*
+     * `ethr_cond_signal()' and `ethr_cond_broadcast()' end up here. If `mtx'
+     * is not currently locked by current thread, we almost certainly have a
+     * hard to debug race condition. There might however be some (strange)
+     * use for it. POSIX also allow a call to `pthread_cond_signal' or
+     * `pthread_cond_broadcast' even though the the associated mutex isn't
+     * locked by the caller. Therefore, we also allow this kind of strange
+     * usage, but optimize for the case where the mutex is locked by the
+     * calling thread.
+     */
+
+    ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck);
+
+    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+    {
+	int dbg_nws__ = 0;
+	ethr_ts_event *dbg_tse__;
+	for (dbg_tse__ = tse_start;
+	     dbg_tse__ != tse_end;
+	     dbg_tse__ = dbg_tse__->next)
+	    dbg_nws__++;
+	mtx->mtxb.ws += dbg_nws__ + 1;
+    }
+#endif
+
+    act = ethr_atomic_read(&mtx->mtxb.flgs);
+    ETHR_ASSERT(act == 0
+		|| act == ETHR_RWMTX_W_FLG__
+		|| act == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__));
+    if (act & ETHR_RWMTX_W_FLG__) {
+	/* The normal sane case */
+	if (!(act & ETHR_RWMTX_W_WAIT_FLG__)) {
+	    ETHR_ASSERT(!mtx->mtxb.q);
+	    act = ethr_atomic_cmpxchg(&mtx->mtxb.flgs,
+				      (ETHR_RWMTX_W_FLG__
+				       | ETHR_RWMTX_W_WAIT_FLG__),
+				      ETHR_RWMTX_W_FLG__);
+	    if (act != ETHR_RWMTX_W_FLG__) {
+		/*
+		 * Sigh... this wasn't so sane after all since, the mutex was
+		 * obviously not locked by the current thread....
+		 */
+		ETHR_ASSERT(act == 0);
+		goto mtx_unlocked;
+	    }
+	}
+
+#ifdef ETHR_DEBUG
+	if (act & ETHR_RWMTX_W_WAIT_FLG__)
+	    ETHR_ASSERT(mtx->mtxb.q);
+	else
+	    ETHR_ASSERT(!mtx->mtxb.q);
+#endif
+
+	enqueue(&mtx->mtxb.q, tse_start, tse_end);
+
+	ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+	ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
+
+    }
+    else {
+	int multi;
+    mtx_unlocked:
+	/* Sigh... mutex isn't locked... */
+
+	multi = tse_start != tse_end;
+
+	while (1) {
+	    long new, exp = act;
+
+	    if (multi || (act & ETHR_RWMTX_W_FLG__))
+		new = ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__;
+	    else
+		new = ETHR_RWMTX_W_FLG__;
+
+	    act = ethr_atomic_cmpxchg(&mtx->mtxb.flgs, new, exp);
+	    if (exp == act) {
+		ETHR_ASSERT(!mtx->mtxb.q);
+		if (act & ETHR_RWMTX_W_FLG__) {
+		    enqueue(&mtx->mtxb.q, tse_start, tse_end);
+
+		    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+		    ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
+
+		}
+		else {
+		    ETHR_ASSERT(!mtx->mtxb.q);
+		    /*
+		     * Acquired the mutex on behalf of the
+		     * first thread in the queue; wake
+		     * it and enqueue the rest...
+		     */
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+		    mtx->mtxb.ws--;
+#endif
+		    if (multi) {
+			enqueue(&mtx->mtxb.q, tse_start->next, tse_end);
+			ETHR_ASSERT(mtx->mtxb.q);
+		    }
+
+		    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+		    ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
+
+		    ethr_atomic_set(&tse_start->uaflgs, 0);
+		    ethr_event_set(&tse_start->event);
+		}
+		break;
+	    }
+	}
+    }
+}
+
+int
+ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt)
+{
+    int res;
+#if ETHR_XCHK
+    if (!cnd) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = ETHR_COND_INITIALIZED;
+#endif
+    ETHR_MTX_HARD_DEBUG_FENCE_INIT(cnd);
+    cnd->q = NULL;
+    if (no_spin) {
+	cnd->main_scnt = 0;
+	cnd->aux_scnt = 0;
+    }
+    else {
+	if (!opt || opt->main_spincount < 0)
+	    cnd->main_scnt = default_cnd_main_spincount;
+	else if (opt->main_spincount > SHRT_MAX)
+	    cnd->main_scnt = SHRT_MAX;
+	else
+	    cnd->main_scnt = (short) opt->main_spincount;
+	if (!opt || opt->aux_spincount < 0)
+	    cnd->aux_scnt = default_cnd_aux_spincount;
+	else if (opt->aux_spincount > SHRT_MAX)
+	    cnd->aux_scnt = SHRT_MAX;
+	else
+	    cnd->aux_scnt = (short) opt->aux_spincount;
+	if (cnd->main_scnt < cnd->aux_scnt)
+	    cnd->main_scnt = cnd->aux_scnt;
+    }
+    ETHR_MTX_QLOCK_INIT(&cnd->qlck);
+    return 0;
+}
+
+int
+ethr_cond_init(ethr_cond *cnd)
+{
+    return ethr_cond_init_opt(cnd, NULL);
+}
+
+int
+ethr_cond_destroy(ethr_cond *cnd)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = 0;
+#endif
+    return ETHR_MTX_QLOCK_DESTROY(&cnd->qlck);
+}
+
+void
+ethr_cond_signal(ethr_cond *cnd)
+{
+    ethr_ts_event *tse;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(!cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+
+    ETHR_MTX_Q_LOCK(&cnd->qlck);
+
+    tse = cnd->q;
+
+    if (!tse) {
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+	ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+    }
+    else {
+	ethr_mutex *mtx = (ethr_mutex *) tse->udata;
+
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+	ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+	ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) == ETHR_CND_WAIT_FLG__);
+
+	ethr_atomic_set(&tse->uaflgs, ETHR_RWMTX_W_WAIT_FLG__);
+
+	dequeue(&cnd->q, tse, tse);
+
+	ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+	tse->next = tse->prev = NULL;
+
+	enqueue_mtx(mtx, tse, tse);
+
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    }
+}
+
+void
+ethr_cond_broadcast(ethr_cond *cnd)
+{
+    int res;
+    int got_all;
+    ethr_ts_event *tse;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(!cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+    do {
+	got_all = 1;
+
+	ETHR_MTX_Q_LOCK(&cnd->qlck);
+
+	tse = cnd->q;
+
+	if (!tse) {
+	    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+	    ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+	}
+	else {
+	    ethr_mutex *mtx = (ethr_mutex *) tse->udata;
+	    ethr_ts_event *tse_tmp, *tse_end;
+
+	    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+	    tse_end = cnd->q->prev;
+
+	    tse_tmp = tse;
+
+	    do {
+
+		if (mtx == (ethr_mutex *) tse_tmp->udata) {
+		    /* The normal case */
+
+		    ETHR_ASSERT(tse_tmp->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+		    ETHR_ASSERT(ethr_atomic_read(&tse_tmp->uaflgs)
+				== ETHR_CND_WAIT_FLG__);
+
+		    ethr_atomic_set(&tse_tmp->uaflgs, ETHR_RWMTX_W_WAIT_FLG__);
+		}
+		else {
+		    /* Should be very unusual */
+		    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+		    tse_end = tse_tmp->prev;
+		    got_all = 0;
+		    break;
+		}
+
+		tse_tmp = tse_tmp->next;
+
+	    } while (tse_tmp != cnd->q);
+
+	    dequeue(&cnd->q, tse, tse_end);
+
+	    ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+	    enqueue_mtx(mtx, tse, tse_end);
+	}
+
+    } while (!got_all);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+}
+
+int
+ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
+{
+    int woken;
+    int scnt;
+    int res;
+    void *udata = NULL;
+    ethr_ts_event *tse;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(!cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+    ETHR_ASSERT(!mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    tse = ethr_get_ts_event();
+
+    scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
+	    ? cnd->main_scnt
+	    : cnd->aux_scnt);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+
+    udata = tse->udata; /* Got to restore udata before returning */
+    tse->udata = (void *) mtx;
+
+    tse->uflgs = ETHR_RWMTX_W_WAIT_FLG__; /* Prep for mutex lock op */
+    ethr_atomic_set(&tse->uaflgs, ETHR_CND_WAIT_FLG__);
+
+    ETHR_MTX_Q_LOCK(&cnd->qlck);
+
+    enqueue(&cnd->q, tse, tse);
+
+    ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+    ethr_mutex_unlock(mtx);
+
+    /* Wait */
+    woken = 0;
+    while (1) {
+	long act;
+
+	ethr_event_reset(&tse->event);
+
+	act = ethr_atomic_read_acqb(&tse->uaflgs);
+	if (!act)
+	    break; /* Mtx locked */
+
+	/* First time, got EINTR, or spurious wakeup... */
+
+	ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__
+		    || act == ETHR_RWMTX_W_WAIT_FLG__);
+
+	if (woken) {
+	    /*
+	     * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already been enqueued
+	     * on the mutex; continue wait until locked...
+	     */
+	    if (act == ETHR_CND_WAIT_FLG__) {
+		ETHR_MTX_Q_LOCK(&cnd->qlck);
+		act = ethr_atomic_read(&tse->uaflgs);
+		ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__
+			    || act == ETHR_RWMTX_W_WAIT_FLG__);
+		/*
+		 * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already
+		 * enqueued on the mutex; continue wait until locked...
+		 */
+		if (act == ETHR_CND_WAIT_FLG__)
+		    dequeue(&cnd->q, tse, tse);
+
+		ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+		if (act == ETHR_CND_WAIT_FLG__) {
+		    tse->udata = udata;
+		    ethr_leave_ts_event(tse);
+		    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+		    ethr_mutex_lock(mtx);
+		    return EINTR;
+		}
+	    }
+	    ETHR_ASSERT(act == ETHR_RWMTX_W_WAIT_FLG__);
+	}
+	ethr_event_swait(&tse->event, scnt);
+	/* swait result: 0 || EINTR */
+	woken = 1;
+    }
+
+    ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+
+    tse->udata = udata;
+    ethr_leave_ts_event(tse);
+    return 0;
+}
+
+#else
+/* -- pthread mutex and condition variables -------------------------------- */
+
+int
+ethr_mutex_init(ethr_mutex *mtx)
+{
+#if ETHR_XCHK
+    if (!mtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    mtx->initialized = ETHR_MUTEX_INITIALIZED;
+#endif
+    return pthread_mutex_init(&mtx->pt_mtx, NULL);
+}
+
+int
+ethr_mutex_destroy(ethr_mutex *mtx)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+#if ETHR_XCHK
+    mtx->initialized = 0;
+#endif
+    return pthread_mutex_destroy(&mtx->pt_mtx);
+}
+
+int
+ethr_cond_init(ethr_cond *cnd)
+{
+#if ETHR_XCHK
+    if (!cnd) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = ETHR_COND_INITIALIZED;
+#endif
+    return pthread_cond_init(&cnd->pt_cnd, NULL);
+}
+
+int
+ethr_cond_destroy(ethr_cond *cnd)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = 0;
+#endif
+    return pthread_cond_destroy(&cnd->pt_cnd);
+}
+
+void
+ethr_cond_signal(ethr_cond *cnd)
+{
+    int res;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+
+    res = pthread_cond_signal(&cnd->pt_cnd);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+void
+ethr_cond_broadcast(ethr_cond *cnd)
+{
+    int res;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+
+    res = pthread_cond_broadcast(&cnd->pt_cnd);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+int
+ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
+{
+    int res;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    res = pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx);
+    if (res != 0 && res != EINTR)
+	ETHR_FATAL_ERROR__(res);
+    return res;
+}
+
+#endif /* pthread_mutex */
+
+/* -- Exported symbols of inline functions --------------------------------- */
+
+int
+ethr_mutex_trylock(ethr_mutex *mtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    return ethr_mutex_trylock__(mtx);
+}
+
+void
+ethr_mutex_lock(ethr_mutex *mtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    ethr_mutex_lock__(mtx);
+}
+
+void
+ethr_mutex_unlock(ethr_mutex *mtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    ethr_mutex_unlock__(mtx);
+}
+
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Read/Write Mutex                                                          *
+\*                                                                           */
+
+static void
+wake_readers(ethr_rwmutex *rwmtx, int rs)
+{
+    ethr_ts_event *tse;
+#ifdef ETHR_DEBUG
+    int drs = 0;
+#endif
+
+    tse = rwmtx->mtxb.q;
+    ETHR_ASSERT(tse);
+    ETHR_ASSERT(rwmtx->rq_end);
+    dequeue(&rwmtx->mtxb.q, tse, rwmtx->rq_end);
+    rwmtx->rq_end->next = NULL;
+    rwmtx->rq_end = NULL;
+
+    ETHR_ASSERT(!rwmtx->mtxb.q
+		|| (ethr_atomic_read(&rwmtx->mtxb.q->uaflgs)
+		    == ETHR_RWMTX_W_WAIT_FLG__));
+
+    ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
+    ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+
+    while (tse) {
+	ethr_ts_event *tse_next;
+
+#ifdef ETHR_DEBUG
+	ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
+	ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs)
+		    == ETHR_RWMTX_R_WAIT_FLG__);
+	drs++;
+#endif
+
+	tse_next = tse->next; /* we aren't allowed to read tse->next
+				 after we have reset uaflgs */
+
+	ethr_atomic_set(&tse->uaflgs, 0);
+	ethr_event_set(&tse->event);
+	tse = tse_next;
+    }
+
+    ETHR_ASSERT(rs == drs);
+}
+
+static ETHR_INLINE int
+is_w_waiter(ethr_ts_event *tse)
+{
+    ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__
+		|| tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
+    return tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__;
+}
+
+static ETHR_INLINE int
+multiple_w_waiters(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(rwmtx->mtxb.q);
+    ETHR_ASSERT(rwmtx->mtxb.q->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+
+    if (!rwmtx->rq_end)
+	return rwmtx->mtxb.q->next != rwmtx->mtxb.q;
+    else {
+	ETHR_ASSERT(rwmtx->mtxb.q->next != rwmtx->mtxb.q);
+	if (rwmtx->mtxb.q->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__)
+	    return 1;
+	ETHR_ASSERT(rwmtx->rq_end->next == rwmtx->mtxb.q
+		    || rwmtx->rq_end->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+	return rwmtx->rq_end->next != rwmtx->mtxb.q;
+    }
+}
+
+int check_readers_array(ethr_rwmutex *rwmtx,
+			int start_rix,
+			int length,
+			int pre_check)
+{
+    int ix = start_rix;
+
+#ifndef ETHR_READ_MEMORY_BARRIER_IS_FULL
+    if (pre_check)
+	ETHR_READ_MEMORY_BARRIER;
+    else
+#endif
+	ETHR_MEMORY_BARRIER;
+
+    do {
+	long act = rwmutex_freqread_rdrs_read(rwmtx, ix);
+	if (act != 0)
+	    return EBUSY;
+	ix++;
+	if (ix == length)
+	    ix = 0;
+    } while (ix != start_rix);
+
+    return 0;
+}
+
+static ETHR_INLINE void
+rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx,
+					 ethr_ts_event *tse)
+{
+    long act;
+    /*
+     * Restore failed increment
+     */
+    act = rwmutex_freqread_rdrs_dec_read(rwmtx, tse);
+
+    ETHR_WRITE_MEMORY_BARRIER;
+
+    if (act == 0) {
+
+#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL
+	ETHR_READ_MEMORY_BARRIER;
+#endif
+
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+
+	if ((act & ETHR_RWMTX_W_FLG__) == 0
+	    && act & (ETHR_RWMTX_WAIT_FLGS__|ETHR_RWMTX_R_PEND_UNLCK_MASK__)) {
+	    /*
+	     * We either got waiters, or someone else trying
+	     * to read unlock which we might have to help.
+	     */
+	    rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 1, 0);
+	}
+    }
+}
+
+static int
+rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
+			     long initial,
+			     ethr_ts_event *tse,
+			     int start_next_ix,
+			     int check_before_try,
+			     int try_write_lock)
+{
+    ethr_ts_event *tse_tmp;
+    long act = initial;
+    int six, res, length;
+
+    tse_tmp = tse;
+    if (!tse_tmp)
+	tse_tmp = ethr_get_ts_event();
+
+    if ((act & ETHR_RWMTX_WAIT_FLGS__)
+	&& (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0)
+	goto check_waiters;
+
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	length = reader_groups_array_size;
+	six = tse_tmp->rgix;
+    }
+    else {
+	length = main_threads_array_size;
+	six = tse_tmp->mtix;
+    }
+    if (start_next_ix) {
+	six++;
+	if (six >= length)
+	    six = 0;
+    }
+
+    if (!tse)
+	ethr_leave_ts_event(tse_tmp);
+
+    if (check_before_try) {
+	res = check_readers_array(rwmtx, six, length, 1);
+	if (res == EBUSY)
+	    return try_write_lock ? EBUSY : 0;
+    }
+
+    while (1) {
+	long exp = act;
+	long new = act+1;
+
+	ETHR_ASSERT((act & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
+		    < ETHR_RWMTX_R_PEND_UNLCK_MASK__);
+
+	act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	if (exp == act) {
+	    act = new;
+	    break;
+	}
+	if (!try_write_lock) {
+	    if (act == ETHR_RWMTX_W_FLG__ || act == 0)
+		return 0;
+	    if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+		if ((act & ETHR_RWMTX_R_FLG__) == 0)
+		    return 0;
+	    }
+	    else if ((act & ETHR_RWMTX_R_FLG__) == 0) {
+		if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
+		    return 0;
+		goto check_waiters;
+	    }
+	}
+	else {
+	    if (act == 0)
+		goto tryrwlock;
+	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__))
+		return EBUSY;
+	}
+    }
+
+    res = check_readers_array(rwmtx, six, length, 0);
+    if (res == EBUSY) {
+	act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs);
+	if (act & ETHR_RWMTX_R_MASK__)
+	    return try_write_lock ? EBUSY : 0;
+    }
+    else {
+	while (1) {
+	    long exp = act;
+	    long new = act;
+	    new &= ~ETHR_RWMTX_R_FLG__;
+	    new--;
+
+	    ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__);
+
+	    act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	    if (exp == act) {
+		if (new & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
+		    return try_write_lock ? EBUSY : 0;
+		act = new;
+		break;
+	    }
+	}
+    }
+
+    /*
+     * Read unlock completed, but we have to check if
+     * threads have to be woken (or if we should try
+     * to write lock it).
+     */
+
+    if (act & ETHR_RWMTX_W_FLG__)
+	return try_write_lock ? EBUSY : 0;
+
+    if (act & ETHR_RWMTX_WAIT_FLGS__) {
+    check_waiters:
+	rwmutex_unlock_wake(rwmtx, 0, act);
+	return try_write_lock ? EBUSY : 0;
+    }
+
+    if (!try_write_lock)
+	return 0;
+
+ tryrwlock:
+    /* Try to write lock it */
+
+    act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+				   ETHR_RWMTX_W_FLG__,
+				   0);
+    return act == 0 ? 0 : EBUSY;
+}
+
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+
+static ETHR_INLINE void
+rwmutex_incdec_restore_failed_tryrlock(ethr_rwmutex *rwmtx)
+{
+    long act;
+    /*
+     * Restore failed increment
+     */
+    act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs);
+    if ((act & ETHR_RWMTX_WAIT_FLGS__)
+	&& (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	rwmutex_unlock_wake(rwmtx, 0, act);
+    }
+}
+
+#endif
+
+static void
+rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
+			  long initial)
+{
+    long act = initial, exp;
+    int scnt, start_scnt;
+    ethr_ts_event *tse = NULL;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    start_scnt = scnt = initial_spincount(&rwmtx->mtxb);
+
+    /*
+     * Spin trying to read lock for a while. If unsuccessful,
+     * wait on event.
+     */
+
+    while (1) {
+
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	rwmutex_incdec_restore_failed_tryrlock(rwmtx);
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+#endif
+
+	while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+	    if (scnt == 0) {
+		tse = ethr_get_ts_event();
+		if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
+		    event_wait(&rwmtx->mtxb, tse, scnt,
+			       ETHR_RWMTX_R_WAIT_FLG__, 1, 0);
+		    goto done;
+		}
+	    }
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		ETHR_YIELD();
+	    }
+	    act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	    scnt--;
+	}
+	exp = act;
+	ETHR_ASSERT(scnt >= 0);
+
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	act = ethr_atomic_inc_read(&rwmtx->mtxb.flgs);
+	if ((act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) == 0)
+	    goto done; /* Got it */
+#else
+	act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+	if (act == exp)
+	    goto done; /* Got it */
+#endif
+    }
+
+ done:
+    if (tse)
+	ethr_leave_ts_event(tse);
+}
+
+static void
+rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
+			    ethr_ts_event *tse,
+			    long initial)
+{
+    long act = initial;
+    int scnt, start_scnt;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    start_scnt = scnt = initial_spincount(&rwmtx->mtxb);
+
+    /*
+     * Spin trying to read lock for a while. If unsuccessful,
+     * wait on event.
+     */
+
+    while (1) {
+
+	rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
+
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+
+	while (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
+	    if (scnt == 0) {
+		if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
+		    event_wait(&rwmtx->mtxb, tse, scnt,
+			       ETHR_RWMTX_R_WAIT_FLG__, 1, 1);
+		    return; /* Got it */
+		}
+	    }
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		ETHR_YIELD();
+	    }
+	    act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	    scnt--;
+	}
+
+	ETHR_ASSERT(scnt >= 0);
+
+	rwmutex_freqread_rdrs_inc(rwmtx, tse);
+
+	ETHR_MEMORY_BARRIER;
+
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+
+	if (act == ETHR_RWMTX_R_FLG__)
+	    return; /* Got it */
+
+	while (1) {
+	    long exp, new;
+
+	    if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__))
+		break; /* Busy (need to restore inc) */
+
+	    if (act & ETHR_RWMTX_R_FLG__)
+		return; /* Got it */
+
+	    exp = act;
+	    new = act | ETHR_RWMTX_R_FLG__;
+	    act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	    if (act == exp)
+		return; /* Got it */
+	}
+    }
+}
+
+static void
+rwmutex_normal_rwlock_wait(ethr_rwmutex *rwmtx, long initial)
+{
+    write_lock_wait(&rwmtx->mtxb, initial, 1, 0);
+}
+
+static void
+rwmutex_freqread_rwlock_wait(ethr_rwmutex *rwmtx, long initial)
+{
+    write_lock_wait(&rwmtx->mtxb, initial, 1, 1);
+}
+
+static ETHR_INLINE void
+rwlock_wake_set_flags(ethr_rwmutex *rwmtx, long new_initial, int act_initial)
+{
+    long act, act_mask;
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
+	/* r pend unlock mask may vary and must be retained */
+	act_mask = ETHR_RWMTX_R_PEND_UNLCK_MASK__;
+    }
+    else {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	/* rs mask may vary and must be retained */
+	act_mask = ETHR_RWMTX_RS_MASK__;
+#else
+	/* rs mask always zero */
+	ETHR_ASSERT((act_initial & ETHR_RWMTX_RS_MASK__) == 0);
+	ethr_atomic_set(&rwmtx->mtxb.flgs, new_initial);
+	return;
+#endif
+    }
+
+    act = act_initial;
+    while (1) {
+	long exp = act;
+	long new = new_initial + (act & act_mask);
+	act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	if (act == exp)
+	    break;
+	exp = act;
+    }
+}
+
+#ifdef ETHR_DEBUG
+
+static void
+dbg_unlock_wake(ethr_rwmutex *rwmtx,
+		int have_w,
+		ethr_ts_event *tse)
+{
+    long exp, act, imask;
+
+    exp = have_w ? ETHR_RWMTX_W_FLG__ : 0;
+
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
+	imask = ETHR_RWMTX_R_PEND_UNLCK_MASK__;
+    else {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	imask = ETHR_RWMTX_RS_MASK__;
+#else
+	imask = 0;
+#endif
+    }
+
+    ETHR_ASSERT(tse);
+
+    if (is_w_waiter(tse)) {
+
+	exp |= ETHR_RWMTX_W_WAIT_FLG__;
+	if (rwmtx->rq_end) {
+	    exp |= ETHR_RWMTX_R_WAIT_FLG__;
+	}
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	ETHR_ASSERT((exp & ~imask) == (act & ~imask));
+
+	ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
+
+    }
+    else {
+
+	exp |= ETHR_RWMTX_R_WAIT_FLG__;
+	if (rwmtx->rq_end->next != rwmtx->mtxb.q)
+	    exp |= ETHR_RWMTX_W_WAIT_FLG__;
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	ETHR_ASSERT((exp & ~imask) == (act & ~imask));
+
+	ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
+
+    }
+}
+
+#endif
+
+static void
+rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
+{
+    long new, act = initial;
+    ethr_ts_event *tse;
+
+    if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	if (!have_w)
+	    return;
+	else {
+	    while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+		long exp = act;
+		new = exp & ~ETHR_RWMTX_W_FLG__;
+		act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+		if (act == exp)
+		    return;
+	    }
+	}
+    }
+
+    ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+    tse = rwmtx->mtxb.q;
+
+    if (!have_w) {
+	if (!tse) {
+#ifdef ETHR_DEBUG
+	    act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	    ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
+#endif
+	    goto already_served;
+	}
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	if (act & ~ETHR_RWMTX_WAIT_FLGS__) {
+	already_served:
+	    ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+	    return;
+	}
+    }
+
+#ifdef ETHR_DEBUG
+    dbg_unlock_wake(rwmtx, have_w, tse);
+#endif
+
+    if (is_w_waiter(tse)) {
+
+	if (!have_w) {
+	    act = ethr_atomic_read_bor(&rwmtx->mtxb.flgs,
+				       ETHR_RWMTX_W_FLG__);
+	    ETHR_ASSERT((act & ~ETHR_RWMTX_WAIT_FLGS__) == 0);
+	    ETHR_ASSERT(act & ETHR_RWMTX_W_WAIT_FLG__);
+	    act |= ETHR_RWMTX_W_FLG__;
+	}
+
+	/*
+	 * If we have multiple write waiters, there
+	 * is no need to modify mtxb->flgs; otherwise,
+	 * we need to clear the write wait bit...
+	 */
+	if (!multiple_w_waiters(rwmtx)) {
+	    new = ETHR_RWMTX_W_FLG__;
+	    if (tse->next != rwmtx->mtxb.q) {
+		ETHR_ASSERT(tse->next->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
+		new |= ETHR_RWMTX_R_WAIT_FLG__;
+	    }
+
+	    rwlock_wake_set_flags(rwmtx, new, act);
+	}
+
+	wake_writer(&rwmtx->mtxb, 1);
+    }
+    else {
+	int rs;
+
+	if (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL) {
+	    rs = rwmtx->tdata.rs;
+	    new = (long) rs;
+	    rwmtx->tdata.rs = 0;
+	}
+	else {
+	    ethr_rwmutex_type type = rwmtx->type;
+	    int length = (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ
+			  ? reader_groups_array_size
+			  : main_threads_array_size);
+	    int ix;
+
+	    rs = 0;
+	    for (ix = 0; ix < length; ix++) {
+		int wrs = rwmtx->tdata.ra[ix].data.waiting_readers;
+		rwmtx->tdata.ra[ix].data.waiting_readers = 0;
+		ETHR_ASSERT(wrs >= 0);
+		if (wrs) {
+		    rs += wrs;
+		    rwmutex_freqread_rdrs_add(rwmtx, type, ix, wrs);
+		}
+	    }
+	    new = ETHR_RWMTX_R_FLG__;
+	}
+
+	if (rwmtx->rq_end->next != rwmtx->mtxb.q)
+	    new |= ETHR_RWMTX_W_WAIT_FLG__;
+
+	rwlock_wake_set_flags(rwmtx, new, act);
+	wake_readers(rwmtx, rs);
+    }
+}
+
+static ethr_rwmtx_readers_array__ *
+alloc_readers_array(int length, ethr_rwmutex_lived lived)
+{
+    ethr_rwmtx_readers_array__ *ra;
+    size_t sz;
+    void *mem;
+
+    sz = sizeof(ethr_rwmtx_readers_array__) * (length + 1);
+
+    switch (lived) {
+    case ETHR_RWMUTEX_LONG_LIVED:
+	mem = ethr_mem__.ll.alloc(sz);
+	break;
+    case ETHR_RWMUTEX_SHORT_LIVED:
+	mem = ethr_mem__.sl.alloc(sz);
+	break;
+    default:
+	mem = ethr_mem__.std.alloc(sz);
+	break;
+    }
+    if (!mem)
+	return NULL;
+
+    if ((((unsigned long) mem) & ETHR_CACHE_LINE_MASK) == 0) {
+	ra = (ethr_rwmtx_readers_array__ *) mem;
+	ra->data.byte_offset = 0;
+    }
+    else {
+	ra = ((ethr_rwmtx_readers_array__ *)
+	      ((((unsigned long) mem) & ~ETHR_CACHE_LINE_MASK)
+	       + ETHR_CACHE_LINE_SIZE));
+	ra->data.byte_offset = (int) ((unsigned long) ra
+				      - (unsigned long) mem);
+    }
+    ra->data.lived = lived;
+    return ra;
+}
+
+static void
+free_readers_array(ethr_rwmtx_readers_array__ *ra)
+{
+    void *ptr = (void *) (((char *) ra) - ra->data.byte_offset);
+    switch (ra->data.lived) {
+    case ETHR_RWMUTEX_LONG_LIVED:
+	ethr_mem__.ll.free(ptr);
+	break;
+    case ETHR_RWMUTEX_SHORT_LIVED:
+	ethr_mem__.sl.free(ptr);
+	break;
+    default:
+	ethr_mem__.std.free(ptr);
+	break;
+    }
+}
+
+int
+ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt)
+{
+    int res;
+    ethr_rwmtx_readers_array__ *ra = NULL;
+#if ETHR_XCHK
+    if (ethr_not_completely_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!rwmtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
+#endif
+    ETHR_MTX_HARD_DEBUG_FENCE_INIT(rwmtx);
+    rwmtx->rq_end = NULL;
+    rwmtx->type = opt ? opt->type : ETHR_RWMUTEX_TYPE_NORMAL;
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+	if (main_threads_array_size <= reader_groups_array_size) {
+	    /* No point using reader groups... */
+	    rwmtx->type = ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ;
+	}
+	/* Fall through */
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	int length;
+
+	length = (rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
+		  ? main_threads_array_size
+		  : reader_groups_array_size);
+
+	if (length == 1) {
+	    /* No point using a frequent reader type... */
+	    rwmtx->type = ETHR_RWMUTEX_TYPE_NORMAL;
+	}
+	else {
+	    int ix;
+	    ra = alloc_readers_array(length,
+				     (opt
+				      ? opt->lived
+				      : ETHR_RWMUTEX_UNKNOWN_LIVED));
+	    if (!ra) {
+		res = ENOMEM;
+		goto error;
+	    }
+
+	    rwmtx->tdata.ra = ra;
+
+	    for (ix = 0; ix < length; ix++) {
+		ethr_atomic_init(&rwmtx->tdata.ra[ix].data.readers, 0);
+		rwmtx->tdata.ra[ix].data.waiting_readers = 0;
+	    }
+	    break;
+	}
+    }
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	rwmtx->tdata.rs = 0;
+	break;
+    default:
+	res = EINVAL;
+	goto error;
+    }
+    res = mtxb_init(&rwmtx->mtxb,
+		    default_rwmtx_main_spincount,
+		    opt ? opt->main_spincount : -1,
+		    default_rwmtx_aux_spincount,
+		    opt ? opt->aux_spincount : -1);
+    if (res == 0)
+	return 0;
+
+ error:
+
+    if (ra)
+	free_readers_array(ra);
+
+#if ETHR_XCHK
+    rwmtx->initialized = 0;
+#endif
+    return res;
+}
+
+int
+ethr_rwmutex_init(ethr_rwmutex *rwmtx)
+{
+    return ethr_rwmutex_init_opt(rwmtx, NULL);
+}
+
+int
+ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
+{
+    int res;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
+	long act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	if (act == ETHR_RWMTX_R_FLG__)
+	    rwmutex_try_complete_runlock(rwmtx, act, NULL, 0, 0, 0);
+    }
+    res = mtxb_destroy(&rwmtx->mtxb);
+    if (res != 0)
+	return res;
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
+	free_readers_array(rwmtx->tdata.ra);
+#if ETHR_XCHK
+    rwmtx->initialized = 0;
+#endif
+    return 0;
+}
+
+#define ETHR_MAX_TRYRLOCK_TRIES 5
+
+int
+ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
+{
+    int res = 0;
+    long act;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL: {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))
+	    res = EBUSY;
+	else {
+	    act = ethr_atomic_inc_read_acqb(&rwmtx->mtxb.flgs);
+	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+		rwmutex_incdec_restore_failed_tryrlock(rwmtx);
+		res = EBUSY;
+	    }
+	}
+#else
+	long exp = 0;
+	int tries = 0;
+
+	while (1) {
+	    act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+	    if (act == exp) {
+		res = 0;
+		break;
+	    }
+	    if (tries > ETHR_MAX_TRYRLOCK_TRIES
+		|| (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))) {
+		res = EBUSY;
+		break;
+	    }
+	    tries++;
+	    exp = act;
+	}
+#endif
+	break;
+    }
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	ethr_ts_event *tse = ethr_get_ts_event();
+
+	rwmutex_freqread_rdrs_inc(rwmtx, tse);
+
+	ETHR_MEMORY_BARRIER;
+
+	act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs);
+
+	if (act != ETHR_RWMTX_R_FLG__) {
+	    while (1) {
+		long exp, new;
+
+		if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
+		    rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
+		    res = EBUSY;
+		    break;
+		}
+
+		if (act & ETHR_RWMTX_R_FLG__)
+		    break;
+
+		exp = act;
+		new = act | ETHR_RWMTX_R_FLG__;
+		act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
+		if (act == exp)
+		    break;
+	    }
+	}
+
+	ethr_leave_ts_event(tse);
+	break;
+    }
+    }
+
+    ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(&rwmtx->mtxb, res);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    return res;
+}
+
+void
+ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
+{
+    long act;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL: {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	act = ethr_atomic_inc_read_acqb(&rwmtx->mtxb.flgs);
+	if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))
+	    rwmutex_normal_rlock_wait(rwmtx, act);
+#else
+	long exp = 0;
+
+	while (1) {
+	    act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+	    if (act == exp) {
+		break;
+	    }
+
+	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+		rwmutex_normal_rlock_wait(rwmtx, act);
+		break;
+	    }
+	    exp = act;
+	}
+#endif
+	break;
+    }
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	ethr_ts_event *tse = ethr_get_ts_event();
+
+	rwmutex_freqread_rdrs_inc(rwmtx, tse);
+
+	ETHR_MEMORY_BARRIER;
+
+	act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs);
+
+	if (act != ETHR_RWMTX_R_FLG__) {
+	    while (1) {
+		long exp, new;
+
+		if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
+		    rwmutex_freqread_rlock_wait(rwmtx, tse, act);
+		    break;
+		}
+
+		if (act & ETHR_RWMTX_R_FLG__)
+		    break;
+
+		exp = act;
+		new = act | ETHR_RWMTX_R_FLG__;
+		act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
+		if (act == exp)
+		    break;
+	    }
+	}
+
+	ethr_leave_ts_event(tse);
+	break;
+    }
+    }
+
+    ETHR_MTX_HARD_DEBUG_LFS_RLOCK(&rwmtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+}
+
+void
+ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
+{
+    long act;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+    ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(&rwmtx->mtxb);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic_dec_read_relb(&rwmtx->mtxb.flgs);
+	if ((act & ETHR_RWMTX_WAIT_FLGS__)
+	    && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	    ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
+	    rwmutex_unlock_wake(rwmtx, 0, act);
+	}
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	ethr_ts_event *tse = ethr_get_ts_event();
+
+	act = rwmutex_freqread_rdrs_dec_read_relb(rwmtx, tse);
+
+	ETHR_ASSERT(act >= 0);
+
+	ETHR_WRITE_MEMORY_BARRIER;
+
+	if (act == 0) {
+
+#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL
+	    ETHR_READ_MEMORY_BARRIER;
+#endif
+	    act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+
+	    if ((act & ETHR_RWMTX_W_FLG__) == 0
+		&& (act & (ETHR_RWMTX_WAIT_FLGS__
+			   | ETHR_RWMTX_R_PEND_UNLCK_MASK__))) {
+		rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+	    }
+
+	}
+
+	ethr_leave_ts_event(tse);
+	break;
+    }
+    }
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+}
+
+int
+ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
+{
+    int res = 0;
+    long act;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+				       ETHR_RWMTX_W_FLG__, 0);
+	if (act != 0)
+	    res = EBUSY;
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+
+	res = 0;
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+
+	do {
+
+	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__)) {
+		res = EBUSY;
+		break;
+	    }
+
+	    if (act & ETHR_RWMTX_R_MASK__) {
+		res = rwmutex_try_complete_runlock(rwmtx, act, NULL,
+						   0, 1, 1);
+		break;
+	    }
+
+	    act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+					   ETHR_RWMTX_W_FLG__, 0);
+
+	} while (act != 0);
+
+	break;
+    }
+
+    ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&rwmtx->mtxb, res);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    return res;
+}
+
+void
+ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
+{
+    long act;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+				       ETHR_RWMTX_W_FLG__, 0);
+	if (act != 0)
+	    rwmutex_normal_rwlock_wait(rwmtx, act);
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+
+	do {
+
+	    if (act != 0) {
+		rwmutex_freqread_rwlock_wait(rwmtx, act);
+		break;
+	    }
+
+	    act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+					   ETHR_RWMTX_W_FLG__, 0);
+
+	} while (act != 0);
+
+	break;
+    }
+
+    ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&rwmtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+}
+
+void
+ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
+{
+    long act;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+    ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&rwmtx->mtxb);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs,
+				       0, ETHR_RWMTX_W_FLG__);
+	if (act != ETHR_RWMTX_W_FLG__)
+	    rwmutex_unlock_wake(rwmtx, 1, act);
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+	act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs, 0,
+				       ETHR_RWMTX_W_FLG__);
+	if (act != ETHR_RWMTX_W_FLG__)
+	    rwmutex_unlock_wake(rwmtx, 1, act);
+	break;
+    }
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+}
+
+#else
+/* -- pthread read/write mutex --------------------------------------------- */
+
+int
+ethr_rwmutex_init(ethr_rwmutex *rwmtx)
+{
+#if ETHR_XCHK
+    if (!rwmtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
+#endif
+    return pthread_rwlock_init(&rwmtx->pt_rwlock, write_pref_attr);
+}
+
+int
+ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt)
+{
+    return ethr_rwmutex_init(rwmtx);
+}
+
+int
+ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
+{
+    int res;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    res = pthread_rwlock_destroy(&rwmtx->pt_rwlock);
+#if ETHR_XCHK
+    rwmtx->initialized = 0;
+#endif
+    return res;
+}
+
+/* -- Exported symbols of inline functions --------------------------------- */
+
+int
+ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    return ethr_rwmutex_tryrlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ethr_rwmutex_rlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ethr_rwmutex_runlock__(rwmtx);
+}
+
+int
+ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    return ethr_rwmutex_tryrwlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    return ethr_rwmutex_rwlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ethr_rwmutex_rwunlock__(rwmtx);
+}
+
+#endif /* pthread */
+
+
+#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)
+
+#ifdef ETHR_MTX_HARD_DEBUG_Q
+static void
+hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
+{
+    int res;
+    long flgs = ethr_atomic_read(&mtxb->flgs);
+
+    ETHR_MTX_HARD_ASSERT(res == 0);
+
+    ETHR_MTX_HARD_ASSERT(!(flgs & ETHR_RWMTX_R_WAIT_FLG__) || is_rwmtx);
+
+    if (!(flgs & ETHR_RWMTX_WAIT_FLGS__)) {
+	ETHR_MTX_HARD_ASSERT(!mtxb->q);
+	if (is_rwmtx) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end);
+	    ETHR_MTX_HARD_ASSERT(!rwmtx->rs);
+	}
+    }
+    else {
+	ethr_ts_event *tse;
+	int ws = 0, rs = 0, rsf = 0, ref = 0;
+
+	ETHR_MTX_HARD_ASSERT(mtxb->q);
+
+	tse = mtxb->q;
+
+	do {
+	    long type;
+
+	    ETHR_MTX_HARD_ASSERT(tse->next->prev == tse);
+	    ETHR_MTX_HARD_ASSERT(tse->prev->next == tse);
+
+	    type = ethr_atomic_read(&tse->uaflgs);
+	    ETHR_MTX_HARD_ASSERT(type == tse->uflgs);
+	    switch (type) {
+	    case ETHR_RWMTX_W_WAIT_FLG__:
+		ws++;
+		break;
+	    case ETHR_RWMTX_R_WAIT_FLG__: {
+		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+		ETHR_MTX_HARD_ASSERT(is_rwmtx);
+		if (!rsf)
+		    rsf = 1;
+		ETHR_MTX_HARD_ASSERT(!ref);
+		if (rwmtx->rq_end == tse) {
+		    ETHR_MTX_HARD_ASSERT(
+			tse->next == rwmtx->mtxb.q
+			|| tse->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+		    ref = 1;
+		}
+		rs++;
+		break;
+	    }
+	    default:
+		ETHR_MTX_HARD_ASSERT(! "invalid wait type found");
+	    }
+
+	    tse = tse->next;
+	} while (tse != mtxb->q);
+
+	if (is_rwmtx) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_MTX_HARD_ASSERT(rs == rwmtx->rs);
+	}
+
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+	ETHR_MTX_HARD_ASSERT(ws == mtxb->ws);
+#endif
+
+	if (flgs & ETHR_RWMTX_W_WAIT_FLG__)
+	    ETHR_MTX_HARD_ASSERT(ws);
+	else
+	    ETHR_MTX_HARD_ASSERT(!ws);
+
+	if (flgs & ETHR_RWMTX_R_WAIT_FLG__) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_MTX_HARD_ASSERT(is_rwmtx);
+	    ETHR_MTX_HARD_ASSERT(rwmtx->rq_end);
+	    ETHR_MTX_HARD_ASSERT(rsf);
+	    ETHR_MTX_HARD_ASSERT(ref);
+	    ETHR_MTX_HARD_ASSERT(rs);
+	}
+	else {
+	    if (is_rwmtx) {
+		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+		ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end);
+	    }
+	    ETHR_MTX_HARD_ASSERT(!rsf);
+	    ETHR_MTX_HARD_ASSERT(!ref);
+	    ETHR_MTX_HARD_ASSERT(!rs);
+	}
+    }
+}
+
+#elif defined(ETHR_MTX_HARD_DEBUG_WSQ)
+
+static void
+hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
+{
+    int ws = 0;
+    int rs = 0;
+
+    if (mtxb->q) {
+	ethr_ts_event *tse = mtxb->q;
+	do {
+	    switch (tse->uflgs) {
+	    case ETHR_RWMTX_W_WAIT_FLG__:
+		ws++;
+		break;
+	    case ETHR_RWMTX_R_WAIT_FLG__:
+		rs++;
+		break;
+	    default:
+		ETHR_MTX_HARD_ASSERT(0);
+		break;
+	    }
+	    tse = tse->next;
+	} while (tse != mtxb->q);
+    }
+
+    ETHR_MTX_HARD_ASSERT(mtxb->ws == ws);
+    if (is_rwmtx) {
+	ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	ETHR_MTX_HARD_ASSERT(rwmtx->rs == rs);
+    }
+}
+
+#endif
+
+#endif
diff --git a/erts/lib_src/common/ethread.c b/erts/lib_src/common/ethread.c
deleted file mode 100644
index 9c88233934..0000000000
--- a/erts/lib_src/common/ethread.c
+++ /dev/null
@@ -1,3369 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 2004-2010. All Rights Reserved.
- *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- *
- * %CopyrightEnd%
- */
-
-/*
- * Description: A Thread library for use in the ERTS and other OTP
- *              applications.
- * Author: Rickard Green
- */
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#undef ETHR_STACK_GUARD_SIZE 
-
-#if defined(ETHR_PTHREADS)
-
-#ifdef ETHR_TIME_WITH_SYS_TIME
-#  include <time.h>
-#  include <sys/time.h>
-#else
-#  ifdef ETHR_HAVE_SYS_TIME_H
-#    include <sys/time.h>
-#  else
-#    include <time.h>
-#  endif
-#endif
-#include <sys/types.h>
-#include <unistd.h>
-#include <signal.h>
-
-#ifdef ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE
-#  define ETHR_STACK_GUARD_SIZE (pagesize)
-#endif
-
-#elif defined(ETHR_WIN32_THREADS)
-
-#undef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-#include <process.h>
-#include <winerror.h>
-
-#else
-#error "Missing thread implementation"
-#endif
-
-#include <limits.h>
-
-#define ETHR_FORCE_INLINE_FUNCS
-#define ETHR_INLINE_FUNC_NAME_(X) X ## __
-#include "ethread.h"
-
-#ifndef ETHR_HAVE_ETHREAD_DEFINES
-#error Missing configure defines
-#endif
-
-/*
- * ----------------------------------------------------------------------------
- * Common stuff
- * ----------------------------------------------------------------------------
- */
-
-#define ETHR_MAX_THREADS 2048 /* Has to be an even power of 2 */
-
-static int ethr_not_inited = 1;
-
-#define ASSERT(A) ETHR_ASSERT((A))
-
-static void *(*allocp)(size_t) = malloc;
-static void *(*reallocp)(void *, size_t) = realloc;
-static void (*freep)(void *) = free;
-
-#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
-#endif
-
-void *(*thread_create_prepare_func)(void) = NULL;
-void (*thread_create_parent_func)(void *) = NULL;
-void (*thread_create_child_func)(void *) = NULL;
-
-typedef struct ethr_xhndl_list_ ethr_xhndl_list;
-struct ethr_xhndl_list_ {
-    ethr_xhndl_list *next;
-    void (*funcp)(void);
-};
-
-static size_t pagesize;
-#define ETHR_PAGE_ALIGN(SZ) (((((size_t) (SZ)) - 1)/pagesize + 1)*pagesize)
-static size_t min_stack_size; /* kilo words */
-static size_t max_stack_size; /* kilo words */
-#define ETHR_B2KW(B) ((((size_t) (B)) - 1)/(sizeof(void *)*1024) + 1)
-#define ETHR_KW2B(KW) (((size_t) (KW))*sizeof(void *)*1024)
-
-ethr_mutex xhndl_mtx;
-ethr_xhndl_list *xhndl_list;
-
-static int
-init_common(ethr_init_data *id)
-{
-    int res;
-    if (id) {
-	allocp				= id->alloc;
-	reallocp			= id->realloc;
-	freep				= id->free;
-	thread_create_prepare_func	= id->thread_create_prepare_func;
-	thread_create_parent_func	= id->thread_create_parent_func;
-	thread_create_child_func	= id->thread_create_child_func;
-    }
-    if (!allocp || !reallocp || !freep)
-	return EINVAL;
-
-#ifdef _SC_PAGESIZE
-    pagesize = (size_t) sysconf(_SC_PAGESIZE);
-#elif defined(HAVE_GETPAGESIZE)
-    pagesize = (size_t) getpagesize();
-#else
-    pagesize = (size_t) 4*1024; /* Guess 4 KB */
-#endif
-
-    /* User needs at least 4 KB */
-    min_stack_size = 4*1024;
-#if SIZEOF_VOID_P == 8
-    /* Double that on 64-bit archs */
-    min_stack_size *= 2;
-#endif
-    /* On some systems as much as about 4 KB is used by the system */
-    min_stack_size += 4*1024;
-    /* There should be room for signal handlers */
-#ifdef SIGSTKSZ
-    min_stack_size += SIGSTKSZ;
-#else
-    min_stack_size += pagesize;
-#endif
-    /* The system may think that we need more stack */
-#if defined(PTHREAD_STACK_MIN)
-    if (min_stack_size < PTHREAD_STACK_MIN)
-	min_stack_size = PTHREAD_STACK_MIN;
-#elif defined(_SC_THREAD_STACK_MIN)
-    {
-	size_t thr_min_stk_sz = (size_t) sysconf(_SC_THREAD_STACK_MIN);
-	if (min_stack_size < thr_min_stk_sz)
-	    min_stack_size = thr_min_stk_sz;
-    }
-#endif
-    /* The guard is at least on some platforms included in the stack size
-       passed when creating threads */
-#ifdef ETHR_STACK_GUARD_SIZE
-    min_stack_size += ETHR_STACK_GUARD_SIZE;
-#endif
-    min_stack_size = ETHR_PAGE_ALIGN(min_stack_size);
-
-    min_stack_size = ETHR_B2KW(min_stack_size);
-
-    max_stack_size = 32*1024*1024;
-#if SIZEOF_VOID_P == 8
-    max_stack_size *= 2;
-#endif
-    max_stack_size = ETHR_B2KW(max_stack_size);
-
-    xhndl_list = NULL;
-
-    res = ethr_mutex_init(&xhndl_mtx);
-    if (res != 0)
-	return res;
-
-    res = ethr_mutex_set_forksafe(&xhndl_mtx);
-    if (res != 0 && res != ENOTSUP)
-	return res;
-
-    return 0;
-}
-
-int
-ethr_install_exit_handler(void (*funcp)(void))
-{
-    ethr_xhndl_list *xhp;
-    int res;
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-
-    if (!funcp)
-	return EINVAL;
-
-    xhp = (ethr_xhndl_list *) (*allocp)(sizeof(ethr_xhndl_list));
-    if (!xhp)
-	return ENOMEM;
-
-    res = ethr_mutex_lock__(&xhndl_mtx);
-    if (res != 0) {
-	(*freep)((void *) xhp);
-	return res;
-    }
-
-    xhp->funcp = funcp;
-    xhp->next = xhndl_list;
-    xhndl_list = xhp;
-
-    res = ethr_mutex_unlock__(&xhndl_mtx);
-    if (res != 0)
-	abort();
-
-    return res;
-}
-
-static void
-run_exit_handlers(void)
-{
-    int res;
-    ethr_xhndl_list *xhp;
-
-    res = ethr_mutex_lock__(&xhndl_mtx);
-    if (res != 0)
-	abort();
-
-    xhp = xhndl_list;
-
-    res = ethr_mutex_unlock__(&xhndl_mtx);
-    if (res != 0)
-	abort();
-
-    for (; xhp; xhp = xhp->next)
-	(*xhp->funcp)();
-}
-
-#if defined(ETHR_PTHREADS)
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * pthread implementation                                                    *
-\*                                                                           */
-
-typedef struct {
-    pthread_mutex_t mtx;
-    pthread_cond_t cnd;
-    int initialized;
-    void *(*thr_func)(void *);
-    void *arg;
-    void *prep_func_res;
-} thr_wrap_data_;
-
-static int no_ethreads;
-static ethr_mutex no_ethrs_mtx;
-
-#ifndef ETHR_HAVE_PTHREAD_ATFORK
-#define ETHR_HAVE_PTHREAD_ATFORK 0
-#endif
-
-#if !ETHR_HAVE_PTHREAD_ATFORK
-#warning "Cannot enforce fork-safety"
-#endif
-
-#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT
-static pthread_rwlockattr_t write_pref_attr_data;
-static pthread_rwlockattr_t *write_pref_attr;
-#endif
-
-/*
- * ----------------------------------------------------------------------------
- * Static functions
- * ----------------------------------------------------------------------------
- */
-
-/*
- * Functions with safe_ prefix aborts on failure. To be used when
- * we cannot recover after failure.
- */
-
-static ETHR_INLINE void
-safe_mutex_lock(pthread_mutex_t *mtxp)
-{
-    int res = pthread_mutex_lock(mtxp);
-    if (res != 0)
-	abort();
-}
-
-static ETHR_INLINE void
-safe_mutex_unlock(pthread_mutex_t *mtxp)
-{
-    int res = pthread_mutex_unlock(mtxp);
-    if (res != 0)
-	abort();
-}
-
-static ETHR_INLINE void
-safe_cond_signal(pthread_cond_t *cndp)
-{
-    int res = pthread_cond_signal(cndp);
-    if (res != 0)
-	abort();
-}
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-
-static volatile int rec_mtx_attr_need_init = 1;
-static pthread_mutexattr_t rec_mtx_attr;
-
-static int init_rec_mtx_attr(void);
-
-#endif
-
-#if ETHR_HAVE_PTHREAD_ATFORK
-
-static ethr_mutex forksafe_mtx = ETHR_MUTEX_INITER;
-
-static void lock_mutexes(void)
-{
-    ethr_mutex *m = &forksafe_mtx;
-    do {
-
-	safe_mutex_lock(&m->pt_mtx);
-
-	m = m->next;
-
-    } while (m != &forksafe_mtx);
-}
-
-static void unlock_mutexes(void)
-{
-    ethr_mutex *m = forksafe_mtx.prev;
-    do {
-
-	safe_mutex_unlock(&m->pt_mtx);
-
-	m = m->prev;
-
-    } while (m->next != &forksafe_mtx);
-}
-
-#if ETHR_INIT_MUTEX_IN_CHILD_AT_FORK
-
-static void reinit_mutexes(void)
-{
-    ethr_mutex *m = forksafe_mtx.prev;
-    do {
-	pthread_mutexattr_t *attrp = NULL;
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-	if (m->is_rec_mtx) {
-	    if (rec_mtx_attr_need_init) {
-		int res = init_rec_mtx_attr();
-		if (res != 0)
-		    abort();
-	    }
-	    attrp = &rec_mtx_attr;
-	}
-#endif
-	if (pthread_mutex_init(&m->pt_mtx, attrp) != 0)
-	    abort();
-
-	m = m->prev;
-
-    } while (m->next != &forksafe_mtx);
-}
-
-#endif
-
-static int
-init_forksafe(void)
-{
-    static int init_done = 0;
-    int res = 0;
-
-    if (init_done)
-	return res;
-
-    forksafe_mtx.prev = &forksafe_mtx;
-    forksafe_mtx.next = &forksafe_mtx;
-
-    res = pthread_atfork(lock_mutexes,
-			 unlock_mutexes,
-#if ETHR_INIT_MUTEX_IN_CHILD_AT_FORK
-			 reinit_mutexes
-#else
-			 unlock_mutexes
-#endif
-	);
-
-    init_done = 1;
-    return res;
-}
-
-#endif
-
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-
-#if defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE)
-
-#define SET_REC_MUTEX_ATTR(AP) \
-  pthread_mutexattr_settype((AP), PTHREAD_MUTEX_RECURSIVE);
-
-#elif defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP)
-
-#define SET_REC_MUTEX_ATTR(AP) \
-  pthread_mutexattr_setkind_np((AP), PTHREAD_MUTEX_RECURSIVE_NP);
-
-#else
-
-#error "Don't know how to set recursive mutex attributes"
-
-#endif
-
-static int
-init_rec_mtx_attr(void)
-{
-    int res, mres;
-    static pthread_mutex_t attrinit_mtx = PTHREAD_MUTEX_INITIALIZER;
-
-    mres = pthread_mutex_lock(&attrinit_mtx);
-    if (mres != 0)
-	return mres;
-    /* Got here under race conditions; check again ... */
-    if (!rec_mtx_attr_need_init)
-	res = 0;
-    else {
-	res = pthread_mutexattr_init(&rec_mtx_attr);
-	if (res == 0) {
-	    res = SET_REC_MUTEX_ATTR(&rec_mtx_attr);
-	    if (res == 0)
-		rec_mtx_attr_need_init = 0;
-	    else
-		(void) pthread_mutexattr_destroy(&rec_mtx_attr);
-	}
-    }
-
-    mres = pthread_mutex_unlock(&attrinit_mtx);
-    if (mres != 0)
-	return mres;
-    return res;
-}
-
-#endif /* #if ETHR_HAVE_ETHR_REC_MUTEX_INIT */
-
-static ETHR_INLINE void thr_exit_cleanup(void)
-{
-    run_exit_handlers();
-    safe_mutex_lock(&no_ethrs_mtx.pt_mtx);
-    ASSERT(no_ethreads > 0);
-    no_ethreads--;
-    safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-}
-
-static void *thr_wrapper(void *vtwd)
-{
-    void *res;
-    thr_wrap_data_ *twd = (thr_wrap_data_ *) vtwd;
-    void *(*thr_func)(void *) = twd->thr_func;
-    void *arg = twd->arg;
-
-    safe_mutex_lock(&twd->mtx);
-
-    if (thread_create_child_func)
-	(*thread_create_child_func)(twd->prep_func_res);
-
-    twd->initialized = 1;
-
-    safe_cond_signal(&twd->cnd);
-    safe_mutex_unlock(&twd->mtx);
-
-    res = (*thr_func)(arg);
-    thr_exit_cleanup();
-    return res;
-}
-
-
-/*
- * ----------------------------------------------------------------------------
- * Exported functions
- * ----------------------------------------------------------------------------
- */
-
-int
-ethr_init(ethr_init_data *id)
-{
-    int res;
-
-    if (!ethr_not_inited)
-	return EINVAL;
-
-    ethr_not_inited = 0;
-
-    res = init_common(id);
-    if (res != 0)
-	goto error;
-
-#if ETHR_HAVE_PTHREAD_ATFORK
-    init_forksafe();
-#endif
-
-    no_ethreads = 1;
-    res = ethr_mutex_init(&no_ethrs_mtx);
-    if (res != 0)
-	goto error;
-    res = ethr_mutex_set_forksafe(&no_ethrs_mtx);
-    if (res != 0 && res != ENOTSUP)
-	goto error;
-
-#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-    {
-	int i;
-	for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) {
-#ifdef ETHR_HAVE_PTHREAD_SPIN_LOCK
-	    res = pthread_spin_init(&ethr_atomic_protection__[i].u.spnlck, 0);
-#else
-	    res = ethr_mutex_init(&ethr_atomic_protection__[i].u.mtx);
-#endif
-	    if (res != 0)
-		goto error;
-	}
-    }
-#endif
-
-#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT
-#if defined(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP) \
-    && defined(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)
-    res = pthread_rwlockattr_init(&write_pref_attr_data);
-    if (res != 0)
-	goto error;
-    res = pthread_rwlockattr_setkind_np(
-	&write_pref_attr_data,
-	PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
-    if (res != 0)
-	goto error;
-    write_pref_attr = &write_pref_attr_data;
-#else
-    write_pref_attr = NULL;
-#endif
-#endif
-
-
-    return 0;
-
- error:
-    ethr_not_inited = 1;
-    return res;
-
-}
-
-int
-ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
-		ethr_thr_opts *opts)
-{
-    thr_wrap_data_ twd;
-    pthread_attr_t attr;
-    int res, dres;
-    int use_stack_size = (opts && opts->suggested_stack_size >= 0
-			  ? opts->suggested_stack_size
-			  : -1 /* Use system default */);
-
-#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE
-    if (use_stack_size < 0)
-	use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE;
-#endif
-
-    twd.initialized = 0;
-    twd.thr_func = func;
-    twd.arg = arg;
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!tid || !func) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    /* Call prepare func if it exist */
-    if (thread_create_prepare_func)
-	twd.prep_func_res = (*thread_create_prepare_func)();
-    else
-	twd.prep_func_res = NULL;
-
-    /* Set som thread attributes */
-    res = pthread_attr_init(&attr);
-    if (res != 0)
-	goto cleanup_parent_func;
-    res = pthread_mutex_init(&twd.mtx, NULL);
-    if (res != 0)
-	goto cleanup_attr_destroy;
-    res = pthread_cond_init(&twd.cnd, NULL);
-    if (res != 0)
-	goto cleanup_mutex_destroy;
-
-    /* Schedule child thread in system scope (if possible) ... */
-    res = pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
-    if (res != 0 && res != ENOTSUP)
-	goto cleanup_cond_destroy;
-
-    if (use_stack_size >= 0) {
-	size_t suggested_stack_size = (size_t) use_stack_size;
-	size_t stack_size;
-#ifdef DEBUG
-	suggested_stack_size /= 2; /* Make sure we got margin */
-#endif
-#ifdef ETHR_STACK_GUARD_SIZE
-	/* The guard is at least on some platforms included in the stack size
-	   passed when creating threads */
-	suggested_stack_size += ETHR_B2KW(ETHR_STACK_GUARD_SIZE);
-#endif
-	if (suggested_stack_size < min_stack_size)
-	    stack_size = ETHR_KW2B(min_stack_size);
-	else if (suggested_stack_size > max_stack_size)
-	    stack_size = ETHR_KW2B(max_stack_size);
-	else
-	    stack_size = ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size));
-	(void) pthread_attr_setstacksize(&attr, stack_size);
-    }
-
-#ifdef ETHR_STACK_GUARD_SIZE
-    (void) pthread_attr_setguardsize(&attr, ETHR_STACK_GUARD_SIZE);
-#endif
-
-    /* Detached or joinable... */
-    res = pthread_attr_setdetachstate(&attr,
-				      (opts && opts->detached
-				       ? PTHREAD_CREATE_DETACHED
-				       : PTHREAD_CREATE_JOINABLE));
-    if (res != 0)
-	goto cleanup_cond_destroy;
-    
-    res = pthread_mutex_lock(&twd.mtx);
-
-    if (res != 0)
-	goto cleanup_cond_destroy;
-
-    safe_mutex_lock(&no_ethrs_mtx.pt_mtx);
-    if (no_ethreads < ETHR_MAX_THREADS) {
-	no_ethreads++;
-	safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-    }
-    else {
-	res = EAGAIN;
-	safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-	goto cleanup_mutex_unlock;
-    }
-
-    res = pthread_create((pthread_t *) tid, &attr, thr_wrapper, (void *) &twd);
-
-    if (res != 0) {
-	safe_mutex_lock(&no_ethrs_mtx.pt_mtx);
-	ASSERT(no_ethreads > 0);
-	no_ethreads--;
-	safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-    }
-    else {
-
-	/* Wait for child to initialize... */
-	while (!twd.initialized) {
-	    res = pthread_cond_wait(&twd.cnd, &twd.mtx);
-	    if (res != 0 && res != EINTR)
-		break;
-	}
-
-    }
-
-    /* Cleanup... */
- cleanup_mutex_unlock:
-    dres = pthread_mutex_unlock(&twd.mtx);
-    if (res == 0)
-	res = dres;
- cleanup_cond_destroy:
-    dres = pthread_cond_destroy(&twd.cnd);
-    if (res == 0)
-	res = dres;
- cleanup_mutex_destroy:
-    dres = pthread_mutex_destroy(&twd.mtx);
-    if (res == 0)
-	res = dres;
- cleanup_attr_destroy:
-    dres = pthread_attr_destroy(&attr);
-    if (res == 0)
-	res = dres;
- cleanup_parent_func:
-    if (thread_create_parent_func)
-	(*thread_create_parent_func)(twd.prep_func_res);
-
-    return res;
-}
-
-int
-ethr_thr_join(ethr_tid tid, void **res)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_join((pthread_t) tid, res);
-}
-
-int
-ethr_thr_detach(ethr_tid tid)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_detach((pthread_t) tid);
-}
-
-void
-ethr_thr_exit(void *res)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return;
-    }
-#endif
-    thr_exit_cleanup();
-    pthread_exit(res);
-}
-
-ethr_tid
-ethr_self(void)
-{
-    return (ethr_tid) pthread_self();
-}
-
-int
-ethr_equal_tids(ethr_tid tid1, ethr_tid tid2)
-{
-    return pthread_equal((pthread_t) tid1, (pthread_t) tid2);
-}
-
-
-/*
- * Mutex functions
- */
-
-
-int
-ethr_mutex_init(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    mtx->initialized = ETHR_MUTEX_INITIALIZED;
-#endif
-    mtx->prev = NULL;
-    mtx->next = NULL;
-    mtx->is_rec_mtx = 0;
-    return pthread_mutex_init(&mtx->pt_mtx, NULL);
-}
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-
-int
-ethr_rec_mutex_init(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    mtx->initialized = ETHR_MUTEX_INITIALIZED;
-#endif
-    if (rec_mtx_attr_need_init)
-	init_rec_mtx_attr();
-
-    mtx->prev = NULL;
-    mtx->next = NULL;
-    mtx->is_rec_mtx = 1;
-    return pthread_mutex_init(&mtx->pt_mtx, &rec_mtx_attr);
-}
-
-#endif /* #if ETHR_HAVE_ETHR_REC_MUTEX_INIT */
-
-int
-ethr_mutex_destroy(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (mtx->next) {
-	ASSERT(mtx->prev);
-	ethr_mutex_unset_forksafe(mtx);
-    }
-#if ETHR_XCHK
-    mtx->initialized = 0;
-#endif
-    return pthread_mutex_destroy(&mtx->pt_mtx);
-}
-
-int ethr_mutex_set_forksafe(ethr_mutex *mtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-#if ETHR_HAVE_PTHREAD_ATFORK
-    res = pthread_mutex_lock(&forksafe_mtx.pt_mtx);
-    if (res != 0)
-	return res;
-    if (!forksafe_mtx.next) {
-	ASSERT(!forksafe_mtx.prev);
-	init_forksafe();
-    }
-    if (mtx->next) {
-	/* forksafe already set for this mutex */ 
-	ASSERT(mtx->prev);
-    }
-    else {
-	mtx->next = forksafe_mtx.next;
-	mtx->prev = &forksafe_mtx;
-	forksafe_mtx.next->prev = mtx;
-	forksafe_mtx.next = mtx;
-    }
-
-    res = pthread_mutex_unlock(&forksafe_mtx.pt_mtx);
-
-#else /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    res = ENOTSUP;
-#endif /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    return res;
-}
-
-int ethr_mutex_unset_forksafe(ethr_mutex *mtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-#if ETHR_HAVE_PTHREAD_ATFORK
-    res = pthread_mutex_lock(&forksafe_mtx.pt_mtx);
-    if (res != 0)
-	return res;
-    if (!forksafe_mtx.next) {
-	ASSERT(!forksafe_mtx.prev);
-	init_forksafe();
-    }
-    if (!mtx->next) {
-	/* forksafe already unset for this mutex */ 
-	ASSERT(!mtx->prev);
-    }
-    else {
-	mtx->prev->next = mtx->next;
-	mtx->next->prev = mtx->prev;
-	mtx->next = NULL;
-	mtx->prev = NULL;
-    }
-    res = pthread_mutex_unlock(&forksafe_mtx.pt_mtx);
-
-#else /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    res = ENOTSUP;
-#endif /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    return res;
-}
-
-int
-ethr_mutex_trylock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_trylock__(mtx);
-}
-
-int
-ethr_mutex_lock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_lock__(mtx);
-}
-
-int
-ethr_mutex_unlock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_unlock__(mtx);
-}
-
-/*
- * Condition variable functions
- */
-
-int
-ethr_cond_init(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    cnd->initialized = ETHR_COND_INITIALIZED;
-#endif
-    return pthread_cond_init(&cnd->pt_cnd, NULL);
-}
-
-int
-ethr_cond_destroy(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    cnd->initialized = 0;
-#endif
-    return pthread_cond_destroy(&cnd->pt_cnd);
-}
-
-int
-ethr_cond_signal(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_cond_signal(&cnd->pt_cnd);
-}
-
-int
-ethr_cond_broadcast(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_cond_broadcast(&cnd->pt_cnd);
-}
-
-int
-ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| cnd->initialized != ETHR_COND_INITIALIZED
-	|| !mtx
-	|| mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx);
-}
-
-int
-ethr_cond_timedwait(ethr_cond *cnd, ethr_mutex *mtx, ethr_timeval *timeout)
-{
-    struct timespec to;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| cnd->initialized != ETHR_COND_INITIALIZED
-	|| !mtx
-	|| mtx->initialized != ETHR_MUTEX_INITIALIZED
-	|| !timeout) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    to.tv_sec = timeout->tv_sec;
-    to.tv_nsec = timeout->tv_nsec;
-
-    return pthread_cond_timedwait(&cnd->pt_cnd, &mtx->pt_mtx, &to);
-}
-
-
-#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT
-
-int
-ethr_rwmutex_init(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
-#endif
-    return pthread_rwlock_init(&rwmtx->pt_rwlock, write_pref_attr);
-}
-
-int
-ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = pthread_rwlock_destroy(&rwmtx->pt_rwlock);
-#if ETHR_XCHK
-    rwmtx->initialized = 0;
-#endif
-    return res;
-}
-
-int
-ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_tryrlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_rlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_runlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_tryrwlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_rwlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_rwunlock__(rwmtx);
-}
-
-#endif /* #ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT */
-
-/*
- * Current time
- */
-
-int
-ethr_time_now(ethr_timeval *time)
-{
-    int res;
-    struct timeval tv;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!time) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    res = gettimeofday(&tv, NULL);
-    time->tv_sec = (long) tv.tv_sec;
-    time->tv_nsec = ((long) tv.tv_usec)*1000;
-    return res;
-}
-
-/*
- * Thread specific data
- */
-
-int
-ethr_tsd_key_create(ethr_tsd_key *keyp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!keyp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_key_create((pthread_key_t *) keyp, NULL);
-}
-
-int
-ethr_tsd_key_delete(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_key_delete((pthread_key_t) key);
-}
-
-int
-ethr_tsd_set(ethr_tsd_key key, void *value)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_setspecific((pthread_key_t) key, value);
-}
-
-void *
-ethr_tsd_get(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return NULL;
-    }
-#endif
-    return pthread_getspecific((pthread_key_t) key);
-}
-
-/*
- * Signal functions
- */
-
-#if ETHR_HAVE_ETHR_SIG_FUNCS
-
-int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!set && !oset) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-  return pthread_sigmask(how, set, oset);
-}
-
-int ethr_sigwait(const sigset_t *set, int *sig)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!set || !sig) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (sigwait(set, sig) < 0)
-	return errno;
-    return 0;
-}
-
-#endif /* #if ETHR_HAVE_ETHR_SIG_FUNCS */
-
-#elif defined(ETHR_WIN32_THREADS)
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * Native win32 threads implementation                                       *
-\*                                                                           */
-
-#define INVALID_TID -1
-
-/* The spin count values are more or less taken out of the blue */
-#define ETHR_MUTEX_SPIN_COUNT	5000
-#define ETHR_COND_SPIN_COUNT	1000
-
-ethr_tid serial_shift; /* Bits to shift serial when constructing a tid */
-ethr_tid last_serial; /* Last thread table serial used */
-ethr_tid last_ix; /* Last thread table index used */
-ethr_tid thr_ix_mask; /* Mask used to mask out thread table index from a tid */
-
-/* Event used for conditional variables. On per thread. */
-/*typedef struct cnd_wait_event__ cnd_wait_event_;*/
-struct cnd_wait_event__ {
-    HANDLE handle;
-    cnd_wait_event_ *prev;
-    cnd_wait_event_ *next;
-    int in_queue;
-};
-
-/* Thread specific data. Stored in the thread table */
-typedef struct {
-    ethr_tid thr_id;
-    HANDLE thr_handle;
-    ethr_tid joiner;
-    void *result;
-    cnd_wait_event_ wait_event;
-} thr_data_;
-
-/* Argument passed to thr_wrapper() */
-typedef struct {
-    void * (*func)(void *);
-    void * arg;
-    thr_data_ *ptd;
-    thr_data_ *td;
-    int res;
-    void *prep_func_res;
-} thr_wrap_data_;
-
-
-static CRITICAL_SECTION thr_table_cs; /* Critical section used to protect
-					 the thread table from concurrent
-					 accesses. */
-static CRITICAL_SECTION fake_static_init_cs; /* Critical section used to protect
-					initialazition of 'statically
-					initialized' mutexes */
-static thr_data_ * thr_table[ETHR_MAX_THREADS]; /* The thread table */
-
-static DWORD tls_own_thr_data;
-
-static thr_data_ main_thr_data;
-
-#define THR_IX(TID)	((TID) & thr_ix_mask)
-#define OWN_THR_DATA	((thr_data_ *) TlsGetValue(tls_own_thr_data))
-
-/*
- * ----------------------------------------------------------------------------
- * Static functions
- * ----------------------------------------------------------------------------
- */
-
-static int
-get_errno(void)
-{
-    switch (GetLastError()) {
-    case ERROR_INVALID_FUNCTION:		return EINVAL;	/* 1	*/
-    case ERROR_FILE_NOT_FOUND:			return ENOENT;	/* 2	*/
-    case ERROR_PATH_NOT_FOUND:			return ENOENT;	/* 3	*/
-    case ERROR_TOO_MANY_OPEN_FILES:		return EMFILE;	/* 4	*/
-    case ERROR_ACCESS_DENIED:			return EACCES;	/* 5	*/
-    case ERROR_INVALID_HANDLE:			return EBADF;	/* 6	*/
-    case ERROR_ARENA_TRASHED:			return ENOMEM;	/* 7	*/
-    case ERROR_NOT_ENOUGH_MEMORY:		return ENOMEM;	/* 8	*/
-    case ERROR_INVALID_BLOCK:			return ENOMEM;	/* 9	*/
-    case ERROR_BAD_ENVIRONMENT:			return E2BIG;	/* 10	*/
-    case ERROR_BAD_FORMAT:			return ENOEXEC;	/* 11	*/
-    case ERROR_INVALID_ACCESS:			return EINVAL;	/* 12	*/
-    case ERROR_INVALID_DATA:			return EINVAL;	/* 13	*/
-    case ERROR_OUTOFMEMORY:			return ENOMEM;	/* 14	*/
-    case ERROR_INVALID_DRIVE:			return ENOENT;	/* 15	*/
-    case ERROR_CURRENT_DIRECTORY:		return EACCES;	/* 16	*/
-    case ERROR_NOT_SAME_DEVICE:			return EXDEV;	/* 17	*/
-    case ERROR_NO_MORE_FILES:			return ENOENT;	/* 18	*/
-    case ERROR_WRITE_PROTECT:			return EACCES;	/* 19	*/
-    case ERROR_BAD_UNIT:			return EACCES;	/* 20	*/
-    case ERROR_NOT_READY:			return EACCES;	/* 21	*/
-    case ERROR_BAD_COMMAND:			return EACCES;	/* 22	*/
-    case ERROR_CRC:				return EACCES;	/* 23	*/
-    case ERROR_BAD_LENGTH:			return EACCES;	/* 24	*/
-    case ERROR_SEEK:				return EACCES;	/* 25	*/
-    case ERROR_NOT_DOS_DISK:			return EACCES;	/* 26	*/
-    case ERROR_SECTOR_NOT_FOUND:		return EACCES;	/* 27	*/
-    case ERROR_OUT_OF_PAPER:			return EACCES;	/* 28	*/
-    case ERROR_WRITE_FAULT:			return EACCES;	/* 29	*/
-    case ERROR_READ_FAULT:			return EACCES;	/* 30	*/
-    case ERROR_GEN_FAILURE:			return EACCES;	/* 31	*/
-    case ERROR_SHARING_VIOLATION:		return EACCES;	/* 32	*/
-    case ERROR_LOCK_VIOLATION:			return EACCES;	/* 33	*/
-    case ERROR_WRONG_DISK:			return EACCES;	/* 34	*/
-    case ERROR_SHARING_BUFFER_EXCEEDED:		return EACCES;	/* 36	*/
-    case ERROR_BAD_NETPATH:			return ENOENT;	/* 53	*/
-    case ERROR_NETWORK_ACCESS_DENIED:		return EACCES;	/* 65	*/
-    case ERROR_BAD_NET_NAME:			return ENOENT;	/* 67	*/
-    case ERROR_FILE_EXISTS:			return EEXIST;	/* 80	*/
-    case ERROR_CANNOT_MAKE:			return EACCES;	/* 82	*/
-    case ERROR_FAIL_I24:			return EACCES;	/* 83	*/
-    case ERROR_INVALID_PARAMETER:		return EINVAL;	/* 87	*/
-    case ERROR_NO_PROC_SLOTS:			return EAGAIN;	/* 89	*/
-    case ERROR_DRIVE_LOCKED:			return EACCES;	/* 108	*/
-    case ERROR_BROKEN_PIPE:			return EPIPE;	/* 109	*/
-    case ERROR_DISK_FULL:			return ENOSPC;	/* 112	*/
-    case ERROR_INVALID_TARGET_HANDLE:		return EBADF;	/* 114	*/
-    case ERROR_WAIT_NO_CHILDREN:		return ECHILD;	/* 128	*/
-    case ERROR_CHILD_NOT_COMPLETE:		return ECHILD;	/* 129	*/
-    case ERROR_DIRECT_ACCESS_HANDLE:		return EBADF;	/* 130	*/
-    case ERROR_NEGATIVE_SEEK:			return EINVAL;	/* 131	*/
-    case ERROR_SEEK_ON_DEVICE:			return EACCES;	/* 132	*/
-    case ERROR_DIR_NOT_EMPTY:			return ENOTEMPTY;/* 145	*/
-    case ERROR_NOT_LOCKED:			return EACCES;	/* 158	*/
-    case ERROR_BAD_PATHNAME:			return ENOENT;	/* 161	*/
-    case ERROR_MAX_THRDS_REACHED:		return EAGAIN;	/* 164	*/
-    case ERROR_LOCK_FAILED:			return EACCES;	/* 167	*/
-    case ERROR_ALREADY_EXISTS:			return EEXIST;	/* 183	*/
-    case ERROR_INVALID_STARTING_CODESEG:	return ENOEXEC;	/* 188	*/
-    case ERROR_INVALID_STACKSEG:		return ENOEXEC;	/* 189	*/
-    case ERROR_INVALID_MODULETYPE:		return ENOEXEC;	/* 190	*/
-    case ERROR_INVALID_EXE_SIGNATURE:		return ENOEXEC;	/* 191	*/
-    case ERROR_EXE_MARKED_INVALID:		return ENOEXEC;	/* 192	*/
-    case ERROR_BAD_EXE_FORMAT:			return ENOEXEC;	/* 193	*/
-    case ERROR_ITERATED_DATA_EXCEEDS_64k:	return ENOEXEC;	/* 194	*/
-    case ERROR_INVALID_MINALLOCSIZE:		return ENOEXEC;	/* 195	*/
-    case ERROR_DYNLINK_FROM_INVALID_RING:	return ENOEXEC;	/* 196	*/
-    case ERROR_IOPL_NOT_ENABLED:		return ENOEXEC;	/* 197	*/
-    case ERROR_INVALID_SEGDPL:			return ENOEXEC;	/* 198	*/
-    case ERROR_AUTODATASEG_EXCEEDS_64k:		return ENOEXEC;	/* 199	*/
-    case ERROR_RING2SEG_MUST_BE_MOVABLE:	return ENOEXEC;	/* 200	*/
-    case ERROR_RELOC_CHAIN_XEEDS_SEGLIM:	return ENOEXEC;	/* 201	*/
-    case ERROR_INFLOOP_IN_RELOC_CHAIN:		return ENOEXEC;	/* 202	*/
-    case ERROR_FILENAME_EXCED_RANGE:		return ENOENT;	/* 206	*/
-    case ERROR_NESTING_NOT_ALLOWED:		return EAGAIN;	/* 215	*/
-    case ERROR_NOT_ENOUGH_QUOTA:		return ENOMEM;	/* 1816	*/
-    default:					return EINVAL;
-    }
-}
-
-static ETHR_INLINE thr_data_ *
-tid2thr(ethr_tid tid)
-{
-    ethr_tid ix;
-    thr_data_ *td;
-    
-    if (tid < 0)
-	return NULL;
-    ix = THR_IX(tid);
-    if (ix >= ETHR_MAX_THREADS)
-	return NULL;
-    td = thr_table[ix];
-    if (!td)
-	return NULL;
-    if (td->thr_id != tid)
-	return NULL;
-    return td;
-}
-
-static ETHR_INLINE void
-new_tid(ethr_tid *new_tid, ethr_tid *new_serial, ethr_tid *new_ix)
-{
-    ethr_tid tmp_serial = last_serial;
-    ethr_tid tmp_ix = last_ix + 1;
-    ethr_tid start_ix = tmp_ix;
-
-
-    do {
-	if (tmp_ix >= ETHR_MAX_THREADS) {
-	    tmp_serial++;
-	    if ((tmp_serial << serial_shift) < 0)
-		tmp_serial = 0;
-	    tmp_ix = 0;
-	}
-	if (!thr_table[tmp_ix]) {
-	    *new_tid	= (tmp_serial << serial_shift) | tmp_ix;
-	    *new_serial	= tmp_serial;
-	    *new_ix	= tmp_ix;
-	    return;
-	}
-	tmp_ix++;
-    } while (tmp_ix != start_ix);
-
-    *new_tid	= INVALID_TID;
-    *new_serial	= INVALID_TID;
-    *new_ix	= INVALID_TID;
-
-}
-
-
-static void thr_exit_cleanup(thr_data_ *td, void *res)
-{
-
-    ASSERT(td == OWN_THR_DATA);
-
-    run_exit_handlers();
-
-    EnterCriticalSection(&thr_table_cs);
-    CloseHandle(td->wait_event.handle);
-    if (td->thr_handle == INVALID_HANDLE_VALUE) {
-	/* We are detached; cleanup thread table */
-	ASSERT(td->joiner == INVALID_TID);
-	ASSERT(td == thr_table[THR_IX(td->thr_id)]);
-	thr_table[THR_IX(td->thr_id)] = NULL;
-	if (td != &main_thr_data)
-	    (*freep)((void *) td);
-    }
-    else {
-	/* Save result and let joining thread cleanup */
-	td->result = res;
-    }
-    LeaveCriticalSection(&thr_table_cs);
-}
-
-static unsigned __stdcall thr_wrapper(LPVOID args)
-{
-    void *(*func)(void*) = ((thr_wrap_data_ *) args)->func;
-    void *arg = ((thr_wrap_data_ *) args)->arg;
-    thr_data_ *td = ((thr_wrap_data_ *) args)->td;
-
-    td->wait_event.handle = CreateEvent(NULL, FALSE, FALSE, NULL);
-    if (td->wait_event.handle == INVALID_HANDLE_VALUE
-	|| !TlsSetValue(tls_own_thr_data, (LPVOID) td)) {
-	((thr_wrap_data_ *) args)->res = get_errno();
-	if (td->wait_event.handle != INVALID_HANDLE_VALUE)
-	    CloseHandle(td->wait_event.handle);
-	SetEvent(((thr_wrap_data_ *) args)->ptd->wait_event.handle);
-	_endthreadex((unsigned) 0);
-	ASSERT(0);
-    }
-
-    td->wait_event.prev = NULL;
-    td->wait_event.next = NULL;
-    td->wait_event.in_queue = 0;
-
-    if (thread_create_child_func)
-	(*thread_create_child_func)(((thr_wrap_data_ *) args)->prep_func_res);
-
-    ASSERT(td == OWN_THR_DATA);
-
-    ((thr_wrap_data_ *) args)->res = 0;
-    SetEvent(((thr_wrap_data_ *) args)->ptd->wait_event.handle);
-
-    thr_exit_cleanup(td, (*func)(arg));
-    return 0;
-}
-
-int
-ethr_fake_static_mutex_init(ethr_mutex *mtx)
-{
-    EnterCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    /* Got here under race conditions; check again... */
-    if (!mtx->initialized) {
-	if (!InitializeCriticalSectionAndSpinCount(&mtx->cs,
-						   ETHR_MUTEX_SPIN_COUNT))
-	    return get_errno();
-	mtx->initialized = ETHR_MUTEX_INITIALIZED;
-    }
-    LeaveCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    return 0;
-}
-
-static int
-fake_static_cond_init(ethr_cond *cnd)
-{
-    EnterCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    /* Got here under race conditions; check again... */
-    if (!cnd->initialized) {
-	if (!InitializeCriticalSectionAndSpinCount(&cnd->cs,
-						   ETHR_COND_SPIN_COUNT))
-	    return get_errno();
-	cnd->queue = NULL;
-	cnd->queue_end = NULL;
-	cnd->initialized = ETHR_COND_INITIALIZED;
-    }
-    LeaveCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    return 0;
-}
-
-#ifdef __GNUC__
-#define LL_LITERAL(X) X##LL
-#else
-#define LL_LITERAL(X) X##i64
-#endif
-
-#define EPOCH_JULIAN_DIFF LL_LITERAL(11644473600)
-
-static ETHR_INLINE void
-get_curr_time(long *sec, long *nsec)
-{
-    SYSTEMTIME t;
-    FILETIME ft;
-    LONGLONG lft;
-
-    GetSystemTime(&t);
-    SystemTimeToFileTime(&t, &ft);
-    memcpy(&lft, &ft, sizeof(lft));
-    *nsec = ((long) (lft % LL_LITERAL(10000000)))*100;
-    *sec = (long) ((lft / LL_LITERAL(10000000)) - EPOCH_JULIAN_DIFF);
-}
-
-static cnd_wait_event_ *cwe_freelist;
-static CRITICAL_SECTION cwe_cs;
-
-static int
-alloc_cwe(cnd_wait_event_ **cwe_res)
-{
-    cnd_wait_event_ *cwe;
-    EnterCriticalSection(&cwe_cs);
-    cwe = cwe_freelist;
-    if (cwe) {
-	cwe_freelist = cwe->next;
-	LeaveCriticalSection(&cwe_cs);
-    }
-    else {
-	LeaveCriticalSection(&cwe_cs);
-	cwe = (*allocp)(sizeof(cnd_wait_event_));
-	if (!cwe)
-	    return ENOMEM;
-	cwe->handle = CreateEvent(NULL, FALSE, FALSE, NULL);
-	if (cwe->handle == INVALID_HANDLE_VALUE) {
-	    int res = get_errno();
-	    (*freep)(cwe);
-	    return res;
-	}
-    }
-    *cwe_res = cwe;
-    return 0;
-}
-
-static
-free_cwe(cnd_wait_event_ *cwe)
-{
-    EnterCriticalSection(&cwe_cs);
-    cwe->next = cwe_freelist;
-    cwe_freelist = cwe;
-    LeaveCriticalSection(&cwe_cs);
-}
-
-static ETHR_INLINE int
-condwait(ethr_cond *cnd,
-	 ethr_mutex *mtx,
-	 int with_timeout,
-	 ethr_timeval *timeout)
-{
-    int res;
-    thr_data_ *td;
-    cnd_wait_event_ *cwe;
-    DWORD code;
-    long time; /* time until timeout in milli seconds */
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-
-    if (!mtx
-	|| mtx->initialized != ETHR_MUTEX_INITIALIZED
-	|| !cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)
-	|| (with_timeout && !timeout)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    td = OWN_THR_DATA;
-    if (td)
-	cwe = &td->wait_event;
-    else { /* A non-ethread thread */
-	res = alloc_cwe(&cwe);
-	if (res != 0)
-	    return res;
-    }
-
-    if (!cnd->initialized)
-	fake_static_cond_init(cnd);
-    EnterCriticalSection(&cnd->cs);
-
-    ASSERT(!cwe->in_queue);
-    if (cnd->queue_end) {
-	ASSERT(cnd->queue);
-	cwe->prev = cnd->queue_end;
-	cwe->next = NULL;
-	cnd->queue_end->next = cwe;
-	cnd->queue_end = cwe;
-    }
-    else {
-	ASSERT(!cnd->queue);
-	cwe->prev = NULL;
-	cwe->next = NULL;
-	cnd->queue = cwe;
-	cnd->queue_end = cwe;
-    }
-    cwe->in_queue = 1;
-
-    LeaveCriticalSection(&cnd->cs);
- 
-    LeaveCriticalSection(&mtx->cs);
-
-    if (!with_timeout)
-	time = INFINITE;
-    else {
-	long sec, nsec;
-	ASSERT(timeout);
-	get_curr_time(&sec, &nsec);
-	time = (timeout->tv_sec - sec)*1000;
-	time += (timeout->tv_nsec - nsec + 500)/1000000;
-	if (time < 0)
-	    time = 0;
-    }
-
-    /* wait for event to signal */
-    code = WaitForSingleObject(cwe->handle, time);
-
-    EnterCriticalSection(&mtx->cs);
-
-    if (code == WAIT_OBJECT_0) {
-	/* We were woken by a signal or a broadcast ... */
-	res = 0;
-
-	/* ... no need to remove event from wait queue since this was
-	   taken care of by the signal or broadcast */
-#ifdef DEBUG
-	EnterCriticalSection(&cnd->cs);
-	ASSERT(!cwe->in_queue);
-	LeaveCriticalSection(&cnd->cs);
-#endif
-
-    }
-    else {
-	/* We timed out... */
-	res = ETIMEDOUT;
-
-	/* ... probably have to remove event from wait queue ... */
-	EnterCriticalSection(&cnd->cs);
-
-	if (cwe->in_queue) { /* ... but we must check that we are in queue
-				since a signal or broadcast after timeout
-				may have removed us from the queue */
-	    if (cwe->prev) {
-		cwe->prev->next = cwe->next;
-	    }
-	    else {
-		ASSERT(cnd->queue == cwe);
-		cnd->queue = cwe->next;
-	    }
-
-	    if (cwe->next) {
-		cwe->next->prev = cwe->prev;
-	    }
-	    else {
-		ASSERT(cnd->queue_end == cwe);
-		cnd->queue_end = cwe->prev;
-	    }
-	    cwe->in_queue = 0;
-	}
-
-	LeaveCriticalSection(&cnd->cs);
-
-    }
-
-    if (!td)
-	free_cwe(cwe);
-
-    return res;
-
-}
-
-
-/*
- * ----------------------------------------------------------------------------
- * Exported functions
- * ----------------------------------------------------------------------------
- */
-
-int
-ethr_init(ethr_init_data *id)
-{
-#ifdef _WIN32_WINNT
-    DWORD major = (_WIN32_WINNT >> 8) & 0xff;
-    DWORD minor = _WIN32_WINNT & 0xff;
-    OSVERSIONINFO os_version;
-#endif
-    int err = 0;
-    thr_data_ *td = &main_thr_data;
-    unsigned long i;
-
-    if (!ethr_not_inited)
-	return EINVAL;
-
-#ifdef _WIN32_WINNT
-    os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
-    GetVersionEx(&os_version);
-    if (os_version.dwPlatformId != VER_PLATFORM_WIN32_NT
-	|| os_version.dwMajorVersion < major
-	|| (os_version.dwMajorVersion == major
-	    && os_version.dwMinorVersion < minor))
-	return ENOTSUP;
-#endif
-
-    ASSERT(ETHR_MAX_THREADS > 0);
-    for (i = ETHR_MAX_THREADS - 1, serial_shift = 0;
-	 i;
-	 serial_shift++, i >>= 1);
-    thr_ix_mask = ~(~((ethr_tid) 0) << serial_shift);
-
-    tls_own_thr_data = TlsAlloc();
-    if (tls_own_thr_data == TLS_OUT_OF_INDEXES)
-	goto error;
-
-    last_serial = 0;
-    last_ix = 0;
-
-    td->thr_id = 0;
-    td->thr_handle = GetCurrentThread();
-    td->joiner = INVALID_TID;
-    td->result = NULL;
-    td->wait_event.handle = CreateEvent(NULL, FALSE, FALSE, NULL);
-    if (td->wait_event.handle == INVALID_HANDLE_VALUE)
-	goto error;
-    td->wait_event.prev = NULL;
-    td->wait_event.next = NULL;
-    td->wait_event.in_queue = 0;
-    thr_table[0] = td;
-
-    if (!TlsSetValue(tls_own_thr_data, (LPVOID) td))
-	goto error;
-
-    ASSERT(td == OWN_THR_DATA);
-
-
-    cwe_freelist = NULL;
-    if (!InitializeCriticalSectionAndSpinCount(&cwe_cs,
-					       ETHR_MUTEX_SPIN_COUNT))
-	goto error;
-
-    for (i = 1; i < ETHR_MAX_THREADS; i++)
-	thr_table[i] = NULL;
-
-    if (!InitializeCriticalSectionAndSpinCount(&thr_table_cs,
-					       ETHR_MUTEX_SPIN_COUNT))
-	goto error;
-    if (!InitializeCriticalSectionAndSpinCount(&fake_static_init_cs,
-					       ETHR_MUTEX_SPIN_COUNT))
-	goto error;
-    ethr_not_inited = 0;
-
-    err = init_common(id);
-    if (err)
-	goto error;
-
-    return 0;
-
- error:
-    ethr_not_inited = 1;
-    if (err == 0)
-	err = get_errno();
-    ASSERT(err != 0);
-    if (td->thr_handle != INVALID_HANDLE_VALUE)
-	CloseHandle(td->thr_handle);
-    if (td->wait_event.handle != INVALID_HANDLE_VALUE)
-	CloseHandle(td->wait_event.handle);
-    return err;
-}
-
-/*
- * Thread functions.
- */
-
-int
-ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
-		ethr_thr_opts *opts)
-{
-    int err = 0;
-    thr_wrap_data_ twd;
-    thr_data_ *my_td, *child_td = NULL;
-    ethr_tid child_tid, child_serial, child_ix;
-    DWORD code;
-    unsigned ID;
-    unsigned stack_size = 0; /* 0 = system default */
-    int use_stack_size = (opts && opts->suggested_stack_size >= 0
-			  ? opts->suggested_stack_size
-			  : -1 /* Use system default */);
-
-#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE
-    if (use_stack_size < 0)
-	use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE;
-#endif
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!tid || !func) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    my_td = OWN_THR_DATA;
-    if (!my_td) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return EACCES;
-    }
-
-    if (use_stack_size >= 0) {
-	size_t suggested_stack_size = (size_t) use_stack_size;
-#ifdef DEBUG
-	suggested_stack_size /= 2; /* Make sure we got margin */
-#endif
-	if (suggested_stack_size < min_stack_size)
-	    stack_size = (unsigned) ETHR_KW2B(min_stack_size);
-	else if (suggested_stack_size > max_stack_size)
-	    stack_size = (unsigned) ETHR_KW2B(max_stack_size);
-	else
-	    stack_size =
-		(unsigned) ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size));
-    }
-
-    EnterCriticalSection(&thr_table_cs);
-
-    /* Call prepare func if it exist */
-    if (thread_create_prepare_func)
-	twd.prep_func_res = (*thread_create_prepare_func)();
-    else
-	twd.prep_func_res = NULL;
-
-    /* Find a new thread id to use */
-    new_tid(&child_tid, &child_serial, &child_ix);
-    if (child_tid == INVALID_TID) {
-	err = EAGAIN;
-	goto error;
-    }
-
-    ASSERT(child_ix == THR_IX(child_tid));
-
-    *tid = child_tid;
-
-    ASSERT(!thr_table[child_ix]);
-
-    /* Alloc thread data */
-    thr_table[child_ix] = child_td = (thr_data_ *) (*allocp)(sizeof(thr_data_));
-    if (!child_td) {
-	err = ENOMEM;
-	goto error;
-    }
-
-    /* Init thread data */
-
-    child_td->thr_id		= child_tid;
-    child_td->thr_handle	= INVALID_HANDLE_VALUE;
-    child_td->joiner		= INVALID_TID;
-    child_td->result		= NULL;
-    /* 'child_td->wait_event' is initialized by child thread */
-
-
-    /* Init thread wrapper data */
-
-    twd.func			= func;
-    twd.arg			= arg;
-    twd.ptd			= my_td;
-    twd.td			= child_td;
-    twd.res			= 0;
-
-    ASSERT(!my_td->wait_event.in_queue);
-
-    /* spawn the thr_wrapper function */
-    child_td->thr_handle = (HANDLE) _beginthreadex(NULL,
-						   stack_size,
-						   thr_wrapper,
-						   (LPVOID) &twd,
-						   0,
-						   &ID);
-    if (child_td->thr_handle == (HANDLE) 0) {
-	child_td->thr_handle = INVALID_HANDLE_VALUE;
-	goto error;
-    }
-
-    ASSERT(child_td->thr_handle != INVALID_HANDLE_VALUE);
-
-    /* Wait for child to finish initialization */
-    code = WaitForSingleObject(my_td->wait_event.handle, INFINITE);
-    if (twd.res || code != WAIT_OBJECT_0) {
-	err = twd.res;
-	goto error;
-    }
-
-    if (opts && opts->detached) {
-	CloseHandle(child_td->thr_handle);
-	child_td->thr_handle = INVALID_HANDLE_VALUE;
-    }
-
-    last_serial	= child_serial;
-    last_ix	= child_ix;
-
-    ASSERT(thr_table[child_ix] == child_td);
-
-    if (thread_create_parent_func)
-	(*thread_create_parent_func)(twd.prep_func_res);
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    return 0;
-
- error:
-
-    if (err == 0)
-	err = get_errno();
-    ASSERT(err != 0);
-
-    if (thread_create_parent_func)
-	(*thread_create_parent_func)(twd.prep_func_res);
-
-    if (child_ix != INVALID_TID) {
-
-	if (child_td) {
-	    ASSERT(thr_table[child_ix] == child_td);
-
-	    if (child_td->thr_handle != INVALID_HANDLE_VALUE) {
-		WaitForSingleObject(child_td->thr_handle, INFINITE);
-		CloseHandle(child_td->thr_handle);
-	    }
-
-	    (*freep)((void *) child_td);
-	    thr_table[child_ix] = NULL;
-	}
-    }
-
-    *tid = INVALID_TID;
-
-    LeaveCriticalSection(&thr_table_cs);
-    return err;
-}
-
-int ethr_thr_join(ethr_tid tid, void **res)
-{
-    int err = 0;
-    DWORD code;
-    thr_data_ *td;
-    thr_data_ *my_td;
-
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-
-    my_td = OWN_THR_DATA;
-
-    if (!my_td) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return EACCES;
-    }
-
-    EnterCriticalSection(&thr_table_cs);
-
-    td = tid2thr(tid);
-    if (!td)
-	err = ESRCH;
-    else if (td->thr_handle == INVALID_HANDLE_VALUE /* i.e. detached */
-	     || td->joiner != INVALID_TID) /* i.e. someone else is joining */
-	err = EINVAL;
-    else if (my_td == td)
-	err = EDEADLK;
-    else
-	td->joiner = my_td->thr_id;
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    if (err)
-	goto error;
-
-    /* Wait for thread to terminate */
-    code = WaitForSingleObject(td->thr_handle, INFINITE);
-    if (code != WAIT_OBJECT_0)
-	goto error;
-
-    EnterCriticalSection(&thr_table_cs);
-
-    ASSERT(td == tid2thr(tid));
-    ASSERT(td->thr_handle != INVALID_HANDLE_VALUE);
-    ASSERT(td->joiner == my_td->thr_id);
-
-    if (res)
-	*res = td->result;
-
-    CloseHandle(td->thr_handle);
-    ASSERT(td == thr_table[THR_IX(td->thr_id)]);
-    thr_table[THR_IX(td->thr_id)] = NULL;
-    if (td != &main_thr_data)
-	(*freep)((void *) td);
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    return 0;
-
- error:
-    if (err == 0)
-	err = get_errno();
-    ASSERT(err != 0);
-    return err;
-}
-
-
-int
-ethr_thr_detach(ethr_tid tid)
-{
-    int res;
-    DWORD code;
-    thr_data_ *td;
-
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-
-    if (!OWN_THR_DATA) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return EACCES;
-    }
-
-    EnterCriticalSection(&thr_table_cs);
-
-    td = tid2thr(tid);
-    if (!td)
-	res = ESRCH;
-    if (td->thr_handle == INVALID_HANDLE_VALUE /* i.e. detached */
-	|| td->joiner != INVALID_TID) /* i.e. someone is joining */
-	res = EINVAL;
-    else {
-	res = 0;
-	CloseHandle(td->thr_handle);
-	td->thr_handle = INVALID_HANDLE_VALUE;
-    }
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    return res;
-}
-
-
-void
-ethr_thr_exit(void *res)
-{
-    thr_data_ *td;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return;
-    }
-#endif
-    td = OWN_THR_DATA;
-    if (!td) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return;
-    }
-    thr_exit_cleanup(td, res);
-    _endthreadex((unsigned) 0);
-}
-
-ethr_tid
-ethr_self(void)
-{
-    thr_data_ *td;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return INVALID_TID;
-    }
-#endif
-    /* It is okay for non-ethreads (i.e. native win32 threads) to call
-       ethr_self(). They will however be returned the INVALID_TID. */
-    td = OWN_THR_DATA;
-    if (!td)
-	return INVALID_TID;
-    return td->thr_id;
-}
-
-int
-ethr_equal_tids(ethr_tid tid1, ethr_tid tid2)
-{
-    /* INVALID_TID does not equal any tid, not even the INVALID_TID */
-    return tid1 == tid2 && tid1 != INVALID_TID;
-}
-
-/*
- * Mutex functions.
- */
-
-int
-ethr_mutex_init(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!InitializeCriticalSectionAndSpinCount(&mtx->cs, ETHR_MUTEX_SPIN_COUNT))
-	return get_errno();
-    mtx->initialized = ETHR_MUTEX_INITIALIZED;
-#if ETHR_XCHK
-    mtx->is_rec_mtx = 0;
-#endif
-    return 0;
-}
-
-int
-ethr_rec_mutex_init(ethr_mutex *mtx)
-{
-    int res;
-    res = ethr_mutex_init(mtx);
-#if ETHR_XCHK
-    mtx->is_rec_mtx = 1;
-#endif
-    return res;
-}
-
-int
-ethr_mutex_destroy(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    DeleteCriticalSection(&mtx->cs);
-    mtx->initialized = 0;
-    return 0;
-}
-
-int ethr_mutex_set_forksafe(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return 0; /* No fork() */
-}
-
-int ethr_mutex_unset_forksafe(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return 0; /* No fork() */
-}
-
-int
-ethr_mutex_trylock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx
-	|| (mtx->initialized && mtx->initialized != ETHR_MUTEX_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!mtx->initialized) {
-	int res = ethr_fake_static_mutex_init(mtx);
-	if (res != 0)
-	    return res;
-    }
-    return ethr_mutex_trylock__(mtx);
-}
-
-int
-ethr_mutex_lock(ethr_mutex *mtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx
-	|| (mtx->initialized && mtx->initialized != ETHR_MUTEX_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_lock__(mtx);
-}
-
-int
-ethr_mutex_unlock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    int res;
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_unlock__(mtx);
-}
-
-/*
- * Condition variable functions.
- */
-
-int
-ethr_cond_init(ethr_cond *cnd)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!InitializeCriticalSectionAndSpinCount(&cnd->cs, ETHR_COND_SPIN_COUNT))
-	return get_errno();
-    cnd->queue = NULL;
-    cnd->queue_end = NULL;
-    cnd->initialized = ETHR_COND_INITIALIZED;
-    return 0;
-}
-
-int
-ethr_cond_destroy(ethr_cond *cnd)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)
-	|| cnd->queue) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    DeleteCriticalSection(&cnd->cs);
-    cnd->initialized = 0;
-    return 0;
-}
-
-int
-ethr_cond_signal(ethr_cond *cnd)
-{
-    cnd_wait_event_ *cwe;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!cnd->initialized) {
-	int res = fake_static_cond_init(cnd);
-	if (res != 0)
-	    return res;
-    }
-    EnterCriticalSection(&cnd->cs);
-    cwe = cnd->queue;
-    if (cwe) {
-	ASSERT(cwe->in_queue);
-	SetEvent(cnd->queue->handle);
-	if (cwe->next)
-	    cwe->next->prev = NULL;
-	else {
-	    ASSERT(cnd->queue_end == cnd->queue);
-	    cnd->queue_end = NULL;
-	}
-	cnd->queue = cwe->next;
-	cwe->in_queue = 0;
-    }
-    LeaveCriticalSection(&cnd->cs);
-    return 0;
-}
-
-int
-ethr_cond_broadcast(ethr_cond *cnd)
-{
-    cnd_wait_event_ *cwe;
-
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!cnd->initialized) {
-	int res = fake_static_cond_init(cnd);
-	if (res != 0)
-	    return res;
-    }
-    EnterCriticalSection(&cnd->cs);
-    for (cwe = cnd->queue; cwe; cwe = cwe->next) {
-	ASSERT(cwe->in_queue);
-	SetEvent(cwe->handle);
-	cwe->in_queue = 0;
-    }
-    cnd->queue = NULL;
-    cnd->queue_end = NULL;
-    LeaveCriticalSection(&cnd->cs);
-    return 0;
-
-}
-
-int
-ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
-{
-    return condwait(cnd, mtx, 0, NULL);
-}
-
-int
-ethr_cond_timedwait(ethr_cond *cnd, ethr_mutex *mtx, ethr_timeval *timeout)
-{
-    return condwait(cnd, mtx, 1, timeout);
-}
-
-int
-ethr_time_now(ethr_timeval *time)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!time) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    get_curr_time(&time->tv_sec, &time->tv_nsec);
-    return 0;
-}
-
-/*
- * Thread specific data
- */
-
-int
-ethr_tsd_key_create(ethr_tsd_key *keyp)
-{
-    DWORD key;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!keyp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    key = TlsAlloc();
-    if (key == TLS_OUT_OF_INDEXES)
-	return get_errno();
-    *keyp = (ethr_tsd_key) key;
-    return 0;
-}
-
-int
-ethr_tsd_key_delete(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    if (!TlsFree((DWORD) key))
-	return get_errno();
-    return 0;
-}
-
-int
-ethr_tsd_set(ethr_tsd_key key, void *value)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    if (!TlsSetValue((DWORD) key, (LPVOID) value))
-	return get_errno();
-    return 0;
-}
-
-void *
-ethr_tsd_get(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return NULL;
-    }
-#endif
-    return (void *) TlsGetValue((DWORD) key);
-}
-
-/* Misc */
-
-#ifndef ETHR_HAVE_OPTIMIZED_LOCKS
-
-int
-ethr_do_spinlock_init(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (InitializeCriticalSectionAndSpinCount(&lock->cs, INT_MAX))
-	return 0;
-    else
-	return get_errno();
-}
-
-int
-ethr_do_rwlock_init(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    lock->counter = 0;
-    if (InitializeCriticalSectionAndSpinCount(&lock->cs, INT_MAX))
-	return 0;
-    else
-	return get_errno();    
-}
-
-#endif /* #ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS */
-
-#else
-#error "Missing thread implementation"
-#endif
-
-/* Atomics */
-
-int
-ethr_atomic_init(ethr_atomic_t *var, long i)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_init__(var, i);
-}
-
-int
-ethr_atomic_set(ethr_atomic_t *var, long i)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_set__(var, i);
-}
-
-int
-ethr_atomic_read(ethr_atomic_t *var, long *i)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var || !i) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_read__(var, i);
-}
-
-
-int
-ethr_atomic_addtest(ethr_atomic_t *var, long incr, long *testp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var || !testp) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_addtest__(var, incr, testp);
-}   
-    
-int
-ethr_atomic_inctest(ethr_atomic_t *incp, long *testp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!incp || !testp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_inctest__(incp, testp);
-}
-
-int
-ethr_atomic_dectest(ethr_atomic_t *decp, long *testp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!decp || !testp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_dectest__(decp, testp);
-}
-
-int
-ethr_atomic_add(ethr_atomic_t *var, long incr)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_add__(var, incr);
-}   
-    
-int 
-ethr_atomic_inc(ethr_atomic_t *incp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!incp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_inc__(incp);
-}
-
-int
-ethr_atomic_dec(ethr_atomic_t *decp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!decp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_dec__(decp);
-}
-
-int
-ethr_atomic_and_old(ethr_atomic_t *var, long mask, long *old)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_and_old__(var, mask, old);
-}
-
-int
-ethr_atomic_or_old(ethr_atomic_t *var, long mask, long *old)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_or_old__(var, mask, old);
-}
-
-int
-ethr_atomic_xchg(ethr_atomic_t *var, long new, long *old)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }  
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }  
-#endif
-    return ethr_atomic_xchg__(var, new, old);
-}   
-
-int
-ethr_atomic_cmpxchg(ethr_atomic_t *var, long new, long expected, long *old)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }  
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }  
-#endif
-    return ethr_atomic_cmpxchg__(var, new, expected, old);
-}
-
-/* Spinlocks and rwspinlocks */
-
-int
-ethr_spinlock_init(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spinlock_init__(lock);
-}
-
-int
-ethr_spinlock_destroy(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spinlock_destroy__(lock);
-}
-
-
-int
-ethr_spin_unlock(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spin_unlock__(lock);
-}
-
-int
-ethr_spin_lock(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spin_lock__(lock);
-}
-
-int
-ethr_rwlock_init(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwlock_init__(lock);
-}
-
-int
-ethr_rwlock_destroy(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwlock_destroy__(lock);
-}
-
-int
-ethr_read_unlock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_read_unlock__(lock);
-}
-
-int
-ethr_read_lock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_read_lock__(lock);
-}
-
-int
-ethr_write_unlock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_write_unlock__(lock);
-}
-
-int
-ethr_write_lock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_write_lock__(lock);
-}
-
-
-int
-ethr_gate_init(ethr_gate *gp)
-{
-    int res;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_init(&gp->mtx);
-    if (res != 0)
-	return res;
-    res = ethr_cond_init(&gp->cnd);
-    if (res != 0) {
-	ethr_mutex_destroy(&gp->mtx);
-	return res;
-    }
-    gp->open = 0;
-    return 0;
-}
-
-int
-ethr_gate_destroy(ethr_gate *gp)
-{
-    int res, dres;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_destroy(&gp->mtx);
-    dres = ethr_cond_destroy(&gp->cnd);
-    if (res == 0)
-	res = dres;
-    gp->open = 0;
-    return res;
-}
-
-int
-ethr_gate_close(ethr_gate *gp)
-{
-    int res;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&gp->mtx);
-    if (res != 0)
-	return res;
-    gp->open = 0;
-    res = ethr_mutex_unlock__(&gp->mtx);
-    return res;
-}
-
-int
-ethr_gate_let_through(ethr_gate *gp, unsigned no)
-{
-    int res, ures;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&gp->mtx);
-    if (res != 0)
-	return res;
-    gp->open += no;
-    res = (gp->open == 1
-	   ? ethr_cond_signal(&gp->cnd)
-	   : ethr_cond_broadcast(&gp->cnd));
-    ures = ethr_mutex_unlock__(&gp->mtx);
-    if (res != 0)
-	res = ures;
-    return res;
-}
-
-int
-ethr_gate_swait(ethr_gate *gp, int spincount)
-{
-    int res, ures, n;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    n = spincount;
-    res = ethr_mutex_lock__(&gp->mtx);
-    if (res != 0)
-	return res;
-    while (n >= 0 && !gp->open) {
-	res = ethr_mutex_unlock__(&gp->mtx);
-	if (res != 0)
-	    return res;
-	res = ethr_mutex_lock__(&gp->mtx);
-	if (res != 0)
-	    return res;
-	n--;
-    }
-    while (!gp->open) {
-	res = ethr_cond_wait(&gp->cnd, &gp->mtx);
-	if (res != 0 && res != EINTR)
-	    goto done;
-    }
-    gp->open--;
- done:
-    ures = ethr_mutex_unlock__(&gp->mtx);
-    if (res == 0)
-	res = ures;
-    return res;
-}
-
-
-int
-ethr_gate_wait(ethr_gate *gp)
-{
-    return ethr_gate_swait(gp, 0);
-}
-
-
-/* rwmutex fallback */
-#ifdef ETHR_USE_RWMTX_FALLBACK
-
-int
-ethr_rwmutex_init(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_init(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    ethr_cond_init(&rwmtx->rcnd);
-    if (res != 0)
-	goto error_cleanup1;
-    res = ethr_cond_init(&rwmtx->wcnd);
-    if (res != 0)
-	goto error_cleanup2;
-    rwmtx->readers = 0;
-    rwmtx->waiting_readers = 0;
-    rwmtx->waiting_writers = 0;
-#if ETHR_XCHK
-    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
-#endif
-    return 0;
- error_cleanup2:
-    ethr_cond_destroy(&rwmtx->rcnd);
- error_cleanup1:
-    ethr_mutex_destroy(&rwmtx->mtx);
-    return res;
-}
-
-int
-ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
-{
-    int res, pres;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    rwmtx->initialized = 0;
-#endif
-    res = ethr_mutex_destroy(&rwmtx->mtx);
-    pres = ethr_cond_destroy(&rwmtx->rcnd);
-    if (res == 0)
-	res = pres;
-    pres = ethr_cond_destroy(&rwmtx->wcnd);
-    if (res == 0)
-	res = pres;
-    return res;
-}
-
-int
-ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_trylock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    if (rwmtx->waiting_writers) {
-	res = ethr_mutex_unlock__(&rwmtx->mtx);
-	if (res == 0)
-	    return EBUSY;
-	return res;
-    }
-    rwmtx->readers++;
-    return ethr_mutex_unlock__(&rwmtx->mtx);
-}
-
-int
-ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    while (rwmtx->waiting_writers) {
-	rwmtx->waiting_readers++;
-	res = ethr_cond_wait(&rwmtx->rcnd, &rwmtx->mtx);
-	rwmtx->waiting_readers--;
-	if (res != 0 && res != EINTR) {
-	    (void) ethr_mutex_unlock__(&rwmtx->mtx);
-	    return res;
-	}
-    }
-    rwmtx->readers++;
-    return ethr_mutex_unlock__(&rwmtx->mtx);
-}
-
-int
-ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
-{
-    int res, ures;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    rwmtx->readers--;
-    if (!rwmtx->readers && rwmtx->waiting_writers)
-	res = ethr_cond_signal(&rwmtx->wcnd);
-    ures = ethr_mutex_unlock__(&rwmtx->mtx);
-    if (res == 0)
-	res = ures;
-    return res;
-}
-
-int
-ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_trylock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    if (!rwmtx->readers && !rwmtx->waiting_writers)
-	return 0;
-    else {
-	res = ethr_mutex_unlock__(&rwmtx->mtx);
-	if (res == 0)
-	    return EBUSY;
-	return res;
-    }
-}
-
-int
-ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    if (!rwmtx->readers && !rwmtx->waiting_writers) 
-	return 0;
-    
-    while (rwmtx->readers) {
-	rwmtx->waiting_writers++;
-	res = ethr_cond_wait(&rwmtx->wcnd, &rwmtx->mtx);
-	rwmtx->waiting_writers--;
-	if (res != 0 && res != EINTR) {
-	    (void) ethr_rwmutex_rwunlock(rwmtx);
-	    return res;
-	}
-    }
-    return 0;
-}
-
-int
-ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
-{
-    int res, ures;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = 0;
-    if (rwmtx->waiting_writers)
-	res = ethr_cond_signal(&rwmtx->wcnd);
-    else if (rwmtx->waiting_readers)
-	res = ethr_cond_broadcast(&rwmtx->rcnd);
-    ures = ethr_mutex_unlock__(&rwmtx->mtx);
-    if (res == 0)
-	res = ures;
-    return res;
-}
-
-#endif /* #ifdef ETHR_USE_RWMTX_FALLBACK */
-
-void
-ethr_compiler_barrier(void)
-{
-
-}
-
-#ifdef DEBUG
-
-#include <stdio.h>
-int ethr_assert_failed(char *f, int l, char *a)
-{
-    fprintf(stderr, "%s:%d: Assertion failed: %s\n", f, l, a);
-    abort();
-    return 0;
-}
-
-#endif
-
-