Merge branch 'rickard/ethread-rewrite/OTP-8544' into dev

* rickard/ethread-rewrite/OTP-8544:
  Rewrite ethread library

5 files changed, 3713 insertions, 3391 deletions

diff --git a/erts/lib_src/common/erl_misc_utils.c b/erts/lib_src/common/erl_misc_utils.c
index f70db86960..d2ef7140a5 100644
--- a/erts/lib_src/common/erl_misc_utils.c
+++ b/erts/lib_src/common/erl_misc_utils.c
@@ -59,8 +59,25 @@
 #  endif
 #endif
 
-#ifdef HAVE_SCHED_xETAFFINITY
+#if defined(HAVE_SCHED_xETAFFINITY)
 #  include <sched.h>
+#  define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__
+#define ERTS_MU_GET_PROC_AFFINITY__(CPUINFOP)				\
+     (sched_getaffinity((CPUINFOP)->pid,				\
+			sizeof(cpu_set_t),				\
+			&(CPUINFOP)->cpuset) != 0 ? -errno : 0)
+#define ERTS_MU_SET_THR_AFFINITY__(SETP)				\
+     (sched_setaffinity(0, sizeof(cpu_set_t), (SETP)) != 0 ? -errno : 0)
+#elif defined(__WIN32__)
+#  define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__
+#  define cpu_set_t DWORD
+#  define CPU_SETSIZE (sizeof(DWORD)*8)
+#  define CPU_ZERO(SETP) (*(SETP) = (DWORD) 0)
+#  define CPU_SET(CPU, SETP) (*(SETP) |= (((DWORD) 1) << (CPU)))
+#  define CPU_CLR(CPU, SETP) (*(SETP) &= ~(((DWORD) 1) << (CPU)))
+#  define CPU_ISSET(CPU, SETP) ((*(SETP) & (((DWORD) 1) << (CPU))) != (DWORD) 0)
+#define ERTS_MU_GET_PROC_AFFINITY__ get_proc_affinity
+#define ERTS_MU_SET_THR_AFFINITY__ set_thr_affinity
 #endif
 #ifdef HAVE_PSET_INFO
 #  include <sys/pset.h>
@@ -105,25 +122,58 @@ struct erts_cpu_info_t_ {
     int available;
     int topology_size;
     erts_cpu_topology_t *topology;
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     char *affinity_str;
     char affinity_str_buf[CPU_SETSIZE/4+2];
     cpu_set_t cpuset;
+#if defined(HAVE_SCHED_xETAFFINITY)
     pid_t pid;
+#endif
 #elif defined(HAVE_PSET_INFO)
     processorid_t *cpuids;
 #endif
 };
 
+#if defined(__WIN32__)
+
+static __forceinline int
+get_proc_affinity(erts_cpu_info_t *cpuinfo)
+{
+    DWORD pamask, samask;
+    if (GetProcessAffinityMask(GetCurrentProcess(), &pamask, &samask)) {
+	cpuinfo->cpuset = (cpu_set_t) pamask;
+	return 0;
+    }
+    else {
+	cpuinfo->cpuset = (cpu_set_t) 0;
+	return -erts_get_last_win_errno();
+    }
+}
+
+static __forceinline int
+set_thr_affinity(cpu_set_t *set)
+{
+    if (*set == (cpu_set_t) 0)
+	return -ENOTSUP;
+    if (SetThreadAffinityMask(GetCurrentThread(), *set) == 0)
+	return -erts_get_last_win_errno();
+    else
+	return 0;
+}
+
+#endif
+
 erts_cpu_info_t *
 erts_cpu_info_create(void)
 {
     erts_cpu_info_t *cpuinfo = malloc(sizeof(erts_cpu_info_t));
     if (!cpuinfo)
 	return NULL;
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     cpuinfo->affinity_str = NULL;
+#if defined(HAVE_SCHED_xETAFFINITY)
     cpuinfo->pid = getpid();
+#endif
 #elif defined(HAVE_PSET_INFO)
     cpuinfo->cpuids = NULL;
 #endif
@@ -162,10 +212,13 @@ erts_cpu_info_update(erts_cpu_info_t *cpuinfo)
 
 #ifdef __WIN32__
     {
+	int i;
 	SYSTEM_INFO sys_info;
 	GetSystemInfo(&sys_info);
 	cpuinfo->configured = (int) sys_info.dwNumberOfProcessors;
-	
+	for (i = 0; i < sizeof(DWORD)*8; i++)
+	    if (sys_info.dwActiveProcessorMask & (((DWORD) 1) << i))
+		cpuinfo->online++;
     }
 #elif !defined(NO_SYSCONF) && (defined(_SC_NPROCESSORS_CONF) \
 			       || defined(_SC_NPROCESSORS_ONLN))
@@ -205,8 +258,8 @@ erts_cpu_info_update(erts_cpu_info_t *cpuinfo)
     if (cpuinfo->online > cpuinfo->configured)
 	cpuinfo->online = cpuinfo->configured;
 
-#ifdef HAVE_SCHED_xETAFFINITY
-    if (sched_getaffinity(cpuinfo->pid, sizeof(cpu_set_t), &cpuinfo->cpuset) == 0) {
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+    if (ERTS_MU_GET_PROC_AFFINITY__(cpuinfo) == 0) {
 	int i, c, cn, si;
 	c = cn = 0;
 	si = sizeof(cpuinfo->affinity_str_buf) - 1;
@@ -289,7 +342,7 @@ erts_get_cpu_available(erts_cpu_info_t *cpuinfo)
 char *
 erts_get_unbind_from_cpu_str(erts_cpu_info_t *cpuinfo)
 {
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     if (!cpuinfo)
 	return "false";
     return cpuinfo->affinity_str;
@@ -303,7 +356,7 @@ erts_get_available_cpu(erts_cpu_info_t *cpuinfo, int no)
 {
     if (!cpuinfo || no < 1 || cpuinfo->available < no)
 	return -EINVAL;
-#ifdef HAVE_SCHED_xETAFFINITY
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     {
 	cpu_set_t *allowed = &cpuinfo->cpuset;
 	int ix, n;
@@ -335,8 +388,8 @@ int
 erts_is_cpu_available(erts_cpu_info_t *cpuinfo, int id)
 {
     if (cpuinfo && 0 <= id) {
-#ifdef HAVE_SCHED_xETAFFINITY
-	if (id <= CPU_SETSIZE)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+	if (id < CPU_SETSIZE)
 	    return CPU_ISSET(id, &cpuinfo->cpuset);
 #elif defined(HAVE_PROCESSOR_BIND)
 	int no;
@@ -388,7 +441,7 @@ erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu)
      */
     if (!cpuinfo)
 	return -EINVAL;
-#ifdef HAVE_SCHED_xETAFFINITY
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     {
 	cpu_set_t bind_set;
 	if (cpu < 0)
@@ -398,9 +451,7 @@ erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu)
 
 	CPU_ZERO(&bind_set);
 	CPU_SET(cpu, &bind_set);
-	if (sched_setaffinity(0, sizeof(cpu_set_t), &bind_set) != 0)
-	    return -errno;
-	return 0;
+	return ERTS_MU_SET_THR_AFFINITY__(&bind_set);
     }
 #elif defined(HAVE_PROCESSOR_BIND)
     if (cpu < 0)
@@ -418,10 +469,8 @@ erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo)
 {
     if (!cpuinfo)
 	return -EINVAL;
-#if defined(HAVE_SCHED_xETAFFINITY)
-    if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuinfo->cpuset) != 0)
-	return -errno;
-    return 0;
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
+    return ERTS_MU_SET_THR_AFFINITY__(&cpuinfo->cpuset);
 #elif defined(HAVE_PROCESSOR_BIND)
     if (processor_bind(P_LWPID, P_MYID, PBIND_NONE, NULL) != 0)
 	return -errno;
@@ -434,7 +483,7 @@ erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo)
 int
 erts_unbind_from_cpu_str(char *str)
 {
-#if defined(HAVE_SCHED_xETAFFINITY)
+#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__)
     char *c = str;
     int cpus = 0;
     int shft = 0;
@@ -486,9 +535,7 @@ erts_unbind_from_cpu_str(char *str)
     if (!cpus)
 	return -EINVAL;
 
-    if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuset) != 0)
-	return -errno;
-    return 0;
+    return ERTS_MU_SET_THR_AFFINITY__(&cpuset);
 #elif defined(HAVE_PROCESSOR_BIND)
     if (processor_bind(P_LWPID, P_MYID, PBIND_NONE, NULL) != 0)
 	return -errno;
@@ -965,3 +1012,91 @@ read_topology(erts_cpu_info_t *cpuinfo)
 }
 
 #endif
+
+#if defined(__WIN32__)
+
+int
+erts_get_last_win_errno(void)
+{
+    switch (GetLastError()) {
+    case ERROR_INVALID_FUNCTION:		return EINVAL;	/* 1	*/
+    case ERROR_FILE_NOT_FOUND:			return ENOENT;	/* 2	*/
+    case ERROR_PATH_NOT_FOUND:			return ENOENT;	/* 3	*/
+    case ERROR_TOO_MANY_OPEN_FILES:		return EMFILE;	/* 4	*/
+    case ERROR_ACCESS_DENIED:			return EACCES;	/* 5	*/
+    case ERROR_INVALID_HANDLE:			return EBADF;	/* 6	*/
+    case ERROR_ARENA_TRASHED:			return ENOMEM;	/* 7	*/
+    case ERROR_NOT_ENOUGH_MEMORY:		return ENOMEM;	/* 8	*/
+    case ERROR_INVALID_BLOCK:			return ENOMEM;	/* 9	*/
+    case ERROR_BAD_ENVIRONMENT:			return E2BIG;	/* 10	*/
+    case ERROR_BAD_FORMAT:			return ENOEXEC;	/* 11	*/
+    case ERROR_INVALID_ACCESS:			return EINVAL;	/* 12	*/
+    case ERROR_INVALID_DATA:			return EINVAL;	/* 13	*/
+    case ERROR_OUTOFMEMORY:			return ENOMEM;	/* 14	*/
+    case ERROR_INVALID_DRIVE:			return ENOENT;	/* 15	*/
+    case ERROR_CURRENT_DIRECTORY:		return EACCES;	/* 16	*/
+    case ERROR_NOT_SAME_DEVICE:			return EXDEV;	/* 17	*/
+    case ERROR_NO_MORE_FILES:			return ENOENT;	/* 18	*/
+    case ERROR_WRITE_PROTECT:			return EACCES;	/* 19	*/
+    case ERROR_BAD_UNIT:			return EACCES;	/* 20	*/
+    case ERROR_NOT_READY:			return EACCES;	/* 21	*/
+    case ERROR_BAD_COMMAND:			return EACCES;	/* 22	*/
+    case ERROR_CRC:				return EACCES;	/* 23	*/
+    case ERROR_BAD_LENGTH:			return EACCES;	/* 24	*/
+    case ERROR_SEEK:				return EACCES;	/* 25	*/
+    case ERROR_NOT_DOS_DISK:			return EACCES;	/* 26	*/
+    case ERROR_SECTOR_NOT_FOUND:		return EACCES;	/* 27	*/
+    case ERROR_OUT_OF_PAPER:			return EACCES;	/* 28	*/
+    case ERROR_WRITE_FAULT:			return EACCES;	/* 29	*/
+    case ERROR_READ_FAULT:			return EACCES;	/* 30	*/
+    case ERROR_GEN_FAILURE:			return EACCES;	/* 31	*/
+    case ERROR_SHARING_VIOLATION:		return EACCES;	/* 32	*/
+    case ERROR_LOCK_VIOLATION:			return EACCES;	/* 33	*/
+    case ERROR_WRONG_DISK:			return EACCES;	/* 34	*/
+    case ERROR_SHARING_BUFFER_EXCEEDED:		return EACCES;	/* 36	*/
+    case ERROR_BAD_NETPATH:			return ENOENT;	/* 53	*/
+    case ERROR_NETWORK_ACCESS_DENIED:		return EACCES;	/* 65	*/
+    case ERROR_BAD_NET_NAME:			return ENOENT;	/* 67	*/
+    case ERROR_FILE_EXISTS:			return EEXIST;	/* 80	*/
+    case ERROR_CANNOT_MAKE:			return EACCES;	/* 82	*/
+    case ERROR_FAIL_I24:			return EACCES;	/* 83	*/
+    case ERROR_INVALID_PARAMETER:		return EINVAL;	/* 87	*/
+    case ERROR_NO_PROC_SLOTS:			return EAGAIN;	/* 89	*/
+    case ERROR_DRIVE_LOCKED:			return EACCES;	/* 108	*/
+    case ERROR_BROKEN_PIPE:			return EPIPE;	/* 109	*/
+    case ERROR_DISK_FULL:			return ENOSPC;	/* 112	*/
+    case ERROR_INVALID_TARGET_HANDLE:		return EBADF;	/* 114	*/
+    case ERROR_WAIT_NO_CHILDREN:		return ECHILD;	/* 128	*/
+    case ERROR_CHILD_NOT_COMPLETE:		return ECHILD;	/* 129	*/
+    case ERROR_DIRECT_ACCESS_HANDLE:		return EBADF;	/* 130	*/
+    case ERROR_NEGATIVE_SEEK:			return EINVAL;	/* 131	*/
+    case ERROR_SEEK_ON_DEVICE:			return EACCES;	/* 132	*/
+    case ERROR_DIR_NOT_EMPTY:			return ENOTEMPTY;/* 145	*/
+    case ERROR_NOT_LOCKED:			return EACCES;	/* 158	*/
+    case ERROR_BAD_PATHNAME:			return ENOENT;	/* 161	*/
+    case ERROR_MAX_THRDS_REACHED:		return EAGAIN;	/* 164	*/
+    case ERROR_LOCK_FAILED:			return EACCES;	/* 167	*/
+    case ERROR_ALREADY_EXISTS:			return EEXIST;	/* 183	*/
+    case ERROR_INVALID_STARTING_CODESEG:	return ENOEXEC;	/* 188	*/
+    case ERROR_INVALID_STACKSEG:		return ENOEXEC;	/* 189	*/
+    case ERROR_INVALID_MODULETYPE:		return ENOEXEC;	/* 190	*/
+    case ERROR_INVALID_EXE_SIGNATURE:		return ENOEXEC;	/* 191	*/
+    case ERROR_EXE_MARKED_INVALID:		return ENOEXEC;	/* 192	*/
+    case ERROR_BAD_EXE_FORMAT:			return ENOEXEC;	/* 193	*/
+    case ERROR_ITERATED_DATA_EXCEEDS_64k:	return ENOEXEC;	/* 194	*/
+    case ERROR_INVALID_MINALLOCSIZE:		return ENOEXEC;	/* 195	*/
+    case ERROR_DYNLINK_FROM_INVALID_RING:	return ENOEXEC;	/* 196	*/
+    case ERROR_IOPL_NOT_ENABLED:		return ENOEXEC;	/* 197	*/
+    case ERROR_INVALID_SEGDPL:			return ENOEXEC;	/* 198	*/
+    case ERROR_AUTODATASEG_EXCEEDS_64k:		return ENOEXEC;	/* 199	*/
+    case ERROR_RING2SEG_MUST_BE_MOVABLE:	return ENOEXEC;	/* 200	*/
+    case ERROR_RELOC_CHAIN_XEEDS_SEGLIM:	return ENOEXEC;	/* 201	*/
+    case ERROR_INFLOOP_IN_RELOC_CHAIN:		return ENOEXEC;	/* 202	*/
+    case ERROR_FILENAME_EXCED_RANGE:		return ENOENT;	/* 206	*/
+    case ERROR_NESTING_NOT_ALLOWED:		return EAGAIN;	/* 215	*/
+    case ERROR_NOT_ENOUGH_QUOTA:		return ENOMEM;	/* 1816	*/
+    default:					return EINVAL;
+    }
+}
+
+#endif
diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c
new file mode 100644
index 0000000000..4db4cffd3a
--- /dev/null
+++ b/erts/lib_src/common/ethr_aux.c
@@ -0,0 +1,762 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: A Thread library for use in the ERTS and other OTP
+ *              applications.
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_AUX_IMPL__
+
+#include "ethread.h"
+#include "ethr_internal.h"
+#include <string.h>
+#include <limits.h>
+
+#ifndef __WIN32__
+#include <unistd.h>
+#endif
+
+#define ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE 100
+#define ERTS_TS_EV_ALLOC_POOL_SIZE 25
+
+erts_cpu_info_t *ethr_cpu_info__;
+
+int ethr_not_completely_inited__ = 1;
+int ethr_not_inited__ = 1;
+
+ethr_memory_allocators ethr_mem__ = ETHR_MEM_ALLOCS_DEF_INITER__;
+
+#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
+ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
+#endif
+
+void *(*ethr_thr_prepare_func__)(void) = NULL;
+void (*ethr_thr_parent_func__)(void *) = NULL;
+void (*ethr_thr_child_func__)(void *) = NULL;
+
+typedef struct ethr_xhndl_list_ ethr_xhndl_list;
+struct ethr_xhndl_list_ {
+    ethr_xhndl_list *next;
+    void (*funcp)(void);
+};
+
+size_t ethr_pagesize__;
+size_t ethr_min_stack_size__; /* kilo words */
+size_t ethr_max_stack_size__; /* kilo words */
+
+ethr_rwmutex xhndl_rwmtx;
+ethr_xhndl_list *xhndl_list;
+
+static int main_threads;
+
+static int init_ts_event_alloc(void);
+
+int
+ethr_init_common__(ethr_init_data *id)
+{
+    int res;
+    if (id) {
+	ethr_thr_prepare_func__	= id->thread_create_prepare_func;
+	ethr_thr_parent_func__	= id->thread_create_parent_func;
+	ethr_thr_child_func__	= id->thread_create_child_func;
+    }
+
+    ethr_cpu_info__ = erts_cpu_info_create();
+    if (!ethr_cpu_info__)
+	return ENOMEM;
+
+#ifdef _SC_PAGESIZE
+    ethr_pagesize__ = (size_t) sysconf(_SC_PAGESIZE);
+#elif defined(HAVE_GETPAGESIZE)
+    ethr_pagesize__ = (size_t) getpagesize();
+#else
+    ethr_pagesize__ = (size_t) 4*1024; /* Guess 4 KB */
+#endif
+
+    /* User needs at least 4 KB */
+    ethr_min_stack_size__ = 4*1024;
+#if SIZEOF_VOID_P == 8
+    /* Double that on 64-bit archs */
+    ethr_min_stack_size__ *= 2;
+#endif
+    /* On some systems as much as about 4 KB is used by the system */
+    ethr_min_stack_size__ += 4*1024;
+    /* There should be room for signal handlers */
+#ifdef SIGSTKSZ
+    ethr_min_stack_size__ += SIGSTKSZ;
+#else
+    ethr_min_stack_size__ += ethr_pagesize__;
+#endif
+    /* The system may think that we need more stack */
+#if defined(PTHREAD_STACK_MIN)
+    if (ethr_min_stack_size__ < PTHREAD_STACK_MIN)
+	ethr_min_stack_size__ = PTHREAD_STACK_MIN;
+#elif defined(_SC_THREAD_STACK_MIN)
+    {
+	size_t thr_min_stk_sz = (size_t) sysconf(_SC_THREAD_STACK_MIN);
+	if (ethr_min_stack_size__ < thr_min_stk_sz)
+	    ethr_min_stack_size__ = thr_min_stk_sz;
+    }
+#endif
+    /* The guard is at least on some platforms included in the stack size
+       passed when creating threads */
+#ifdef ETHR_STACK_GUARD_SIZE
+    ethr_min_stack_size__ += ETHR_STACK_GUARD_SIZE;
+#endif
+    ethr_min_stack_size__ = ETHR_PAGE_ALIGN(ethr_min_stack_size__);
+
+    ethr_min_stack_size__ = ETHR_B2KW(ethr_min_stack_size__);
+
+    ethr_max_stack_size__ = 32*1024*1024;
+#if SIZEOF_VOID_P == 8
+    ethr_max_stack_size__ *= 2;
+#endif
+    ethr_max_stack_size__ = ETHR_B2KW(ethr_max_stack_size__);
+
+#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
+    {
+	int i;
+	for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) {
+	    res = ethr_spinlock_init(&ethr_atomic_protection__[i].u.lck);
+	    if (res != 0)
+		return res;
+	}
+    }
+#endif
+
+    res = ethr_mutex_lib_init(erts_get_cpu_configured(ethr_cpu_info__));
+    if (res != 0)
+	return res;
+
+    xhndl_list = NULL;
+
+    return 0;
+}
+
+int
+ethr_late_init_common__(ethr_late_init_data *lid)
+{
+    ethr_ts_event *tsep = NULL;
+    int reader_groups;
+    int res;
+    int i;
+    ethr_memory_allocator *m[] = {&ethr_mem__.std,
+				  &ethr_mem__.sl,
+				  &ethr_mem__.ll};
+    if (lid)
+	ethr_mem__ = lid->mem;
+    if (!ethr_mem__.std.alloc
+	|| !ethr_mem__.std.realloc
+	|| !ethr_mem__.std.free) {
+	ethr_mem__.std.alloc = malloc;
+	ethr_mem__.std.realloc = realloc;
+	ethr_mem__.std.free = free;
+    }
+    for (i = 0; i < sizeof(m)/sizeof(m[0]); i++) {
+	if (!m[i]->alloc || !m[i]->realloc || !m[i]->free) {
+	    m[i]->alloc = ethr_mem__.std.alloc;
+	    m[i]->realloc = ethr_mem__.std.realloc;
+	    m[i]->free = ethr_mem__.std.free;
+	}
+
+    }
+    res = init_ts_event_alloc();
+    if (res != 0)
+	return res;
+    res = ethr_make_ts_event__(&tsep);
+    if (res == 0)
+	tsep->iflgs |= ETHR_TS_EV_ETHREAD;
+    if (!lid) {
+	main_threads = 0;
+	reader_groups = 0;
+    }
+    else {
+	if (lid->main_threads < 0 || USHRT_MAX < lid->main_threads)
+	    return res;
+	main_threads = lid->main_threads;
+	reader_groups = lid->reader_groups;
+    }
+    res = ethr_mutex_lib_late_init(reader_groups, main_threads);
+    if (res != 0)
+	return res;
+    ethr_not_completely_inited__ = 0; /* Need it for
+					 rwmutex_init */
+    res = ethr_rwmutex_init(&xhndl_rwmtx);
+    ethr_not_completely_inited__ = 1;
+    if (res != 0)
+	return res;
+    return 0;
+}
+
+int
+ethr_install_exit_handler(void (*funcp)(void))
+{
+    ethr_xhndl_list *xhp;
+
+#if ETHR_XCHK
+    if (ethr_not_completely_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+
+    if (!funcp)
+	return EINVAL;
+
+    xhp = (ethr_xhndl_list *) ethr_mem__.std.alloc(sizeof(ethr_xhndl_list));
+    if (!xhp)
+	return ENOMEM;
+
+    ethr_rwmutex_rwlock(&xhndl_rwmtx);
+
+    xhp->funcp = funcp;
+    xhp->next = xhndl_list;
+    xhndl_list = xhp;
+
+    ethr_rwmutex_rwunlock(&xhndl_rwmtx);
+
+    return 0;
+}
+
+void
+ethr_run_exit_handlers__(void)
+{
+    ethr_xhndl_list *xhp;
+
+    ethr_rwmutex_rlock(&xhndl_rwmtx);
+
+    xhp = xhndl_list;
+
+    ethr_rwmutex_runlock(&xhndl_rwmtx);
+
+    for (; xhp; xhp = xhp->next)
+	(*xhp->funcp)();
+}
+
+/*
+ * Thread specific event alloc, etc.
+ *
+ * Note that we don't know when it is safe to destroy an event, but
+ * we know when it is safe to reuse it. ts_event_free() therefore
+ * never destroys an event (but makes freed events available for
+ * reuse).
+ *
+ * We could easily keep track of the usage of events, and by this
+ * make it possible to destroy events. We would however suffer a
+ * performance penalty for this and save very little memory.
+ */
+
+typedef union {
+    ethr_ts_event ts_ev;
+    char align[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_ts_event))];
+} ethr_aligned_ts_event;
+
+static ethr_spinlock_t ts_ev_alloc_lock;
+static ethr_ts_event *free_ts_ev;
+
+#if SIZEOF_VOID_P == SIZEOF_INT
+typedef unsigned int EthrPtrSzUInt;
+#elif  SIZEOF_VOID_P == SIZEOF_LONG
+typedef unsigned long EthrPtrSzUInt;
+#else
+#error No pointer sized integer type
+#endif
+
+static ethr_ts_event *ts_event_pool(int size, ethr_ts_event **endpp)
+{
+    int i;
+    ethr_aligned_ts_event *atsev;
+    atsev = ethr_mem__.std.alloc(sizeof(ethr_aligned_ts_event) * size
+				 + ETHR_CACHE_LINE_SIZE);
+    if (!atsev)
+	return NULL;
+    if ((((EthrPtrSzUInt) atsev) & ETHR_CACHE_LINE_MASK) == 0)
+	atsev = ((ethr_aligned_ts_event *)
+		 ((((EthrPtrSzUInt) atsev) & ~ETHR_CACHE_LINE_MASK)
+		  + ETHR_CACHE_LINE_SIZE));
+    for (i = 1; i < size; i++) {
+	atsev[i-1].ts_ev.next = &atsev[i].ts_ev;
+	ethr_atomic_init(&atsev[i-1].ts_ev.uaflgs, 0);
+	atsev[i-1].ts_ev.iflgs = 0;
+    }
+    ethr_atomic_init(&atsev[size-1].ts_ev.uaflgs, 0);
+    atsev[size-1].ts_ev.iflgs = 0;
+    atsev[size-1].ts_ev.next = NULL;
+    if (endpp)
+	*endpp = &atsev[size-1].ts_ev;
+    return &atsev[0].ts_ev;
+}
+
+static int init_ts_event_alloc(void)
+{
+    free_ts_ev = ts_event_pool(ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE,
+			       NULL);
+    if (!free_ts_ev)
+	return ENOMEM;
+    return ethr_spinlock_init(&ts_ev_alloc_lock);
+}
+
+static ethr_ts_event *ts_event_alloc(void)
+{
+    ethr_ts_event *ts_ev;
+    ethr_spin_lock(&ts_ev_alloc_lock);
+    if (free_ts_ev) {
+	ts_ev = free_ts_ev;
+	free_ts_ev = ts_ev->next;
+	ethr_spin_unlock(&ts_ev_alloc_lock);
+    }
+    else {
+	ethr_ts_event *ts_ev_pool_end;
+	ethr_spin_unlock(&ts_ev_alloc_lock);
+
+	ts_ev = ts_event_pool(ERTS_TS_EV_ALLOC_POOL_SIZE, &ts_ev_pool_end);
+	if (!ts_ev)
+	    return NULL;
+
+	ethr_spin_lock(&ts_ev_alloc_lock);
+	ts_ev_pool_end->next = free_ts_ev;
+	free_ts_ev = ts_ev->next;
+	ethr_spin_unlock(&ts_ev_alloc_lock);
+    }
+    return ts_ev;
+}
+
+static void ts_event_free(ethr_ts_event *ts_ev)
+{
+    ETHR_ASSERT(!ts_ev->udata);
+    ethr_spin_lock(&ts_ev_alloc_lock);
+    ts_ev->next = free_ts_ev;
+    free_ts_ev = ts_ev;
+    ethr_spin_unlock(&ts_ev_alloc_lock);
+}
+
+int ethr_make_ts_event__(ethr_ts_event **tsepp)
+{
+    int res;
+    ethr_ts_event *tsep = *tsepp;
+
+    if (!tsep) {
+	tsep = ts_event_alloc();
+	if (!tsep)
+	    return ENOMEM;
+    }
+
+    if ((tsep->iflgs & ETHR_TS_EV_INITED) == 0) {
+	res = ethr_event_init(&tsep->event);
+	if (res != 0) {
+	    ts_event_free(tsep);
+	    return res;
+	}
+    }
+
+    tsep->iflgs = ETHR_TS_EV_INITED;
+    tsep->udata = NULL;
+    tsep->rgix = 0;
+    tsep->mtix = 0;
+
+    res = ethr_set_tse__(tsep);
+    if (res != 0 && tsepp && *tsepp) {
+	ts_event_free(tsep);
+	return res;
+    }
+
+    if (tsepp)
+	*tsepp = tsep;
+
+    return 0;
+}
+
+int ethr_get_tmp_ts_event__(ethr_ts_event **tsepp)
+{
+    int res;
+    ethr_ts_event *tsep = *tsepp;
+
+    if (!tsep) {
+	tsep = ts_event_alloc();
+	if (!tsep)
+	    return ENOMEM;
+    }
+
+    if ((tsep->iflgs & ETHR_TS_EV_INITED) == 0) {
+	res = ethr_event_init(&tsep->event);
+	if (res != 0) {
+	    ts_event_free(tsep);
+	    return res;
+	}
+    }
+
+    tsep->iflgs = ETHR_TS_EV_INITED|ETHR_TS_EV_TMP;
+    tsep->udata = NULL;
+
+    if (tsepp)
+	*tsepp = tsep;
+
+    return 0;
+}
+
+int ethr_free_ts_event__(ethr_ts_event *tsep)
+{
+    ts_event_free(tsep);
+    return 0;
+}
+
+void ethr_ts_event_destructor__(void *vtsep)
+{
+    if (vtsep) {
+	ethr_ts_event *tsep = (ethr_ts_event *) vtsep;
+	ts_event_free(tsep);
+	ethr_set_tse__(NULL);
+    }
+}
+
+int ethr_set_main_thr_status(int on, int no)
+{
+    ethr_ts_event *tsep = ethr_get_tse__();
+    if (!tsep)
+	return EINVAL;
+    if (on) {
+	if (no < 1 || main_threads < no)
+	    return EINVAL;
+	tsep->mtix = (unsigned short) no;
+	tsep->iflgs |= ETHR_TS_EV_MAIN_THR;
+    }
+    else {
+	tsep->iflgs &= ~ETHR_TS_EV_MAIN_THR;
+	tsep->mtix = (unsigned short) 0;
+    }
+    return 0;
+}
+
+int ethr_get_main_thr_status(int *on)
+{
+    ethr_ts_event *tsep = ethr_get_tse__();
+    if (!tsep)
+	*on = 0;
+    else {
+	if (tsep->iflgs & ETHR_TS_EV_MAIN_THR)
+	    *on = 1;
+	else
+	    *on = 0;
+    }
+    return 0;
+}
+
+
+/* Atomics */
+
+void
+ethr_atomic_init(ethr_atomic_t *var, long i)
+{
+    ETHR_ASSERT(var);
+    ethr_atomic_init__(var, i);
+}
+
+void
+ethr_atomic_set(ethr_atomic_t *var, long i)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_set__(var, i);
+}
+
+long
+ethr_atomic_read(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_read__(var);
+}
+
+
+long
+ethr_atomic_add_read(ethr_atomic_t *var, long incr)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_add_read__(var, incr);
+}   
+
+long
+ethr_atomic_inc_read(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_inc_read__(var);
+}
+
+long
+ethr_atomic_dec_read(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_dec_read__(var);
+}
+
+void
+ethr_atomic_add(ethr_atomic_t *var, long incr)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_add__(var, incr);
+}   
+    
+void
+ethr_atomic_inc(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_inc__(var);
+}
+
+void
+ethr_atomic_dec(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_dec__(var);
+}
+
+long
+ethr_atomic_read_band(ethr_atomic_t *var, long mask)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_read_band__(var, mask);
+}
+
+long
+ethr_atomic_read_bor(ethr_atomic_t *var, long mask)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_read_bor__(var, mask);
+}
+
+long
+ethr_atomic_xchg(ethr_atomic_t *var, long new)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_xchg__(var, new);
+}   
+
+long
+ethr_atomic_cmpxchg(ethr_atomic_t *var, long new, long expected)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_cmpxchg__(var, new, expected);
+}
+
+long
+ethr_atomic_read_acqb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_read_acqb__(var);
+}
+
+long
+ethr_atomic_inc_read_acqb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_inc_read_acqb__(var);
+}
+
+void
+ethr_atomic_set_relb(ethr_atomic_t *var, long i)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_set_relb__(var, i);
+}
+
+void
+ethr_atomic_dec_relb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    ethr_atomic_dec_relb__(var);
+}
+
+long
+ethr_atomic_dec_read_relb(ethr_atomic_t *var)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_dec_read_relb__(var);
+}
+
+long
+ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, long new, long exp)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_cmpxchg_acqb__(var, new, exp);
+}
+
+long
+ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, long new, long exp)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(var);
+    return ethr_atomic_cmpxchg_relb__(var, new, exp);
+}
+
+
+/* Spinlocks and rwspinlocks */
+
+int
+ethr_spinlock_init(ethr_spinlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_spinlock_init__(lock);
+}
+
+int
+ethr_spinlock_destroy(ethr_spinlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_spinlock_destroy__(lock);
+}
+
+void
+ethr_spin_unlock(ethr_spinlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_spin_unlock__(lock);
+}
+
+void
+ethr_spin_lock(ethr_spinlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_spin_lock__(lock);
+}
+
+int
+ethr_rwlock_init(ethr_rwlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_rwlock_init__(lock);
+}
+
+int
+ethr_rwlock_destroy(ethr_rwlock_t *lock)
+{
+#if ETHR_XCHK 
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!lock) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return ethr_rwlock_destroy__(lock);
+}
+
+void
+ethr_read_unlock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_read_unlock__(lock);
+}
+
+void
+ethr_read_lock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_read_lock__(lock);
+}
+
+void
+ethr_write_unlock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_write_unlock__(lock);
+}
+
+void
+ethr_write_lock(ethr_rwlock_t *lock)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(lock);
+    ethr_write_lock__(lock);
+}
+
+ETHR_IMPL_NORETURN__ ethr_fatal_error__(const char *file,
+					int line,
+					const char *func,
+					int err)
+{
+    char *errstr;
+    if (err == ENOTSUP)
+	errstr = "Operation not supported";
+    else {
+	errstr = strerror(err);
+	if (!errstr)
+	    errstr = "Unknown error";
+    }
+    fprintf(stderr, "%s:%d: Fatal error in %s(): %s (%d)\n",
+	    file, line, func, errstr, err);
+    ethr_abort__();
+}
+
+int ethr_assert_failed(const char *file, int line, const char *func, char *a)
+{
+    fprintf(stderr, "%s:%d: %s(): Assertion failed: %s\n", file, line, func, a);
+    ethr_abort__();
+    return 0;
+}
diff --git a/erts/lib_src/common/ethr_cbf.c b/erts/lib_src/common/ethr_cbf.c
new file mode 100644
index 0000000000..04feceec89
--- /dev/null
+++ b/erts/lib_src/common/ethr_cbf.c
@@ -0,0 +1,36 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+
+/*
+ * We keep this function alone in a separate file so the
+ * compiler wont optimize it away.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "ethread.h"
+
+void
+ethr_compiler_barrier_fallback(void)
+{
+
+}
diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c
new file mode 100644
index 0000000000..f918bba81d
--- /dev/null
+++ b/erts/lib_src/common/ethr_mutex.c
@@ -0,0 +1,2758 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Mutex, rwmutex and condition variable implementation
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_MUTEX_IMPL__
+
+#include <limits.h>
+#include "ethread.h"
+#include "ethr_internal.h"
+
+#define ETHR_SPIN_WITH_WAITERS 1
+
+#define ETHR_MTX_MAX_FLGS_SPIN 1000
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+static int default_rwmtx_main_spincount;
+static int default_rwmtx_aux_spincount;
+#endif
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+static int default_mtx_main_spincount;
+static int default_mtx_aux_spincount;
+static int default_cnd_main_spincount;
+static int default_cnd_aux_spincount;
+#endif
+
+static int no_spin;
+
+#ifndef ETHR_USE_OWN_RWMTX_IMPL__
+static pthread_rwlockattr_t write_pref_attr_data;
+static pthread_rwlockattr_t *write_pref_attr;
+#endif
+
+#if defined(ETHR_MTX_Q_LOCK_SPINLOCK__)
+#  define ETHR_MTX_QLOCK_INIT ethr_spinlock_init
+#  define ETHR_MTX_QLOCK_DESTROY ethr_spinlock_destroy
+#  define ETHR_MTX_Q_LOCK ethr_spin_lock
+#  define ETHR_MTX_Q_UNLOCK ethr_spin_unlock
+#elif defined(ETHR_MTX_Q_LOCK_PTHREAD_MUTEX__)
+#  define ETHR_MTX_QLOCK_INIT(QL) pthread_mutex_init((QL), NULL)
+#  define ETHR_MTX_QLOCK_DESTROY pthread_mutex_destroy
+#  define ETHR_MTX_Q_LOCK(L)					\
+do {								\
+    int res__ = pthread_mutex_lock(L);				\
+    if (res__ != 0)						\
+	ETHR_FATAL_ERROR__(res__);				\
+} while (0)
+#  define ETHR_MTX_Q_UNLOCK(L)					\
+do {								\
+    int res__ = pthread_mutex_unlock(L);			\
+    if (res__ != 0)						\
+	ETHR_FATAL_ERROR__(res__);				\
+} while (0)
+#elif defined(ETHR_MTX_Q_LOCK_CRITICAL_SECTION__)
+#  define ETHR_MTX_QLOCK_INIT(QL) (InitializeCriticalSection((QL)), 0)
+#  define ETHR_MTX_QLOCK_DESTROY(QL) (DeleteCriticalSection((QL)), 0)
+#  define ETHR_MTX_Q_LOCK(QL) EnterCriticalSection((QL))
+#  define ETHR_MTX_Q_UNLOCK(QL) LeaveCriticalSection((QL))
+#endif
+
+int
+ethr_mutex_lib_init(int cpu_conf)
+{
+    int res = 0;
+
+    no_spin = cpu_conf == 1;
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+    default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT;
+    default_mtx_aux_spincount = ETHR_MTX_DEFAULT_AUX_SPINCOUNT;
+    default_cnd_main_spincount = ETHR_CND_DEFAULT_MAIN_SPINCOUNT;
+    default_cnd_aux_spincount = ETHR_CND_DEFAULT_AUX_SPINCOUNT;
+#endif
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+    default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT;
+    default_rwmtx_aux_spincount = ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT;
+
+#else
+
+#if defined(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP) \
+    && defined(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)
+    res = pthread_rwlockattr_init(&write_pref_attr_data);
+    if (res != 0)
+	return res;
+    res = pthread_rwlockattr_setkind_np(
+	&write_pref_attr_data,
+	PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
+    write_pref_attr = &write_pref_attr_data;
+#else
+    write_pref_attr = NULL;
+#endif
+
+#endif
+
+    return res;
+}
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+
+#ifdef ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS
+#if 0 /*
+       * When inc and dec are real atomic instructions as on x86, the
+       * ETHR_RLOCK_WITH_INC_DEC implementations performs better with
+       * lots of read locks compared to the cmpxchg based implementation.
+       * It, however, performs worse with lots of mixed reads and writes.
+       * It could be used for rwlocks that are known to be read locked
+       * much, but the readers array based implementation outperforms it
+       * by far. Therefore, it has been disabled, and will probably be
+       * removed some time in the future.
+       */
+#  define ETHR_RLOCK_WITH_INC_DEC
+#endif
+#endif
+
+static int reader_groups_array_size = 0;
+static int main_threads_array_size = 0;
+
+#endif
+
+int
+ethr_mutex_lib_late_init(int no_reader_groups, int no_main_threads)
+{
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+    reader_groups_array_size = (no_reader_groups <= 1
+				? 1
+				: no_reader_groups + 1);
+    main_threads_array_size = (no_main_threads <= 1
+			       ? 1
+			       : no_main_threads + 1);
+#endif
+    return 0;
+}
+
+int
+ethr_rwmutex_set_reader_group(int ix)
+{
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+    ethr_ts_event *tse;
+
+    if (ix < 0 || reader_groups_array_size <= ix)
+	return EINVAL;
+
+    tse = ethr_get_ts_event();
+
+    if ((tse->iflgs & ETHR_TS_EV_ETHREAD) == 0) {
+	ethr_leave_ts_event(tse);
+	return EINVAL;
+    }
+
+    tse->rgix = ix;
+    
+    ethr_leave_ts_event(tse);
+#endif
+    return 0;
+}
+
+#if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ)
+static void hard_debug_chk_q__(struct ethr_mutex_base_ *, int);
+#define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX) hard_debug_chk_q__(&(RWMTX)->mtxb,1)
+#define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX) hard_debug_chk_q__(&(MTX)->mtxb, 0)
+#else
+#define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX)
+#define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX)
+#endif
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+static void
+rwmutex_unlock_wake(ethr_rwmutex *rwmtx,
+		    int have_w,
+		    long initial);
+static int
+rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
+			     long initial,
+			     ethr_ts_event *tse,
+			     int start_next_ix,
+			     int check_before_try,
+			     int try_write_lock);
+#endif
+
+#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)
+
+/* -- Utilities operating both on ordinary mutexes and read write mutexes -- */
+
+static ETHR_INLINE void
+rwmutex_freqread_wtng_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix = (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ
+	      ? tse->rgix
+	      : tse->mtix);
+    rwmtx->tdata.ra[ix].data.waiting_readers++;
+}
+
+static ETHR_INLINE void
+rwmutex_freqread_rdrs_add(ethr_rwmutex *rwmtx,
+			  ethr_rwmutex_type type,
+			  int ix,
+			  int inc)
+{
+    if (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ || ix == 0)
+	ethr_atomic_add(&rwmtx->tdata.ra[ix].data.readers, inc);
+    else {
+	ETHR_ASSERT(type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
+	ETHR_ASSERT(inc == 1);
+	ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 1);
+    }
+}
+
+static ETHR_INLINE void
+rwmutex_freqread_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_inc:
+	ethr_atomic_inc(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_inc;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 0);
+	ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 1);
+    }
+}
+
+static ETHR_INLINE void
+rwmutex_freqread_rdrs_dec(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_dec:
+	ethr_atomic_dec(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_dec;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+	ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 0);
+    }
+}
+
+static ETHR_INLINE long
+rwmutex_freqread_rdrs_dec_read(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_dec_read:
+	return ethr_atomic_dec_read(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_dec_read;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+	ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 0);
+	return (long) 0;
+    }
+}
+
+static ETHR_INLINE long
+rwmutex_freqread_rdrs_dec_read_relb(ethr_rwmutex *rwmtx, ethr_ts_event *tse)
+{
+    int ix;
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	ix = tse->rgix;
+    atomic_dec_read:
+	return ethr_atomic_dec_read_relb(&rwmtx->tdata.ra[ix].data.readers);
+    }
+    else {
+	ix = tse->mtix;
+	if (ix == 0)
+	    goto atomic_dec_read;
+	ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ);
+	ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1);
+	ethr_atomic_set_relb(&rwmtx->tdata.ra[ix].data.readers, (long) 0);
+	return (long) 0;
+    }
+}
+
+static ETHR_INLINE long
+rwmutex_freqread_rdrs_read(ethr_rwmutex *rwmtx, int ix)
+{
+    long res = ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers);
+#ifdef ETHR_DEBUG
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+	ETHR_ASSERT(res >= 0);
+	break;
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+	ETHR_ASSERT(res == 0 || res == 1);
+	break;
+    default:
+	ETHR_ASSERT(0);
+	break;
+    }
+#endif
+    return res;
+}
+
+
+static ETHR_INLINE void
+enqueue(ethr_ts_event **queue,
+	ethr_ts_event *tse_start,
+	ethr_ts_event *tse_end)
+{
+    if (!*queue) {
+	*queue = tse_start;
+	tse_start->prev = tse_end;
+	tse_end->next = tse_start;
+    }
+    else {
+	tse_end->next = *queue;
+	tse_start->prev = (*queue)->prev;
+	(*queue)->prev->next = tse_start;
+	(*queue)->prev = tse_end;
+    }
+}
+
+static ETHR_INLINE void
+insert(ethr_ts_event *tse_pred, ethr_ts_event *tse)
+{
+    tse->next = tse_pred->next;
+    tse->prev = tse_pred;
+    tse_pred->next->prev = tse;
+    tse_pred->next = tse;
+}
+
+static ETHR_INLINE void
+dequeue(ethr_ts_event **queue,
+	ethr_ts_event *tse_start,
+	ethr_ts_event *tse_end)
+{
+    if (tse_start->prev == tse_end) {
+	ETHR_ASSERT(*queue == tse_start && tse_end->next == tse_start);
+	*queue = NULL;
+    }
+    else {
+	if (*queue == tse_start)
+	    *queue = tse_end->next;
+	tse_end->next->prev = tse_start->prev;
+	tse_start->prev->next = tse_end->next;
+    }
+}
+
+static void
+event_wait(struct ethr_mutex_base_ *mtxb,
+	   ethr_ts_event *tse,
+	   int spincount,
+	   long type,
+	   int is_rwmtx,
+	   int is_freq_read)
+{
+    int locked = 0;
+    long act;
+    int need_try_complete_runlock = 0;
+
+    /* Need to enqueue and wait... */
+
+    tse->uflgs = type;
+    ethr_atomic_set(&tse->uaflgs, type);
+
+    ETHR_MTX_Q_LOCK(&mtxb->qlck);
+    locked = 1;
+
+#ifdef ETHR_MTX_HARD_DEBUG_Q
+    hard_debug_chk_q__(mtxb, is_rwmtx);
+#endif
+
+    act = ethr_atomic_read(&mtxb->flgs);
+
+    if (act & type) {
+
+	/* Wait bit already there; enqueue... */
+
+	ETHR_ASSERT(mtxb->q);
+	if (type == ETHR_RWMTX_W_WAIT_FLG__) {
+	    enqueue(&mtxb->q, tse, tse);
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+	    mtxb->ws++;
+#endif
+	}
+	else {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_ASSERT(is_rwmtx);
+	    ETHR_ASSERT(rwmtx->rq_end);
+	    insert(rwmtx->rq_end, tse);
+	    rwmtx->rq_end = tse;
+	    if (is_freq_read)
+		rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse);
+	    else
+		rwmtx->tdata.rs++;
+	}
+    }
+    else {
+
+	/* Set wait bit */
+
+	while (1) {
+	    long new, exp = act;
+	    int freqread_tryrlock = 0;
+	    need_try_complete_runlock = 0;
+
+	    if (type == ETHR_RWMTX_W_WAIT_FLG__) {
+		if (is_freq_read && act == ETHR_RWMTX_R_FLG__)
+		    need_try_complete_runlock = 1;
+		if (act != 0)
+		    new = act | ETHR_RWMTX_W_WAIT_FLG__;
+		else
+		    new = ETHR_RWMTX_W_FLG__; /* Try to get it */
+	    }
+	    else {
+		ETHR_ASSERT(is_rwmtx);
+
+		if (!is_freq_read) {
+		    if (act & (ETHR_RWMTX_W_FLG__| ETHR_RWMTX_W_WAIT_FLG__))
+			new = act | ETHR_RWMTX_R_WAIT_FLG__;
+		    else
+			new = act + 1; /* Try to get it */
+		}
+		else {
+		    if (act & ~ETHR_RWMTX_R_FLG__)
+			new = act | ETHR_RWMTX_R_WAIT_FLG__;
+		    else { /* Try to get it */
+			ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+			rwmutex_freqread_rdrs_inc(rwmtx, tse);
+			ETHR_MEMORY_BARRIER;
+			new = act | ETHR_RWMTX_R_FLG__;
+			freqread_tryrlock = 1;
+		    }
+		}
+	    }
+
+	    act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs, new, exp);
+	    if (exp == act) {
+		if (new & type) {
+		    act = new;
+		    break;
+		}
+		else {
+		    /* Got it */
+		    goto done;
+		}
+	    }
+
+	    if (freqread_tryrlock) {
+		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+
+		/* We didn't set ETHR_RWMTX_R_FLG__, however someone
+		   else might have */
+		if (act == ETHR_RWMTX_R_FLG__)
+		    goto done; /* Got it by help from someone else */
+
+		ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
+		/*
+		 * We know that no waiter flags have been set, i.e.,
+		 * we cannot get into a situation where we need to wake
+		 * someone up here. Just restore the readers counter
+		 * and do it over again...
+		 */
+		rwmutex_freqread_rdrs_dec(rwmtx, tse);
+	    }
+	}
+
+	/* Enqueue */
+
+	if (type == ETHR_RWMTX_R_WAIT_FLG__) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_ASSERT(is_rwmtx);
+	    ETHR_ASSERT(!rwmtx->rq_end);
+	    rwmtx->rq_end =  tse;
+	    if (is_freq_read)
+		rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse);
+	    else
+		rwmtx->tdata.rs++;
+	}
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+	else {
+	    mtxb->ws++;
+	}
+#endif
+
+	enqueue(&mtxb->q, tse, tse);
+    }
+
+#ifdef ETHR_MTX_HARD_DEBUG_Q
+    hard_debug_chk_q__(mtxb, is_rwmtx);
+#endif
+
+    /* Wait */
+    locked = 0;
+    ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+
+    if (need_try_complete_runlock) {
+	ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type
+		    != ETHR_RWMUTEX_TYPE_NORMAL);
+	/*
+	 * We were the only one in queue when we enqueued, and it
+	 * was seemingly read locked. We need to try to complete a
+	 * runlock otherwise we might be hanging forever. If the
+	 * runlock could be completed we will be dequeued and
+	 * woken by ourselves.
+	 */
+	rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb,
+				     act, tse, 0, 1, 0);
+    }
+
+    while (1) {
+	ethr_event_reset(&tse->event);
+
+	act = ethr_atomic_read_acqb(&tse->uaflgs);
+	if (!act)
+	    goto done; /* Got it */
+
+	ETHR_ASSERT(act == type);
+	ethr_event_swait(&tse->event, spincount);
+	/* swait result: 0 || EINTR */
+
+	act = ethr_atomic_read_acqb(&tse->uaflgs);
+	if (!act)
+	    goto done; /* Got it */
+    }
+
+ done:
+    if (locked)
+	ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+}
+
+static void
+wake_writer(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
+{
+    ethr_ts_event *tse;
+
+    tse = mtxb->q;
+    ETHR_ASSERT(tse);
+    dequeue(&mtxb->q, tse, tse);
+
+    ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+    ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__);
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+    mtxb->ws--;
+#endif
+#if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ)
+    hard_debug_chk_q__(mtxb, is_rwmtx);
+#endif
+
+    ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+
+    ethr_atomic_set(&tse->uaflgs, 0);
+    ethr_event_set(&tse->event);
+}
+
+static ETHR_INLINE int
+initial_spincount(struct ethr_mutex_base_ *mtxb)
+{
+    return (mtxb->aux_scnt < ETHR_MTX_MAX_FLGS_SPIN
+	    ? mtxb->aux_scnt
+	    : ETHR_MTX_MAX_FLGS_SPIN);
+}
+
+static ETHR_INLINE int
+update_spincount(struct ethr_mutex_base_ *mtxb,
+		 ethr_ts_event *tse,
+		 int *start_scnt,
+		 int *scnt)
+{
+    int sscnt = *start_scnt;
+    if (sscnt < 0) {
+	*scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
+		 ? mtxb->main_scnt
+		 : mtxb->aux_scnt);
+	*scnt -= ETHR_MTX_MAX_FLGS_SPIN;
+    }
+    else {
+	*scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
+		 ? mtxb->main_scnt
+		 : mtxb->aux_scnt);
+	*scnt -= sscnt;
+	if (*scnt > 0 && sscnt < ETHR_MTX_MAX_FLGS_SPIN) {
+	    *scnt = ETHR_MTX_MAX_FLGS_SPIN - sscnt;
+	    *start_scnt = -1;
+	    return 0;
+	}
+    }
+    return 1;
+}
+
+int check_readers_array(ethr_rwmutex *rwmtx,
+			int start_rix,
+			int length,
+			int pre_check);
+
+static ETHR_INLINE void
+write_lock_wait(struct ethr_mutex_base_ *mtxb,
+		long initial,
+		int is_rwmtx,
+		int is_freq_read)
+{
+    long act = initial;
+    int scnt, start_scnt;
+    ethr_ts_event *tse = NULL;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+    int res;
+    int freq_read_size = -1;
+    int freq_read_start_ix = -1;
+
+    ETHR_ASSERT(!is_freq_read || is_rwmtx);
+
+    start_scnt = scnt = initial_spincount(mtxb);
+
+    /*
+     * Spin trying to write lock for a while. If unsuccessful,
+     * wait on event.
+     */
+
+    while (1) {
+	long exp;
+
+	while (act != 0) {
+
+	    if (is_freq_read && act == ETHR_RWMTX_R_FLG__) {
+		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+		if (!tse)
+		    tse = ethr_get_ts_event();
+		if (freq_read_size < 0) {
+		    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+			freq_read_size = reader_groups_array_size;
+			freq_read_start_ix = tse->rgix;
+		    }
+		    else {
+			freq_read_size = main_threads_array_size;
+			freq_read_start_ix = tse->mtix;
+		    }
+		}
+		res = check_readers_array(rwmtx,
+					  freq_read_start_ix,
+					  freq_read_size,
+					  1);
+		scnt--;
+		if (res == 0) {
+		    act = ethr_atomic_read(&mtxb->flgs);
+		    if (act & ETHR_RWMTX_R_MASK__) {
+			res = rwmutex_try_complete_runlock(rwmtx, act,
+							   tse, 0, 0,
+							   1);
+			if (res != EBUSY)
+			    goto done; /* Got it */
+		    }
+		    if (scnt <= 0)
+			goto chk_spin;
+		    if (--until_yield == 0) {
+			until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+			ETHR_YIELD();
+		    }
+		    continue;
+		}
+	    }
+
+	    if (scnt <= 0) {
+	    chk_spin:
+		scnt = 0;
+
+		if (!tse)
+		    tse = ethr_get_ts_event();
+		if (update_spincount(mtxb, tse, &start_scnt, &scnt)) {
+		    event_wait(mtxb, tse, scnt, ETHR_RWMTX_W_WAIT_FLG__,
+			       is_rwmtx, is_freq_read);
+		    goto done;
+		}
+	    }
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		ETHR_YIELD();
+	    }
+	    act = ethr_atomic_read(&mtxb->flgs);
+	    scnt--;
+	}
+	ETHR_ASSERT(scnt >= 0);
+
+	exp = act;
+
+	act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs,
+				       ETHR_RWMTX_W_FLG__,
+				       exp);
+	if (act == 0)
+	    goto done; /* Got it */
+    }
+
+ done:
+    if (tse)
+	ethr_leave_ts_event(tse);
+}
+
+static int
+mtxb_init(struct ethr_mutex_base_ *mtxb,
+	  int def_main_scnt,
+	  int main_scnt,
+	  int def_aux_scnt,
+	  int aux_scnt)
+{
+    ETHR_MTX_HARD_DEBUG_LFS_INIT(mtxb);
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+    mtxb->ws = 0;
+#endif
+    if (no_spin) {
+	mtxb->main_scnt = 0;
+	mtxb->aux_scnt = 0;
+    }
+    else {
+	if (main_scnt > SHRT_MAX)
+	    mtxb->main_scnt = SHRT_MAX;
+	else if (main_scnt < 0)
+	    mtxb->main_scnt = def_main_scnt;
+	else
+	    mtxb->main_scnt = (short) main_scnt;
+	if (aux_scnt > SHRT_MAX)
+	    mtxb->aux_scnt = SHRT_MAX;
+	else if (aux_scnt < 0)
+	    mtxb->aux_scnt = def_aux_scnt;
+	else
+	    mtxb->aux_scnt = (short) aux_scnt;
+	if (mtxb->main_scnt < mtxb->aux_scnt)
+	    mtxb->main_scnt = mtxb->aux_scnt;
+
+    }
+    mtxb->q = NULL;
+    ethr_atomic_init(&mtxb->flgs, 0);
+    return ETHR_MTX_QLOCK_INIT(&mtxb->qlck);
+}
+
+static int
+mtxb_destroy(struct ethr_mutex_base_ *mtxb)
+{
+    long act;
+    ETHR_MTX_Q_LOCK(&mtxb->qlck);
+    act = ethr_atomic_read(&mtxb->flgs);
+    ETHR_MTX_Q_UNLOCK(&mtxb->qlck);
+    if (act != 0)
+	return EINVAL;
+    return ETHR_MTX_QLOCK_DESTROY(&mtxb->qlck);
+}
+
+
+#endif /* ETHR_USE_OWN_RWMTX_IMPL__ || ETHR_USE_OWN_MTX_IMPL__ */
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Mutex and condition variable implementation                               *
+\*                                                                           */
+
+#ifdef ETHR_USE_OWN_MTX_IMPL__
+
+/* -- Mutex ---------------------------------------------------------------- */
+
+int
+ethr_mutex_init_opt(ethr_mutex *mtx, ethr_mutex_opt *opt)
+{
+    int res;
+#if ETHR_XCHK
+    if (!mtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    mtx->initialized = ETHR_MUTEX_INITIALIZED;
+#endif
+    ETHR_MTX_HARD_DEBUG_FENCE_INIT(mtx);
+    res = mtxb_init(&mtx->mtxb,
+		    default_mtx_main_spincount,
+		    opt ? opt->main_spincount : -1,
+		    default_mtx_aux_spincount,
+		    opt ? opt->aux_spincount : -1);
+#if ETHR_XCHK
+    if (res != 0)
+	mtx->initialized = 0;
+#endif
+    return res;
+}
+
+int
+ethr_mutex_init(ethr_mutex *mtx)
+{
+    return ethr_mutex_init_opt(mtx, NULL);
+}
+
+int
+ethr_mutex_destroy(ethr_mutex *mtx)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!mtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    mtx->initialized = 0;
+#endif
+    return mtxb_destroy(&mtx->mtxb);
+}
+
+void
+ethr_mutex_lock_wait__(ethr_mutex *mtx, long initial)
+{
+    write_lock_wait(&mtx->mtxb, initial, 0, 0);
+}
+
+void
+ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial)
+{
+    ethr_ts_event *tse;
+
+    ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck);
+    tse = mtx->mtxb.q;
+
+    ETHR_ASSERT(tse);
+    ETHR_ASSERT(ethr_atomic_read(&mtx->mtxb.flgs)
+		== (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__));
+    ETHR_ASSERT(initial & ETHR_RWMTX_W_WAIT_FLG__);
+    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+
+    /*
+     * If we have multiple waiters, there is no need to modify
+     * mtxb->flgs; otherwise, we need to clear the write wait bit...
+     */
+    if (tse->next == mtx->mtxb.q)
+	ethr_atomic_set(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__);
+
+    wake_writer(&mtx->mtxb, 0);
+}
+
+/* -- Condition variables -------------------------------------------------- */
+
+static void
+enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end)
+{
+    long act;
+
+    /*
+     * `ethr_cond_signal()' and `ethr_cond_broadcast()' end up here. If `mtx'
+     * is not currently locked by current thread, we almost certainly have a
+     * hard to debug race condition. There might however be some (strange)
+     * use for it. POSIX also allow a call to `pthread_cond_signal' or
+     * `pthread_cond_broadcast' even though the the associated mutex isn't
+     * locked by the caller. Therefore, we also allow this kind of strange
+     * usage, but optimize for the case where the mutex is locked by the
+     * calling thread.
+     */
+
+    ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck);
+
+    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+    {
+	int dbg_nws__ = 0;
+	ethr_ts_event *dbg_tse__;
+	for (dbg_tse__ = tse_start;
+	     dbg_tse__ != tse_end;
+	     dbg_tse__ = dbg_tse__->next)
+	    dbg_nws__++;
+	mtx->mtxb.ws += dbg_nws__ + 1;
+    }
+#endif
+
+    act = ethr_atomic_read(&mtx->mtxb.flgs);
+    ETHR_ASSERT(act == 0
+		|| act == ETHR_RWMTX_W_FLG__
+		|| act == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__));
+    if (act & ETHR_RWMTX_W_FLG__) {
+	/* The normal sane case */
+	if (!(act & ETHR_RWMTX_W_WAIT_FLG__)) {
+	    ETHR_ASSERT(!mtx->mtxb.q);
+	    act = ethr_atomic_cmpxchg(&mtx->mtxb.flgs,
+				      (ETHR_RWMTX_W_FLG__
+				       | ETHR_RWMTX_W_WAIT_FLG__),
+				      ETHR_RWMTX_W_FLG__);
+	    if (act != ETHR_RWMTX_W_FLG__) {
+		/*
+		 * Sigh... this wasn't so sane after all since, the mutex was
+		 * obviously not locked by the current thread....
+		 */
+		ETHR_ASSERT(act == 0);
+		goto mtx_unlocked;
+	    }
+	}
+
+#ifdef ETHR_DEBUG
+	if (act & ETHR_RWMTX_W_WAIT_FLG__)
+	    ETHR_ASSERT(mtx->mtxb.q);
+	else
+	    ETHR_ASSERT(!mtx->mtxb.q);
+#endif
+
+	enqueue(&mtx->mtxb.q, tse_start, tse_end);
+
+	ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+	ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
+
+    }
+    else {
+	int multi;
+    mtx_unlocked:
+	/* Sigh... mutex isn't locked... */
+
+	multi = tse_start != tse_end;
+
+	while (1) {
+	    long new, exp = act;
+
+	    if (multi || (act & ETHR_RWMTX_W_FLG__))
+		new = ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__;
+	    else
+		new = ETHR_RWMTX_W_FLG__;
+
+	    act = ethr_atomic_cmpxchg(&mtx->mtxb.flgs, new, exp);
+	    if (exp == act) {
+		ETHR_ASSERT(!mtx->mtxb.q);
+		if (act & ETHR_RWMTX_W_FLG__) {
+		    enqueue(&mtx->mtxb.q, tse_start, tse_end);
+
+		    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+		    ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
+
+		}
+		else {
+		    ETHR_ASSERT(!mtx->mtxb.q);
+		    /*
+		     * Acquired the mutex on behalf of the
+		     * first thread in the queue; wake
+		     * it and enqueue the rest...
+		     */
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+		    mtx->mtxb.ws--;
+#endif
+		    if (multi) {
+			enqueue(&mtx->mtxb.q, tse_start->next, tse_end);
+			ETHR_ASSERT(mtx->mtxb.q);
+		    }
+
+		    ETHR_MTX_HARD_DEBUG_CHK_Q(mtx);
+		    ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck);
+
+		    ethr_atomic_set(&tse_start->uaflgs, 0);
+		    ethr_event_set(&tse_start->event);
+		}
+		break;
+	    }
+	}
+    }
+}
+
+int
+ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt)
+{
+    int res;
+#if ETHR_XCHK
+    if (!cnd) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = ETHR_COND_INITIALIZED;
+#endif
+    ETHR_MTX_HARD_DEBUG_FENCE_INIT(cnd);
+    cnd->q = NULL;
+    if (no_spin) {
+	cnd->main_scnt = 0;
+	cnd->aux_scnt = 0;
+    }
+    else {
+	if (!opt || opt->main_spincount < 0)
+	    cnd->main_scnt = default_cnd_main_spincount;
+	else if (opt->main_spincount > SHRT_MAX)
+	    cnd->main_scnt = SHRT_MAX;
+	else
+	    cnd->main_scnt = (short) opt->main_spincount;
+	if (!opt || opt->aux_spincount < 0)
+	    cnd->aux_scnt = default_cnd_aux_spincount;
+	else if (opt->aux_spincount > SHRT_MAX)
+	    cnd->aux_scnt = SHRT_MAX;
+	else
+	    cnd->aux_scnt = (short) opt->aux_spincount;
+	if (cnd->main_scnt < cnd->aux_scnt)
+	    cnd->main_scnt = cnd->aux_scnt;
+    }
+    ETHR_MTX_QLOCK_INIT(&cnd->qlck);
+    return 0;
+}
+
+int
+ethr_cond_init(ethr_cond *cnd)
+{
+    return ethr_cond_init_opt(cnd, NULL);
+}
+
+int
+ethr_cond_destroy(ethr_cond *cnd)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = 0;
+#endif
+    return ETHR_MTX_QLOCK_DESTROY(&cnd->qlck);
+}
+
+void
+ethr_cond_signal(ethr_cond *cnd)
+{
+    ethr_ts_event *tse;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(!cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+
+    ETHR_MTX_Q_LOCK(&cnd->qlck);
+
+    tse = cnd->q;
+
+    if (!tse) {
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+	ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+    }
+    else {
+	ethr_mutex *mtx = (ethr_mutex *) tse->udata;
+
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+	ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+	ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) == ETHR_CND_WAIT_FLG__);
+
+	ethr_atomic_set(&tse->uaflgs, ETHR_RWMTX_W_WAIT_FLG__);
+
+	dequeue(&cnd->q, tse, tse);
+
+	ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+	tse->next = tse->prev = NULL;
+
+	enqueue_mtx(mtx, tse, tse);
+
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+	ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+    }
+}
+
+void
+ethr_cond_broadcast(ethr_cond *cnd)
+{
+    int res;
+    int got_all;
+    ethr_ts_event *tse;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(!cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+    do {
+	got_all = 1;
+
+	ETHR_MTX_Q_LOCK(&cnd->qlck);
+
+	tse = cnd->q;
+
+	if (!tse) {
+	    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+	    ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+	}
+	else {
+	    ethr_mutex *mtx = (ethr_mutex *) tse->udata;
+	    ethr_ts_event *tse_tmp, *tse_end;
+
+	    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+	    tse_end = cnd->q->prev;
+
+	    tse_tmp = tse;
+
+	    do {
+
+		if (mtx == (ethr_mutex *) tse_tmp->udata) {
+		    /* The normal case */
+
+		    ETHR_ASSERT(tse_tmp->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+		    ETHR_ASSERT(ethr_atomic_read(&tse_tmp->uaflgs)
+				== ETHR_CND_WAIT_FLG__);
+
+		    ethr_atomic_set(&tse_tmp->uaflgs, ETHR_RWMTX_W_WAIT_FLG__);
+		}
+		else {
+		    /* Should be very unusual */
+		    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+		    tse_end = tse_tmp->prev;
+		    got_all = 0;
+		    break;
+		}
+
+		tse_tmp = tse_tmp->next;
+
+	    } while (tse_tmp != cnd->q);
+
+	    dequeue(&cnd->q, tse, tse_end);
+
+	    ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+	    enqueue_mtx(mtx, tse, tse_end);
+	}
+
+    } while (!got_all);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+}
+
+int
+ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
+{
+    int woken;
+    int scnt;
+    int res;
+    void *udata = NULL;
+    ethr_ts_event *tse;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(!cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+    ETHR_ASSERT(!mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    tse = ethr_get_ts_event();
+
+    scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR)
+	    ? cnd->main_scnt
+	    : cnd->aux_scnt);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+
+    udata = tse->udata; /* Got to restore udata before returning */
+    tse->udata = (void *) mtx;
+
+    tse->uflgs = ETHR_RWMTX_W_WAIT_FLG__; /* Prep for mutex lock op */
+    ethr_atomic_set(&tse->uaflgs, ETHR_CND_WAIT_FLG__);
+
+    ETHR_MTX_Q_LOCK(&cnd->qlck);
+
+    enqueue(&cnd->q, tse, tse);
+
+    ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+    ethr_mutex_unlock(mtx);
+
+    /* Wait */
+    woken = 0;
+    while (1) {
+	long act;
+
+	ethr_event_reset(&tse->event);
+
+	act = ethr_atomic_read_acqb(&tse->uaflgs);
+	if (!act)
+	    break; /* Mtx locked */
+
+	/* First time, got EINTR, or spurious wakeup... */
+
+	ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__
+		    || act == ETHR_RWMTX_W_WAIT_FLG__);
+
+	if (woken) {
+	    /*
+	     * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already been enqueued
+	     * on the mutex; continue wait until locked...
+	     */
+	    if (act == ETHR_CND_WAIT_FLG__) {
+		ETHR_MTX_Q_LOCK(&cnd->qlck);
+		act = ethr_atomic_read(&tse->uaflgs);
+		ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__
+			    || act == ETHR_RWMTX_W_WAIT_FLG__);
+		/*
+		 * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already
+		 * enqueued on the mutex; continue wait until locked...
+		 */
+		if (act == ETHR_CND_WAIT_FLG__)
+		    dequeue(&cnd->q, tse, tse);
+
+		ETHR_MTX_Q_UNLOCK(&cnd->qlck);
+
+		if (act == ETHR_CND_WAIT_FLG__) {
+		    tse->udata = udata;
+		    ethr_leave_ts_event(tse);
+		    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+		    ethr_mutex_lock(mtx);
+		    return EINTR;
+		}
+	    }
+	    ETHR_ASSERT(act == ETHR_RWMTX_W_WAIT_FLG__);
+	}
+	ethr_event_swait(&tse->event, scnt);
+	/* swait result: 0 || EINTR */
+	woken = 1;
+    }
+
+    ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx);
+
+    tse->udata = udata;
+    ethr_leave_ts_event(tse);
+    return 0;
+}
+
+#else
+/* -- pthread mutex and condition variables -------------------------------- */
+
+int
+ethr_mutex_init(ethr_mutex *mtx)
+{
+#if ETHR_XCHK
+    if (!mtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    mtx->initialized = ETHR_MUTEX_INITIALIZED;
+#endif
+    return pthread_mutex_init(&mtx->pt_mtx, NULL);
+}
+
+int
+ethr_mutex_destroy(ethr_mutex *mtx)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+#if ETHR_XCHK
+    mtx->initialized = 0;
+#endif
+    return pthread_mutex_destroy(&mtx->pt_mtx);
+}
+
+int
+ethr_cond_init(ethr_cond *cnd)
+{
+#if ETHR_XCHK
+    if (!cnd) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = ETHR_COND_INITIALIZED;
+#endif
+    return pthread_cond_init(&cnd->pt_cnd, NULL);
+}
+
+int
+ethr_cond_destroy(ethr_cond *cnd)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    cnd->initialized = 0;
+#endif
+    return pthread_cond_destroy(&cnd->pt_cnd);
+}
+
+void
+ethr_cond_signal(ethr_cond *cnd)
+{
+    int res;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+
+    res = pthread_cond_signal(&cnd->pt_cnd);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+void
+ethr_cond_broadcast(ethr_cond *cnd)
+{
+    int res;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+
+    res = pthread_cond_broadcast(&cnd->pt_cnd);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+}
+
+int
+ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
+{
+    int res;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(cnd);
+    ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    res = pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx);
+    if (res != 0 && res != EINTR)
+	ETHR_FATAL_ERROR__(res);
+    return res;
+}
+
+#endif /* pthread_mutex */
+
+/* -- Exported symbols of inline functions --------------------------------- */
+
+int
+ethr_mutex_trylock(ethr_mutex *mtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    return ethr_mutex_trylock__(mtx);
+}
+
+void
+ethr_mutex_lock(ethr_mutex *mtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    ethr_mutex_lock__(mtx);
+}
+
+void
+ethr_mutex_unlock(ethr_mutex *mtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(mtx);
+    ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED);
+
+    ethr_mutex_unlock__(mtx);
+}
+
+
+#ifdef ETHR_USE_OWN_RWMTX_IMPL__
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
+ * Read/Write Mutex                                                          *
+\*                                                                           */
+
+static void
+wake_readers(ethr_rwmutex *rwmtx, int rs)
+{
+    ethr_ts_event *tse;
+#ifdef ETHR_DEBUG
+    int drs = 0;
+#endif
+
+    tse = rwmtx->mtxb.q;
+    ETHR_ASSERT(tse);
+    ETHR_ASSERT(rwmtx->rq_end);
+    dequeue(&rwmtx->mtxb.q, tse, rwmtx->rq_end);
+    rwmtx->rq_end->next = NULL;
+    rwmtx->rq_end = NULL;
+
+    ETHR_ASSERT(!rwmtx->mtxb.q
+		|| (ethr_atomic_read(&rwmtx->mtxb.q->uaflgs)
+		    == ETHR_RWMTX_W_WAIT_FLG__));
+
+    ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
+    ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+
+    while (tse) {
+	ethr_ts_event *tse_next;
+
+#ifdef ETHR_DEBUG
+	ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
+	ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs)
+		    == ETHR_RWMTX_R_WAIT_FLG__);
+	drs++;
+#endif
+
+	tse_next = tse->next; /* we aren't allowed to read tse->next
+				 after we have reset uaflgs */
+
+	ethr_atomic_set(&tse->uaflgs, 0);
+	ethr_event_set(&tse->event);
+	tse = tse_next;
+    }
+
+    ETHR_ASSERT(rs == drs);
+}
+
+static ETHR_INLINE int
+is_w_waiter(ethr_ts_event *tse)
+{
+    ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__
+		|| tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
+    return tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__;
+}
+
+static ETHR_INLINE int
+multiple_w_waiters(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(rwmtx->mtxb.q);
+    ETHR_ASSERT(rwmtx->mtxb.q->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+
+    if (!rwmtx->rq_end)
+	return rwmtx->mtxb.q->next != rwmtx->mtxb.q;
+    else {
+	ETHR_ASSERT(rwmtx->mtxb.q->next != rwmtx->mtxb.q);
+	if (rwmtx->mtxb.q->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__)
+	    return 1;
+	ETHR_ASSERT(rwmtx->rq_end->next == rwmtx->mtxb.q
+		    || rwmtx->rq_end->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+	return rwmtx->rq_end->next != rwmtx->mtxb.q;
+    }
+}
+
+int check_readers_array(ethr_rwmutex *rwmtx,
+			int start_rix,
+			int length,
+			int pre_check)
+{
+    int ix = start_rix;
+
+#ifndef ETHR_READ_MEMORY_BARRIER_IS_FULL
+    if (pre_check)
+	ETHR_READ_MEMORY_BARRIER;
+    else
+#endif
+	ETHR_MEMORY_BARRIER;
+
+    do {
+	long act = rwmutex_freqread_rdrs_read(rwmtx, ix);
+	if (act != 0)
+	    return EBUSY;
+	ix++;
+	if (ix == length)
+	    ix = 0;
+    } while (ix != start_rix);
+
+    return 0;
+}
+
+static ETHR_INLINE void
+rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx,
+					 ethr_ts_event *tse)
+{
+    long act;
+    /*
+     * Restore failed increment
+     */
+    act = rwmutex_freqread_rdrs_dec_read(rwmtx, tse);
+
+    ETHR_WRITE_MEMORY_BARRIER;
+
+    if (act == 0) {
+
+#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL
+	ETHR_READ_MEMORY_BARRIER;
+#endif
+
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+
+	if ((act & ETHR_RWMTX_W_FLG__) == 0
+	    && act & (ETHR_RWMTX_WAIT_FLGS__|ETHR_RWMTX_R_PEND_UNLCK_MASK__)) {
+	    /*
+	     * We either got waiters, or someone else trying
+	     * to read unlock which we might have to help.
+	     */
+	    rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 1, 0);
+	}
+    }
+}
+
+static int
+rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx,
+			     long initial,
+			     ethr_ts_event *tse,
+			     int start_next_ix,
+			     int check_before_try,
+			     int try_write_lock)
+{
+    ethr_ts_event *tse_tmp;
+    long act = initial;
+    int six, res, length;
+
+    tse_tmp = tse;
+    if (!tse_tmp)
+	tse_tmp = ethr_get_ts_event();
+
+    if ((act & ETHR_RWMTX_WAIT_FLGS__)
+	&& (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0)
+	goto check_waiters;
+
+    if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) {
+	length = reader_groups_array_size;
+	six = tse_tmp->rgix;
+    }
+    else {
+	length = main_threads_array_size;
+	six = tse_tmp->mtix;
+    }
+    if (start_next_ix) {
+	six++;
+	if (six >= length)
+	    six = 0;
+    }
+
+    if (!tse)
+	ethr_leave_ts_event(tse_tmp);
+
+    if (check_before_try) {
+	res = check_readers_array(rwmtx, six, length, 1);
+	if (res == EBUSY)
+	    return try_write_lock ? EBUSY : 0;
+    }
+
+    while (1) {
+	long exp = act;
+	long new = act+1;
+
+	ETHR_ASSERT((act & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
+		    < ETHR_RWMTX_R_PEND_UNLCK_MASK__);
+
+	act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	if (exp == act) {
+	    act = new;
+	    break;
+	}
+	if (!try_write_lock) {
+	    if (act == ETHR_RWMTX_W_FLG__ || act == 0)
+		return 0;
+	    if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+		if ((act & ETHR_RWMTX_R_FLG__) == 0)
+		    return 0;
+	    }
+	    else if ((act & ETHR_RWMTX_R_FLG__) == 0) {
+		if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
+		    return 0;
+		goto check_waiters;
+	    }
+	}
+	else {
+	    if (act == 0)
+		goto tryrwlock;
+	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__))
+		return EBUSY;
+	}
+    }
+
+    res = check_readers_array(rwmtx, six, length, 0);
+    if (res == EBUSY) {
+	act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs);
+	if (act & ETHR_RWMTX_R_MASK__)
+	    return try_write_lock ? EBUSY : 0;
+    }
+    else {
+	while (1) {
+	    long exp = act;
+	    long new = act;
+	    new &= ~ETHR_RWMTX_R_FLG__;
+	    new--;
+
+	    ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__);
+
+	    act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	    if (exp == act) {
+		if (new & ETHR_RWMTX_R_PEND_UNLCK_MASK__)
+		    return try_write_lock ? EBUSY : 0;
+		act = new;
+		break;
+	    }
+	}
+    }
+
+    /*
+     * Read unlock completed, but we have to check if
+     * threads have to be woken (or if we should try
+     * to write lock it).
+     */
+
+    if (act & ETHR_RWMTX_W_FLG__)
+	return try_write_lock ? EBUSY : 0;
+
+    if (act & ETHR_RWMTX_WAIT_FLGS__) {
+    check_waiters:
+	rwmutex_unlock_wake(rwmtx, 0, act);
+	return try_write_lock ? EBUSY : 0;
+    }
+
+    if (!try_write_lock)
+	return 0;
+
+ tryrwlock:
+    /* Try to write lock it */
+
+    act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+				   ETHR_RWMTX_W_FLG__,
+				   0);
+    return act == 0 ? 0 : EBUSY;
+}
+
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+
+static ETHR_INLINE void
+rwmutex_incdec_restore_failed_tryrlock(ethr_rwmutex *rwmtx)
+{
+    long act;
+    /*
+     * Restore failed increment
+     */
+    act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs);
+    if ((act & ETHR_RWMTX_WAIT_FLGS__)
+	&& (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	rwmutex_unlock_wake(rwmtx, 0, act);
+    }
+}
+
+#endif
+
+static void
+rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx,
+			  long initial)
+{
+    long act = initial, exp;
+    int scnt, start_scnt;
+    ethr_ts_event *tse = NULL;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    start_scnt = scnt = initial_spincount(&rwmtx->mtxb);
+
+    /*
+     * Spin trying to read lock for a while. If unsuccessful,
+     * wait on event.
+     */
+
+    while (1) {
+
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	rwmutex_incdec_restore_failed_tryrlock(rwmtx);
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+#endif
+
+	while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+	    if (scnt == 0) {
+		tse = ethr_get_ts_event();
+		if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
+		    event_wait(&rwmtx->mtxb, tse, scnt,
+			       ETHR_RWMTX_R_WAIT_FLG__, 1, 0);
+		    goto done;
+		}
+	    }
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		ETHR_YIELD();
+	    }
+	    act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	    scnt--;
+	}
+	exp = act;
+	ETHR_ASSERT(scnt >= 0);
+
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	act = ethr_atomic_inc_read(&rwmtx->mtxb.flgs);
+	if ((act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) == 0)
+	    goto done; /* Got it */
+#else
+	act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+	if (act == exp)
+	    goto done; /* Got it */
+#endif
+    }
+
+ done:
+    if (tse)
+	ethr_leave_ts_event(tse);
+}
+
+static void
+rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx,
+			    ethr_ts_event *tse,
+			    long initial)
+{
+    long act = initial;
+    int scnt, start_scnt;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    start_scnt = scnt = initial_spincount(&rwmtx->mtxb);
+
+    /*
+     * Spin trying to read lock for a while. If unsuccessful,
+     * wait on event.
+     */
+
+    while (1) {
+
+	rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
+
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+
+	while (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
+	    if (scnt == 0) {
+		if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) {
+		    event_wait(&rwmtx->mtxb, tse, scnt,
+			       ETHR_RWMTX_R_WAIT_FLG__, 1, 1);
+		    return; /* Got it */
+		}
+	    }
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		ETHR_YIELD();
+	    }
+	    act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	    scnt--;
+	}
+
+	ETHR_ASSERT(scnt >= 0);
+
+	rwmutex_freqread_rdrs_inc(rwmtx, tse);
+
+	ETHR_MEMORY_BARRIER;
+
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+
+	if (act == ETHR_RWMTX_R_FLG__)
+	    return; /* Got it */
+
+	while (1) {
+	    long exp, new;
+
+	    if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__))
+		break; /* Busy (need to restore inc) */
+
+	    if (act & ETHR_RWMTX_R_FLG__)
+		return; /* Got it */
+
+	    exp = act;
+	    new = act | ETHR_RWMTX_R_FLG__;
+	    act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	    if (act == exp)
+		return; /* Got it */
+	}
+    }
+}
+
+static void
+rwmutex_normal_rwlock_wait(ethr_rwmutex *rwmtx, long initial)
+{
+    write_lock_wait(&rwmtx->mtxb, initial, 1, 0);
+}
+
+static void
+rwmutex_freqread_rwlock_wait(ethr_rwmutex *rwmtx, long initial)
+{
+    write_lock_wait(&rwmtx->mtxb, initial, 1, 1);
+}
+
+static ETHR_INLINE void
+rwlock_wake_set_flags(ethr_rwmutex *rwmtx, long new_initial, int act_initial)
+{
+    long act, act_mask;
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
+	/* r pend unlock mask may vary and must be retained */
+	act_mask = ETHR_RWMTX_R_PEND_UNLCK_MASK__;
+    }
+    else {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	/* rs mask may vary and must be retained */
+	act_mask = ETHR_RWMTX_RS_MASK__;
+#else
+	/* rs mask always zero */
+	ETHR_ASSERT((act_initial & ETHR_RWMTX_RS_MASK__) == 0);
+	ethr_atomic_set(&rwmtx->mtxb.flgs, new_initial);
+	return;
+#endif
+    }
+
+    act = act_initial;
+    while (1) {
+	long exp = act;
+	long new = new_initial + (act & act_mask);
+	act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+	if (act == exp)
+	    break;
+	exp = act;
+    }
+}
+
+#ifdef ETHR_DEBUG
+
+static void
+dbg_unlock_wake(ethr_rwmutex *rwmtx,
+		int have_w,
+		ethr_ts_event *tse)
+{
+    long exp, act, imask;
+
+    exp = have_w ? ETHR_RWMTX_W_FLG__ : 0;
+
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
+	imask = ETHR_RWMTX_R_PEND_UNLCK_MASK__;
+    else {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	imask = ETHR_RWMTX_RS_MASK__;
+#else
+	imask = 0;
+#endif
+    }
+
+    ETHR_ASSERT(tse);
+
+    if (is_w_waiter(tse)) {
+
+	exp |= ETHR_RWMTX_W_WAIT_FLG__;
+	if (rwmtx->rq_end) {
+	    exp |= ETHR_RWMTX_R_WAIT_FLG__;
+	}
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	ETHR_ASSERT((exp & ~imask) == (act & ~imask));
+
+	ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
+
+    }
+    else {
+
+	exp |= ETHR_RWMTX_R_WAIT_FLG__;
+	if (rwmtx->rq_end->next != rwmtx->mtxb.q)
+	    exp |= ETHR_RWMTX_W_WAIT_FLG__;
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	ETHR_ASSERT((exp & ~imask) == (act & ~imask));
+
+	ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx);
+
+    }
+}
+
+#endif
+
+static void
+rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial)
+{
+    long new, act = initial;
+    ethr_ts_event *tse;
+
+    if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	if (!have_w)
+	    return;
+	else {
+	    while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) {
+		long exp = act;
+		new = exp & ~ETHR_RWMTX_W_FLG__;
+		act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp);
+		if (act == exp)
+		    return;
+	    }
+	}
+    }
+
+    ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck);
+    tse = rwmtx->mtxb.q;
+
+    if (!have_w) {
+	if (!tse) {
+#ifdef ETHR_DEBUG
+	    act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	    ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0);
+#endif
+	    goto already_served;
+	}
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	if (act & ~ETHR_RWMTX_WAIT_FLGS__) {
+	already_served:
+	    ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck);
+	    return;
+	}
+    }
+
+#ifdef ETHR_DEBUG
+    dbg_unlock_wake(rwmtx, have_w, tse);
+#endif
+
+    if (is_w_waiter(tse)) {
+
+	if (!have_w) {
+	    act = ethr_atomic_read_bor(&rwmtx->mtxb.flgs,
+				       ETHR_RWMTX_W_FLG__);
+	    ETHR_ASSERT((act & ~ETHR_RWMTX_WAIT_FLGS__) == 0);
+	    ETHR_ASSERT(act & ETHR_RWMTX_W_WAIT_FLG__);
+	    act |= ETHR_RWMTX_W_FLG__;
+	}
+
+	/*
+	 * If we have multiple write waiters, there
+	 * is no need to modify mtxb->flgs; otherwise,
+	 * we need to clear the write wait bit...
+	 */
+	if (!multiple_w_waiters(rwmtx)) {
+	    new = ETHR_RWMTX_W_FLG__;
+	    if (tse->next != rwmtx->mtxb.q) {
+		ETHR_ASSERT(tse->next->uflgs == ETHR_RWMTX_R_WAIT_FLG__);
+		new |= ETHR_RWMTX_R_WAIT_FLG__;
+	    }
+
+	    rwlock_wake_set_flags(rwmtx, new, act);
+	}
+
+	wake_writer(&rwmtx->mtxb, 1);
+    }
+    else {
+	int rs;
+
+	if (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL) {
+	    rs = rwmtx->tdata.rs;
+	    new = (long) rs;
+	    rwmtx->tdata.rs = 0;
+	}
+	else {
+	    ethr_rwmutex_type type = rwmtx->type;
+	    int length = (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ
+			  ? reader_groups_array_size
+			  : main_threads_array_size);
+	    int ix;
+
+	    rs = 0;
+	    for (ix = 0; ix < length; ix++) {
+		int wrs = rwmtx->tdata.ra[ix].data.waiting_readers;
+		rwmtx->tdata.ra[ix].data.waiting_readers = 0;
+		ETHR_ASSERT(wrs >= 0);
+		if (wrs) {
+		    rs += wrs;
+		    rwmutex_freqread_rdrs_add(rwmtx, type, ix, wrs);
+		}
+	    }
+	    new = ETHR_RWMTX_R_FLG__;
+	}
+
+	if (rwmtx->rq_end->next != rwmtx->mtxb.q)
+	    new |= ETHR_RWMTX_W_WAIT_FLG__;
+
+	rwlock_wake_set_flags(rwmtx, new, act);
+	wake_readers(rwmtx, rs);
+    }
+}
+
+static ethr_rwmtx_readers_array__ *
+alloc_readers_array(int length, ethr_rwmutex_lived lived)
+{
+    ethr_rwmtx_readers_array__ *ra;
+    size_t sz;
+    void *mem;
+
+    sz = sizeof(ethr_rwmtx_readers_array__) * (length + 1);
+
+    switch (lived) {
+    case ETHR_RWMUTEX_LONG_LIVED:
+	mem = ethr_mem__.ll.alloc(sz);
+	break;
+    case ETHR_RWMUTEX_SHORT_LIVED:
+	mem = ethr_mem__.sl.alloc(sz);
+	break;
+    default:
+	mem = ethr_mem__.std.alloc(sz);
+	break;
+    }
+    if (!mem)
+	return NULL;
+
+    if ((((unsigned long) mem) & ETHR_CACHE_LINE_MASK) == 0) {
+	ra = (ethr_rwmtx_readers_array__ *) mem;
+	ra->data.byte_offset = 0;
+    }
+    else {
+	ra = ((ethr_rwmtx_readers_array__ *)
+	      ((((unsigned long) mem) & ~ETHR_CACHE_LINE_MASK)
+	       + ETHR_CACHE_LINE_SIZE));
+	ra->data.byte_offset = (int) ((unsigned long) ra
+				      - (unsigned long) mem);
+    }
+    ra->data.lived = lived;
+    return ra;
+}
+
+static void
+free_readers_array(ethr_rwmtx_readers_array__ *ra)
+{
+    void *ptr = (void *) (((char *) ra) - ra->data.byte_offset);
+    switch (ra->data.lived) {
+    case ETHR_RWMUTEX_LONG_LIVED:
+	ethr_mem__.ll.free(ptr);
+	break;
+    case ETHR_RWMUTEX_SHORT_LIVED:
+	ethr_mem__.sl.free(ptr);
+	break;
+    default:
+	ethr_mem__.std.free(ptr);
+	break;
+    }
+}
+
+int
+ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt)
+{
+    int res;
+    ethr_rwmtx_readers_array__ *ra = NULL;
+#if ETHR_XCHK
+    if (ethr_not_completely_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!rwmtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
+#endif
+    ETHR_MTX_HARD_DEBUG_FENCE_INIT(rwmtx);
+    rwmtx->rq_end = NULL;
+    rwmtx->type = opt ? opt->type : ETHR_RWMUTEX_TYPE_NORMAL;
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+	if (main_threads_array_size <= reader_groups_array_size) {
+	    /* No point using reader groups... */
+	    rwmtx->type = ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ;
+	}
+	/* Fall through */
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	int length;
+
+	length = (rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ
+		  ? main_threads_array_size
+		  : reader_groups_array_size);
+
+	if (length == 1) {
+	    /* No point using a frequent reader type... */
+	    rwmtx->type = ETHR_RWMUTEX_TYPE_NORMAL;
+	}
+	else {
+	    int ix;
+	    ra = alloc_readers_array(length,
+				     (opt
+				      ? opt->lived
+				      : ETHR_RWMUTEX_UNKNOWN_LIVED));
+	    if (!ra) {
+		res = ENOMEM;
+		goto error;
+	    }
+
+	    rwmtx->tdata.ra = ra;
+
+	    for (ix = 0; ix < length; ix++) {
+		ethr_atomic_init(&rwmtx->tdata.ra[ix].data.readers, 0);
+		rwmtx->tdata.ra[ix].data.waiting_readers = 0;
+	    }
+	    break;
+	}
+    }
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	rwmtx->tdata.rs = 0;
+	break;
+    default:
+	res = EINVAL;
+	goto error;
+    }
+    res = mtxb_init(&rwmtx->mtxb,
+		    default_rwmtx_main_spincount,
+		    opt ? opt->main_spincount : -1,
+		    default_rwmtx_aux_spincount,
+		    opt ? opt->aux_spincount : -1);
+    if (res == 0)
+	return 0;
+
+ error:
+
+    if (ra)
+	free_readers_array(ra);
+
+#if ETHR_XCHK
+    rwmtx->initialized = 0;
+#endif
+    return res;
+}
+
+int
+ethr_rwmutex_init(ethr_rwmutex *rwmtx)
+{
+    return ethr_rwmutex_init_opt(rwmtx, NULL);
+}
+
+int
+ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
+{
+    int res;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) {
+	long act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	if (act == ETHR_RWMTX_R_FLG__)
+	    rwmutex_try_complete_runlock(rwmtx, act, NULL, 0, 0, 0);
+    }
+    res = mtxb_destroy(&rwmtx->mtxb);
+    if (res != 0)
+	return res;
+    if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL)
+	free_readers_array(rwmtx->tdata.ra);
+#if ETHR_XCHK
+    rwmtx->initialized = 0;
+#endif
+    return 0;
+}
+
+#define ETHR_MAX_TRYRLOCK_TRIES 5
+
+int
+ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
+{
+    int res = 0;
+    long act;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL: {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+	if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))
+	    res = EBUSY;
+	else {
+	    act = ethr_atomic_inc_read_acqb(&rwmtx->mtxb.flgs);
+	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+		rwmutex_incdec_restore_failed_tryrlock(rwmtx);
+		res = EBUSY;
+	    }
+	}
+#else
+	long exp = 0;
+	int tries = 0;
+
+	while (1) {
+	    act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+	    if (act == exp) {
+		res = 0;
+		break;
+	    }
+	    if (tries > ETHR_MAX_TRYRLOCK_TRIES
+		|| (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))) {
+		res = EBUSY;
+		break;
+	    }
+	    tries++;
+	    exp = act;
+	}
+#endif
+	break;
+    }
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	ethr_ts_event *tse = ethr_get_ts_event();
+
+	rwmutex_freqread_rdrs_inc(rwmtx, tse);
+
+	ETHR_MEMORY_BARRIER;
+
+	act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs);
+
+	if (act != ETHR_RWMTX_R_FLG__) {
+	    while (1) {
+		long exp, new;
+
+		if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
+		    rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse);
+		    res = EBUSY;
+		    break;
+		}
+
+		if (act & ETHR_RWMTX_R_FLG__)
+		    break;
+
+		exp = act;
+		new = act | ETHR_RWMTX_R_FLG__;
+		act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
+		if (act == exp)
+		    break;
+	    }
+	}
+
+	ethr_leave_ts_event(tse);
+	break;
+    }
+    }
+
+    ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(&rwmtx->mtxb, res);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    return res;
+}
+
+void
+ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
+{
+    long act;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL: {
+#ifdef ETHR_RLOCK_WITH_INC_DEC
+	act = ethr_atomic_inc_read_acqb(&rwmtx->mtxb.flgs);
+	if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))
+	    rwmutex_normal_rlock_wait(rwmtx, act);
+#else
+	long exp = 0;
+
+	while (1) {
+	    act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp);
+	    if (act == exp) {
+		break;
+	    }
+
+	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) {
+		rwmutex_normal_rlock_wait(rwmtx, act);
+		break;
+	    }
+	    exp = act;
+	}
+#endif
+	break;
+    }
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	ethr_ts_event *tse = ethr_get_ts_event();
+
+	rwmutex_freqread_rdrs_inc(rwmtx, tse);
+
+	ETHR_MEMORY_BARRIER;
+
+	act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs);
+
+	if (act != ETHR_RWMTX_R_FLG__) {
+	    while (1) {
+		long exp, new;
+
+		if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) {
+		    rwmutex_freqread_rlock_wait(rwmtx, tse, act);
+		    break;
+		}
+
+		if (act & ETHR_RWMTX_R_FLG__)
+		    break;
+
+		exp = act;
+		new = act | ETHR_RWMTX_R_FLG__;
+		act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp);
+		if (act == exp)
+		    break;
+	    }
+	}
+
+	ethr_leave_ts_event(tse);
+	break;
+    }
+    }
+
+    ETHR_MTX_HARD_DEBUG_LFS_RLOCK(&rwmtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+}
+
+void
+ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
+{
+    long act;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+    ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(&rwmtx->mtxb);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic_dec_read_relb(&rwmtx->mtxb.flgs);
+	if ((act & ETHR_RWMTX_WAIT_FLGS__)
+	    && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) {
+	    ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0);
+	    rwmutex_unlock_wake(rwmtx, 0, act);
+	}
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: {
+	ethr_ts_event *tse = ethr_get_ts_event();
+
+	act = rwmutex_freqread_rdrs_dec_read_relb(rwmtx, tse);
+
+	ETHR_ASSERT(act >= 0);
+
+	ETHR_WRITE_MEMORY_BARRIER;
+
+	if (act == 0) {
+
+#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL
+	    ETHR_READ_MEMORY_BARRIER;
+#endif
+	    act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+
+	    if ((act & ETHR_RWMTX_W_FLG__) == 0
+		&& (act & (ETHR_RWMTX_WAIT_FLGS__
+			   | ETHR_RWMTX_R_PEND_UNLCK_MASK__))) {
+		rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0);
+	    }
+
+	}
+
+	ethr_leave_ts_event(tse);
+	break;
+    }
+    }
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+}
+
+int
+ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
+{
+    int res = 0;
+    long act;
+
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+				       ETHR_RWMTX_W_FLG__, 0);
+	if (act != 0)
+	    res = EBUSY;
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+
+	res = 0;
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+
+	do {
+
+	    if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__)) {
+		res = EBUSY;
+		break;
+	    }
+
+	    if (act & ETHR_RWMTX_R_MASK__) {
+		res = rwmutex_try_complete_runlock(rwmtx, act, NULL,
+						   0, 1, 1);
+		break;
+	    }
+
+	    act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+					   ETHR_RWMTX_W_FLG__, 0);
+
+	} while (act != 0);
+
+	break;
+    }
+
+    ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&rwmtx->mtxb, res);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    return res;
+}
+
+void
+ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
+{
+    long act;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+				       ETHR_RWMTX_W_FLG__, 0);
+	if (act != 0)
+	    rwmutex_normal_rwlock_wait(rwmtx, act);
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+
+	act = ethr_atomic_read(&rwmtx->mtxb.flgs);
+
+	do {
+
+	    if (act != 0) {
+		rwmutex_freqread_rwlock_wait(rwmtx, act);
+		break;
+	    }
+
+	    act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs,
+					   ETHR_RWMTX_W_FLG__, 0);
+
+	} while (act != 0);
+
+	break;
+    }
+
+    ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&rwmtx->mtxb);
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+
+}
+
+void
+ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
+{
+    long act;
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+    ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&rwmtx->mtxb);
+
+    switch (rwmtx->type) {
+    case ETHR_RWMUTEX_TYPE_NORMAL:
+	act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs,
+				       0, ETHR_RWMTX_W_FLG__);
+	if (act != ETHR_RWMTX_W_FLG__)
+	    rwmutex_unlock_wake(rwmtx, 1, act);
+	break;
+
+    case ETHR_RWMUTEX_TYPE_FREQUENT_READ:
+    case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ:
+	act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs, 0,
+				       ETHR_RWMTX_W_FLG__);
+	if (act != ETHR_RWMTX_W_FLG__)
+	    rwmutex_unlock_wake(rwmtx, 1, act);
+	break;
+    }
+
+    ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx);
+}
+
+#else
+/* -- pthread read/write mutex --------------------------------------------- */
+
+int
+ethr_rwmutex_init(ethr_rwmutex *rwmtx)
+{
+#if ETHR_XCHK
+    if (!rwmtx) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
+#endif
+    return pthread_rwlock_init(&rwmtx->pt_rwlock, write_pref_attr);
+}
+
+int
+ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt)
+{
+    return ethr_rwmutex_init(rwmtx);
+}
+
+int
+ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
+{
+    int res;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    res = pthread_rwlock_destroy(&rwmtx->pt_rwlock);
+#if ETHR_XCHK
+    rwmtx->initialized = 0;
+#endif
+    return res;
+}
+
+/* -- Exported symbols of inline functions --------------------------------- */
+
+int
+ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    return ethr_rwmutex_tryrlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ethr_rwmutex_rlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ethr_rwmutex_runlock__(rwmtx);
+}
+
+int
+ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    return ethr_rwmutex_tryrwlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    return ethr_rwmutex_rwlock__(rwmtx);
+}
+
+void
+ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
+{
+    ETHR_ASSERT(!ethr_not_inited__);
+    ETHR_ASSERT(rwmtx);
+    ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED);
+
+    ethr_rwmutex_rwunlock__(rwmtx);
+}
+
+#endif /* pthread */
+
+
+#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__)
+
+#ifdef ETHR_MTX_HARD_DEBUG_Q
+static void
+hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
+{
+    int res;
+    long flgs = ethr_atomic_read(&mtxb->flgs);
+
+    ETHR_MTX_HARD_ASSERT(res == 0);
+
+    ETHR_MTX_HARD_ASSERT(!(flgs & ETHR_RWMTX_R_WAIT_FLG__) || is_rwmtx);
+
+    if (!(flgs & ETHR_RWMTX_WAIT_FLGS__)) {
+	ETHR_MTX_HARD_ASSERT(!mtxb->q);
+	if (is_rwmtx) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end);
+	    ETHR_MTX_HARD_ASSERT(!rwmtx->rs);
+	}
+    }
+    else {
+	ethr_ts_event *tse;
+	int ws = 0, rs = 0, rsf = 0, ref = 0;
+
+	ETHR_MTX_HARD_ASSERT(mtxb->q);
+
+	tse = mtxb->q;
+
+	do {
+	    long type;
+
+	    ETHR_MTX_HARD_ASSERT(tse->next->prev == tse);
+	    ETHR_MTX_HARD_ASSERT(tse->prev->next == tse);
+
+	    type = ethr_atomic_read(&tse->uaflgs);
+	    ETHR_MTX_HARD_ASSERT(type == tse->uflgs);
+	    switch (type) {
+	    case ETHR_RWMTX_W_WAIT_FLG__:
+		ws++;
+		break;
+	    case ETHR_RWMTX_R_WAIT_FLG__: {
+		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+		ETHR_MTX_HARD_ASSERT(is_rwmtx);
+		if (!rsf)
+		    rsf = 1;
+		ETHR_MTX_HARD_ASSERT(!ref);
+		if (rwmtx->rq_end == tse) {
+		    ETHR_MTX_HARD_ASSERT(
+			tse->next == rwmtx->mtxb.q
+			|| tse->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__);
+		    ref = 1;
+		}
+		rs++;
+		break;
+	    }
+	    default:
+		ETHR_MTX_HARD_ASSERT(! "invalid wait type found");
+	    }
+
+	    tse = tse->next;
+	} while (tse != mtxb->q);
+
+	if (is_rwmtx) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_MTX_HARD_ASSERT(rs == rwmtx->rs);
+	}
+
+#ifdef ETHR_MTX_HARD_DEBUG_WSQ
+	ETHR_MTX_HARD_ASSERT(ws == mtxb->ws);
+#endif
+
+	if (flgs & ETHR_RWMTX_W_WAIT_FLG__)
+	    ETHR_MTX_HARD_ASSERT(ws);
+	else
+	    ETHR_MTX_HARD_ASSERT(!ws);
+
+	if (flgs & ETHR_RWMTX_R_WAIT_FLG__) {
+	    ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	    ETHR_MTX_HARD_ASSERT(is_rwmtx);
+	    ETHR_MTX_HARD_ASSERT(rwmtx->rq_end);
+	    ETHR_MTX_HARD_ASSERT(rsf);
+	    ETHR_MTX_HARD_ASSERT(ref);
+	    ETHR_MTX_HARD_ASSERT(rs);
+	}
+	else {
+	    if (is_rwmtx) {
+		ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+		ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end);
+	    }
+	    ETHR_MTX_HARD_ASSERT(!rsf);
+	    ETHR_MTX_HARD_ASSERT(!ref);
+	    ETHR_MTX_HARD_ASSERT(!rs);
+	}
+    }
+}
+
+#elif defined(ETHR_MTX_HARD_DEBUG_WSQ)
+
+static void
+hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx)
+{
+    int ws = 0;
+    int rs = 0;
+
+    if (mtxb->q) {
+	ethr_ts_event *tse = mtxb->q;
+	do {
+	    switch (tse->uflgs) {
+	    case ETHR_RWMTX_W_WAIT_FLG__:
+		ws++;
+		break;
+	    case ETHR_RWMTX_R_WAIT_FLG__:
+		rs++;
+		break;
+	    default:
+		ETHR_MTX_HARD_ASSERT(0);
+		break;
+	    }
+	    tse = tse->next;
+	} while (tse != mtxb->q);
+    }
+
+    ETHR_MTX_HARD_ASSERT(mtxb->ws == ws);
+    if (is_rwmtx) {
+	ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb;
+	ETHR_MTX_HARD_ASSERT(rwmtx->rs == rs);
+    }
+}
+
+#endif
+
+#endif
diff --git a/erts/lib_src/common/ethread.c b/erts/lib_src/common/ethread.c
deleted file mode 100644
index 9c88233934..0000000000
--- a/erts/lib_src/common/ethread.c
+++ /dev/null
@@ -1,3369 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 2004-2010. All Rights Reserved.
- *
- * The contents of this file are subject to the Erlang Public License,
- * Version 1.1, (the "License"); you may not use this file except in
- * compliance with the License. You should have received a copy of the
- * Erlang Public License along with this software. If not, it can be
- * retrieved online at http://www.erlang.org/.
- *
- * Software distributed under the License is distributed on an "AS IS"
- * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
- * the License for the specific language governing rights and limitations
- * under the License.
- *
- * %CopyrightEnd%
- */
-
-/*
- * Description: A Thread library for use in the ERTS and other OTP
- *              applications.
- * Author: Rickard Green
- */
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#undef ETHR_STACK_GUARD_SIZE 
-
-#if defined(ETHR_PTHREADS)
-
-#ifdef ETHR_TIME_WITH_SYS_TIME
-#  include <time.h>
-#  include <sys/time.h>
-#else
-#  ifdef ETHR_HAVE_SYS_TIME_H
-#    include <sys/time.h>
-#  else
-#    include <time.h>
-#  endif
-#endif
-#include <sys/types.h>
-#include <unistd.h>
-#include <signal.h>
-
-#ifdef ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE
-#  define ETHR_STACK_GUARD_SIZE (pagesize)
-#endif
-
-#elif defined(ETHR_WIN32_THREADS)
-
-#undef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-#include <process.h>
-#include <winerror.h>
-
-#else
-#error "Missing thread implementation"
-#endif
-
-#include <limits.h>
-
-#define ETHR_FORCE_INLINE_FUNCS
-#define ETHR_INLINE_FUNC_NAME_(X) X ## __
-#include "ethread.h"
-
-#ifndef ETHR_HAVE_ETHREAD_DEFINES
-#error Missing configure defines
-#endif
-
-/*
- * ----------------------------------------------------------------------------
- * Common stuff
- * ----------------------------------------------------------------------------
- */
-
-#define ETHR_MAX_THREADS 2048 /* Has to be an even power of 2 */
-
-static int ethr_not_inited = 1;
-
-#define ASSERT(A) ETHR_ASSERT((A))
-
-static void *(*allocp)(size_t) = malloc;
-static void *(*reallocp)(void *, size_t) = realloc;
-static void (*freep)(void *) = free;
-
-#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS];
-#endif
-
-void *(*thread_create_prepare_func)(void) = NULL;
-void (*thread_create_parent_func)(void *) = NULL;
-void (*thread_create_child_func)(void *) = NULL;
-
-typedef struct ethr_xhndl_list_ ethr_xhndl_list;
-struct ethr_xhndl_list_ {
-    ethr_xhndl_list *next;
-    void (*funcp)(void);
-};
-
-static size_t pagesize;
-#define ETHR_PAGE_ALIGN(SZ) (((((size_t) (SZ)) - 1)/pagesize + 1)*pagesize)
-static size_t min_stack_size; /* kilo words */
-static size_t max_stack_size; /* kilo words */
-#define ETHR_B2KW(B) ((((size_t) (B)) - 1)/(sizeof(void *)*1024) + 1)
-#define ETHR_KW2B(KW) (((size_t) (KW))*sizeof(void *)*1024)
-
-ethr_mutex xhndl_mtx;
-ethr_xhndl_list *xhndl_list;
-
-static int
-init_common(ethr_init_data *id)
-{
-    int res;
-    if (id) {
-	allocp				= id->alloc;
-	reallocp			= id->realloc;
-	freep				= id->free;
-	thread_create_prepare_func	= id->thread_create_prepare_func;
-	thread_create_parent_func	= id->thread_create_parent_func;
-	thread_create_child_func	= id->thread_create_child_func;
-    }
-    if (!allocp || !reallocp || !freep)
-	return EINVAL;
-
-#ifdef _SC_PAGESIZE
-    pagesize = (size_t) sysconf(_SC_PAGESIZE);
-#elif defined(HAVE_GETPAGESIZE)
-    pagesize = (size_t) getpagesize();
-#else
-    pagesize = (size_t) 4*1024; /* Guess 4 KB */
-#endif
-
-    /* User needs at least 4 KB */
-    min_stack_size = 4*1024;
-#if SIZEOF_VOID_P == 8
-    /* Double that on 64-bit archs */
-    min_stack_size *= 2;
-#endif
-    /* On some systems as much as about 4 KB is used by the system */
-    min_stack_size += 4*1024;
-    /* There should be room for signal handlers */
-#ifdef SIGSTKSZ
-    min_stack_size += SIGSTKSZ;
-#else
-    min_stack_size += pagesize;
-#endif
-    /* The system may think that we need more stack */
-#if defined(PTHREAD_STACK_MIN)
-    if (min_stack_size < PTHREAD_STACK_MIN)
-	min_stack_size = PTHREAD_STACK_MIN;
-#elif defined(_SC_THREAD_STACK_MIN)
-    {
-	size_t thr_min_stk_sz = (size_t) sysconf(_SC_THREAD_STACK_MIN);
-	if (min_stack_size < thr_min_stk_sz)
-	    min_stack_size = thr_min_stk_sz;
-    }
-#endif
-    /* The guard is at least on some platforms included in the stack size
-       passed when creating threads */
-#ifdef ETHR_STACK_GUARD_SIZE
-    min_stack_size += ETHR_STACK_GUARD_SIZE;
-#endif
-    min_stack_size = ETHR_PAGE_ALIGN(min_stack_size);
-
-    min_stack_size = ETHR_B2KW(min_stack_size);
-
-    max_stack_size = 32*1024*1024;
-#if SIZEOF_VOID_P == 8
-    max_stack_size *= 2;
-#endif
-    max_stack_size = ETHR_B2KW(max_stack_size);
-
-    xhndl_list = NULL;
-
-    res = ethr_mutex_init(&xhndl_mtx);
-    if (res != 0)
-	return res;
-
-    res = ethr_mutex_set_forksafe(&xhndl_mtx);
-    if (res != 0 && res != ENOTSUP)
-	return res;
-
-    return 0;
-}
-
-int
-ethr_install_exit_handler(void (*funcp)(void))
-{
-    ethr_xhndl_list *xhp;
-    int res;
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-
-    if (!funcp)
-	return EINVAL;
-
-    xhp = (ethr_xhndl_list *) (*allocp)(sizeof(ethr_xhndl_list));
-    if (!xhp)
-	return ENOMEM;
-
-    res = ethr_mutex_lock__(&xhndl_mtx);
-    if (res != 0) {
-	(*freep)((void *) xhp);
-	return res;
-    }
-
-    xhp->funcp = funcp;
-    xhp->next = xhndl_list;
-    xhndl_list = xhp;
-
-    res = ethr_mutex_unlock__(&xhndl_mtx);
-    if (res != 0)
-	abort();
-
-    return res;
-}
-
-static void
-run_exit_handlers(void)
-{
-    int res;
-    ethr_xhndl_list *xhp;
-
-    res = ethr_mutex_lock__(&xhndl_mtx);
-    if (res != 0)
-	abort();
-
-    xhp = xhndl_list;
-
-    res = ethr_mutex_unlock__(&xhndl_mtx);
-    if (res != 0)
-	abort();
-
-    for (; xhp; xhp = xhp->next)
-	(*xhp->funcp)();
-}
-
-#if defined(ETHR_PTHREADS)
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * pthread implementation                                                    *
-\*                                                                           */
-
-typedef struct {
-    pthread_mutex_t mtx;
-    pthread_cond_t cnd;
-    int initialized;
-    void *(*thr_func)(void *);
-    void *arg;
-    void *prep_func_res;
-} thr_wrap_data_;
-
-static int no_ethreads;
-static ethr_mutex no_ethrs_mtx;
-
-#ifndef ETHR_HAVE_PTHREAD_ATFORK
-#define ETHR_HAVE_PTHREAD_ATFORK 0
-#endif
-
-#if !ETHR_HAVE_PTHREAD_ATFORK
-#warning "Cannot enforce fork-safety"
-#endif
-
-#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT
-static pthread_rwlockattr_t write_pref_attr_data;
-static pthread_rwlockattr_t *write_pref_attr;
-#endif
-
-/*
- * ----------------------------------------------------------------------------
- * Static functions
- * ----------------------------------------------------------------------------
- */
-
-/*
- * Functions with safe_ prefix aborts on failure. To be used when
- * we cannot recover after failure.
- */
-
-static ETHR_INLINE void
-safe_mutex_lock(pthread_mutex_t *mtxp)
-{
-    int res = pthread_mutex_lock(mtxp);
-    if (res != 0)
-	abort();
-}
-
-static ETHR_INLINE void
-safe_mutex_unlock(pthread_mutex_t *mtxp)
-{
-    int res = pthread_mutex_unlock(mtxp);
-    if (res != 0)
-	abort();
-}
-
-static ETHR_INLINE void
-safe_cond_signal(pthread_cond_t *cndp)
-{
-    int res = pthread_cond_signal(cndp);
-    if (res != 0)
-	abort();
-}
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-
-static volatile int rec_mtx_attr_need_init = 1;
-static pthread_mutexattr_t rec_mtx_attr;
-
-static int init_rec_mtx_attr(void);
-
-#endif
-
-#if ETHR_HAVE_PTHREAD_ATFORK
-
-static ethr_mutex forksafe_mtx = ETHR_MUTEX_INITER;
-
-static void lock_mutexes(void)
-{
-    ethr_mutex *m = &forksafe_mtx;
-    do {
-
-	safe_mutex_lock(&m->pt_mtx);
-
-	m = m->next;
-
-    } while (m != &forksafe_mtx);
-}
-
-static void unlock_mutexes(void)
-{
-    ethr_mutex *m = forksafe_mtx.prev;
-    do {
-
-	safe_mutex_unlock(&m->pt_mtx);
-
-	m = m->prev;
-
-    } while (m->next != &forksafe_mtx);
-}
-
-#if ETHR_INIT_MUTEX_IN_CHILD_AT_FORK
-
-static void reinit_mutexes(void)
-{
-    ethr_mutex *m = forksafe_mtx.prev;
-    do {
-	pthread_mutexattr_t *attrp = NULL;
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-	if (m->is_rec_mtx) {
-	    if (rec_mtx_attr_need_init) {
-		int res = init_rec_mtx_attr();
-		if (res != 0)
-		    abort();
-	    }
-	    attrp = &rec_mtx_attr;
-	}
-#endif
-	if (pthread_mutex_init(&m->pt_mtx, attrp) != 0)
-	    abort();
-
-	m = m->prev;
-
-    } while (m->next != &forksafe_mtx);
-}
-
-#endif
-
-static int
-init_forksafe(void)
-{
-    static int init_done = 0;
-    int res = 0;
-
-    if (init_done)
-	return res;
-
-    forksafe_mtx.prev = &forksafe_mtx;
-    forksafe_mtx.next = &forksafe_mtx;
-
-    res = pthread_atfork(lock_mutexes,
-			 unlock_mutexes,
-#if ETHR_INIT_MUTEX_IN_CHILD_AT_FORK
-			 reinit_mutexes
-#else
-			 unlock_mutexes
-#endif
-	);
-
-    init_done = 1;
-    return res;
-}
-
-#endif
-
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-
-#if defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE)
-
-#define SET_REC_MUTEX_ATTR(AP) \
-  pthread_mutexattr_settype((AP), PTHREAD_MUTEX_RECURSIVE);
-
-#elif defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP)
-
-#define SET_REC_MUTEX_ATTR(AP) \
-  pthread_mutexattr_setkind_np((AP), PTHREAD_MUTEX_RECURSIVE_NP);
-
-#else
-
-#error "Don't know how to set recursive mutex attributes"
-
-#endif
-
-static int
-init_rec_mtx_attr(void)
-{
-    int res, mres;
-    static pthread_mutex_t attrinit_mtx = PTHREAD_MUTEX_INITIALIZER;
-
-    mres = pthread_mutex_lock(&attrinit_mtx);
-    if (mres != 0)
-	return mres;
-    /* Got here under race conditions; check again ... */
-    if (!rec_mtx_attr_need_init)
-	res = 0;
-    else {
-	res = pthread_mutexattr_init(&rec_mtx_attr);
-	if (res == 0) {
-	    res = SET_REC_MUTEX_ATTR(&rec_mtx_attr);
-	    if (res == 0)
-		rec_mtx_attr_need_init = 0;
-	    else
-		(void) pthread_mutexattr_destroy(&rec_mtx_attr);
-	}
-    }
-
-    mres = pthread_mutex_unlock(&attrinit_mtx);
-    if (mres != 0)
-	return mres;
-    return res;
-}
-
-#endif /* #if ETHR_HAVE_ETHR_REC_MUTEX_INIT */
-
-static ETHR_INLINE void thr_exit_cleanup(void)
-{
-    run_exit_handlers();
-    safe_mutex_lock(&no_ethrs_mtx.pt_mtx);
-    ASSERT(no_ethreads > 0);
-    no_ethreads--;
-    safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-}
-
-static void *thr_wrapper(void *vtwd)
-{
-    void *res;
-    thr_wrap_data_ *twd = (thr_wrap_data_ *) vtwd;
-    void *(*thr_func)(void *) = twd->thr_func;
-    void *arg = twd->arg;
-
-    safe_mutex_lock(&twd->mtx);
-
-    if (thread_create_child_func)
-	(*thread_create_child_func)(twd->prep_func_res);
-
-    twd->initialized = 1;
-
-    safe_cond_signal(&twd->cnd);
-    safe_mutex_unlock(&twd->mtx);
-
-    res = (*thr_func)(arg);
-    thr_exit_cleanup();
-    return res;
-}
-
-
-/*
- * ----------------------------------------------------------------------------
- * Exported functions
- * ----------------------------------------------------------------------------
- */
-
-int
-ethr_init(ethr_init_data *id)
-{
-    int res;
-
-    if (!ethr_not_inited)
-	return EINVAL;
-
-    ethr_not_inited = 0;
-
-    res = init_common(id);
-    if (res != 0)
-	goto error;
-
-#if ETHR_HAVE_PTHREAD_ATFORK
-    init_forksafe();
-#endif
-
-    no_ethreads = 1;
-    res = ethr_mutex_init(&no_ethrs_mtx);
-    if (res != 0)
-	goto error;
-    res = ethr_mutex_set_forksafe(&no_ethrs_mtx);
-    if (res != 0 && res != ENOTSUP)
-	goto error;
-
-#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS
-    {
-	int i;
-	for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) {
-#ifdef ETHR_HAVE_PTHREAD_SPIN_LOCK
-	    res = pthread_spin_init(&ethr_atomic_protection__[i].u.spnlck, 0);
-#else
-	    res = ethr_mutex_init(&ethr_atomic_protection__[i].u.mtx);
-#endif
-	    if (res != 0)
-		goto error;
-	}
-    }
-#endif
-
-#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT
-#if defined(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP) \
-    && defined(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)
-    res = pthread_rwlockattr_init(&write_pref_attr_data);
-    if (res != 0)
-	goto error;
-    res = pthread_rwlockattr_setkind_np(
-	&write_pref_attr_data,
-	PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP);
-    if (res != 0)
-	goto error;
-    write_pref_attr = &write_pref_attr_data;
-#else
-    write_pref_attr = NULL;
-#endif
-#endif
-
-
-    return 0;
-
- error:
-    ethr_not_inited = 1;
-    return res;
-
-}
-
-int
-ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
-		ethr_thr_opts *opts)
-{
-    thr_wrap_data_ twd;
-    pthread_attr_t attr;
-    int res, dres;
-    int use_stack_size = (opts && opts->suggested_stack_size >= 0
-			  ? opts->suggested_stack_size
-			  : -1 /* Use system default */);
-
-#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE
-    if (use_stack_size < 0)
-	use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE;
-#endif
-
-    twd.initialized = 0;
-    twd.thr_func = func;
-    twd.arg = arg;
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!tid || !func) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    /* Call prepare func if it exist */
-    if (thread_create_prepare_func)
-	twd.prep_func_res = (*thread_create_prepare_func)();
-    else
-	twd.prep_func_res = NULL;
-
-    /* Set som thread attributes */
-    res = pthread_attr_init(&attr);
-    if (res != 0)
-	goto cleanup_parent_func;
-    res = pthread_mutex_init(&twd.mtx, NULL);
-    if (res != 0)
-	goto cleanup_attr_destroy;
-    res = pthread_cond_init(&twd.cnd, NULL);
-    if (res != 0)
-	goto cleanup_mutex_destroy;
-
-    /* Schedule child thread in system scope (if possible) ... */
-    res = pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
-    if (res != 0 && res != ENOTSUP)
-	goto cleanup_cond_destroy;
-
-    if (use_stack_size >= 0) {
-	size_t suggested_stack_size = (size_t) use_stack_size;
-	size_t stack_size;
-#ifdef DEBUG
-	suggested_stack_size /= 2; /* Make sure we got margin */
-#endif
-#ifdef ETHR_STACK_GUARD_SIZE
-	/* The guard is at least on some platforms included in the stack size
-	   passed when creating threads */
-	suggested_stack_size += ETHR_B2KW(ETHR_STACK_GUARD_SIZE);
-#endif
-	if (suggested_stack_size < min_stack_size)
-	    stack_size = ETHR_KW2B(min_stack_size);
-	else if (suggested_stack_size > max_stack_size)
-	    stack_size = ETHR_KW2B(max_stack_size);
-	else
-	    stack_size = ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size));
-	(void) pthread_attr_setstacksize(&attr, stack_size);
-    }
-
-#ifdef ETHR_STACK_GUARD_SIZE
-    (void) pthread_attr_setguardsize(&attr, ETHR_STACK_GUARD_SIZE);
-#endif
-
-    /* Detached or joinable... */
-    res = pthread_attr_setdetachstate(&attr,
-				      (opts && opts->detached
-				       ? PTHREAD_CREATE_DETACHED
-				       : PTHREAD_CREATE_JOINABLE));
-    if (res != 0)
-	goto cleanup_cond_destroy;
-    
-    res = pthread_mutex_lock(&twd.mtx);
-
-    if (res != 0)
-	goto cleanup_cond_destroy;
-
-    safe_mutex_lock(&no_ethrs_mtx.pt_mtx);
-    if (no_ethreads < ETHR_MAX_THREADS) {
-	no_ethreads++;
-	safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-    }
-    else {
-	res = EAGAIN;
-	safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-	goto cleanup_mutex_unlock;
-    }
-
-    res = pthread_create((pthread_t *) tid, &attr, thr_wrapper, (void *) &twd);
-
-    if (res != 0) {
-	safe_mutex_lock(&no_ethrs_mtx.pt_mtx);
-	ASSERT(no_ethreads > 0);
-	no_ethreads--;
-	safe_mutex_unlock(&no_ethrs_mtx.pt_mtx);
-    }
-    else {
-
-	/* Wait for child to initialize... */
-	while (!twd.initialized) {
-	    res = pthread_cond_wait(&twd.cnd, &twd.mtx);
-	    if (res != 0 && res != EINTR)
-		break;
-	}
-
-    }
-
-    /* Cleanup... */
- cleanup_mutex_unlock:
-    dres = pthread_mutex_unlock(&twd.mtx);
-    if (res == 0)
-	res = dres;
- cleanup_cond_destroy:
-    dres = pthread_cond_destroy(&twd.cnd);
-    if (res == 0)
-	res = dres;
- cleanup_mutex_destroy:
-    dres = pthread_mutex_destroy(&twd.mtx);
-    if (res == 0)
-	res = dres;
- cleanup_attr_destroy:
-    dres = pthread_attr_destroy(&attr);
-    if (res == 0)
-	res = dres;
- cleanup_parent_func:
-    if (thread_create_parent_func)
-	(*thread_create_parent_func)(twd.prep_func_res);
-
-    return res;
-}
-
-int
-ethr_thr_join(ethr_tid tid, void **res)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_join((pthread_t) tid, res);
-}
-
-int
-ethr_thr_detach(ethr_tid tid)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_detach((pthread_t) tid);
-}
-
-void
-ethr_thr_exit(void *res)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return;
-    }
-#endif
-    thr_exit_cleanup();
-    pthread_exit(res);
-}
-
-ethr_tid
-ethr_self(void)
-{
-    return (ethr_tid) pthread_self();
-}
-
-int
-ethr_equal_tids(ethr_tid tid1, ethr_tid tid2)
-{
-    return pthread_equal((pthread_t) tid1, (pthread_t) tid2);
-}
-
-
-/*
- * Mutex functions
- */
-
-
-int
-ethr_mutex_init(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    mtx->initialized = ETHR_MUTEX_INITIALIZED;
-#endif
-    mtx->prev = NULL;
-    mtx->next = NULL;
-    mtx->is_rec_mtx = 0;
-    return pthread_mutex_init(&mtx->pt_mtx, NULL);
-}
-
-#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT
-
-int
-ethr_rec_mutex_init(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    mtx->initialized = ETHR_MUTEX_INITIALIZED;
-#endif
-    if (rec_mtx_attr_need_init)
-	init_rec_mtx_attr();
-
-    mtx->prev = NULL;
-    mtx->next = NULL;
-    mtx->is_rec_mtx = 1;
-    return pthread_mutex_init(&mtx->pt_mtx, &rec_mtx_attr);
-}
-
-#endif /* #if ETHR_HAVE_ETHR_REC_MUTEX_INIT */
-
-int
-ethr_mutex_destroy(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (mtx->next) {
-	ASSERT(mtx->prev);
-	ethr_mutex_unset_forksafe(mtx);
-    }
-#if ETHR_XCHK
-    mtx->initialized = 0;
-#endif
-    return pthread_mutex_destroy(&mtx->pt_mtx);
-}
-
-int ethr_mutex_set_forksafe(ethr_mutex *mtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-#if ETHR_HAVE_PTHREAD_ATFORK
-    res = pthread_mutex_lock(&forksafe_mtx.pt_mtx);
-    if (res != 0)
-	return res;
-    if (!forksafe_mtx.next) {
-	ASSERT(!forksafe_mtx.prev);
-	init_forksafe();
-    }
-    if (mtx->next) {
-	/* forksafe already set for this mutex */ 
-	ASSERT(mtx->prev);
-    }
-    else {
-	mtx->next = forksafe_mtx.next;
-	mtx->prev = &forksafe_mtx;
-	forksafe_mtx.next->prev = mtx;
-	forksafe_mtx.next = mtx;
-    }
-
-    res = pthread_mutex_unlock(&forksafe_mtx.pt_mtx);
-
-#else /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    res = ENOTSUP;
-#endif /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    return res;
-}
-
-int ethr_mutex_unset_forksafe(ethr_mutex *mtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-#if ETHR_HAVE_PTHREAD_ATFORK
-    res = pthread_mutex_lock(&forksafe_mtx.pt_mtx);
-    if (res != 0)
-	return res;
-    if (!forksafe_mtx.next) {
-	ASSERT(!forksafe_mtx.prev);
-	init_forksafe();
-    }
-    if (!mtx->next) {
-	/* forksafe already unset for this mutex */ 
-	ASSERT(!mtx->prev);
-    }
-    else {
-	mtx->prev->next = mtx->next;
-	mtx->next->prev = mtx->prev;
-	mtx->next = NULL;
-	mtx->prev = NULL;
-    }
-    res = pthread_mutex_unlock(&forksafe_mtx.pt_mtx);
-
-#else /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    res = ENOTSUP;
-#endif /* #if ETHR_HAVE_PTHREAD_ATFORK */
-    return res;
-}
-
-int
-ethr_mutex_trylock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_trylock__(mtx);
-}
-
-int
-ethr_mutex_lock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_lock__(mtx);
-}
-
-int
-ethr_mutex_unlock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_unlock__(mtx);
-}
-
-/*
- * Condition variable functions
- */
-
-int
-ethr_cond_init(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    cnd->initialized = ETHR_COND_INITIALIZED;
-#endif
-    return pthread_cond_init(&cnd->pt_cnd, NULL);
-}
-
-int
-ethr_cond_destroy(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    cnd->initialized = 0;
-#endif
-    return pthread_cond_destroy(&cnd->pt_cnd);
-}
-
-int
-ethr_cond_signal(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_cond_signal(&cnd->pt_cnd);
-}
-
-int
-ethr_cond_broadcast(ethr_cond *cnd)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_cond_broadcast(&cnd->pt_cnd);
-}
-
-int
-ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| cnd->initialized != ETHR_COND_INITIALIZED
-	|| !mtx
-	|| mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx);
-}
-
-int
-ethr_cond_timedwait(ethr_cond *cnd, ethr_mutex *mtx, ethr_timeval *timeout)
-{
-    struct timespec to;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| cnd->initialized != ETHR_COND_INITIALIZED
-	|| !mtx
-	|| mtx->initialized != ETHR_MUTEX_INITIALIZED
-	|| !timeout) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    to.tv_sec = timeout->tv_sec;
-    to.tv_nsec = timeout->tv_nsec;
-
-    return pthread_cond_timedwait(&cnd->pt_cnd, &mtx->pt_mtx, &to);
-}
-
-
-#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT
-
-int
-ethr_rwmutex_init(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
-#endif
-    return pthread_rwlock_init(&rwmtx->pt_rwlock, write_pref_attr);
-}
-
-int
-ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = pthread_rwlock_destroy(&rwmtx->pt_rwlock);
-#if ETHR_XCHK
-    rwmtx->initialized = 0;
-#endif
-    return res;
-}
-
-int
-ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_tryrlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_rlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_runlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_tryrwlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_rwlock__(rwmtx);
-}
-
-int
-ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwmutex_rwunlock__(rwmtx);
-}
-
-#endif /* #ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT */
-
-/*
- * Current time
- */
-
-int
-ethr_time_now(ethr_timeval *time)
-{
-    int res;
-    struct timeval tv;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!time) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    res = gettimeofday(&tv, NULL);
-    time->tv_sec = (long) tv.tv_sec;
-    time->tv_nsec = ((long) tv.tv_usec)*1000;
-    return res;
-}
-
-/*
- * Thread specific data
- */
-
-int
-ethr_tsd_key_create(ethr_tsd_key *keyp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!keyp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return pthread_key_create((pthread_key_t *) keyp, NULL);
-}
-
-int
-ethr_tsd_key_delete(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_key_delete((pthread_key_t) key);
-}
-
-int
-ethr_tsd_set(ethr_tsd_key key, void *value)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return pthread_setspecific((pthread_key_t) key, value);
-}
-
-void *
-ethr_tsd_get(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return NULL;
-    }
-#endif
-    return pthread_getspecific((pthread_key_t) key);
-}
-
-/*
- * Signal functions
- */
-
-#if ETHR_HAVE_ETHR_SIG_FUNCS
-
-int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!set && !oset) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-  return pthread_sigmask(how, set, oset);
-}
-
-int ethr_sigwait(const sigset_t *set, int *sig)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!set || !sig) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (sigwait(set, sig) < 0)
-	return errno;
-    return 0;
-}
-
-#endif /* #if ETHR_HAVE_ETHR_SIG_FUNCS */
-
-#elif defined(ETHR_WIN32_THREADS)
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
- * Native win32 threads implementation                                       *
-\*                                                                           */
-
-#define INVALID_TID -1
-
-/* The spin count values are more or less taken out of the blue */
-#define ETHR_MUTEX_SPIN_COUNT	5000
-#define ETHR_COND_SPIN_COUNT	1000
-
-ethr_tid serial_shift; /* Bits to shift serial when constructing a tid */
-ethr_tid last_serial; /* Last thread table serial used */
-ethr_tid last_ix; /* Last thread table index used */
-ethr_tid thr_ix_mask; /* Mask used to mask out thread table index from a tid */
-
-/* Event used for conditional variables. On per thread. */
-/*typedef struct cnd_wait_event__ cnd_wait_event_;*/
-struct cnd_wait_event__ {
-    HANDLE handle;
-    cnd_wait_event_ *prev;
-    cnd_wait_event_ *next;
-    int in_queue;
-};
-
-/* Thread specific data. Stored in the thread table */
-typedef struct {
-    ethr_tid thr_id;
-    HANDLE thr_handle;
-    ethr_tid joiner;
-    void *result;
-    cnd_wait_event_ wait_event;
-} thr_data_;
-
-/* Argument passed to thr_wrapper() */
-typedef struct {
-    void * (*func)(void *);
-    void * arg;
-    thr_data_ *ptd;
-    thr_data_ *td;
-    int res;
-    void *prep_func_res;
-} thr_wrap_data_;
-
-
-static CRITICAL_SECTION thr_table_cs; /* Critical section used to protect
-					 the thread table from concurrent
-					 accesses. */
-static CRITICAL_SECTION fake_static_init_cs; /* Critical section used to protect
-					initialazition of 'statically
-					initialized' mutexes */
-static thr_data_ * thr_table[ETHR_MAX_THREADS]; /* The thread table */
-
-static DWORD tls_own_thr_data;
-
-static thr_data_ main_thr_data;
-
-#define THR_IX(TID)	((TID) & thr_ix_mask)
-#define OWN_THR_DATA	((thr_data_ *) TlsGetValue(tls_own_thr_data))
-
-/*
- * ----------------------------------------------------------------------------
- * Static functions
- * ----------------------------------------------------------------------------
- */
-
-static int
-get_errno(void)
-{
-    switch (GetLastError()) {
-    case ERROR_INVALID_FUNCTION:		return EINVAL;	/* 1	*/
-    case ERROR_FILE_NOT_FOUND:			return ENOENT;	/* 2	*/
-    case ERROR_PATH_NOT_FOUND:			return ENOENT;	/* 3	*/
-    case ERROR_TOO_MANY_OPEN_FILES:		return EMFILE;	/* 4	*/
-    case ERROR_ACCESS_DENIED:			return EACCES;	/* 5	*/
-    case ERROR_INVALID_HANDLE:			return EBADF;	/* 6	*/
-    case ERROR_ARENA_TRASHED:			return ENOMEM;	/* 7	*/
-    case ERROR_NOT_ENOUGH_MEMORY:		return ENOMEM;	/* 8	*/
-    case ERROR_INVALID_BLOCK:			return ENOMEM;	/* 9	*/
-    case ERROR_BAD_ENVIRONMENT:			return E2BIG;	/* 10	*/
-    case ERROR_BAD_FORMAT:			return ENOEXEC;	/* 11	*/
-    case ERROR_INVALID_ACCESS:			return EINVAL;	/* 12	*/
-    case ERROR_INVALID_DATA:			return EINVAL;	/* 13	*/
-    case ERROR_OUTOFMEMORY:			return ENOMEM;	/* 14	*/
-    case ERROR_INVALID_DRIVE:			return ENOENT;	/* 15	*/
-    case ERROR_CURRENT_DIRECTORY:		return EACCES;	/* 16	*/
-    case ERROR_NOT_SAME_DEVICE:			return EXDEV;	/* 17	*/
-    case ERROR_NO_MORE_FILES:			return ENOENT;	/* 18	*/
-    case ERROR_WRITE_PROTECT:			return EACCES;	/* 19	*/
-    case ERROR_BAD_UNIT:			return EACCES;	/* 20	*/
-    case ERROR_NOT_READY:			return EACCES;	/* 21	*/
-    case ERROR_BAD_COMMAND:			return EACCES;	/* 22	*/
-    case ERROR_CRC:				return EACCES;	/* 23	*/
-    case ERROR_BAD_LENGTH:			return EACCES;	/* 24	*/
-    case ERROR_SEEK:				return EACCES;	/* 25	*/
-    case ERROR_NOT_DOS_DISK:			return EACCES;	/* 26	*/
-    case ERROR_SECTOR_NOT_FOUND:		return EACCES;	/* 27	*/
-    case ERROR_OUT_OF_PAPER:			return EACCES;	/* 28	*/
-    case ERROR_WRITE_FAULT:			return EACCES;	/* 29	*/
-    case ERROR_READ_FAULT:			return EACCES;	/* 30	*/
-    case ERROR_GEN_FAILURE:			return EACCES;	/* 31	*/
-    case ERROR_SHARING_VIOLATION:		return EACCES;	/* 32	*/
-    case ERROR_LOCK_VIOLATION:			return EACCES;	/* 33	*/
-    case ERROR_WRONG_DISK:			return EACCES;	/* 34	*/
-    case ERROR_SHARING_BUFFER_EXCEEDED:		return EACCES;	/* 36	*/
-    case ERROR_BAD_NETPATH:			return ENOENT;	/* 53	*/
-    case ERROR_NETWORK_ACCESS_DENIED:		return EACCES;	/* 65	*/
-    case ERROR_BAD_NET_NAME:			return ENOENT;	/* 67	*/
-    case ERROR_FILE_EXISTS:			return EEXIST;	/* 80	*/
-    case ERROR_CANNOT_MAKE:			return EACCES;	/* 82	*/
-    case ERROR_FAIL_I24:			return EACCES;	/* 83	*/
-    case ERROR_INVALID_PARAMETER:		return EINVAL;	/* 87	*/
-    case ERROR_NO_PROC_SLOTS:			return EAGAIN;	/* 89	*/
-    case ERROR_DRIVE_LOCKED:			return EACCES;	/* 108	*/
-    case ERROR_BROKEN_PIPE:			return EPIPE;	/* 109	*/
-    case ERROR_DISK_FULL:			return ENOSPC;	/* 112	*/
-    case ERROR_INVALID_TARGET_HANDLE:		return EBADF;	/* 114	*/
-    case ERROR_WAIT_NO_CHILDREN:		return ECHILD;	/* 128	*/
-    case ERROR_CHILD_NOT_COMPLETE:		return ECHILD;	/* 129	*/
-    case ERROR_DIRECT_ACCESS_HANDLE:		return EBADF;	/* 130	*/
-    case ERROR_NEGATIVE_SEEK:			return EINVAL;	/* 131	*/
-    case ERROR_SEEK_ON_DEVICE:			return EACCES;	/* 132	*/
-    case ERROR_DIR_NOT_EMPTY:			return ENOTEMPTY;/* 145	*/
-    case ERROR_NOT_LOCKED:			return EACCES;	/* 158	*/
-    case ERROR_BAD_PATHNAME:			return ENOENT;	/* 161	*/
-    case ERROR_MAX_THRDS_REACHED:		return EAGAIN;	/* 164	*/
-    case ERROR_LOCK_FAILED:			return EACCES;	/* 167	*/
-    case ERROR_ALREADY_EXISTS:			return EEXIST;	/* 183	*/
-    case ERROR_INVALID_STARTING_CODESEG:	return ENOEXEC;	/* 188	*/
-    case ERROR_INVALID_STACKSEG:		return ENOEXEC;	/* 189	*/
-    case ERROR_INVALID_MODULETYPE:		return ENOEXEC;	/* 190	*/
-    case ERROR_INVALID_EXE_SIGNATURE:		return ENOEXEC;	/* 191	*/
-    case ERROR_EXE_MARKED_INVALID:		return ENOEXEC;	/* 192	*/
-    case ERROR_BAD_EXE_FORMAT:			return ENOEXEC;	/* 193	*/
-    case ERROR_ITERATED_DATA_EXCEEDS_64k:	return ENOEXEC;	/* 194	*/
-    case ERROR_INVALID_MINALLOCSIZE:		return ENOEXEC;	/* 195	*/
-    case ERROR_DYNLINK_FROM_INVALID_RING:	return ENOEXEC;	/* 196	*/
-    case ERROR_IOPL_NOT_ENABLED:		return ENOEXEC;	/* 197	*/
-    case ERROR_INVALID_SEGDPL:			return ENOEXEC;	/* 198	*/
-    case ERROR_AUTODATASEG_EXCEEDS_64k:		return ENOEXEC;	/* 199	*/
-    case ERROR_RING2SEG_MUST_BE_MOVABLE:	return ENOEXEC;	/* 200	*/
-    case ERROR_RELOC_CHAIN_XEEDS_SEGLIM:	return ENOEXEC;	/* 201	*/
-    case ERROR_INFLOOP_IN_RELOC_CHAIN:		return ENOEXEC;	/* 202	*/
-    case ERROR_FILENAME_EXCED_RANGE:		return ENOENT;	/* 206	*/
-    case ERROR_NESTING_NOT_ALLOWED:		return EAGAIN;	/* 215	*/
-    case ERROR_NOT_ENOUGH_QUOTA:		return ENOMEM;	/* 1816	*/
-    default:					return EINVAL;
-    }
-}
-
-static ETHR_INLINE thr_data_ *
-tid2thr(ethr_tid tid)
-{
-    ethr_tid ix;
-    thr_data_ *td;
-    
-    if (tid < 0)
-	return NULL;
-    ix = THR_IX(tid);
-    if (ix >= ETHR_MAX_THREADS)
-	return NULL;
-    td = thr_table[ix];
-    if (!td)
-	return NULL;
-    if (td->thr_id != tid)
-	return NULL;
-    return td;
-}
-
-static ETHR_INLINE void
-new_tid(ethr_tid *new_tid, ethr_tid *new_serial, ethr_tid *new_ix)
-{
-    ethr_tid tmp_serial = last_serial;
-    ethr_tid tmp_ix = last_ix + 1;
-    ethr_tid start_ix = tmp_ix;
-
-
-    do {
-	if (tmp_ix >= ETHR_MAX_THREADS) {
-	    tmp_serial++;
-	    if ((tmp_serial << serial_shift) < 0)
-		tmp_serial = 0;
-	    tmp_ix = 0;
-	}
-	if (!thr_table[tmp_ix]) {
-	    *new_tid	= (tmp_serial << serial_shift) | tmp_ix;
-	    *new_serial	= tmp_serial;
-	    *new_ix	= tmp_ix;
-	    return;
-	}
-	tmp_ix++;
-    } while (tmp_ix != start_ix);
-
-    *new_tid	= INVALID_TID;
-    *new_serial	= INVALID_TID;
-    *new_ix	= INVALID_TID;
-
-}
-
-
-static void thr_exit_cleanup(thr_data_ *td, void *res)
-{
-
-    ASSERT(td == OWN_THR_DATA);
-
-    run_exit_handlers();
-
-    EnterCriticalSection(&thr_table_cs);
-    CloseHandle(td->wait_event.handle);
-    if (td->thr_handle == INVALID_HANDLE_VALUE) {
-	/* We are detached; cleanup thread table */
-	ASSERT(td->joiner == INVALID_TID);
-	ASSERT(td == thr_table[THR_IX(td->thr_id)]);
-	thr_table[THR_IX(td->thr_id)] = NULL;
-	if (td != &main_thr_data)
-	    (*freep)((void *) td);
-    }
-    else {
-	/* Save result and let joining thread cleanup */
-	td->result = res;
-    }
-    LeaveCriticalSection(&thr_table_cs);
-}
-
-static unsigned __stdcall thr_wrapper(LPVOID args)
-{
-    void *(*func)(void*) = ((thr_wrap_data_ *) args)->func;
-    void *arg = ((thr_wrap_data_ *) args)->arg;
-    thr_data_ *td = ((thr_wrap_data_ *) args)->td;
-
-    td->wait_event.handle = CreateEvent(NULL, FALSE, FALSE, NULL);
-    if (td->wait_event.handle == INVALID_HANDLE_VALUE
-	|| !TlsSetValue(tls_own_thr_data, (LPVOID) td)) {
-	((thr_wrap_data_ *) args)->res = get_errno();
-	if (td->wait_event.handle != INVALID_HANDLE_VALUE)
-	    CloseHandle(td->wait_event.handle);
-	SetEvent(((thr_wrap_data_ *) args)->ptd->wait_event.handle);
-	_endthreadex((unsigned) 0);
-	ASSERT(0);
-    }
-
-    td->wait_event.prev = NULL;
-    td->wait_event.next = NULL;
-    td->wait_event.in_queue = 0;
-
-    if (thread_create_child_func)
-	(*thread_create_child_func)(((thr_wrap_data_ *) args)->prep_func_res);
-
-    ASSERT(td == OWN_THR_DATA);
-
-    ((thr_wrap_data_ *) args)->res = 0;
-    SetEvent(((thr_wrap_data_ *) args)->ptd->wait_event.handle);
-
-    thr_exit_cleanup(td, (*func)(arg));
-    return 0;
-}
-
-int
-ethr_fake_static_mutex_init(ethr_mutex *mtx)
-{
-    EnterCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    /* Got here under race conditions; check again... */
-    if (!mtx->initialized) {
-	if (!InitializeCriticalSectionAndSpinCount(&mtx->cs,
-						   ETHR_MUTEX_SPIN_COUNT))
-	    return get_errno();
-	mtx->initialized = ETHR_MUTEX_INITIALIZED;
-    }
-    LeaveCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    return 0;
-}
-
-static int
-fake_static_cond_init(ethr_cond *cnd)
-{
-    EnterCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    /* Got here under race conditions; check again... */
-    if (!cnd->initialized) {
-	if (!InitializeCriticalSectionAndSpinCount(&cnd->cs,
-						   ETHR_COND_SPIN_COUNT))
-	    return get_errno();
-	cnd->queue = NULL;
-	cnd->queue_end = NULL;
-	cnd->initialized = ETHR_COND_INITIALIZED;
-    }
-    LeaveCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs);
-    return 0;
-}
-
-#ifdef __GNUC__
-#define LL_LITERAL(X) X##LL
-#else
-#define LL_LITERAL(X) X##i64
-#endif
-
-#define EPOCH_JULIAN_DIFF LL_LITERAL(11644473600)
-
-static ETHR_INLINE void
-get_curr_time(long *sec, long *nsec)
-{
-    SYSTEMTIME t;
-    FILETIME ft;
-    LONGLONG lft;
-
-    GetSystemTime(&t);
-    SystemTimeToFileTime(&t, &ft);
-    memcpy(&lft, &ft, sizeof(lft));
-    *nsec = ((long) (lft % LL_LITERAL(10000000)))*100;
-    *sec = (long) ((lft / LL_LITERAL(10000000)) - EPOCH_JULIAN_DIFF);
-}
-
-static cnd_wait_event_ *cwe_freelist;
-static CRITICAL_SECTION cwe_cs;
-
-static int
-alloc_cwe(cnd_wait_event_ **cwe_res)
-{
-    cnd_wait_event_ *cwe;
-    EnterCriticalSection(&cwe_cs);
-    cwe = cwe_freelist;
-    if (cwe) {
-	cwe_freelist = cwe->next;
-	LeaveCriticalSection(&cwe_cs);
-    }
-    else {
-	LeaveCriticalSection(&cwe_cs);
-	cwe = (*allocp)(sizeof(cnd_wait_event_));
-	if (!cwe)
-	    return ENOMEM;
-	cwe->handle = CreateEvent(NULL, FALSE, FALSE, NULL);
-	if (cwe->handle == INVALID_HANDLE_VALUE) {
-	    int res = get_errno();
-	    (*freep)(cwe);
-	    return res;
-	}
-    }
-    *cwe_res = cwe;
-    return 0;
-}
-
-static
-free_cwe(cnd_wait_event_ *cwe)
-{
-    EnterCriticalSection(&cwe_cs);
-    cwe->next = cwe_freelist;
-    cwe_freelist = cwe;
-    LeaveCriticalSection(&cwe_cs);
-}
-
-static ETHR_INLINE int
-condwait(ethr_cond *cnd,
-	 ethr_mutex *mtx,
-	 int with_timeout,
-	 ethr_timeval *timeout)
-{
-    int res;
-    thr_data_ *td;
-    cnd_wait_event_ *cwe;
-    DWORD code;
-    long time; /* time until timeout in milli seconds */
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-
-    if (!mtx
-	|| mtx->initialized != ETHR_MUTEX_INITIALIZED
-	|| !cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)
-	|| (with_timeout && !timeout)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    td = OWN_THR_DATA;
-    if (td)
-	cwe = &td->wait_event;
-    else { /* A non-ethread thread */
-	res = alloc_cwe(&cwe);
-	if (res != 0)
-	    return res;
-    }
-
-    if (!cnd->initialized)
-	fake_static_cond_init(cnd);
-    EnterCriticalSection(&cnd->cs);
-
-    ASSERT(!cwe->in_queue);
-    if (cnd->queue_end) {
-	ASSERT(cnd->queue);
-	cwe->prev = cnd->queue_end;
-	cwe->next = NULL;
-	cnd->queue_end->next = cwe;
-	cnd->queue_end = cwe;
-    }
-    else {
-	ASSERT(!cnd->queue);
-	cwe->prev = NULL;
-	cwe->next = NULL;
-	cnd->queue = cwe;
-	cnd->queue_end = cwe;
-    }
-    cwe->in_queue = 1;
-
-    LeaveCriticalSection(&cnd->cs);
- 
-    LeaveCriticalSection(&mtx->cs);
-
-    if (!with_timeout)
-	time = INFINITE;
-    else {
-	long sec, nsec;
-	ASSERT(timeout);
-	get_curr_time(&sec, &nsec);
-	time = (timeout->tv_sec - sec)*1000;
-	time += (timeout->tv_nsec - nsec + 500)/1000000;
-	if (time < 0)
-	    time = 0;
-    }
-
-    /* wait for event to signal */
-    code = WaitForSingleObject(cwe->handle, time);
-
-    EnterCriticalSection(&mtx->cs);
-
-    if (code == WAIT_OBJECT_0) {
-	/* We were woken by a signal or a broadcast ... */
-	res = 0;
-
-	/* ... no need to remove event from wait queue since this was
-	   taken care of by the signal or broadcast */
-#ifdef DEBUG
-	EnterCriticalSection(&cnd->cs);
-	ASSERT(!cwe->in_queue);
-	LeaveCriticalSection(&cnd->cs);
-#endif
-
-    }
-    else {
-	/* We timed out... */
-	res = ETIMEDOUT;
-
-	/* ... probably have to remove event from wait queue ... */
-	EnterCriticalSection(&cnd->cs);
-
-	if (cwe->in_queue) { /* ... but we must check that we are in queue
-				since a signal or broadcast after timeout
-				may have removed us from the queue */
-	    if (cwe->prev) {
-		cwe->prev->next = cwe->next;
-	    }
-	    else {
-		ASSERT(cnd->queue == cwe);
-		cnd->queue = cwe->next;
-	    }
-
-	    if (cwe->next) {
-		cwe->next->prev = cwe->prev;
-	    }
-	    else {
-		ASSERT(cnd->queue_end == cwe);
-		cnd->queue_end = cwe->prev;
-	    }
-	    cwe->in_queue = 0;
-	}
-
-	LeaveCriticalSection(&cnd->cs);
-
-    }
-
-    if (!td)
-	free_cwe(cwe);
-
-    return res;
-
-}
-
-
-/*
- * ----------------------------------------------------------------------------
- * Exported functions
- * ----------------------------------------------------------------------------
- */
-
-int
-ethr_init(ethr_init_data *id)
-{
-#ifdef _WIN32_WINNT
-    DWORD major = (_WIN32_WINNT >> 8) & 0xff;
-    DWORD minor = _WIN32_WINNT & 0xff;
-    OSVERSIONINFO os_version;
-#endif
-    int err = 0;
-    thr_data_ *td = &main_thr_data;
-    unsigned long i;
-
-    if (!ethr_not_inited)
-	return EINVAL;
-
-#ifdef _WIN32_WINNT
-    os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
-    GetVersionEx(&os_version);
-    if (os_version.dwPlatformId != VER_PLATFORM_WIN32_NT
-	|| os_version.dwMajorVersion < major
-	|| (os_version.dwMajorVersion == major
-	    && os_version.dwMinorVersion < minor))
-	return ENOTSUP;
-#endif
-
-    ASSERT(ETHR_MAX_THREADS > 0);
-    for (i = ETHR_MAX_THREADS - 1, serial_shift = 0;
-	 i;
-	 serial_shift++, i >>= 1);
-    thr_ix_mask = ~(~((ethr_tid) 0) << serial_shift);
-
-    tls_own_thr_data = TlsAlloc();
-    if (tls_own_thr_data == TLS_OUT_OF_INDEXES)
-	goto error;
-
-    last_serial = 0;
-    last_ix = 0;
-
-    td->thr_id = 0;
-    td->thr_handle = GetCurrentThread();
-    td->joiner = INVALID_TID;
-    td->result = NULL;
-    td->wait_event.handle = CreateEvent(NULL, FALSE, FALSE, NULL);
-    if (td->wait_event.handle == INVALID_HANDLE_VALUE)
-	goto error;
-    td->wait_event.prev = NULL;
-    td->wait_event.next = NULL;
-    td->wait_event.in_queue = 0;
-    thr_table[0] = td;
-
-    if (!TlsSetValue(tls_own_thr_data, (LPVOID) td))
-	goto error;
-
-    ASSERT(td == OWN_THR_DATA);
-
-
-    cwe_freelist = NULL;
-    if (!InitializeCriticalSectionAndSpinCount(&cwe_cs,
-					       ETHR_MUTEX_SPIN_COUNT))
-	goto error;
-
-    for (i = 1; i < ETHR_MAX_THREADS; i++)
-	thr_table[i] = NULL;
-
-    if (!InitializeCriticalSectionAndSpinCount(&thr_table_cs,
-					       ETHR_MUTEX_SPIN_COUNT))
-	goto error;
-    if (!InitializeCriticalSectionAndSpinCount(&fake_static_init_cs,
-					       ETHR_MUTEX_SPIN_COUNT))
-	goto error;
-    ethr_not_inited = 0;
-
-    err = init_common(id);
-    if (err)
-	goto error;
-
-    return 0;
-
- error:
-    ethr_not_inited = 1;
-    if (err == 0)
-	err = get_errno();
-    ASSERT(err != 0);
-    if (td->thr_handle != INVALID_HANDLE_VALUE)
-	CloseHandle(td->thr_handle);
-    if (td->wait_event.handle != INVALID_HANDLE_VALUE)
-	CloseHandle(td->wait_event.handle);
-    return err;
-}
-
-/*
- * Thread functions.
- */
-
-int
-ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
-		ethr_thr_opts *opts)
-{
-    int err = 0;
-    thr_wrap_data_ twd;
-    thr_data_ *my_td, *child_td = NULL;
-    ethr_tid child_tid, child_serial, child_ix;
-    DWORD code;
-    unsigned ID;
-    unsigned stack_size = 0; /* 0 = system default */
-    int use_stack_size = (opts && opts->suggested_stack_size >= 0
-			  ? opts->suggested_stack_size
-			  : -1 /* Use system default */);
-
-#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE
-    if (use_stack_size < 0)
-	use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE;
-#endif
-
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!tid || !func) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-
-    my_td = OWN_THR_DATA;
-    if (!my_td) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return EACCES;
-    }
-
-    if (use_stack_size >= 0) {
-	size_t suggested_stack_size = (size_t) use_stack_size;
-#ifdef DEBUG
-	suggested_stack_size /= 2; /* Make sure we got margin */
-#endif
-	if (suggested_stack_size < min_stack_size)
-	    stack_size = (unsigned) ETHR_KW2B(min_stack_size);
-	else if (suggested_stack_size > max_stack_size)
-	    stack_size = (unsigned) ETHR_KW2B(max_stack_size);
-	else
-	    stack_size =
-		(unsigned) ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size));
-    }
-
-    EnterCriticalSection(&thr_table_cs);
-
-    /* Call prepare func if it exist */
-    if (thread_create_prepare_func)
-	twd.prep_func_res = (*thread_create_prepare_func)();
-    else
-	twd.prep_func_res = NULL;
-
-    /* Find a new thread id to use */
-    new_tid(&child_tid, &child_serial, &child_ix);
-    if (child_tid == INVALID_TID) {
-	err = EAGAIN;
-	goto error;
-    }
-
-    ASSERT(child_ix == THR_IX(child_tid));
-
-    *tid = child_tid;
-
-    ASSERT(!thr_table[child_ix]);
-
-    /* Alloc thread data */
-    thr_table[child_ix] = child_td = (thr_data_ *) (*allocp)(sizeof(thr_data_));
-    if (!child_td) {
-	err = ENOMEM;
-	goto error;
-    }
-
-    /* Init thread data */
-
-    child_td->thr_id		= child_tid;
-    child_td->thr_handle	= INVALID_HANDLE_VALUE;
-    child_td->joiner		= INVALID_TID;
-    child_td->result		= NULL;
-    /* 'child_td->wait_event' is initialized by child thread */
-
-
-    /* Init thread wrapper data */
-
-    twd.func			= func;
-    twd.arg			= arg;
-    twd.ptd			= my_td;
-    twd.td			= child_td;
-    twd.res			= 0;
-
-    ASSERT(!my_td->wait_event.in_queue);
-
-    /* spawn the thr_wrapper function */
-    child_td->thr_handle = (HANDLE) _beginthreadex(NULL,
-						   stack_size,
-						   thr_wrapper,
-						   (LPVOID) &twd,
-						   0,
-						   &ID);
-    if (child_td->thr_handle == (HANDLE) 0) {
-	child_td->thr_handle = INVALID_HANDLE_VALUE;
-	goto error;
-    }
-
-    ASSERT(child_td->thr_handle != INVALID_HANDLE_VALUE);
-
-    /* Wait for child to finish initialization */
-    code = WaitForSingleObject(my_td->wait_event.handle, INFINITE);
-    if (twd.res || code != WAIT_OBJECT_0) {
-	err = twd.res;
-	goto error;
-    }
-
-    if (opts && opts->detached) {
-	CloseHandle(child_td->thr_handle);
-	child_td->thr_handle = INVALID_HANDLE_VALUE;
-    }
-
-    last_serial	= child_serial;
-    last_ix	= child_ix;
-
-    ASSERT(thr_table[child_ix] == child_td);
-
-    if (thread_create_parent_func)
-	(*thread_create_parent_func)(twd.prep_func_res);
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    return 0;
-
- error:
-
-    if (err == 0)
-	err = get_errno();
-    ASSERT(err != 0);
-
-    if (thread_create_parent_func)
-	(*thread_create_parent_func)(twd.prep_func_res);
-
-    if (child_ix != INVALID_TID) {
-
-	if (child_td) {
-	    ASSERT(thr_table[child_ix] == child_td);
-
-	    if (child_td->thr_handle != INVALID_HANDLE_VALUE) {
-		WaitForSingleObject(child_td->thr_handle, INFINITE);
-		CloseHandle(child_td->thr_handle);
-	    }
-
-	    (*freep)((void *) child_td);
-	    thr_table[child_ix] = NULL;
-	}
-    }
-
-    *tid = INVALID_TID;
-
-    LeaveCriticalSection(&thr_table_cs);
-    return err;
-}
-
-int ethr_thr_join(ethr_tid tid, void **res)
-{
-    int err = 0;
-    DWORD code;
-    thr_data_ *td;
-    thr_data_ *my_td;
-
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-
-    my_td = OWN_THR_DATA;
-
-    if (!my_td) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return EACCES;
-    }
-
-    EnterCriticalSection(&thr_table_cs);
-
-    td = tid2thr(tid);
-    if (!td)
-	err = ESRCH;
-    else if (td->thr_handle == INVALID_HANDLE_VALUE /* i.e. detached */
-	     || td->joiner != INVALID_TID) /* i.e. someone else is joining */
-	err = EINVAL;
-    else if (my_td == td)
-	err = EDEADLK;
-    else
-	td->joiner = my_td->thr_id;
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    if (err)
-	goto error;
-
-    /* Wait for thread to terminate */
-    code = WaitForSingleObject(td->thr_handle, INFINITE);
-    if (code != WAIT_OBJECT_0)
-	goto error;
-
-    EnterCriticalSection(&thr_table_cs);
-
-    ASSERT(td == tid2thr(tid));
-    ASSERT(td->thr_handle != INVALID_HANDLE_VALUE);
-    ASSERT(td->joiner == my_td->thr_id);
-
-    if (res)
-	*res = td->result;
-
-    CloseHandle(td->thr_handle);
-    ASSERT(td == thr_table[THR_IX(td->thr_id)]);
-    thr_table[THR_IX(td->thr_id)] = NULL;
-    if (td != &main_thr_data)
-	(*freep)((void *) td);
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    return 0;
-
- error:
-    if (err == 0)
-	err = get_errno();
-    ASSERT(err != 0);
-    return err;
-}
-
-
-int
-ethr_thr_detach(ethr_tid tid)
-{
-    int res;
-    DWORD code;
-    thr_data_ *td;
-
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-
-    if (!OWN_THR_DATA) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return EACCES;
-    }
-
-    EnterCriticalSection(&thr_table_cs);
-
-    td = tid2thr(tid);
-    if (!td)
-	res = ESRCH;
-    if (td->thr_handle == INVALID_HANDLE_VALUE /* i.e. detached */
-	|| td->joiner != INVALID_TID) /* i.e. someone is joining */
-	res = EINVAL;
-    else {
-	res = 0;
-	CloseHandle(td->thr_handle);
-	td->thr_handle = INVALID_HANDLE_VALUE;
-    }
-
-    LeaveCriticalSection(&thr_table_cs);
-
-    return res;
-}
-
-
-void
-ethr_thr_exit(void *res)
-{
-    thr_data_ *td;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return;
-    }
-#endif
-    td = OWN_THR_DATA;
-    if (!td) {
-	/* Only ethreads are allowed to call this function */
-	ASSERT(0);
-	return;
-    }
-    thr_exit_cleanup(td, res);
-    _endthreadex((unsigned) 0);
-}
-
-ethr_tid
-ethr_self(void)
-{
-    thr_data_ *td;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return INVALID_TID;
-    }
-#endif
-    /* It is okay for non-ethreads (i.e. native win32 threads) to call
-       ethr_self(). They will however be returned the INVALID_TID. */
-    td = OWN_THR_DATA;
-    if (!td)
-	return INVALID_TID;
-    return td->thr_id;
-}
-
-int
-ethr_equal_tids(ethr_tid tid1, ethr_tid tid2)
-{
-    /* INVALID_TID does not equal any tid, not even the INVALID_TID */
-    return tid1 == tid2 && tid1 != INVALID_TID;
-}
-
-/*
- * Mutex functions.
- */
-
-int
-ethr_mutex_init(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!InitializeCriticalSectionAndSpinCount(&mtx->cs, ETHR_MUTEX_SPIN_COUNT))
-	return get_errno();
-    mtx->initialized = ETHR_MUTEX_INITIALIZED;
-#if ETHR_XCHK
-    mtx->is_rec_mtx = 0;
-#endif
-    return 0;
-}
-
-int
-ethr_rec_mutex_init(ethr_mutex *mtx)
-{
-    int res;
-    res = ethr_mutex_init(mtx);
-#if ETHR_XCHK
-    mtx->is_rec_mtx = 1;
-#endif
-    return res;
-}
-
-int
-ethr_mutex_destroy(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    DeleteCriticalSection(&mtx->cs);
-    mtx->initialized = 0;
-    return 0;
-}
-
-int ethr_mutex_set_forksafe(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return 0; /* No fork() */
-}
-
-int ethr_mutex_unset_forksafe(ethr_mutex *mtx)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    return 0; /* No fork() */
-}
-
-int
-ethr_mutex_trylock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx
-	|| (mtx->initialized && mtx->initialized != ETHR_MUTEX_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!mtx->initialized) {
-	int res = ethr_fake_static_mutex_init(mtx);
-	if (res != 0)
-	    return res;
-    }
-    return ethr_mutex_trylock__(mtx);
-}
-
-int
-ethr_mutex_lock(ethr_mutex *mtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx
-	|| (mtx->initialized && mtx->initialized != ETHR_MUTEX_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_lock__(mtx);
-}
-
-int
-ethr_mutex_unlock(ethr_mutex *mtx)
-{
-#if ETHR_XCHK
-    int res;
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_mutex_unlock__(mtx);
-}
-
-/*
- * Condition variable functions.
- */
-
-int
-ethr_cond_init(ethr_cond *cnd)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!InitializeCriticalSectionAndSpinCount(&cnd->cs, ETHR_COND_SPIN_COUNT))
-	return get_errno();
-    cnd->queue = NULL;
-    cnd->queue_end = NULL;
-    cnd->initialized = ETHR_COND_INITIALIZED;
-    return 0;
-}
-
-int
-ethr_cond_destroy(ethr_cond *cnd)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)
-	|| cnd->queue) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    DeleteCriticalSection(&cnd->cs);
-    cnd->initialized = 0;
-    return 0;
-}
-
-int
-ethr_cond_signal(ethr_cond *cnd)
-{
-    cnd_wait_event_ *cwe;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!cnd->initialized) {
-	int res = fake_static_cond_init(cnd);
-	if (res != 0)
-	    return res;
-    }
-    EnterCriticalSection(&cnd->cs);
-    cwe = cnd->queue;
-    if (cwe) {
-	ASSERT(cwe->in_queue);
-	SetEvent(cnd->queue->handle);
-	if (cwe->next)
-	    cwe->next->prev = NULL;
-	else {
-	    ASSERT(cnd->queue_end == cnd->queue);
-	    cnd->queue_end = NULL;
-	}
-	cnd->queue = cwe->next;
-	cwe->in_queue = 0;
-    }
-    LeaveCriticalSection(&cnd->cs);
-    return 0;
-}
-
-int
-ethr_cond_broadcast(ethr_cond *cnd)
-{
-    cnd_wait_event_ *cwe;
-
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!cnd
-	|| (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (!cnd->initialized) {
-	int res = fake_static_cond_init(cnd);
-	if (res != 0)
-	    return res;
-    }
-    EnterCriticalSection(&cnd->cs);
-    for (cwe = cnd->queue; cwe; cwe = cwe->next) {
-	ASSERT(cwe->in_queue);
-	SetEvent(cwe->handle);
-	cwe->in_queue = 0;
-    }
-    cnd->queue = NULL;
-    cnd->queue_end = NULL;
-    LeaveCriticalSection(&cnd->cs);
-    return 0;
-
-}
-
-int
-ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx)
-{
-    return condwait(cnd, mtx, 0, NULL);
-}
-
-int
-ethr_cond_timedwait(ethr_cond *cnd, ethr_mutex *mtx, ethr_timeval *timeout)
-{
-    return condwait(cnd, mtx, 1, timeout);
-}
-
-int
-ethr_time_now(ethr_timeval *time)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!time) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    get_curr_time(&time->tv_sec, &time->tv_nsec);
-    return 0;
-}
-
-/*
- * Thread specific data
- */
-
-int
-ethr_tsd_key_create(ethr_tsd_key *keyp)
-{
-    DWORD key;
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!keyp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    key = TlsAlloc();
-    if (key == TLS_OUT_OF_INDEXES)
-	return get_errno();
-    *keyp = (ethr_tsd_key) key;
-    return 0;
-}
-
-int
-ethr_tsd_key_delete(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    if (!TlsFree((DWORD) key))
-	return get_errno();
-    return 0;
-}
-
-int
-ethr_tsd_set(ethr_tsd_key key, void *value)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-#endif
-    if (!TlsSetValue((DWORD) key, (LPVOID) value))
-	return get_errno();
-    return 0;
-}
-
-void *
-ethr_tsd_get(ethr_tsd_key key)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return NULL;
-    }
-#endif
-    return (void *) TlsGetValue((DWORD) key);
-}
-
-/* Misc */
-
-#ifndef ETHR_HAVE_OPTIMIZED_LOCKS
-
-int
-ethr_do_spinlock_init(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    if (InitializeCriticalSectionAndSpinCount(&lock->cs, INT_MAX))
-	return 0;
-    else
-	return get_errno();
-}
-
-int
-ethr_do_rwlock_init(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    lock->counter = 0;
-    if (InitializeCriticalSectionAndSpinCount(&lock->cs, INT_MAX))
-	return 0;
-    else
-	return get_errno();    
-}
-
-#endif /* #ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS */
-
-#else
-#error "Missing thread implementation"
-#endif
-
-/* Atomics */
-
-int
-ethr_atomic_init(ethr_atomic_t *var, long i)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_init__(var, i);
-}
-
-int
-ethr_atomic_set(ethr_atomic_t *var, long i)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_set__(var, i);
-}
-
-int
-ethr_atomic_read(ethr_atomic_t *var, long *i)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var || !i) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_read__(var, i);
-}
-
-
-int
-ethr_atomic_addtest(ethr_atomic_t *var, long incr, long *testp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var || !testp) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_addtest__(var, incr, testp);
-}   
-    
-int
-ethr_atomic_inctest(ethr_atomic_t *incp, long *testp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!incp || !testp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_inctest__(incp, testp);
-}
-
-int
-ethr_atomic_dectest(ethr_atomic_t *decp, long *testp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!decp || !testp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_dectest__(decp, testp);
-}
-
-int
-ethr_atomic_add(ethr_atomic_t *var, long incr)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-       ASSERT(0);
-       return EACCES;
-    }  
-    if (!var) {
-       ASSERT(0);
-       return EINVAL;
-    }  
-#endif
-    return ethr_atomic_add__(var, incr);
-}   
-    
-int 
-ethr_atomic_inc(ethr_atomic_t *incp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!incp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_inc__(incp);
-}
-
-int
-ethr_atomic_dec(ethr_atomic_t *decp)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!decp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_dec__(decp);
-}
-
-int
-ethr_atomic_and_old(ethr_atomic_t *var, long mask, long *old)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_and_old__(var, mask, old);
-}
-
-int
-ethr_atomic_or_old(ethr_atomic_t *var, long mask, long *old)
-{
-#if ETHR_XCHK
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_atomic_or_old__(var, mask, old);
-}
-
-int
-ethr_atomic_xchg(ethr_atomic_t *var, long new, long *old)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }  
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }  
-#endif
-    return ethr_atomic_xchg__(var, new, old);
-}   
-
-int
-ethr_atomic_cmpxchg(ethr_atomic_t *var, long new, long expected, long *old)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }  
-    if (!var || !old) {
-	ASSERT(0);
-	return EINVAL;
-    }  
-#endif
-    return ethr_atomic_cmpxchg__(var, new, expected, old);
-}
-
-/* Spinlocks and rwspinlocks */
-
-int
-ethr_spinlock_init(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spinlock_init__(lock);
-}
-
-int
-ethr_spinlock_destroy(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spinlock_destroy__(lock);
-}
-
-
-int
-ethr_spin_unlock(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spin_unlock__(lock);
-}
-
-int
-ethr_spin_lock(ethr_spinlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_spin_lock__(lock);
-}
-
-int
-ethr_rwlock_init(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwlock_init__(lock);
-}
-
-int
-ethr_rwlock_destroy(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_rwlock_destroy__(lock);
-}
-
-int
-ethr_read_unlock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_read_unlock__(lock);
-}
-
-int
-ethr_read_lock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_read_lock__(lock);
-}
-
-int
-ethr_write_unlock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_write_unlock__(lock);
-}
-
-int
-ethr_write_lock(ethr_rwlock_t *lock)
-{
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!lock) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    return ethr_write_lock__(lock);
-}
-
-
-int
-ethr_gate_init(ethr_gate *gp)
-{
-    int res;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_init(&gp->mtx);
-    if (res != 0)
-	return res;
-    res = ethr_cond_init(&gp->cnd);
-    if (res != 0) {
-	ethr_mutex_destroy(&gp->mtx);
-	return res;
-    }
-    gp->open = 0;
-    return 0;
-}
-
-int
-ethr_gate_destroy(ethr_gate *gp)
-{
-    int res, dres;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_destroy(&gp->mtx);
-    dres = ethr_cond_destroy(&gp->cnd);
-    if (res == 0)
-	res = dres;
-    gp->open = 0;
-    return res;
-}
-
-int
-ethr_gate_close(ethr_gate *gp)
-{
-    int res;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&gp->mtx);
-    if (res != 0)
-	return res;
-    gp->open = 0;
-    res = ethr_mutex_unlock__(&gp->mtx);
-    return res;
-}
-
-int
-ethr_gate_let_through(ethr_gate *gp, unsigned no)
-{
-    int res, ures;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&gp->mtx);
-    if (res != 0)
-	return res;
-    gp->open += no;
-    res = (gp->open == 1
-	   ? ethr_cond_signal(&gp->cnd)
-	   : ethr_cond_broadcast(&gp->cnd));
-    ures = ethr_mutex_unlock__(&gp->mtx);
-    if (res != 0)
-	res = ures;
-    return res;
-}
-
-int
-ethr_gate_swait(ethr_gate *gp, int spincount)
-{
-    int res, ures, n;
-#if ETHR_XCHK 
-    if (ethr_not_inited) {
-	ASSERT(0);
-	return EACCES;
-    }
-    if (!gp) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    n = spincount;
-    res = ethr_mutex_lock__(&gp->mtx);
-    if (res != 0)
-	return res;
-    while (n >= 0 && !gp->open) {
-	res = ethr_mutex_unlock__(&gp->mtx);
-	if (res != 0)
-	    return res;
-	res = ethr_mutex_lock__(&gp->mtx);
-	if (res != 0)
-	    return res;
-	n--;
-    }
-    while (!gp->open) {
-	res = ethr_cond_wait(&gp->cnd, &gp->mtx);
-	if (res != 0 && res != EINTR)
-	    goto done;
-    }
-    gp->open--;
- done:
-    ures = ethr_mutex_unlock__(&gp->mtx);
-    if (res == 0)
-	res = ures;
-    return res;
-}
-
-
-int
-ethr_gate_wait(ethr_gate *gp)
-{
-    return ethr_gate_swait(gp, 0);
-}
-
-
-/* rwmutex fallback */
-#ifdef ETHR_USE_RWMTX_FALLBACK
-
-int
-ethr_rwmutex_init(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_init(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    ethr_cond_init(&rwmtx->rcnd);
-    if (res != 0)
-	goto error_cleanup1;
-    res = ethr_cond_init(&rwmtx->wcnd);
-    if (res != 0)
-	goto error_cleanup2;
-    rwmtx->readers = 0;
-    rwmtx->waiting_readers = 0;
-    rwmtx->waiting_writers = 0;
-#if ETHR_XCHK
-    rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED;
-#endif
-    return 0;
- error_cleanup2:
-    ethr_cond_destroy(&rwmtx->rcnd);
- error_cleanup1:
-    ethr_mutex_destroy(&rwmtx->mtx);
-    return res;
-}
-
-int
-ethr_rwmutex_destroy(ethr_rwmutex *rwmtx)
-{
-    int res, pres;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-    rwmtx->initialized = 0;
-#endif
-    res = ethr_mutex_destroy(&rwmtx->mtx);
-    pres = ethr_cond_destroy(&rwmtx->rcnd);
-    if (res == 0)
-	res = pres;
-    pres = ethr_cond_destroy(&rwmtx->wcnd);
-    if (res == 0)
-	res = pres;
-    return res;
-}
-
-int
-ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_trylock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    if (rwmtx->waiting_writers) {
-	res = ethr_mutex_unlock__(&rwmtx->mtx);
-	if (res == 0)
-	    return EBUSY;
-	return res;
-    }
-    rwmtx->readers++;
-    return ethr_mutex_unlock__(&rwmtx->mtx);
-}
-
-int
-ethr_rwmutex_rlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    while (rwmtx->waiting_writers) {
-	rwmtx->waiting_readers++;
-	res = ethr_cond_wait(&rwmtx->rcnd, &rwmtx->mtx);
-	rwmtx->waiting_readers--;
-	if (res != 0 && res != EINTR) {
-	    (void) ethr_mutex_unlock__(&rwmtx->mtx);
-	    return res;
-	}
-    }
-    rwmtx->readers++;
-    return ethr_mutex_unlock__(&rwmtx->mtx);
-}
-
-int
-ethr_rwmutex_runlock(ethr_rwmutex *rwmtx)
-{
-    int res, ures;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    rwmtx->readers--;
-    if (!rwmtx->readers && rwmtx->waiting_writers)
-	res = ethr_cond_signal(&rwmtx->wcnd);
-    ures = ethr_mutex_unlock__(&rwmtx->mtx);
-    if (res == 0)
-	res = ures;
-    return res;
-}
-
-int
-ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_trylock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    if (!rwmtx->readers && !rwmtx->waiting_writers)
-	return 0;
-    else {
-	res = ethr_mutex_unlock__(&rwmtx->mtx);
-	if (res == 0)
-	    return EBUSY;
-	return res;
-    }
-}
-
-int
-ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx)
-{
-    int res;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = ethr_mutex_lock__(&rwmtx->mtx);
-    if (res != 0)
-	return res;
-    if (!rwmtx->readers && !rwmtx->waiting_writers) 
-	return 0;
-    
-    while (rwmtx->readers) {
-	rwmtx->waiting_writers++;
-	res = ethr_cond_wait(&rwmtx->wcnd, &rwmtx->mtx);
-	rwmtx->waiting_writers--;
-	if (res != 0 && res != EINTR) {
-	    (void) ethr_rwmutex_rwunlock(rwmtx);
-	    return res;
-	}
-    }
-    return 0;
-}
-
-int
-ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx)
-{
-    int res, ures;
-#if ETHR_XCHK
-    if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) {
-	ASSERT(0);
-	return EINVAL;
-    }
-#endif
-    res = 0;
-    if (rwmtx->waiting_writers)
-	res = ethr_cond_signal(&rwmtx->wcnd);
-    else if (rwmtx->waiting_readers)
-	res = ethr_cond_broadcast(&rwmtx->rcnd);
-    ures = ethr_mutex_unlock__(&rwmtx->mtx);
-    if (res == 0)
-	res = ures;
-    return res;
-}
-
-#endif /* #ifdef ETHR_USE_RWMTX_FALLBACK */
-
-void
-ethr_compiler_barrier(void)
-{
-
-}
-
-#ifdef DEBUG
-
-#include <stdio.h>
-int ethr_assert_failed(char *f, int l, char *a)
-{
-    fprintf(stderr, "%s:%d: Assertion failed: %s\n", f, l, a);
-    abort();
-    return 0;
-}
-
-#endif
-
-