Rewrite ethread library

Large parts of the ethread library have been rewritten. The
ethread library is an Erlang runtime system internal, portable
thread library used by the runtime system itself.

Most notable improvement is a reader optimized rwlock
implementation which dramatically improve the performance of
read-lock/read-unlock operations on multi processor systems by
avoiding ping-ponging of the rwlock cache lines. The reader
optimized rwlock implementation is used by miscellaneous
rwlocks in the runtime system that are known to be read-locked
frequently, and can be enabled on ETS tables by passing the
`{read_concurrency, true}' option upon table creation. See the
documentation of `ets:new/2' for more information.

The ethread library can now also use the libatomic_ops library
for atomic memory accesses. This makes it possible for the
Erlang runtime system to utilize optimized atomic operations
on more platforms than before. Use the
`--with-libatomic_ops=PATH' configure command line argument
when specifying where the libatomic_ops installation is
located. The libatomic_ops library can be downloaded from:
http://www.hpl.hp.com/research/linux/atomic_ops/

The changed API of the ethread library has also caused
modifications in the Erlang runtime system. Preparations for
the to come "delayed deallocation" feature has also been done
since it depends on the ethread library.

Note: When building for x86, the ethread library will now use
instructions that first appeared on the pentium 4 processor. If
you want the runtime system to be compatible with older
processors (back to 486) you need to pass the
`--enable-ethread-pre-pentium4-compatibility' configure command
line argument when configuring the system.

2 files changed, 696 insertions, 0 deletions

diff --git a/erts/lib_src/pthread/ethr_event.c b/erts/lib_src/pthread/ethr_event.c
new file mode 100644
index 0000000000..6731c0eb46
--- /dev/null
+++ b/erts/lib_src/pthread/ethr_event.c
@@ -0,0 +1,219 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2009-2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Author: Rickard Green
+ */
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHR_EVENT_IMPL__
+
+#include "ethread.h"
+
+#if defined(ETHR_LINUX_FUTEX_IMPL__)
+/* --- Linux futex implementation of ethread events ------------------------- */
+
+#include <sched.h>
+#include <errno.h>
+
+#define ETHR_YIELD_AFTER_BUSY_LOOPS 50
+
+int
+ethr_event_init(ethr_event *e)
+{
+    ethr_atomic_init(&e->futex, ETHR_EVENT_OFF__);
+    return 0;
+}
+
+int
+ethr_event_destroy(ethr_event *e)
+{
+    return 0;
+}
+
+static ETHR_INLINE int
+wait__(ethr_event *e, int spincount)
+{
+    unsigned sc = spincount;
+    int res;
+    long val;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    if (spincount < 0)
+	ETHR_FATAL_ERROR__(EINVAL);
+
+    while (1) {
+	while (1) {
+	    val = ethr_atomic_read(&e->futex);
+	    if (val == ETHR_EVENT_ON__)
+		return 0;
+	    if (sc == 0)
+		break;
+	    sc--;
+	    ETHR_SPIN_BODY;
+	    if (--until_yield == 0) {
+		until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+		res = ETHR_YIELD();
+		if (res != 0)
+		    ETHR_FATAL_ERROR__(res);
+	    }
+	}
+
+	if (val != ETHR_EVENT_OFF_WAITER__) {
+	    val = ethr_atomic_cmpxchg(&e->futex,
+				      ETHR_EVENT_OFF_WAITER__,
+				      ETHR_EVENT_OFF__);
+
+	    if (val == ETHR_EVENT_ON__)
+		return 0;
+	    ETHR_ASSERT(val == ETHR_EVENT_OFF__);
+	}
+
+	res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAIT__, ETHR_EVENT_OFF_WAITER__);
+	if (res == EINTR)
+	    break;
+	if (res != 0 && res != EWOULDBLOCK)
+	    ETHR_FATAL_ERROR__(res);
+    }
+
+    return res;
+}
+
+#elif defined(ETHR_PTHREADS)
+/* --- Posix mutex/cond implementation of events ---------------------------- */
+
+int
+ethr_event_init(ethr_event *e)
+{
+    int res;
+    ethr_atomic_init(&e->state, ETHR_EVENT_OFF__);
+    res = pthread_mutex_init(&e->mtx, NULL);
+    if (res != 0)
+	return res;
+    res = pthread_cond_init(&e->cnd, NULL);
+    if (res != 0) {
+	pthread_mutex_destroy(&e->mtx);
+	return res;
+    }
+    return 0;
+}
+
+int
+ethr_event_destroy(ethr_event *e)
+{
+    int res;
+    res = pthread_mutex_destroy(&e->mtx);
+    if (res != 0)
+	return res;
+    res = pthread_cond_destroy(&e->cnd);
+    if (res != 0)
+	return res;
+    return 0;
+}
+
+static ETHR_INLINE int
+wait__(ethr_event *e, int spincount)
+{
+    int sc = spincount;
+    long val;
+    int res, ulres;
+    int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+
+    if (spincount < 0)
+	ETHR_FATAL_ERROR__(EINVAL);
+
+    while (1) {
+	val = ethr_atomic_read(&e->state);
+	if (val == ETHR_EVENT_ON__)
+	    return 0;
+	if (sc == 0)
+	    break;
+	sc--;
+	ETHR_SPIN_BODY;
+	if (--until_yield == 0) {
+	    until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS;
+	    res = ETHR_YIELD();
+	    if (res != 0)
+		ETHR_FATAL_ERROR__(res);
+	}
+    }
+
+    if (val != ETHR_EVENT_OFF_WAITER__) {
+	val = ethr_atomic_cmpxchg(&e->state,
+				  ETHR_EVENT_OFF_WAITER__,
+				  ETHR_EVENT_OFF__);
+	if (val == ETHR_EVENT_ON__)
+	    return 0;
+	ETHR_ASSERT(val == ETHR_EVENT_OFF__);
+    }
+
+    ETHR_ASSERT(val == ETHR_EVENT_OFF_WAITER__
+		|| val == ETHR_EVENT_OFF__);
+
+    res = pthread_mutex_lock(&e->mtx);
+    if (res != 0)
+	ETHR_FATAL_ERROR__(res);
+
+    while (1) {
+
+	val = ethr_atomic_read(&e->state);
+	if (val == ETHR_EVENT_ON__)
+	    break;
+
+	res = pthread_cond_wait(&e->cnd, &e->mtx);
+	if (res == EINTR)
+	    break;
+	if (res != 0)
+	    ETHR_FATAL_ERROR__(res);
+    }
+
+    ulres = pthread_mutex_unlock(&e->mtx);
+    if (ulres != 0)
+	ETHR_FATAL_ERROR__(ulres);
+
+    return res; /* 0 || EINTR */
+}
+
+#else
+#error No ethread event implementation
+#endif
+
+void
+ethr_event_reset(ethr_event *e)
+{
+    ethr_event_reset__(e);
+}
+
+void
+ethr_event_set(ethr_event *e)
+{
+    ethr_event_set__(e);
+}
+
+int
+ethr_event_wait(ethr_event *e)
+{
+    return wait__(e, 0);
+}
+
+int
+ethr_event_swait(ethr_event *e, int spincount)
+{
+    return wait__(e, spincount);
+}
diff --git a/erts/lib_src/pthread/ethread.c b/erts/lib_src/pthread/ethread.c
new file mode 100644
index 0000000000..ea1d9d43f0
--- /dev/null
+++ b/erts/lib_src/pthread/ethread.c
@@ -0,0 +1,477 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2010. All Rights Reserved.
+ *
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Description: Pthread implementation of the ethread library
+ * Author: Rickard Green
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#define ETHR_CHILD_WAIT_SPIN_COUNT 4000
+
+#include <stdio.h>
+#ifdef ETHR_TIME_WITH_SYS_TIME
+#  include <time.h>
+#  include <sys/time.h>
+#else
+#  ifdef ETHR_HAVE_SYS_TIME_H
+#    include <sys/time.h>
+#  else
+#    include <time.h>
+#  endif
+#endif
+#include <sys/types.h>
+#include <unistd.h>
+#include <signal.h>
+
+#include <limits.h>
+
+#define ETHR_INLINE_FUNC_NAME_(X) X ## __
+#define ETHREAD_IMPL__
+
+#include "ethread.h"
+#include "ethr_internal.h"
+
+#ifndef ETHR_HAVE_ETHREAD_DEFINES
+#error Missing configure defines
+#endif
+
+pthread_key_t ethr_ts_event_key__;
+static int child_wait_spin_count;
+
+/*
+ * --------------------------------------------------------------------------
+ * Static functions
+ * --------------------------------------------------------------------------
+ */
+
+static void thr_exit_cleanup(void)
+{
+    ethr_run_exit_handlers__();
+}
+
+
+/* Argument passed to thr_wrapper() */
+typedef struct {
+    ethr_atomic_t result;
+    ethr_ts_event *tse;
+    void *(*thr_func)(void *);
+    void *arg;
+    void *prep_func_res;
+} ethr_thr_wrap_data__;
+
+static void *thr_wrapper(void *vtwd)
+{
+    long result;
+    void *res;
+    ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd;
+    void *(*thr_func)(void *) = twd->thr_func;
+    void *arg = twd->arg;
+    ethr_ts_event *tsep = NULL;
+
+    result = (long) ethr_make_ts_event__(&tsep);
+
+    if (result == 0) {
+	tsep->iflgs |= ETHR_TS_EV_ETHREAD;
+	if (ethr_thr_child_func__)
+	    ethr_thr_child_func__(twd->prep_func_res);
+    }
+
+    tsep = twd->tse; /* We aren't allowed to follow twd after
+			result has been set! */
+
+    ethr_atomic_set(&twd->result, result);
+
+    ethr_event_set(&tsep->event);
+
+    res = result == 0 ? (*thr_func)(arg) : NULL;
+
+    thr_exit_cleanup();
+    return res;
+}
+
+/* internal exports */
+
+int ethr_set_tse__(ethr_ts_event *tsep)
+{
+    return pthread_setspecific(ethr_ts_event_key__, (void *) tsep);
+}
+
+ethr_ts_event *ethr_get_tse__(void)
+{
+    return pthread_getspecific(ethr_ts_event_key__);
+}
+
+/*
+ * --------------------------------------------------------------------------
+ * Exported functions
+ * --------------------------------------------------------------------------
+ */
+
+int
+ethr_init(ethr_init_data *id)
+{
+    int res;
+
+    if (!ethr_not_inited__)
+	return EINVAL;
+
+    ethr_not_inited__ = 0;
+
+    res = ethr_init_common__(id);
+    if (res != 0)
+	goto error;
+
+    child_wait_spin_count = ETHR_CHILD_WAIT_SPIN_COUNT;
+    if (erts_get_cpu_configured(ethr_cpu_info__) == 1)
+	child_wait_spin_count = 0;
+
+    res = pthread_key_create(&ethr_ts_event_key__, ethr_ts_event_destructor__);
+
+    return 0;
+ error:
+    ethr_not_inited__ = 1;
+    return res;
+
+}
+
+int
+ethr_late_init(ethr_late_init_data *id)
+{
+    int res = ethr_late_init_common__(id);
+    if (res != 0)
+	return res;
+    ethr_not_completely_inited__ = 0;
+    return res;
+}
+
+int
+ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg,
+		ethr_thr_opts *opts)
+{
+    ethr_thr_wrap_data__ twd;
+    pthread_attr_t attr;
+    int res, dres;
+    int use_stack_size = (opts && opts->suggested_stack_size >= 0
+			  ? opts->suggested_stack_size
+			  : -1 /* Use system default */);
+
+#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE
+    if (use_stack_size < 0)
+	use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE;
+#endif
+
+#if ETHR_XCHK
+    if (ethr_not_completely_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!tid || !func) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+
+    ethr_atomic_init(&twd.result, -1);
+    twd.tse = ethr_get_ts_event();
+    twd.thr_func = func;
+    twd.arg = arg;
+
+    res = pthread_attr_init(&attr);
+    if (res != 0)
+	return res;
+
+    /* Error cleanup needed after this point */
+
+    /* Schedule child thread in system scope (if possible) ... */
+    res = pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM);
+    if (res != 0 && res != ENOTSUP)
+	goto error;
+
+    if (use_stack_size >= 0) {
+	size_t suggested_stack_size = (size_t) use_stack_size;
+	size_t stack_size;
+#ifdef ETHR_DEBUG
+	suggested_stack_size /= 2; /* Make sure we got margin */
+#endif
+#ifdef ETHR_STACK_GUARD_SIZE
+	/* The guard is at least on some platforms included in the stack size
+	   passed when creating threads */
+	suggested_stack_size += ETHR_B2KW(ETHR_STACK_GUARD_SIZE);
+#endif
+	if (suggested_stack_size < ethr_min_stack_size__)
+	    stack_size = ETHR_KW2B(ethr_min_stack_size__);
+	else if (suggested_stack_size > ethr_max_stack_size__)
+	    stack_size = ETHR_KW2B(ethr_max_stack_size__);
+	else
+	    stack_size = ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size));
+	(void) pthread_attr_setstacksize(&attr, stack_size);
+    }
+
+#ifdef ETHR_STACK_GUARD_SIZE
+    (void) pthread_attr_setguardsize(&attr, ETHR_STACK_GUARD_SIZE);
+#endif
+
+    /* Detached or joinable... */
+    res = pthread_attr_setdetachstate(&attr,
+				      (opts && opts->detached
+				       ? PTHREAD_CREATE_DETACHED
+				       : PTHREAD_CREATE_JOINABLE));
+    if (res != 0)
+	goto error;
+
+    /* Call prepare func if it exist */
+    if (ethr_thr_prepare_func__)
+	twd.prep_func_res = ethr_thr_prepare_func__();
+    else
+	twd.prep_func_res = NULL;
+
+    res = pthread_create((pthread_t *) tid, &attr, thr_wrapper, (void*) &twd);
+
+    if (res == 0) {
+	int spin_count = child_wait_spin_count;
+
+	/* Wait for child to initialize... */
+	while (1) {
+	    long result;
+	    ethr_event_reset(&twd.tse->event);
+
+	    result = ethr_atomic_read(&twd.result);
+	    if (result == 0)
+		break;
+
+	    if (result > 0) {
+		res = (int) result;
+		goto error;
+	    }
+
+	    res = ethr_event_swait(&twd.tse->event, spin_count);
+	    if (res != 0 && res != EINTR)
+		goto error;
+	    spin_count = 0;
+	}
+    }
+
+    /* Cleanup... */
+
+ error:
+    dres = pthread_attr_destroy(&attr);
+    if (res == 0)
+	res = dres;
+    if (ethr_thr_parent_func__)
+	ethr_thr_parent_func__(twd.prep_func_res);
+    return res;
+}
+
+int
+ethr_thr_join(ethr_tid tid, void **res)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    return pthread_join((pthread_t) tid, res);
+}
+
+int
+ethr_thr_detach(ethr_tid tid)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    return pthread_detach((pthread_t) tid);
+}
+
+void
+ethr_thr_exit(void *res)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return;
+    }
+#endif
+    thr_exit_cleanup();
+    pthread_exit(res);
+}
+
+ethr_tid
+ethr_self(void)
+{
+    return (ethr_tid) pthread_self();
+}
+
+int
+ethr_equal_tids(ethr_tid tid1, ethr_tid tid2)
+{
+    return pthread_equal((pthread_t) tid1, (pthread_t) tid2);
+}
+
+
+/*
+ * Thread specific events
+ */
+
+ethr_ts_event *
+ethr_get_ts_event(void)
+{
+    return ethr_get_ts_event__();
+}
+
+void
+ethr_leave_ts_event(ethr_ts_event *tsep)
+{
+    ethr_leave_ts_event__(tsep);
+}
+
+/*
+ * Current time
+ */
+
+int
+ethr_time_now(ethr_timeval *time)
+{
+    int res;
+    struct timeval tv;
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!time) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+
+    res = gettimeofday(&tv, NULL);
+    time->tv_sec = (long) tv.tv_sec;
+    time->tv_nsec = ((long) tv.tv_usec)*1000;
+    return res;
+}
+
+/*
+ * Thread specific data
+ */
+
+int
+ethr_tsd_key_create(ethr_tsd_key *keyp)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!keyp) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    return pthread_key_create((pthread_key_t *) keyp, NULL);
+}
+
+int
+ethr_tsd_key_delete(ethr_tsd_key key)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    return pthread_key_delete((pthread_key_t) key);
+}
+
+int
+ethr_tsd_set(ethr_tsd_key key, void *value)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+#endif
+    return pthread_setspecific((pthread_key_t) key, value);
+}
+
+void *
+ethr_tsd_get(ethr_tsd_key key)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return NULL;
+    }
+#endif
+    return pthread_getspecific((pthread_key_t) key);
+}
+
+/*
+ * Signal functions
+ */
+
+#if ETHR_HAVE_ETHR_SIG_FUNCS
+
+int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!set && !oset) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+  return pthread_sigmask(how, set, oset);
+}
+
+int ethr_sigwait(const sigset_t *set, int *sig)
+{
+#if ETHR_XCHK
+    if (ethr_not_inited__) {
+	ETHR_ASSERT(0);
+	return EACCES;
+    }
+    if (!set || !sig) {
+	ETHR_ASSERT(0);
+	return EINVAL;
+    }
+#endif
+    if (sigwait(set, sig) < 0)
+	return errno;
+    return 0;
+}
+
+#endif /* #if ETHR_HAVE_ETHR_SIG_FUNCS */
+
+ETHR_IMPL_NORETURN__
+ethr_abort__(void)
+{
+    abort();
+}