The R13B03 release.OTP_R13B03

4 files changed, 415 insertions, 0 deletions

diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h
new file mode 100644
index 0000000000..3291ad38e5
--- /dev/null
+++ b/erts/include/internal/i386/atomic.h
@@ -0,0 +1,155 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+/*
+ * Native ethread atomics on x86/x86-64.
+ * Author: Mikael Pettersson.
+ *
+ * This code requires a 486 or newer processor.
+ */
+#ifndef ETHREAD_I386_ATOMIC_H
+#define ETHREAD_I386_ATOMIC_H
+
+/* An atomic is an aligned long accessed via locked operations.
+ */
+typedef struct {
+    volatile long counter;
+} ethr_native_atomic_t;
+
+#ifdef ETHR_TRY_INLINE_FUNCS
+
+#ifdef __x86_64__
+#define LONG_SUFFIX "q"
+#else
+#define LONG_SUFFIX "l"
+#endif
+
+static ETHR_INLINE void
+ethr_native_atomic_init(ethr_native_atomic_t *var, long i)
+{
+    var->counter = i;
+}
+#define ethr_native_atomic_set(v, i)	ethr_native_atomic_init((v), (i))
+
+static ETHR_INLINE long
+ethr_native_atomic_read(ethr_native_atomic_t *var)
+{
+    return var->counter;
+}
+
+static ETHR_INLINE void
+ethr_native_atomic_add(ethr_native_atomic_t *var, long incr)
+{
+    __asm__ __volatile__(
+       "lock; add" LONG_SUFFIX " %1, %0"
+       : "=m"(var->counter)
+       : "ir"(incr), "m"(var->counter));
+}      
+       
+static ETHR_INLINE void
+ethr_native_atomic_inc(ethr_native_atomic_t *var)
+{
+    __asm__ __volatile__(
+	"lock; inc" LONG_SUFFIX " %0"
+	: "=m"(var->counter)
+	: "m"(var->counter));
+}
+
+static ETHR_INLINE void
+ethr_native_atomic_dec(ethr_native_atomic_t *var)
+{
+    __asm__ __volatile__(
+	"lock; dec" LONG_SUFFIX " %0"
+	: "=m"(var->counter)
+	: "m"(var->counter));
+}
+
+static ETHR_INLINE long
+ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr)
+{
+    long tmp;
+
+    tmp = incr;
+    __asm__ __volatile__(
+	"lock; xadd" LONG_SUFFIX " %0, %1" /* xadd didn't exist prior to the 486 */
+	: "=r"(tmp)
+	: "m"(var->counter), "0"(tmp));
+    /* now tmp is the atomic's previous value */
+    return tmp + incr;
+}
+
+#define ethr_native_atomic_inc_return(var) ethr_native_atomic_add_return((var), 1)
+#define ethr_native_atomic_dec_return(var) ethr_native_atomic_add_return((var), -1)
+
+static ETHR_INLINE long
+ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old)
+{
+    __asm__ __volatile__(
+      "lock; cmpxchg" LONG_SUFFIX " %2, %3"
+      : "=a"(old), "=m"(var->counter)
+      : "r"(new), "m"(var->counter), "0"(old)
+      : "cc", "memory"); /* full memory clobber to make this a compiler barrier */
+    return old;
+}
+
+static ETHR_INLINE long
+ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask)
+{
+    long tmp, old;
+
+    tmp = var->counter;
+    do {
+	old = tmp;
+        tmp = ethr_native_atomic_cmpxchg(var, tmp & mask, tmp);
+    } while (__builtin_expect(tmp != old, 0));
+    /* now tmp is the atomic's previous value */
+    return tmp;
+}
+
+static ETHR_INLINE long
+ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask)
+{
+    long tmp, old;
+
+    tmp = var->counter;
+    do {
+	old = tmp;
+        tmp = ethr_native_atomic_cmpxchg(var, tmp | mask, tmp);
+    } while (__builtin_expect(tmp != old, 0));
+    /* now tmp is the atomic's previous value */
+    return tmp;
+}
+
+static ETHR_INLINE long
+ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val)
+{   
+    long tmp = val;
+    __asm__ __volatile__(
+	"xchg" LONG_SUFFIX " %0, %1"
+	: "=r"(tmp)
+	: "m"(var->counter), "0"(tmp));
+    /* now tmp is the atomic's previous value */ 
+    return tmp;
+} 
+
+#undef LONG_SUFFIX
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#endif /* ETHREAD_I386_ATOMIC_H */
diff --git a/erts/include/internal/i386/ethread.h b/erts/include/internal/i386/ethread.h
new file mode 100644
index 0000000000..fad8b108fa
--- /dev/null
+++ b/erts/include/internal/i386/ethread.h
@@ -0,0 +1,34 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+/*
+ * Low-level ethread support on x86/x86-64.
+ * Author: Mikael Pettersson.
+ */
+#ifndef ETHREAD_I386_ETHREAD_H
+#define ETHREAD_I386_ETHREAD_H
+
+#include "atomic.h"
+#include "spinlock.h"
+#include "rwlock.h"
+
+#define ETHR_HAVE_NATIVE_ATOMICS 1
+#define ETHR_HAVE_NATIVE_LOCKS 1
+
+#endif /* ETHREAD_I386_ETHREAD_H */
diff --git a/erts/include/internal/i386/rwlock.h b/erts/include/internal/i386/rwlock.h
new file mode 100644
index 0000000000..c009be8ef1
--- /dev/null
+++ b/erts/include/internal/i386/rwlock.h
@@ -0,0 +1,134 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+/*
+ * Native ethread rwlocks on x86/x86-64.
+ * Author: Mikael Pettersson.
+ *
+ * This code requires a 486 or newer processor.
+ */
+#ifndef ETHREAD_I386_RWLOCK_H
+#define ETHREAD_I386_RWLOCK_H
+
+/* XXX: describe the algorithm */
+typedef struct {
+    volatile int lock;
+} ethr_native_rwlock_t;
+
+#ifdef ETHR_TRY_INLINE_FUNCS
+
+#define ETHR_RWLOCK_OFFSET	(1<<24)
+
+static ETHR_INLINE void
+ethr_native_rwlock_init(ethr_native_rwlock_t *lock)
+{
+    lock->lock = 0;
+}
+
+static ETHR_INLINE void
+ethr_native_read_unlock(ethr_native_rwlock_t *lock)
+{
+    __asm__ __volatile__(
+	"lock; decl %0"
+	: "=m"(lock->lock)
+	: "m"(lock->lock));
+}
+
+static ETHR_INLINE int
+ethr_native_read_trylock(ethr_native_rwlock_t *lock)
+{
+    int tmp;
+
+    tmp = 1;
+    __asm__ __volatile__(
+	"lock; xaddl %0, %1"
+	: "=r"(tmp)
+	: "m"(lock->lock), "0"(tmp));
+    /* tmp is now the lock's previous value */
+    if (__builtin_expect(tmp >= 0, 1))
+	return 1;
+    ethr_native_read_unlock(lock);
+    return 0;
+}
+
+static ETHR_INLINE int
+ethr_native_read_is_locked(ethr_native_rwlock_t *lock)
+{
+    return lock->lock < 0;
+}
+
+static ETHR_INLINE void
+ethr_native_read_lock(ethr_native_rwlock_t *lock)
+{
+    for(;;) {
+	if (__builtin_expect(ethr_native_read_trylock(lock) != 0, 1))
+	    break;
+	do {
+	    __asm__ __volatile__("rep;nop" : "=m"(lock->lock) : : "memory");
+	} while (ethr_native_read_is_locked(lock));
+    }
+}
+
+static ETHR_INLINE void
+ethr_native_write_unlock(ethr_native_rwlock_t *lock)
+{
+    __asm__ __volatile__(
+	"lock; addl %2,%0"
+	: "=m"(lock->lock)
+	: "m"(lock->lock), "i"(ETHR_RWLOCK_OFFSET));
+}
+
+static ETHR_INLINE int
+ethr_native_write_trylock(ethr_native_rwlock_t *lock)
+{
+    int tmp;
+
+    tmp = -ETHR_RWLOCK_OFFSET;
+    __asm__ __volatile__(
+	"lock; xaddl %0, %1"
+	: "=r"(tmp)
+	: "m"(lock->lock), "0"(tmp));
+    /* tmp is now the lock's previous value */
+    if (__builtin_expect(tmp == 0, 1))
+	return 1;
+    ethr_native_write_unlock(lock);
+    return 0;
+}
+
+static ETHR_INLINE int
+ethr_native_write_is_locked(ethr_native_rwlock_t *lock)
+{
+    return lock->lock != 0;
+}
+
+static ETHR_INLINE void
+ethr_native_write_lock(ethr_native_rwlock_t *lock)
+{
+    for(;;) {
+	if (__builtin_expect(ethr_native_write_trylock(lock) != 0, 1))
+	    break;
+	do {
+	    __asm__ __volatile__("rep;nop" : "=m"(lock->lock) : : "memory");
+	} while (ethr_native_write_is_locked(lock));
+    }
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#endif /* ETHREAD_I386_RWLOCK_H */
diff --git a/erts/include/internal/i386/spinlock.h b/erts/include/internal/i386/spinlock.h
new file mode 100644
index 0000000000..2b4832e26a
--- /dev/null
+++ b/erts/include/internal/i386/spinlock.h
@@ -0,0 +1,92 @@
+/*
+ * %CopyrightBegin%
+ * 
+ * Copyright Ericsson AB 2005-2009. All Rights Reserved.
+ * 
+ * The contents of this file are subject to the Erlang Public License,
+ * Version 1.1, (the "License"); you may not use this file except in
+ * compliance with the License. You should have received a copy of the
+ * Erlang Public License along with this software. If not, it can be
+ * retrieved online at http://www.erlang.org/.
+ * 
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
+ * the License for the specific language governing rights and limitations
+ * under the License.
+ * 
+ * %CopyrightEnd%
+ */
+
+/*
+ * Native ethread spinlocks on x86/x86-64.
+ * Author: Mikael Pettersson.
+ */
+#ifndef ETHREAD_I386_SPINLOCK_H
+#define ETHREAD_I386_SPINLOCK_H
+
+/* A spinlock is the low byte of an aligned 32-bit integer.
+ * A non-zero value means that the lock is locked.
+ */
+typedef struct {
+    volatile unsigned int lock;
+} ethr_native_spinlock_t;
+
+#ifdef ETHR_TRY_INLINE_FUNCS
+
+static ETHR_INLINE void
+ethr_native_spinlock_init(ethr_native_spinlock_t *lock)
+{
+    lock->lock = 0;
+}
+
+static ETHR_INLINE void
+ethr_native_spin_unlock(ethr_native_spinlock_t *lock)
+{
+    /* To unlock we move 0 to the lock.
+     * On i386 this needs to be a locked operation
+     * to avoid Pentium Pro errata 66 and 92.
+     */
+#if defined(__x86_64__)
+    __asm__ __volatile__("" : : : "memory");
+    *(unsigned char*)&lock->lock = 0;
+#else
+    char tmp = 0;
+    __asm__ __volatile__(
+	"xchgb %b0, %1"
+	: "=q"(tmp), "=m"(lock->lock)
+	: "0"(tmp) : "memory");
+#endif
+}
+
+static ETHR_INLINE int
+ethr_native_spin_trylock(ethr_native_spinlock_t *lock)
+{
+    char tmp = 1;
+    __asm__ __volatile__(
+	"xchgb %b0, %1"
+	: "=q"(tmp), "=m"(lock->lock)
+	: "0"(tmp) : "memory");
+    return tmp == 0;
+}
+
+static ETHR_INLINE int
+ethr_native_spin_is_locked(ethr_native_spinlock_t *lock)
+{
+    return *(volatile unsigned char*)&lock->lock != 0;
+}
+
+static ETHR_INLINE void
+ethr_native_spin_lock(ethr_native_spinlock_t *lock)
+{
+    for(;;) {
+	if (__builtin_expect(ethr_native_spin_trylock(lock) != 0, 1))
+	    break;
+	do {
+	    __asm__ __volatile__("rep;nop" : "=m"(lock->lock) : : "memory");
+	} while (ethr_native_spin_is_locked(lock));
+    }
+}
+
+#endif /* ETHR_TRY_INLINE_FUNCS */
+
+#endif /* ETHREAD_I386_SPINLOCK_H */