Improve ethread atomics

The ethread atomics API now also provide double word size atomics.
Double word size atomics are implemented using native atomic
instructions on x86 (when the cmpxchg8b instruction is available)
and on x86_64 (when the cmpxchg16b instruction is available). On
other hardware where 32-bit atomics or word size atomics are
available, an optimized fallback is used; otherwise, a spinlock,
or a mutex based fallback is used.

The ethread library now performs runtime tests for presence of
hardware features, such as for example SSE2 instructions, instead
of requiring this to be determined at compile time.

There are now functions implementing each atomic operation with the
following implied memory barrier semantics: none, read, write,
acquire, release, and full. Some of the operation-barrier
combinations aren't especially useful. But instead of filtering
useful ones out, and potentially miss a useful one, we implement
them all.

A much smaller set of functionality for native atomics are required
to be implemented than before. More or less only cmpxchg and a
membar macro are required to be implemented for each atomic size.
Other functions will automatically be constructed from these. It is,
of course, often wise to implement more that this if possible from a
performance perspective.

1 files changed, 102 insertions, 46 deletions

diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h
index 522f433649..6001620677 100644
--- a/erts/include/internal/ppc32/atomic.h
+++ b/erts/include/internal/ppc32/atomic.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -29,27 +29,31 @@
 #define ETHREAD_PPC_ATOMIC_H
 
 #define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATIVE_ATOMIC32_IMPL "ethread"
 
 typedef struct {
     volatile ethr_sint32_t counter;
 } ethr_native_atomic32_t;
 
-#define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory")
-
 #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
+
 static ETHR_INLINE ethr_sint32_t *
 ethr_native_atomic32_addr(ethr_native_atomic32_t *var)
 {
     return (ethr_sint32_t *) &var->counter;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
+
 static ETHR_INLINE void
-ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i)
+ethr_native_atomic32_set(ethr_native_atomic32_t *var, ethr_sint32_t i)
 {
     var->counter = i;
 }
-#define ethr_native_atomic32_set(v, i)	ethr_native_atomic32_init((v), (i))
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
 
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_read(ethr_native_atomic32_t *var)
@@ -57,57 +61,68 @@ ethr_native_atomic32_read(ethr_native_atomic32_t *var)
     return var->counter;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1
+
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr)
 {
     ethr_sint32_t tmp;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%1\n\t"
 	"add	%0,%2,%0\n\t"
 	"stwcx.	%0,0,%1\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(tmp)
 	: "r"(&var->counter), "r"(incr)
 	: "cc", "memory");
     return tmp;
 }
 
-static ETHR_INLINE void
-ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_add_return_acqb(ethr_native_atomic32_t *var, ethr_sint32_t incr)
 {
-    /* XXX: could use weaker version here w/o eieio+isync */
-    (void)ethr_native_atomic32_add_return(var, incr);
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_add_return(var, incr);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN 1
+
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var)
 {
     ethr_sint32_t tmp;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%1\n\t"
 	"addic	%0,%0,1\n\t" /* due to addi's (rA|0) behaviour */
 	"stwcx.	%0,0,%1\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(tmp)
 	: "r"(&var->counter)
 	: "cc", "memory");
     return tmp;
 }
 
-static ETHR_INLINE void
-ethr_native_atomic32_inc(ethr_native_atomic32_t *var)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var)
 {
-    /* XXX: could use weaker version here w/o eieio+isync */
-    (void)ethr_native_atomic32_inc_return(var);
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_inc_return(var);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
 }
+    
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN 1
 
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var)
@@ -115,82 +130,120 @@ ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var)
     ethr_sint32_t tmp;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%1\n\t"
 	"addic	%0,%0,-1\n\t"
 	"stwcx.	%0,0,%1\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(tmp)
 	: "r"(&var->counter)
 	: "cc", "memory");
     return tmp;
 }
 
-static ETHR_INLINE void
-ethr_native_atomic32_dec(ethr_native_atomic32_t *var)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_dec_return_acqb(ethr_native_atomic32_t *var)
 {
-    /* XXX: could use weaker version here w/o eieio+isync */
-    (void)ethr_native_atomic32_dec_return(var);
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_dec_return(var);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD 1
+
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
 {
     ethr_sint32_t old, new;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%2\n\t"
 	"and	%1,%0,%3\n\t"
 	"stwcx.	%1,0,%2\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(old), "=&r"(new)
 	: "r"(&var->counter), "r"(mask)
 	: "cc", "memory");
     return old;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_and_retold_acqb(ethr_native_atomic32_t *var, ethr_sint32_t mask)
+{
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_and_retold(var, mask);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD 1
+
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
 {
     ethr_sint32_t old, new;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%2\n\t"
 	"or	%1,%0,%3\n\t"
 	"stwcx.	%1,0,%2\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(old), "=&r"(new)
 	: "r"(&var->counter), "r"(mask)
 	: "cc", "memory");
     return old;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_or_retold_acqb(ethr_native_atomic32_t *var, ethr_sint32_t mask)
+{
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_or_retold(var, mask);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
+}
+
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG 1
+
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val)
 {
     ethr_sint32_t tmp;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%1\n\t"
 	"stwcx.	%2,0,%1\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(tmp)
 	: "r"(&var->counter), "r"(val)
 	: "cc", "memory");
     return tmp;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_xchg_acqb(ethr_native_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_xchg(var, val);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1
+
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var,
 			     ethr_sint32_t new,
@@ -199,14 +252,12 @@ ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var,
   ethr_sint32_t old;
 
   __asm__ __volatile__(
-    "eieio\n\t"
     "1:\t"
     "lwarx	%0,0,%2\n\t"
     "cmpw	0,%0,%3\n\t"
     "bne	2f\n\t"
     "stwcx.	%1,0,%2\n\t"
     "bne-	1b\n\t"
-    "isync\n"
     "2:"
     : "=&r"(old)
     : "r"(new), "r"(&var->counter), "r"(expected)
@@ -215,25 +266,30 @@ ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var,
     return old;
 }
 
-/*
- * Atomic ops with at least specified barriers.
- */
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
 
-static ETHR_INLINE long
-ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_cmpxchg_acqb(ethr_native_atomic32_t *var,
+				  ethr_sint32_t new,
+				  ethr_sint32_t expected)
 {
-    long res = ethr_native_atomic32_read(var);
-    ETHR_MEMORY_BARRIER;
-    return res;
-}
+  ethr_sint32_t old;
 
-#define ethr_native_atomic32_set_relb ethr_native_atomic32_xchg
-#define ethr_native_atomic32_inc_return_acqb ethr_native_atomic32_inc_return
-#define ethr_native_atomic32_dec_relb ethr_native_atomic32_dec_return
-#define ethr_native_atomic32_dec_return_relb ethr_native_atomic32_dec_return
+  __asm__ __volatile__(
+    "1:\t"
+    "lwarx	%0,0,%2\n\t"
+    "cmpw	0,%0,%3\n\t"
+    "bne	2f\n\t"
+    "stwcx.	%1,0,%2\n\t"
+    "bne-	1b\n\t"
+    "isync\n"
+    "2:"
+    : "=&r"(old)
+    : "r"(new), "r"(&var->counter), "r"(expected)
+    : "cc", "memory");
 
-#define ethr_native_atomic32_cmpxchg_acqb ethr_native_atomic32_cmpxchg
-#define ethr_native_atomic32_cmpxchg_relb ethr_native_atomic32_cmpxchg
+    return old;
+}
 
 #endif /* ETHR_TRY_INLINE_FUNCS */