Merge branch 'rickard/atomics-api/OTP-9014' into major

* rickard/atomics-api/OTP-9014:
  Use new atomic API in runtime system
  Improve ethread atomics

1 files changed, 102 insertions, 46 deletions

diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h
index 522f433649..6001620677 100644
--- a/erts/include/internal/ppc32/atomic.h
+++ b/erts/include/internal/ppc32/atomic.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2005-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2011. All Rights Reserved.
  *
  * The contents of this file are subject to the Erlang Public License,
  * Version 1.1, (the "License"); you may not use this file except in
@@ -29,27 +29,31 @@
 #define ETHREAD_PPC_ATOMIC_H
 
 #define ETHR_HAVE_NATIVE_ATOMIC32 1
+#define ETHR_NATIVE_ATOMIC32_IMPL "ethread"
 
 typedef struct {
     volatile ethr_sint32_t counter;
 } ethr_native_atomic32_t;
 
-#define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory")
-
 #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__)
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR 1
+
 static ETHR_INLINE ethr_sint32_t *
 ethr_native_atomic32_addr(ethr_native_atomic32_t *var)
 {
     return (ethr_sint32_t *) &var->counter;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1
+
 static ETHR_INLINE void
-ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i)
+ethr_native_atomic32_set(ethr_native_atomic32_t *var, ethr_sint32_t i)
 {
     var->counter = i;
 }
-#define ethr_native_atomic32_set(v, i)	ethr_native_atomic32_init((v), (i))
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1
 
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_read(ethr_native_atomic32_t *var)
@@ -57,57 +61,68 @@ ethr_native_atomic32_read(ethr_native_atomic32_t *var)
     return var->counter;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN 1
+
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr)
 {
     ethr_sint32_t tmp;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%1\n\t"
 	"add	%0,%2,%0\n\t"
 	"stwcx.	%0,0,%1\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(tmp)
 	: "r"(&var->counter), "r"(incr)
 	: "cc", "memory");
     return tmp;
 }
 
-static ETHR_INLINE void
-ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_add_return_acqb(ethr_native_atomic32_t *var, ethr_sint32_t incr)
 {
-    /* XXX: could use weaker version here w/o eieio+isync */
-    (void)ethr_native_atomic32_add_return(var, incr);
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_add_return(var, incr);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN 1
+
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var)
 {
     ethr_sint32_t tmp;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%1\n\t"
 	"addic	%0,%0,1\n\t" /* due to addi's (rA|0) behaviour */
 	"stwcx.	%0,0,%1\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(tmp)
 	: "r"(&var->counter)
 	: "cc", "memory");
     return tmp;
 }
 
-static ETHR_INLINE void
-ethr_native_atomic32_inc(ethr_native_atomic32_t *var)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var)
 {
-    /* XXX: could use weaker version here w/o eieio+isync */
-    (void)ethr_native_atomic32_inc_return(var);
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_inc_return(var);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
 }
+    
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN 1
 
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var)
@@ -115,82 +130,120 @@ ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var)
     ethr_sint32_t tmp;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%1\n\t"
 	"addic	%0,%0,-1\n\t"
 	"stwcx.	%0,0,%1\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(tmp)
 	: "r"(&var->counter)
 	: "cc", "memory");
     return tmp;
 }
 
-static ETHR_INLINE void
-ethr_native_atomic32_dec(ethr_native_atomic32_t *var)
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_dec_return_acqb(ethr_native_atomic32_t *var)
 {
-    /* XXX: could use weaker version here w/o eieio+isync */
-    (void)ethr_native_atomic32_dec_return(var);
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_dec_return(var);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD 1
+
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
 {
     ethr_sint32_t old, new;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%2\n\t"
 	"and	%1,%0,%3\n\t"
 	"stwcx.	%1,0,%2\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(old), "=&r"(new)
 	: "r"(&var->counter), "r"(mask)
 	: "cc", "memory");
     return old;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_and_retold_acqb(ethr_native_atomic32_t *var, ethr_sint32_t mask)
+{
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_and_retold(var, mask);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD 1
+
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
 {
     ethr_sint32_t old, new;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%2\n\t"
 	"or	%1,%0,%3\n\t"
 	"stwcx.	%1,0,%2\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(old), "=&r"(new)
 	: "r"(&var->counter), "r"(mask)
 	: "cc", "memory");
     return old;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_or_retold_acqb(ethr_native_atomic32_t *var, ethr_sint32_t mask)
+{
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_or_retold(var, mask);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
+}
+
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG 1
+
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val)
 {
     ethr_sint32_t tmp;
 
     __asm__ __volatile__(
-	"eieio\n\t"
 	"1:\t"
 	"lwarx	%0,0,%1\n\t"
 	"stwcx.	%2,0,%1\n\t"
 	"bne-	1b\n\t"
-	"isync"
 	: "=&r"(tmp)
 	: "r"(&var->counter), "r"(val)
 	: "cc", "memory");
     return tmp;
 }
 
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB 1
+
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_xchg_acqb(ethr_native_atomic32_t *var, ethr_sint32_t val)
+{
+    ethr_sint32_t res;
+    res = ethr_native_atomic32_xchg(var, val);
+    __asm__ __volatile("isync\n\t" : : : "memory");
+    return res;
+}
+
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1
+
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var,
 			     ethr_sint32_t new,
@@ -199,14 +252,12 @@ ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var,
   ethr_sint32_t old;
 
   __asm__ __volatile__(
-    "eieio\n\t"
     "1:\t"
     "lwarx	%0,0,%2\n\t"
     "cmpw	0,%0,%3\n\t"
     "bne	2f\n\t"
     "stwcx.	%1,0,%2\n\t"
     "bne-	1b\n\t"
-    "isync\n"
     "2:"
     : "=&r"(old)
     : "r"(new), "r"(&var->counter), "r"(expected)
@@ -215,25 +266,30 @@ ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var,
     return old;
 }
 
-/*
- * Atomic ops with at least specified barriers.
- */
+#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1
 
-static ETHR_INLINE long
-ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var)
+static ETHR_INLINE ethr_sint32_t
+ethr_native_atomic32_cmpxchg_acqb(ethr_native_atomic32_t *var,
+				  ethr_sint32_t new,
+				  ethr_sint32_t expected)
 {
-    long res = ethr_native_atomic32_read(var);
-    ETHR_MEMORY_BARRIER;
-    return res;
-}
+  ethr_sint32_t old;
 
-#define ethr_native_atomic32_set_relb ethr_native_atomic32_xchg
-#define ethr_native_atomic32_inc_return_acqb ethr_native_atomic32_inc_return
-#define ethr_native_atomic32_dec_relb ethr_native_atomic32_dec_return
-#define ethr_native_atomic32_dec_return_relb ethr_native_atomic32_dec_return
+  __asm__ __volatile__(
+    "1:\t"
+    "lwarx	%0,0,%2\n\t"
+    "cmpw	0,%0,%3\n\t"
+    "bne	2f\n\t"
+    "stwcx.	%1,0,%2\n\t"
+    "bne-	1b\n\t"
+    "isync\n"
+    "2:"
+    : "=&r"(old)
+    : "r"(new), "r"(&var->counter), "r"(expected)
+    : "cc", "memory");
 
-#define ethr_native_atomic32_cmpxchg_acqb ethr_native_atomic32_cmpxchg
-#define ethr_native_atomic32_cmpxchg_relb ethr_native_atomic32_cmpxchg
+    return old;
+}
 
 #endif /* ETHR_TRY_INLINE_FUNCS */