From 5bcdb3ac4ea27ca47e18628aa147e7544043fa84 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 May 2011 11:45:30 +0200
Subject: Homogenize memory barriers on atomics

Atomic operations with specified barriers have specified barrier semantics.
Set and read operations have undefined barrier semantics. All other atomic
operations implied full memory barriers, except when using the libatomic_ops
library and the tilera atomics api.

Some code in the runtime system assumed that all operations used (except for
set, read and specified) implied full memory barriers. The use of the
libatomic_ops library and the tilera atomics api have therefore been modified
to behave as the other implementations.

Some atomic operations with specified barrier semantics on sparc32 have also
been been relaxed in this commit.
---
 erts/aclocal.m4                                   |  2 +-
 erts/include/internal/libatomic_ops/ethr_atomic.h | 26 ++++++++----------
 erts/include/internal/sparc32/atomic.h            | 29 ++++++++------------
 erts/include/internal/tile/atomic.h               | 33 ++++++++++++++++-------
 4 files changed, 47 insertions(+), 43 deletions(-)

(limited to 'erts')

diff --git a/erts/aclocal.m4 b/erts/aclocal.m4
index a1211bbf0c..f7e5c31910 100644
--- a/erts/aclocal.m4
+++ b/erts/aclocal.m4
@@ -1153,7 +1153,7 @@ case "$THR_LIB_NAME" in
 			AO_nop_full();
 			AO_store(&x, (AO_t) 0);
 			z = AO_load(&x);
-			z = AO_compare_and_swap(&x, (AO_t) 0, (AO_t) 1);
+			z = AO_compare_and_swap_full(&x, (AO_t) 0, (AO_t) 1);
 		    ],
 		    [ethr_have_native_atomics=yes
 		     ethr_have_libatomic_ops=yes])
diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h
index d56693dbf8..2fc82c99a8 100644
--- a/erts/include/internal/libatomic_ops/ethr_atomic.h
+++ b/erts/include/internal/libatomic_ops/ethr_atomic.h
@@ -146,13 +146,13 @@ ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var)
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
 {
-#ifdef AO_HAVE_fetch_and_add
-    return ((ETHR_AINT_T__) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr;
+#ifdef AO_HAVE_fetch_and_add_full
+    return ((ETHR_AINT_T__) AO_fetch_and_add_full(&var->counter, (AO_t) incr)) + incr;
 #else
     while (1) {
 	AO_t exp = AO_load(&var->counter);
 	AO_t new = exp + (AO_t) incr;
-	if (AO_compare_and_swap(&var->counter, exp, new))
+	if (AO_compare_and_swap_full(&var->counter, exp, new))
 	    return (ETHR_AINT_T__) new;
     }
 #endif
@@ -167,8 +167,8 @@ ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var)
 {
-#ifdef AO_HAVE_fetch_and_add1
-    return ((ETHR_AINT_T__) AO_fetch_and_add1(&var->counter)) + 1;
+#ifdef AO_HAVE_fetch_and_add1_full
+    return ((ETHR_AINT_T__) AO_fetch_and_add1_full(&var->counter)) + 1;
 #else
     return ETHR_NATMC_FUNC__(add_return)(var, 1);
 #endif
@@ -183,8 +183,8 @@ ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var)
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var)
 {
-#ifdef AO_HAVE_fetch_and_sub1
-    return ((ETHR_AINT_T__) AO_fetch_and_sub1(&var->counter)) - 1;
+#ifdef AO_HAVE_fetch_and_sub1_full
+    return ((ETHR_AINT_T__) AO_fetch_and_sub1_full(&var->counter)) - 1;
 #else
     return ETHR_NATMC_FUNC__(add_return)(var, -1);
 #endif
@@ -202,7 +202,7 @@ ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
     while (1) {
 	AO_t exp = AO_load(&var->counter);
 	AO_t new = exp & ((AO_t) mask);
-	if (AO_compare_and_swap(&var->counter, exp, new))
+	if (AO_compare_and_swap_full(&var->counter, exp, new))
 	    return (ETHR_AINT_T__) exp;
     }
 }
@@ -213,7 +213,7 @@ ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
     while (1) {
 	AO_t exp = AO_load(&var->counter);
 	AO_t new = exp | ((AO_t) mask);
-	if (AO_compare_and_swap(&var->counter, exp, new))
+	if (AO_compare_and_swap_full(&var->counter, exp, new))
 	    return (ETHR_AINT_T__) exp;
     }
 }
@@ -225,7 +225,7 @@ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var,
 {
     ETHR_AINT_T__ act;
     do {
-	if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new))
+	if (AO_compare_and_swap_full(&var->counter, (AO_t) exp, (AO_t) new))
 	    return exp;
 	act = (ETHR_AINT_T__) AO_load(&var->counter);
     } while (act == exp);
@@ -237,7 +237,7 @@ ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new)
 {
     while (1) {
 	AO_t exp = AO_load(&var->counter);
-	if (AO_compare_and_swap(&var->counter, exp, (AO_t) new))
+	if (AO_compare_and_swap_full(&var->counter, exp, (AO_t) new))
 	    return (ETHR_AINT_T__) exp;
     }
 }
@@ -265,7 +265,6 @@ ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
     return ((ETHR_AINT_T__) AO_fetch_and_add1_acquire(&var->counter)) + 1;
 #else
     ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(add_return)(var, 1);
-    ETHR_MEMORY_BARRIER;
     return res;
 #endif
 }
@@ -287,7 +286,6 @@ ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
 #ifdef AO_HAVE_fetch_and_sub1_release
     return ((ETHR_AINT_T__) AO_fetch_and_sub1_release(&var->counter)) - 1;
 #else
-    ETHR_MEMORY_BARRIER;
     return ETHR_NATMC_FUNC__(dec_return)(var);
 #endif
 }
@@ -314,7 +312,6 @@ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var,
     return act;
 #else
     ETHR_AINT_T__ act = ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp);
-    ETHR_MEMORY_BARRIER;
     return act;
 #endif
 }
@@ -333,7 +330,6 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var,
     } while (act == exp);
     return act;
 #else
-    ETHR_MEMORY_BARRIER;
     return ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp);
 #endif
 }
diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h
index 00380dbf07..16182f8b01 100644
--- a/erts/include/internal/sparc32/atomic.h
+++ b/erts/include/internal/sparc32/atomic.h
@@ -95,7 +95,7 @@ ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
 {
     ETHR_AINT_T__ old, tmp;
 
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n");
+    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory");
     do {
 	old = var->counter;
 	tmp = old+incr;
@@ -105,7 +105,7 @@ ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr)
 	    : "r"(old), "r"(&var->counter), "0"(tmp)
 	    : "memory");
     } while (__builtin_expect(old != tmp, 0));
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+    __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
     return old+incr;
 }
 
@@ -144,7 +144,7 @@ ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
 {
     ETHR_AINT_T__ old, tmp;
 
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n");
+    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory");
     do {
 	old = var->counter;
 	tmp = old & mask;
@@ -154,7 +154,7 @@ ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
 	    : "r"(old), "r"(&var->counter), "0"(tmp)
 	    : "memory");
     } while (__builtin_expect(old != tmp, 0));
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+    __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
     return old;
 }
 
@@ -163,7 +163,7 @@ ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
 {
     ETHR_AINT_T__ old, tmp;
 
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n");
+    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory");
     do {
 	old = var->counter;
 	tmp = old | mask;
@@ -173,7 +173,7 @@ ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask)
 	    : "r"(old), "r"(&var->counter), "0"(tmp)
 	    : "memory");
     } while (__builtin_expect(old != tmp, 0));
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+    __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
     return old;
 }
 
@@ -182,7 +182,7 @@ ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val)
 {
     ETHR_AINT_T__ old, new;
 
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad");
+    __asm__ __volatile__("membar #LoadLoad|#StoreLoad" : : : "memory");
     do {
 	old = var->counter;
 	new = val;
@@ -192,20 +192,20 @@ ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val)
 	    : "r"(old), "r"(&var->counter), "0"(new)
 	    : "memory");
     } while (__builtin_expect(old != new, 0));
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+    __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
     return old;
 }
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old)
 {
-    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n");
+    __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n" : : : "memory");
     __asm__ __volatile__(
       ETHR_CAS__ " [%2], %1, %0"
       : "=&r"(new)
       : "r"(old), "r"(&var->counter), "0"(new)
       : "memory");
-    __asm__ __volatile__("membar #StoreLoad|#StoreStore");
+    __asm__ __volatile__("membar #StoreLoad|#StoreStore" : : : "memory");
     return new;
 }
 
@@ -213,13 +213,11 @@ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__
  * Atomic ops with at least specified barriers.
  */
 
-/* TODO: relax acquire barriers */
-
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var)
 {
     ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var);
-    __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore" : : : "memory");
+    __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory");
     return res;
 }
 
@@ -234,21 +232,18 @@ static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var)
 {
     ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var);
-    __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory");
     return res;
 }
 
 static ETHR_INLINE void
 ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var)
 {
-    __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory");
     ETHR_NATMC_FUNC__(dec)(var);
 }
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var)
 {
-    __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory");
     return ETHR_NATMC_FUNC__(dec_return)(var);
 }
 
@@ -256,14 +251,12 @@ static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old)
 {
     ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
-    __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory");
     return res;
 }
 
 static ETHR_INLINE ETHR_AINT_T__
 ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old)
 {
-    __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory");
     return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old);
 }
 
diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h
index 48e4c0c6c8..0c7b597a6b 100644
--- a/erts/include/internal/tile/atomic.h
+++ b/erts/include/internal/tile/atomic.h
@@ -65,25 +65,35 @@ ethr_native_atomic32_read(ethr_native_atomic32_t *var)
 static ETHR_INLINE void
 ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr)
 {
+    ETHR_MEMORY_BARRIER;
     atomic_add(&var->counter, incr);
+    ETHR_MEMORY_BARRIER;
 }      
        
 static ETHR_INLINE void
 ethr_native_atomic32_inc(ethr_native_atomic32_t *var)
 {
+    ETHR_MEMORY_BARRIER;
     atomic_increment(&var->counter);
+    ETHR_MEMORY_BARRIER;
 }
 
 static ETHR_INLINE void
 ethr_native_atomic32_dec(ethr_native_atomic32_t *var)
 {
+    ETHR_MEMORY_BARRIER;
     atomic_decrement(&var->counter);
+    ETHR_MEMORY_BARRIER;
 }
 
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr)
 {
-    return atomic_exchange_and_add(&var->counter, incr) + incr;
+    ethr_sint32_t res;
+    ETHR_MEMORY_BARRIER;
+    res = atomic_exchange_and_add(&var->counter, incr) + incr;
+    ETHR_MEMORY_BARRIER;
+    return res;
 }
 
 static ETHR_INLINE ethr_sint32_t
@@ -101,18 +111,27 @@ ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var)
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
 {
-    return atomic_and_val(&var->counter, mask);
+    ethr_sint32_t res;
+    ETHR_MEMORY_BARRIER;
+    res = atomic_and_val(&var->counter, mask);
+    ETHR_MEMORY_BARRIER;
+    return res;
 }
 
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask)
 {
-    return atomic_or_val(&var->counter, mask);
+    ethr_sint32_t res;
+    ETHR_MEMORY_BARRIER;
+    res = atomic_or_val(&var->counter, mask);
+    ETHR_MEMORY_BARRIER;
+    return res;
 }
 
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val)
 {   
+    ETHR_MEMORY_BARRIER;
     return atomic_exchange_acq(&var->counter, val);
 } 
 
@@ -121,6 +140,7 @@ ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var,
 			     ethr_sint32_t new,
 			     ethr_sint32_t expected)
 {
+    ETHR_MEMORY_BARRIER;
     return atomic_compare_and_exchange_val_acq(&var->counter, new, expected);
 }
 
@@ -139,9 +159,7 @@ ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var)
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var)
 {
-    ethr_sint32_t res = ethr_native_atomic32_inc_return(var);
-    ETHR_MEMORY_BARRIER;
-    return res;
+    return ethr_native_atomic32_inc_return(var);
 }
 
 static ETHR_INLINE void
@@ -154,14 +172,12 @@ ethr_native_atomic32_set_relb(ethr_native_atomic32_t *var, ethr_sint32_t val)
 static ETHR_INLINE void
 ethr_native_atomic32_dec_relb(ethr_native_atomic32_t *var)
 {
-    ETHR_MEMORY_BARRIER;
     ethr_native_atomic32_dec(var);
 }
 
 static ETHR_INLINE ethr_sint32_t
 ethr_native_atomic32_dec_return_relb(ethr_native_atomic32_t *var)
 {
-    ETHR_MEMORY_BARRIER;
     return ethr_native_atomic32_dec_return(var);
 }
 
@@ -178,7 +194,6 @@ ethr_native_atomic32_cmpxchg_relb(ethr_native_atomic32_t *var,
 				  ethr_sint32_t new,
 				  ethr_sint32_t exp)
 {
-    ETHR_MEMORY_BARRIER;
     return ethr_native_atomic32_cmpxchg(var, new, exp);
 }
 
-- 
cgit v1.2.3


From 498a1d56be241458c85231b5f9da43f4eac0b033 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 May 2011 14:26:44 +0200
Subject: Add needed barriers for write_concurrency tables

Ets tables using the write_concurrency option could potentially get
into an internally inconsistent state.
---
 erts/emulator/beam/erl_db_hash.c | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

(limited to 'erts')

diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 9ef990cc4f..9092fa8785 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -105,7 +105,7 @@
 #define NSEG_2   256 /* Size of second segment table */
 #define NSEG_INC 128 /* Number of segments to grow after that */
 
-#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read(&(tb)->segtab))
+#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_acqb(&(tb)->segtab))
 #define NACTIVE(tb) ((int)erts_smp_atomic_read(&(tb)->nactive))
 #define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems))
 
@@ -122,8 +122,8 @@
 */
 static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval)
 {
-    Uint mask = erts_smp_atomic_read(&tb->szm);
-    Uint ix = hval & mask; 
+    Uint mask = erts_smp_atomic_read_acqb(&tb->szm);
+    Uint ix = hval & mask;
     if (ix >= erts_smp_atomic_read(&tb->nactive)) {
 	ix &= mask>>1;
 	ASSERT(ix < erts_smp_atomic_read(&tb->nactive));
@@ -668,6 +668,7 @@ int db_create_hash(Process *p, DbTable *tbl)
     else { /* coarse locking */
 	tb->locks = NULL;
     }
+    ERTS_THR_MEMORY_BARRIER;
 #endif /* ERST_SMP */
     return DB_ERROR_NONE;
 }
@@ -2342,7 +2343,7 @@ static int alloc_seg(DbTableHash *tb)
 	    struct ext_segment* eseg;
 	    eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1];
 	    MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment);
-	    erts_smp_atomic_set(&tb->segtab, (erts_aint_t) eseg->segtab);
+	    erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t) eseg->segtab);
 	    tb->nsegs = eseg->nsegs;
 	}
 	ASSERT(seg_ix < tb->nsegs);
@@ -2414,7 +2415,7 @@ static int free_seg(DbTableHash *tb, int free_records)
 	    MY_ASSERT(newtop->s.is_ext_segment);
 	    if (newtop->prev_segtab != NULL) {
 		/* Time to use a smaller segtab */
-		erts_smp_atomic_set(&tb->segtab, (erts_aint_t)newtop->prev_segtab);
+		erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)newtop->prev_segtab);
 		tb->nsegs = seg_ix;
 		ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs);
 	    }
@@ -2431,7 +2432,7 @@ static int free_seg(DbTableHash *tb, int free_records)
     if (seg_ix > 0) {
 	if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL;
     } else {
-	erts_smp_atomic_set(&tb->segtab, (erts_aint_t)NULL);
+	erts_smp_atomic_set_relb(&tb->segtab, (erts_aint_t)NULL);
     }
 #endif
     tb->nslots -= SEGSZ;
@@ -2526,9 +2527,9 @@ static void grow(DbTableHash* tb, int nactive)
     }
     erts_smp_atomic_inc(&tb->nactive);
     if (from_ix == 0) {
-	erts_smp_atomic_set(&tb->szm, szm);
+	erts_smp_atomic_set_relb(&tb->szm, szm);
     }
-    erts_smp_atomic_set(&tb->is_resizing, 0);
+    erts_smp_atomic_set_relb(&tb->is_resizing, 0);
 
     /* Finally, let's split the bucket. We try to do it in a smart way
        to keep link order and avoid unnecessary updates of next-pointers */
@@ -2560,7 +2561,7 @@ static void grow(DbTableHash* tb, int nactive)
     return;
    
 abort:
-    erts_smp_atomic_set(&tb->is_resizing, 0);
+    erts_smp_atomic_set_relb(&tb->is_resizing, 0);
 }
 
 
@@ -2604,7 +2605,7 @@ static void shrink(DbTableHash* tb, int nactive)
 	    
 	    erts_smp_atomic_set(&tb->nactive, src_ix);
 	    if (dst_ix == 0) {
-		erts_smp_atomic_set(&tb->szm, low_szm);
+		erts_smp_atomic_set_relb(&tb->szm, low_szm);
 	    }
 	    WUNLOCK_HASH(lck);
 	    
@@ -2618,7 +2619,7 @@ static void shrink(DbTableHash* tb, int nactive)
 
     }
     /*else already done */
-    erts_smp_atomic_set(&tb->is_resizing, 0);
+    erts_smp_atomic_set_relb(&tb->is_resizing, 0);
 }
 
 
-- 
cgit v1.2.3


From 434cab885e50dd72f3c4f87f30d9ee21085cc657 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 May 2011 20:12:02 +0200
Subject: Ensure that stack updates are seen when stack is released

Ets tables using ordered_set could potentially get into an
internally inconsistent state.
---
 erts/emulator/beam/erl_db_tree.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts')

diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c
index a59c0c258d..d5b02584a5 100644
--- a/erts/emulator/beam/erl_db_tree.c
+++ b/erts/emulator/beam/erl_db_tree.c
@@ -111,7 +111,7 @@ static void release_stack(DbTableTree* tb, DbTreeStack* stack)
 {
     if (stack == &tb->static_stack) {
 	ASSERT(erts_smp_atomic_read(&tb->is_stack_busy) == 1);
-	erts_smp_atomic_set(&tb->is_stack_busy, 0);
+	erts_smp_atomic_set_relb(&tb->is_stack_busy, 0);
     }
     else {
 	erts_db_free(ERTS_ALC_T_DB_STK, (DbTable *) tb,
-- 
cgit v1.2.3


From 673f5d1bf684f15ac5526d8a21552eca9a0c9052 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 May 2011 20:25:53 +0200
Subject: Ensure that all rehashing information are seen when done

This is not a bugfix. The change is done in order to avoid a
future bug.
---
 erts/emulator/beam/safe_hash.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts')

diff --git a/erts/emulator/beam/safe_hash.c b/erts/emulator/beam/safe_hash.c
index 21d6ce9304..3e9243c77d 100644
--- a/erts/emulator/beam/safe_hash.c
+++ b/erts/emulator/beam/safe_hash.c
@@ -99,7 +99,7 @@ static void rehash(SafeHash* h, int grow_limit)
 	erts_free(h->type, (void *) old_tab);
     }
     /*else already done */
-    erts_smp_atomic_set(&h->is_rehashing, 0);
+    erts_smp_atomic_set_relb(&h->is_rehashing, 0);
 }
 
 
-- 
cgit v1.2.3


From 425e282be62f8205b1ba262b112f38688b421c49 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 May 2011 20:43:11 +0200
Subject: Ensure quick break

Make sure that we don't have to wait in poll before break
handling is done.
---
 erts/emulator/sys/common/erl_check_io.c | 5 +++++
 erts/emulator/sys/win32/erl_poll.c      | 2 ++
 2 files changed, 7 insertions(+)

(limited to 'erts')

diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index 218bd79584..71b374527e 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -1137,6 +1137,11 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait)
 
  restart:
 
+#ifdef ERTS_BREAK_REQUESTED
+    if (ERTS_BREAK_REQUESTED)
+	erts_do_break_handling();
+#endif
+
     /* Figure out timeout value */
     if (do_wait) {
 	erts_time_remaining(&wait_time);
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index 7662f190ef..074e2e247f 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -693,6 +693,7 @@ static void *break_waiter(void *param)
 	    ResetEvent(harr[0]);
 	    erts_mtx_lock(&break_waiter_lock);
 	    erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_BREAK);
+	    ERTS_THR_MEMORY_BARRIER;
 	    SetEvent(break_happened_event);
 	    erts_mtx_unlock(&break_waiter_lock);
 	    break;
@@ -700,6 +701,7 @@ static void *break_waiter(void *param)
 	    ResetEvent(harr[1]);
 	    erts_mtx_lock(&break_waiter_lock);
 	    erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_HALT);
+	    ERTS_THR_MEMORY_BARRIER;
 	    SetEvent(break_happened_event);
 	    erts_mtx_unlock(&break_waiter_lock);
 	    break;
-- 
cgit v1.2.3


From 3da283a85f73132e73ab911154c2c0ff8797d61d Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 May 2011 20:59:36 +0200
Subject: Remove pointless erts_ports_alive variable

---
 erts/emulator/beam/bif.c           | 22 ++++++++++------------
 erts/emulator/beam/erl_port_task.c |  3 ---
 erts/emulator/beam/global.h        |  1 -
 erts/emulator/beam/io.c            |  3 ---
 4 files changed, 10 insertions(+), 19 deletions(-)

(limited to 'erts')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index b3325d635b..fe8dcc4d8b 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3270,6 +3270,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0)
     Eterm* dead_ports;
     int alive, dead;
     Uint32 next_ss;
+    int i;
 
     /* To get a consistent snapshot... 
      * We add alive ports from start of the buffer
@@ -3283,19 +3284,16 @@ BIF_RETTYPE ports_0(BIF_ALIST_0)
 
     next_ss = erts_smp_atomic_inctest(&erts_ports_snapshot);
 
-    if (erts_smp_atomic_read(&erts_ports_alive) > 0) {
-	erts_aint_t i;
-	for (i = erts_max_ports-1; i >= 0; i--) {
-	    Port* prt = &erts_port[i];
-	    erts_smp_port_state_lock(prt);
-	    if (!(prt->status & ERTS_PORT_SFLGS_DEAD)
-		&& prt->snapshot != next_ss) {
-		ASSERT(prt->snapshot == next_ss - 1);
-		*pp++ = prt->id;		
-		prt->snapshot = next_ss; /* Consumed by this snapshot */
-	    }
-	    erts_smp_port_state_unlock(prt);
+    for (i = erts_max_ports-1; i >= 0; i--) {
+	Port* prt = &erts_port[i];
+	erts_smp_port_state_lock(prt);
+	if (!(prt->status & ERTS_PORT_SFLGS_DEAD)
+	    && prt->snapshot != next_ss) {
+	    ASSERT(prt->snapshot == next_ss - 1);
+	    *pp++ = prt->id;		
+	    prt->snapshot = next_ss; /* Consumed by this snapshot */
 	}
+	erts_smp_port_state_unlock(prt);
     }
 
     dead_ports = (Eterm*)erts_smp_atomic_xchg(&erts_dead_ports_ptr,
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 1b07024ca1..326021643f 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -658,8 +658,6 @@ erts_port_task_free_port(Port *pp)
 	   when scheduled out... */
 	ErtsPortTask *ptp = port_task_alloc();
 	erts_smp_port_state_lock(pp);
-	ASSERT(erts_smp_atomic_read(&erts_ports_alive) > 0);
-	erts_smp_atomic_dec(&erts_ports_alive);
 	pp->status &= ~ERTS_PORT_SFLG_CLOSING;
 	pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED;
 	erts_may_save_closed_port(pp);
@@ -681,7 +679,6 @@ erts_port_task_free_port(Port *pp)
 	    port_is_dequeued = 1;
 	}
 	erts_smp_port_state_lock(pp);
-	erts_smp_atomic_dec(&erts_ports_alive);
 	pp->status &= ~ERTS_PORT_SFLG_CLOSING;
 	pp->status |= ERTS_PORT_SFLG_FREE_SCHEDULED;
 	erts_may_save_closed_port(pp);
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 96da894d90..18dc9f75d5 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -527,7 +527,6 @@ union erl_off_heap_ptr {
 
 /* arrays that get malloced at startup */
 extern Port* erts_port;
-extern erts_smp_atomic_t erts_ports_alive;
 
 extern Uint erts_max_ports;
 extern Uint erts_port_tab_index_mask;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index f619c6f88b..86f550310c 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -56,7 +56,6 @@ static erts_smp_tsd_key_t driver_list_last_error_key;  /* Save last DDLL error o
 							  per thread basis (for BC interfaces) */
 
 Port*      erts_port; /* The port table */
-erts_smp_atomic_t erts_ports_alive;
 erts_smp_atomic_t erts_bytes_out;	/* No bytes sent out of the system */
 erts_smp_atomic_t erts_bytes_in;	/* No bytes gotten into the system */
 
@@ -421,7 +420,6 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver,
     new_name = (char*) erts_alloc(ERTS_ALC_T_PORT_NAME, sys_strlen(name)+1);
     sys_strcpy(new_name, name);
     erts_smp_runq_lock(runq);
-    erts_smp_atomic_inc(&erts_ports_alive);
     erts_smp_port_state_lock(prt);    
     prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus;
     prt->snapshot = (Uint32) erts_smp_atomic_read(&erts_ports_snapshot);    
@@ -1274,7 +1272,6 @@ void init_io(void)
 
     erts_smp_atomic_init(&erts_bytes_out, 0);
     erts_smp_atomic_init(&erts_bytes_in, 0);
-    erts_smp_atomic_init(&erts_ports_alive, 0);
 
     for (i = 0; i < erts_max_ports; i++) {
 	erts_port_task_init_sched(&erts_port[i].sched);
-- 
cgit v1.2.3


From 78ebe8aa3754fc8837ab3a6b0bc11d1e78275eef Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 May 2011 21:12:05 +0200
Subject: Use 32-bit atomic for port snapshot

---
 erts/emulator/beam/bif.c    | 2 +-
 erts/emulator/beam/global.h | 8 ++++----
 erts/emulator/beam/io.c     | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

(limited to 'erts')

diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index fe8dcc4d8b..fe21e30d82 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3282,7 +3282,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0)
     erts_smp_atomic_set(&erts_dead_ports_ptr,
 			(erts_aint_t) (port_buf + erts_max_ports));
 
-    next_ss = erts_smp_atomic_inctest(&erts_ports_snapshot);
+    next_ss = erts_smp_atomic32_inctest(&erts_ports_snapshot);
 
     for (i = erts_max_ports-1; i >= 0; i--) {
 	Port* prt = &erts_port[i];
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 18dc9f75d5..7dc5aa85e9 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -183,7 +183,7 @@ struct port {
 				    process to get (line oriented I/O)*/
     Uint32 status;		 /* Status and type flags */
     int control_flags;		 /* Flags for port_control()  */
-    Uint32 snapshot;             /* Next snapshot that port should be part of */
+    erts_aint32_t snapshot;      /* Next snapshot that port should be part of */
     struct reg_proc *reg;
     ErlDrvPDL port_data_lock;
 
@@ -530,7 +530,7 @@ extern Port* erts_port;
 
 extern Uint erts_max_ports;
 extern Uint erts_port_tab_index_mask;
-extern erts_smp_atomic_t erts_ports_snapshot;
+extern erts_smp_atomic32_t erts_ports_snapshot;
 extern erts_smp_atomic_t erts_dead_ports_ptr;
 
 ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt);
@@ -540,12 +540,12 @@ ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt);
 ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt)
 {
     ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&prt->state_lck));
-    if (prt->snapshot != erts_smp_atomic_read(&erts_ports_snapshot)) {
+    if (prt->snapshot != erts_smp_atomic32_read_acqb(&erts_ports_snapshot)) {
 	/* Dead ports are added from the end of the snapshot buffer */
 	Eterm* tombstone = (Eterm*) erts_smp_atomic_addtest(&erts_dead_ports_ptr,
 							    -(erts_aint_t)sizeof(Eterm));
 	ASSERT(tombstone+1 != NULL);
-	ASSERT(prt->snapshot == (Uint32) erts_smp_atomic_read(&erts_ports_snapshot) - 1);
+	ASSERT(prt->snapshot == erts_smp_atomic32_read(&erts_ports_snapshot) - 1);
 	*tombstone = prt->id;
     }
     /*else no ongoing snapshot or port was already included or created after snapshot */
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 86f550310c..8c1126aa6e 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -189,7 +189,7 @@ typedef struct line_buf_context {
 static erts_smp_spinlock_t get_free_port_lck;
 static Uint last_port_num;
 static Uint port_num_mask;
-erts_smp_atomic_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */
+erts_smp_atomic32_t erts_ports_snapshot; /* Identifies the _next_ snapshot (not the ongoing) */
 
 
 static ERTS_INLINE void
@@ -422,7 +422,7 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver,
     erts_smp_runq_lock(runq);
     erts_smp_port_state_lock(prt);    
     prt->status = ERTS_PORT_SFLG_CONNECTED | xstatus;
-    prt->snapshot = (Uint32) erts_smp_atomic_read(&erts_ports_snapshot);    
+    prt->snapshot = erts_smp_atomic32_read(&erts_ports_snapshot);    
     old_name = prt->name;
     prt->name = new_name;
 #ifdef ERTS_SMP
@@ -1293,7 +1293,7 @@ void init_io(void)
 	erts_port[i].port_data_lock = NULL;
     }
 
-    erts_smp_atomic_init(&erts_ports_snapshot, (erts_aint_t) 0);
+    erts_smp_atomic32_init(&erts_ports_snapshot, (erts_aint32_t) 0);
     last_port_num = 0;
     erts_smp_spinlock_init(&get_free_port_lck, "get_free_port");
 
-- 
cgit v1.2.3


From 150e88657f617d652d0ab5e4b45409edeb529c50 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 May 2011 21:35:30 +0200
Subject: Reduce number of atomic ops

Counters for active, and used schedulers have been coalesced in
order to reduce the amount of atomic operations needed.

Some currently not strictly necessary barriers have also been added
in order to be future proof.
---
 erts/emulator/beam/erl_process.c | 135 +++++++++++++++++++++++++++++----------
 1 file changed, 100 insertions(+), 35 deletions(-)

(limited to 'erts')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 31f23d3978..559224be58 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -166,9 +166,8 @@ static struct {
 
 static struct {
     erts_smp_mtx_t update_mtx;
-    erts_smp_atomic32_t active_runqs;
+    erts_smp_atomic32_t no_runqs;
     int last_active_runqs;
-    erts_smp_atomic32_t used_runqs;
     int forced_check_balance;
     erts_smp_atomic32_t checking_balance;
     int halftime;
@@ -965,7 +964,7 @@ sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount)
     erts_aint32_t flgs;
 
     do {
-	flgs = erts_smp_atomic32_read(&ssi->flags);
+	flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
 	if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING))
 	    != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) {
 	    break;
@@ -1114,7 +1113,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
     else {
 	erts_aint_t dt;
 
-	erts_smp_atomic32_set(&function_calls, 0);
+	erts_smp_atomic32_set_relb(&function_calls, 0);
 	*fcalls = 0;
 
 	sched_waiting_sys(esdp->no, rq);
@@ -1147,7 +1146,7 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    nonblockable_aux_work(esdp, ssi, aux_work);
 #endif
 
-	    flgs = erts_smp_atomic32_read(&ssi->flags);
+	    flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
 	    if (!(flgs & ERTS_SSI_FLG_WAITING)) {
 		ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING));
 		goto sys_woken;
@@ -1333,6 +1332,76 @@ wake_all_schedulers(void)
     }
 }
 
+#define ERTS_NO_USED_RUNQS_SHIFT 16
+#define ERTS_NO_RUNQS_MASK 0xffff
+
+#if ERTS_MAX_NO_OF_SCHEDULERS > ERTS_NO_RUNQS_MASK
+#  error "Too large amount of schedulers allowed"
+#endif
+
+static ERTS_INLINE void
+init_no_runqs(int active, int used)
+{
+    erts_aint32_t no_runqs = (erts_aint32_t) (active & ERTS_NO_RUNQS_MASK);
+    no_runqs |= (erts_aint32_t) ((used & ERTS_NO_RUNQS_MASK) << ERTS_NO_USED_RUNQS_SHIFT);
+    erts_smp_atomic32_init(&balance_info.no_runqs, no_runqs);
+}
+
+static ERTS_INLINE void
+get_no_runqs(int *active, int *used)
+{
+    erts_aint32_t no_runqs = erts_smp_atomic32_read(&balance_info.no_runqs);
+    if (active)
+	*active = (int) (no_runqs & ERTS_NO_RUNQS_MASK);
+    if (used)
+	*used = (int) ((no_runqs >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK);
+}
+
+static ERTS_INLINE void
+set_no_used_runqs(int used)
+{
+    erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs);
+    while (1) {
+	erts_aint32_t act, new;
+	new = (used << ERTS_NO_USED_RUNQS_SHIFT) | (exp & ERTS_NO_RUNQS_MASK);
+	act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp);
+	if (act == exp)
+	    break;
+	exp = act;
+    }
+}
+
+static ERTS_INLINE void
+set_no_active_runqs(int active)
+{
+    erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs);
+    while (1) {
+	erts_aint32_t act, new;
+	new = (exp & (ERTS_NO_RUNQS_MASK << ERTS_NO_USED_RUNQS_SHIFT)) | active;
+	act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp);
+	if (act == exp)
+	    break;
+	exp = act;
+    }
+}
+
+static ERTS_INLINE int
+try_inc_no_active_runqs(int active)
+{
+    erts_aint32_t exp = erts_smp_atomic32_read(&balance_info.no_runqs);
+    if (((exp >> ERTS_NO_USED_RUNQS_SHIFT) & ERTS_NO_RUNQS_MASK) < active)
+	return 0;
+    if ((exp & ERTS_NO_RUNQS_MASK) + 1 == active) {
+	erts_aint32_t new, act;
+	new = (exp & ~ERTS_NO_RUNQS_MASK) | active;
+	act = erts_smp_atomic32_cmpxchg(&balance_info.no_runqs, new, exp);
+	if (act == exp)
+	    return 1;
+    }
+    return 0;
+}
+
+
 static ERTS_INLINE int
 chk_wake_sched(ErtsRunQueue *crq, int ix, int activate)
 {
@@ -1344,9 +1413,7 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate)
     iflgs = erts_smp_atomic32_read(&wrq->info_flags);
     if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) {
 	if (activate) {
-	    if (ix == erts_smp_atomic32_cmpxchg(&balance_info.active_runqs,
-						ix+1,
-						ix)) {
+	    if (try_inc_no_active_runqs(ix+1)) {
 		erts_smp_xrunq_lock(crq, wrq);
 		wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE;
 		erts_smp_xrunq_unlock(crq, wrq);
@@ -1363,8 +1430,9 @@ wake_scheduler_on_empty_runq(ErtsRunQueue *crq)
 {
     int ix = crq->ix;
     int stop_ix = ix;
-    int active_ix = erts_smp_atomic32_read(&balance_info.active_runqs);
-    int balance_ix = erts_smp_atomic32_read(&balance_info.used_runqs);
+    int active_ix, balance_ix;
+
+    get_no_runqs(&active_ix, &balance_ix);
 
     if (active_ix > balance_ix)
 	active_ix = balance_ix;
@@ -1416,7 +1484,7 @@ erts_sched_notify_check_cpu_bind(void)
     int ix;
     if (erts_common_run_queue) {
 	for (ix = 0; ix < erts_no_schedulers; ix++)
-	    erts_smp_atomic32_set(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1);
+	    erts_smp_atomic32_set_relb(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1);
 	wake_all_schedulers();
     }
     else {
@@ -1871,8 +1939,7 @@ try_steal_task(ErtsRunQueue *rq)
 
     ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, rq_locked);
 
-    active_rqs = erts_smp_atomic32_read(&balance_info.active_runqs);
-    blnc_rqs = erts_smp_atomic32_read(&balance_info.used_runqs);
+    get_no_runqs(&active_rqs, &blnc_rqs);
 
     if (active_rqs > blnc_rqs)
 	active_rqs = blnc_rqs;
@@ -1883,7 +1950,7 @@ try_steal_task(ErtsRunQueue *rq)
 	if (active_rqs < blnc_rqs) {
 	    int no = blnc_rqs - active_rqs;
 	    int stop_ix = vix = active_rqs + rq->ix % no;
-	    while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) {
+	    while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) {
 		res = check_possible_steal_victim(rq, &rq_locked, vix);
 		if (res)
 		    goto done;
@@ -1898,7 +1965,7 @@ try_steal_task(ErtsRunQueue *rq)
 	vix = rq->ix;
 
 	/* ... then try to steal a job from another active queue... */
-	while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) {
+	while (erts_smp_atomic32_read_acqb(&no_empty_run_queues) < blnc_rqs) {
 	    vix++;
 	    if (vix >= active_rqs)
 		vix = 0;
@@ -1999,7 +2066,7 @@ check_balance(ErtsRunQueue *c_rq)
 	return;
     }
 
-    blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs);
+    get_no_runqs(NULL, &blnc_no_rqs);
     if (blnc_no_rqs == 1) {
 	c_rq->check_balance_reds = INT_MAX;
 	erts_smp_atomic32_set(&balance_info.checking_balance, 0);
@@ -2038,7 +2105,8 @@ check_balance(ErtsRunQueue *c_rq)
     forced = balance_info.forced_check_balance;
     balance_info.forced_check_balance = 0;
 
-    blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs);
+    get_no_runqs(&current_active, &blnc_no_rqs);
+
     if (blnc_no_rqs == 1) {
 	erts_smp_mtx_unlock(&balance_info.update_mtx);
 	erts_smp_runq_lock(c_rq);
@@ -2052,8 +2120,6 @@ check_balance(ErtsRunQueue *c_rq)
     if (balance_info.full_reds_history_index >= ERTS_FULL_REDS_HISTORY_SIZE)
 	balance_info.full_reds_history_index = 0;
 
-    current_active = erts_smp_atomic32_read(&balance_info.active_runqs);
-
     /* Read balance information for all run queues */
     for (qix = 0; qix < blnc_no_rqs; qix++) {
 	ErtsRunQueue *rq = ERTS_RUNQ_IX(qix);
@@ -2387,7 +2453,7 @@ erts_fprintf(stderr, "--------------------------------\n");
     }
 
     balance_info.last_active_runqs = active;
-    erts_smp_atomic32_set(&balance_info.active_runqs, active);
+    set_no_active_runqs(active);
 
     balance_info.halftime = 1;
     erts_smp_atomic32_set(&balance_info.checking_balance, 0);
@@ -2695,9 +2761,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online)
     erts_smp_atomic32_init(&schdlr_sspnd.msb.ongoing, 0);
     erts_smp_atomic32_init(&schdlr_sspnd.active, no_schedulers);
     schdlr_sspnd.msb.procs = NULL;
-    erts_smp_atomic32_set(&balance_info.used_runqs,
-			  erts_common_run_queue ? 1 : no_schedulers_online);
-    erts_smp_atomic32_init(&balance_info.active_runqs, no_schedulers);
+    init_no_runqs(no_schedulers,
+		  erts_common_run_queue ? 1 : no_schedulers_online);
     balance_info.last_active_runqs = no_schedulers;
     erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update");
     balance_info.forced_check_balance = 0;
@@ -2939,7 +3004,7 @@ sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount)
     erts_aint32_t flgs;
 
     do {
-	flgs = erts_smp_atomic32_read(&ssi->flags);
+	flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
 	if ((flgs & (ERTS_SSI_FLG_SLEEPING
 		     | ERTS_SSI_FLG_WAITING
 		     | ERTS_SSI_FLG_SUSPENDED))
@@ -3068,7 +3133,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 		wake = 0;
 	    }
 
-	    flgs = erts_smp_atomic32_read(&ssi->flags);
+	    flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
 	    if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
 		break;
 	    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
@@ -3292,7 +3357,7 @@ erts_set_schedulers_online(Process *p,
 			ErtsRunQueue *to_rq = ERTS_RUNQ_IX(ix % no);
 			evacuate_run_queue(from_rq, to_rq);
 		    }
-		    erts_smp_atomic32_set(&balance_info.used_runqs, no);
+		    set_no_used_runqs(no);
 		    erts_smp_mtx_unlock(&balance_info.update_mtx);
 		    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 		}
@@ -3346,7 +3411,7 @@ erts_set_schedulers_online(Process *p,
 		    for (ix = erts_no_run_queues-1; ix >= no; ix--)
 			evacuate_run_queue(ERTS_RUNQ_IX(ix),
 					   ERTS_RUNQ_IX(ix % no));
-		    erts_smp_atomic32_set(&balance_info.used_runqs, no);
+		    set_no_used_runqs(no);
 		    erts_smp_mtx_unlock(&balance_info.update_mtx);
 		    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 		    for (ix = no; ix < online; ix++) {
@@ -3443,7 +3508,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		else {
 		    erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 		    erts_smp_mtx_lock(&balance_info.update_mtx);
-		    erts_smp_atomic32_set(&balance_info.used_runqs, 1);
+		    set_no_used_runqs(1);
 		    for (ix = 0; ix < online; ix++) {
 			ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
 			erts_smp_runq_lock(rq);
@@ -3580,7 +3645,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all)
 		    evacuate_run_queue(ERTS_RUNQ_IX(ix),
 				       ERTS_RUNQ_IX(ix % online));
 
-		erts_smp_atomic32_set(&balance_info.used_runqs, online);
+		set_no_used_runqs(online);
 		/* Make sure that we balance soon... */
 		balance_info.forced_check_balance = 1;
 		erts_smp_runq_lock(ERTS_RUNQ_IX(0));
@@ -5097,7 +5162,7 @@ Process *schedule(Process *p, int calls)
 	esdp = erts_get_scheduler_data();
 	rq = erts_get_runq_current(esdp);
 	ASSERT(esdp);
-	fcalls = (int) erts_smp_atomic32_read(&function_calls);
+	fcalls = (int) erts_smp_atomic32_read_acqb(&function_calls);
 	actual_reds = reds = 0;
 	erts_smp_runq_lock(rq);
     } else {
@@ -5248,14 +5313,14 @@ Process *schedule(Process *p, int calls)
 			 | ERTS_RUNQ_FLG_CHK_CPU_BIND
 			 | ERTS_RUNQ_FLG_SUSPENDED)) {
 	    if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
-		|| (erts_smp_atomic32_read(&esdp->ssi->flags)
+		|| (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
 		    & ERTS_SSI_FLG_SUSPENDED)) {
 		ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags)
 		       & ERTS_SSI_FLG_SUSPENDED);
 		suspend_scheduler(esdp);
 	    }
 	    if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND)
-		|| erts_smp_atomic32_read(&esdp->chk_cpu_bind)) {
+		|| erts_smp_atomic32_read_acqb(&esdp->chk_cpu_bind)) {
 		erts_sched_check_cpu_bind(esdp);
 	    }
 	}
@@ -5306,7 +5371,7 @@ Process *schedule(Process *p, int calls)
 	    if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ
 			     | ERTS_RUNQ_FLG_SUSPENDED)) {
 		if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED)
-		    || (erts_smp_atomic32_read(&esdp->ssi->flags)
+		    || (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
 			& ERTS_SSI_FLG_SUSPENDED)) {
 		    ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags)
 			   & ERTS_SSI_FLG_SUSPENDED);
@@ -5350,7 +5415,7 @@ Process *schedule(Process *p, int calls)
 	     * Schedule system-level activities.
 	     */
 
-	    erts_smp_atomic32_set(&function_calls, 0);
+	    erts_smp_atomic32_set_relb(&function_calls, 0);
 	    fcalls = 0;
 
 	    ASSERT(!erts_port_task_have_outstanding_io_tasks());
@@ -5392,7 +5457,7 @@ Process *schedule(Process *p, int calls)
 			if (erts_common_run_queue->waiting)
 			    wake_scheduler(erts_common_run_queue, 0, 1);
 		    }
-		    else if (erts_smp_atomic32_read(&no_empty_run_queues) != 0) {
+		    else if (erts_smp_atomic32_read_acqb(&no_empty_run_queues) != 0) {
 			wake_scheduler_on_empty_runq(rq);
 			rq->wakeup_other = 0;
 		    }
-- 
cgit v1.2.3


From 6af0f286a3a46a7e2faf722306e7be57bf3de687 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 May 2011 21:11:41 +0200
Subject: Fix build with hipe on amd64

---
 erts/emulator/beam/atom.names | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 327620772f..de76cb9680 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -76,6 +76,7 @@ atom allocator_sizes
 atom alloc_util_allocators
 atom allow_passive_connect
 atom already_loaded
+atom amd64
 atom anchored
 atom and
 atom andalso
-- 
cgit v1.2.3


From 139fa05489a6ba3e4384e6f20ea3f943741449d5 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 May 2011 14:37:49 +0200
Subject: Silence warnings

---
 erts/emulator/beam/beam_emu.c       | 2 +-
 erts/emulator/beam/erl_drv_thread.c | 7 +++++++
 erts/emulator/sys/common/erl_poll.c | 7 ++++---
 3 files changed, 12 insertions(+), 4 deletions(-)

(limited to 'erts')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 32ea8588d2..99bf7e0439 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5322,7 +5322,7 @@ void process_main(void)
 	 ep->code[3] = (BeamInstr) OpCode(apply_bif);
 	 ep->code[4] = (BeamInstr) bif_table[i].f;
 	 /* XXX: set func info for bifs */
-	 ((BeamInstr*)ep->code + 3)[-5] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
+	 ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI);
      }
 
      return;
diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 39bbe9633b..dc578f6d2a 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -700,6 +700,13 @@ erl_drv_thread_join(ErlDrvTid tid, void **respp)
 extern int erts_darwin_main_thread_pipe[2];
 extern int erts_darwin_main_thread_result_pipe[2];
 
+int erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp);
+int erl_drv_steal_main_thread(char *name,
+			      ErlDrvTid *dtid,
+			      void* (*func)(void*),
+			      void* arg,
+			      ErlDrvThreadOpts *opts);
+
 
 int
 erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp)
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 3ae5b8d747..f5c785d683 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -766,7 +766,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
 		short filter;
 		int fd = (int) ebuf[i].ident;
 
-		switch ((int) ebuf[i].udata) {
+		switch ((int) (long) ebuf[i].udata) {
 
 		    /*
 		     * Since we use a lazy update approach EV_DELETE will
@@ -805,7 +805,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
 			if (fd == (int) ebuf[j].ident) {
 			    ebuf[j].udata = (void *) ERTS_POLL_KQ_OP_HANDLED;
 			    if (!(ebuf[j].flags & EV_ERROR)) {
-				switch ((int) ebuf[j].udata) {
+				switch ((int) (long) ebuf[j].udata) {
 				case ERTS_POLL_KQ_OP_ADD2_W:
 				    filter = EVFILT_WRITE;
 				    goto rm_add_fb;
@@ -823,7 +823,8 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp)
 			}
 		    }
 		    /* The other add succeded... */
-		    filter = (((int) ebuf[i].udata == ERTS_POLL_KQ_OP_ADD2_W)
+		    filter = ((((int) (long) ebuf[i].udata)
+			       == ERTS_POLL_KQ_OP_ADD2_W)
 			      ? EVFILT_READ
 			      : EVFILT_WRITE);
 		rm_add_fb:
-- 
cgit v1.2.3