From 59eb7be1022a1841aa1153258e80d59794f077a6 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Sat, 11 Dec 2010 08:20:28 +0100 Subject: Add 32-bit atomics to emulator APIs --- erts/emulator/beam/erl_smp.h | 279 +++++++++++++++++++++++++++++++++++++-- erts/emulator/beam/erl_threads.h | 254 ++++++++++++++++++++++++++++++++++- 2 files changed, 522 insertions(+), 11 deletions(-) diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 7d5c4ce554..287327bfe1 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -54,7 +54,8 @@ typedef erts_cnd_t erts_smp_cnd_t; typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t; typedef erts_rwmtx_t erts_smp_rwmtx_t; typedef erts_tsd_key_t erts_smp_tsd_key_t; -typedef ethr_atomic_t erts_smp_atomic_t; +typedef erts_atomic_t erts_smp_atomic_t; +typedef erts_atomic32_t erts_smp_atomic32_t; typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ @@ -83,6 +84,7 @@ typedef struct { typedef int erts_smp_rwmtx_t; typedef int erts_smp_tsd_key_t; typedef SWord erts_smp_atomic_t; +typedef Uint32 erts_smp_atomic32_t; #if __GNUC__ > 2 typedef struct { } erts_smp_spinlock_t; typedef struct { } erts_smp_rwlock_t; @@ -190,6 +192,50 @@ ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchg ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, erts_aint_t new, erts_aint_t exp); +ERTS_GLB_INLINE void +erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE void +erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read(erts_smp_atomic32_t *var); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE void +erts_smp_atomic32_inc(erts_smp_atomic32_t *incp); +ERTS_GLB_INLINE void +erts_smp_atomic32_dec(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i); +ERTS_GLB_INLINE void +erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var); +ERTS_GLB_INLINE void +erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE void +erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra); @@ -772,7 +818,8 @@ erts_smp_atomic_set_relb(erts_smp_atomic_t *var, erts_aint_t i) #endif } -ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp) +ERTS_GLB_INLINE void +erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp) { #ifdef ERTS_SMP erts_atomic_dec_relb(decp); @@ -791,9 +838,10 @@ erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp) #endif } -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp) { #ifdef ERTS_SMP return erts_atomic_cmpxchg_acqb(xchgp, new, exp); @@ -805,9 +853,10 @@ ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchg #endif } -ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, - erts_aint_t new, - erts_aint_t exp) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp) { #ifdef ERTS_SMP return erts_atomic_cmpxchg_relb(xchgp, new, exp); @@ -819,6 +868,220 @@ ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchg #endif } +ERTS_GLB_INLINE void +erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_init(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_set(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read(erts_smp_atomic32_t *var) +{ +#ifdef ERTS_SMP + return erts_atomic32_read(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp) +{ +#ifdef ERTS_SMP + return erts_atomic32_inctest(incp); +#else + return ++(*incp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + return erts_atomic32_dectest(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_inc(erts_smp_atomic32_t *incp) +{ +#ifdef ERTS_SMP + erts_atomic32_inc(incp); +#else + ++(*incp); +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_dec(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + erts_atomic32_dec(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef ERTS_SMP + return erts_atomic32_addtest(addp, i); +#else + return *addp += i; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_add(addp, i); +#else + *addp += i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new) +{ +#ifdef ERTS_SMP + return erts_atomic32_xchg(xchgp, new); +#else + erts_aint32_t old; + old = *xchgp; + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected) +{ +#ifdef ERTS_SMP + return erts_atomic32_cmpxchg(xchgp, new, expected); +#else + erts_aint32_t old = *xchgp; + if (old == expected) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask) +{ +#ifdef ERTS_SMP + return erts_atomic32_bor(var, mask); +#else + erts_aint32_t old; + old = *var; + *var |= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask) +{ +#ifdef ERTS_SMP + return erts_atomic32_band(var, mask); +#else + erts_aint32_t old; + old = *var; + *var &= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var) +{ +#ifdef ERTS_SMP + return erts_atomic32_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + erts_atomic32_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + return erts_atomic32_dectest_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef ERTS_SMP + return erts_atomic32_cmpxchg_acqb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef ERTS_SMP + return erts_atomic32_cmpxchg_relb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra) { diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index f61baab039..84a20b51f2 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -91,6 +91,8 @@ typedef ethr_tsd_key erts_tsd_key_t; typedef ethr_ts_event erts_tse_t; typedef ethr_sint_t erts_aint_t; typedef ethr_atomic_t erts_atomic_t; +typedef ethr_sint32_t erts_aint32_t; +typedef ethr_atomic32_t erts_atomic32_t; /* spinlock */ typedef struct { @@ -154,6 +156,8 @@ typedef int erts_tsd_key_t; typedef int erts_tse_t; typedef SWord erts_aint_t; typedef SWord erts_atomic_t; +typedef SWord erts_aint32_t; +typedef SWord erts_atomic32_t; #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; typedef struct { } erts_rwlock_t; @@ -259,6 +263,36 @@ ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, erts_aint_t new, erts_aint_t exp); +ERTS_GLB_INLINE void erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE void erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read(erts_atomic32_t *var); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_inctest(erts_atomic32_t *incp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest(erts_atomic32_t *decp); +ERTS_GLB_INLINE void erts_atomic32_inc(erts_atomic32_t *incp); +ERTS_GLB_INLINE void erts_atomic32_dec(erts_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_addtest(erts_atomic32_t *addp, + erts_aint32_t i); +ERTS_GLB_INLINE void erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_xchg(erts_atomic32_t *xchgp, + erts_aint32_t new); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_bor(erts_atomic32_t *var, + erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_band(erts_atomic32_t *var, + erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_acqb(erts_atomic32_t *var); +ERTS_GLB_INLINE void erts_atomic32_set_relb(erts_atomic32_t *var, + erts_aint32_t i); +ERTS_GLB_INLINE void erts_atomic32_dec_relb(erts_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest_relb(erts_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra, @@ -1017,14 +1051,13 @@ erts_atomic_add(erts_atomic_t *addp, erts_aint_t i) ERTS_GLB_INLINE erts_aint_t erts_atomic_xchg(erts_atomic_t *xchgp, erts_aint_t new) { - erts_aint_t old; #ifdef USE_THREADS return ethr_atomic_xchg(xchgp, new); #else - old = *xchgp; + erts_aint_t old = *xchgp; *xchgp = new; -#endif return old; +#endif } ERTS_GLB_INLINE erts_aint_t @@ -1134,6 +1167,221 @@ ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, #endif } +/* atomic32 */ + +ERTS_GLB_INLINE void +erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_init(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_set(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read(erts_atomic32_t *var) +{ +#ifdef USE_THREADS + return ethr_atomic32_read(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_inctest(erts_atomic32_t *incp) +{ +#ifdef USE_THREADS + return ethr_atomic32_inc_read(incp); +#else + return ++(*incp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_dectest(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + return ethr_atomic32_dec_read(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_inc(erts_atomic32_t *incp) +{ +#ifdef USE_THREADS + ethr_atomic32_inc(incp); +#else + ++(*incp); +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_dec(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + ethr_atomic32_dec(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_addtest(erts_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef USE_THREADS + return ethr_atomic32_add_read(addp, i); +#else + return *addp += i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_add(addp, i); +#else + *addp += i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_xchg(erts_atomic32_t *xchgp, erts_aint32_t new) +{ +#ifdef USE_THREADS + return ethr_atomic32_xchg(xchgp, new); +#else + erts_aint32_t old = *xchgp; + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_cmpxchg(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected) +{ +#ifdef USE_THREADS + return ethr_atomic32_cmpxchg(xchgp, new, expected); +#else + erts_aint32_t old = *xchgp; + if (old == expected) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_bor(erts_atomic32_t *var, erts_aint32_t mask) +{ +#ifdef USE_THREADS + return ethr_atomic32_read_bor(var, mask); +#else + erts_aint32_t old; + old = *var; + *var |= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_band(erts_atomic32_t *var, erts_aint32_t mask) +{ +#ifdef USE_THREADS + return ethr_atomic32_read_band(var, mask); +#else + erts_aint32_t old; + old = *var; + *var &= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_acqb(erts_atomic32_t *var) +{ +#ifdef USE_THREADS + return ethr_atomic32_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_set_relb(erts_atomic32_t *var, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_dec_relb(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + ethr_atomic32_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_dectest_relb(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + return ethr_atomic32_dec_read_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef USE_THREADS + return ethr_atomic32_cmpxchg_acqb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef USE_THREADS + return ethr_atomic32_cmpxchg_relb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + /* spinlock */ ERTS_GLB_INLINE void -- cgit v1.2.3