diff options
| -rw-r--r-- | erts/emulator/beam/erl_smp.h | 222 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_threads.h | 679 |
2 files changed, 901 insertions, 0 deletions
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index c38ef47d87..6c40edeb3e 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -60,6 +60,7 @@ typedef erts_tsd_key_t erts_smp_tsd_key_t; #define erts_smp_dw_atomic_t erts_dw_atomic_t #define erts_smp_atomic_t erts_atomic_t #define erts_smp_atomic32_t erts_atomic32_t +#define erts_smp_atomic64_t erts_atomic64_t typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ @@ -95,6 +96,7 @@ typedef int erts_smp_tsd_key_t; #define erts_smp_dw_atomic_t erts_no_dw_atomic_t #define erts_smp_atomic_t erts_no_atomic_t #define erts_smp_atomic32_t erts_no_atomic32_t +#define erts_smp_atomic64_t erts_no_atomic64_t #if __GNUC__ > 2 typedef struct { } erts_smp_spinlock_t; typedef struct { } erts_smp_rwlock_t; @@ -489,6 +491,116 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty #define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty +/* 64-bit atomics */ + +#define erts_smp_atomic64_init_nob erts_atomic64_init_nob +#define erts_smp_atomic64_set_nob erts_atomic64_set_nob +#define erts_smp_atomic64_read_nob erts_atomic64_read_nob +#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob +#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob +#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob +#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob +#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob +#define erts_smp_atomic64_add_nob erts_atomic64_add_nob +#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob +#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob +#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob +#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob +#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob + +#define erts_smp_atomic64_init_mb erts_atomic64_init_mb +#define erts_smp_atomic64_set_mb erts_atomic64_set_mb +#define erts_smp_atomic64_read_mb erts_atomic64_read_mb +#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb +#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb +#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb +#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb +#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb +#define erts_smp_atomic64_add_mb erts_atomic64_add_mb +#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb +#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb +#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb +#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb +#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb + +#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb +#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb +#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb +#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb +#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb +#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb +#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb +#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb +#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb +#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb +#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb +#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb +#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb +#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb + +#define erts_smp_atomic64_init_relb erts_atomic64_init_relb +#define erts_smp_atomic64_set_relb erts_atomic64_set_relb +#define erts_smp_atomic64_read_relb erts_atomic64_read_relb +#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb +#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb +#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb +#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb +#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb +#define erts_smp_atomic64_add_relb erts_atomic64_add_relb +#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb +#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb +#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb +#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb +#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb + +#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb +#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb +#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb +#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb +#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb +#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb +#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb +#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb +#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb +#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb +#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb +#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb +#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb +#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb + +#define erts_smp_atomic64_init_rb erts_atomic64_init_rb +#define erts_smp_atomic64_set_rb erts_atomic64_set_rb +#define erts_smp_atomic64_read_rb erts_atomic64_read_rb +#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb +#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb +#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb +#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb +#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb +#define erts_smp_atomic64_add_rb erts_atomic64_add_rb +#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb +#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb +#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb +#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb +#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb + +#define erts_smp_atomic64_init_wb erts_atomic64_init_wb +#define erts_smp_atomic64_set_wb erts_atomic64_set_wb +#define erts_smp_atomic64_read_wb erts_atomic64_read_wb +#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb +#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb +#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb +#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb +#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb +#define erts_smp_atomic64_add_wb erts_atomic64_add_wb +#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb +#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb +#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb +#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb +#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb + +#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty +#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty + #else /* !ERTS_SMP */ /* Double word size atomics */ @@ -751,6 +863,116 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_set_dirty erts_no_atomic32_set #define erts_smp_atomic32_read_dirty erts_no_atomic32_read +/* 64-bit atomics */ + +#define erts_smp_atomic64_init_nob erts_no_atomic64_set +#define erts_smp_atomic64_set_nob erts_no_atomic64_set +#define erts_smp_atomic64_read_nob erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc +#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read +#define erts_smp_atomic64_add_nob erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_mb erts_no_atomic64_set +#define erts_smp_atomic64_set_mb erts_no_atomic64_set +#define erts_smp_atomic64_read_mb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_mb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_acqb erts_no_atomic64_set +#define erts_smp_atomic64_set_acqb erts_no_atomic64_set +#define erts_smp_atomic64_read_acqb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_acqb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_relb erts_no_atomic64_set +#define erts_smp_atomic64_set_relb erts_no_atomic64_set +#define erts_smp_atomic64_read_relb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_relb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set +#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set +#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_rb erts_no_atomic64_set +#define erts_smp_atomic64_set_rb erts_no_atomic64_set +#define erts_smp_atomic64_read_rb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_rb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_wb erts_no_atomic64_set +#define erts_smp_atomic64_set_wb erts_no_atomic64_set +#define erts_smp_atomic64_read_wb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_wb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_set_dirty erts_no_atomic64_set +#define erts_smp_atomic64_read_dirty erts_no_atomic64_read + #endif /* !ERTS_SMP */ #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 80026104db..7214f3ea33 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -344,6 +344,16 @@ typedef ethr_ts_event erts_tse_t; #define erts_aint32_t ethr_sint32_t #define erts_atomic32_t ethr_atomic32_t +#if defined(ARCH_32) +# define erts_atomic64_t ethr_dw_atomic_t +# define erts_aint64_t ethr_sint64_t +#elif defined(ARCH_64) +# define erts_atomic64_t ethr_atomic_t +# define erts_aint64_t ethr_sint_t +#else +# error "Not supported architecture" +#endif + #define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD #define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD @@ -414,10 +424,12 @@ typedef int erts_tse_t; typedef struct { SWord sint[2]; } erts_dw_aint_t; typedef SWord erts_aint_t; typedef Sint32 erts_aint32_t; +typedef Sint64 erts_aint64_t; #define erts_dw_atomic_t erts_dw_aint_t #define erts_atomic_t erts_aint_t #define erts_atomic32_t erts_aint32_t +#define erts_atomic64_t erts_aint64_t #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; @@ -446,6 +458,7 @@ typedef struct { int gcc_is_buggy; } erts_rwlock_t; #define erts_no_dw_atomic_t erts_dw_aint_t #define erts_no_atomic_t erts_aint_t #define erts_no_atomic32_t erts_aint32_t +#define erts_no_atomic64_t erts_aint64_t #define ERTS_AINT_NULL ((erts_aint_t) NULL) @@ -570,6 +583,29 @@ ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set); +ERTS_GLB_INLINE void erts_no_atomic64_set(erts_no_atomic64_t *var, + erts_aint64_t i); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read(erts_no_atomic64_t *var); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_inc_read(erts_no_atomic64_t *incp); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_dec_read(erts_no_atomic64_t *decp); +ERTS_GLB_INLINE void erts_no_atomic64_inc(erts_no_atomic64_t *incp); +ERTS_GLB_INLINE void erts_no_atomic64_dec(erts_no_atomic64_t *decp); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_add_read(erts_no_atomic64_t *addp, + erts_aint64_t i); +ERTS_GLB_INLINE void erts_no_atomic64_add(erts_no_atomic64_t *addp, + erts_aint64_t i); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bor(erts_no_atomic64_t *var, + erts_aint64_t mask); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_band(erts_no_atomic64_t *var, + erts_aint64_t mask); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, + erts_aint64_t new); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp, + erts_aint64_t new, + erts_aint64_t expected); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bset(erts_no_atomic64_t *var, + erts_aint64_t mask, + erts_aint64_t set); ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, @@ -1200,6 +1236,441 @@ erts_atomic32_read_dirty(erts_atomic32_t *var) #endif +/* 64-bit atomics */ + +#if defined(ARCH_64) + +#define erts_atomic64_init_nob ethr_atomic_init +#define erts_atomic64_set_nob ethr_atomic_set +#define erts_atomic64_read_nob ethr_atomic_read +#define erts_atomic64_inc_read_nob ethr_atomic_inc_read +#define erts_atomic64_dec_read_nob ethr_atomic_dec_read +#define erts_atomic64_inc_nob ethr_atomic_inc +#define erts_atomic64_dec_nob ethr_atomic_dec +#define erts_atomic64_add_read_nob ethr_atomic_add_read +#define erts_atomic64_add_nob ethr_atomic_add +#define erts_atomic64_read_bor_nob ethr_atomic_read_bor +#define erts_atomic64_read_band_nob ethr_atomic_read_band +#define erts_atomic64_xchg_nob ethr_atomic_xchg +#define erts_atomic64_cmpxchg_nob ethr_atomic_cmpxchg +#define erts_atomic64_read_bset_nob erts_atomic_read_bset_nob + +#define erts_atomic64_init_mb ethr_atomic_init_mb +#define erts_atomic64_set_mb ethr_atomic_set_mb +#define erts_atomic64_read_mb ethr_atomic_read_mb +#define erts_atomic64_inc_read_mb ethr_atomic_inc_read_mb +#define erts_atomic64_dec_read_mb ethr_atomic_dec_read_mb +#define erts_atomic64_inc_mb ethr_atomic_inc_mb +#define erts_atomic64_dec_mb ethr_atomic_dec_mb +#define erts_atomic64_add_read_mb ethr_atomic_add_read_mb +#define erts_atomic64_add_mb ethr_atomic_add_mb +#define erts_atomic64_read_bor_mb ethr_atomic_read_bor_mb +#define erts_atomic64_read_band_mb ethr_atomic_read_band_mb +#define erts_atomic64_xchg_mb ethr_atomic_xchg_mb +#define erts_atomic64_cmpxchg_mb ethr_atomic_cmpxchg_mb +#define erts_atomic64_read_bset_mb erts_atomic_read_bset_mb + +#define erts_atomic64_init_acqb ethr_atomic_init_acqb +#define erts_atomic64_set_acqb ethr_atomic_set_acqb +#define erts_atomic64_read_acqb ethr_atomic_read_acqb +#define erts_atomic64_inc_read_acqb ethr_atomic_inc_read_acqb +#define erts_atomic64_dec_read_acqb ethr_atomic_dec_read_acqb +#define erts_atomic64_inc_acqb ethr_atomic_inc_acqb +#define erts_atomic64_dec_acqb ethr_atomic_dec_acqb +#define erts_atomic64_add_read_acqb ethr_atomic_add_read_acqb +#define erts_atomic64_add_acqb ethr_atomic_add_acqb +#define erts_atomic64_read_bor_acqb ethr_atomic_read_bor_acqb +#define erts_atomic64_read_band_acqb ethr_atomic_read_band_acqb +#define erts_atomic64_xchg_acqb ethr_atomic_xchg_acqb +#define erts_atomic64_cmpxchg_acqb ethr_atomic_cmpxchg_acqb +#define erts_atomic64_read_bset_acqb erts_atomic_read_bset_acqb + +#define erts_atomic64_init_relb ethr_atomic_init_relb +#define erts_atomic64_set_relb ethr_atomic_set_relb +#define erts_atomic64_read_relb ethr_atomic_read_relb +#define erts_atomic64_inc_read_relb ethr_atomic_inc_read_relb +#define erts_atomic64_dec_read_relb ethr_atomic_dec_read_relb +#define erts_atomic64_inc_relb ethr_atomic_inc_relb +#define erts_atomic64_dec_relb ethr_atomic_dec_relb +#define erts_atomic64_add_read_relb ethr_atomic_add_read_relb +#define erts_atomic64_add_relb ethr_atomic_add_relb +#define erts_atomic64_read_bor_relb ethr_atomic_read_bor_relb +#define erts_atomic64_read_band_relb ethr_atomic_read_band_relb +#define erts_atomic64_xchg_relb ethr_atomic_xchg_relb +#define erts_atomic64_cmpxchg_relb ethr_atomic_cmpxchg_relb +#define erts_atomic64_read_bset_relb erts_atomic_read_bset_relb + +#define erts_atomic64_init_ddrb ethr_atomic_init_ddrb +#define erts_atomic64_set_ddrb ethr_atomic_set_ddrb +#define erts_atomic64_read_ddrb ethr_atomic_read_ddrb +#define erts_atomic64_inc_read_ddrb ethr_atomic_inc_read_ddrb +#define erts_atomic64_dec_read_ddrb ethr_atomic_dec_read_ddrb +#define erts_atomic64_inc_ddrb ethr_atomic_inc_ddrb +#define erts_atomic64_dec_ddrb ethr_atomic_dec_ddrb +#define erts_atomic64_add_read_ddrb ethr_atomic_add_read_ddrb +#define erts_atomic64_add_ddrb ethr_atomic_add_ddrb +#define erts_atomic64_read_bor_ddrb ethr_atomic_read_bor_ddrb +#define erts_atomic64_read_band_ddrb ethr_atomic_read_band_ddrb +#define erts_atomic64_xchg_ddrb ethr_atomic_xchg_ddrb +#define erts_atomic64_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb +#define erts_atomic64_read_bset_ddrb erts_atomic_read_bset_ddrb + +#define erts_atomic64_init_rb ethr_atomic_init_rb +#define erts_atomic64_set_rb ethr_atomic_set_rb +#define erts_atomic64_read_rb ethr_atomic_read_rb +#define erts_atomic64_inc_read_rb ethr_atomic_inc_read_rb +#define erts_atomic64_dec_read_rb ethr_atomic_dec_read_rb +#define erts_atomic64_inc_rb ethr_atomic_inc_rb +#define erts_atomic64_dec_rb ethr_atomic_dec_rb +#define erts_atomic64_add_read_rb ethr_atomic_add_read_rb +#define erts_atomic64_add_rb ethr_atomic_add_rb +#define erts_atomic64_read_bor_rb ethr_atomic_read_bor_rb +#define erts_atomic64_read_band_rb ethr_atomic_read_band_rb +#define erts_atomic64_xchg_rb ethr_atomic_xchg_rb +#define erts_atomic64_cmpxchg_rb ethr_atomic_cmpxchg_rb +#define erts_atomic64_read_bset_rb erts_atomic_read_bset_rb + +#define erts_atomic64_init_wb ethr_atomic_init_wb +#define erts_atomic64_set_wb ethr_atomic_set_wb +#define erts_atomic64_read_wb ethr_atomic_read_wb +#define erts_atomic64_inc_read_wb ethr_atomic_inc_read_wb +#define erts_atomic64_dec_read_wb ethr_atomic_dec_read_wb +#define erts_atomic64_inc_wb ethr_atomic_inc_wb +#define erts_atomic64_dec_wb ethr_atomic_dec_wb +#define erts_atomic64_add_read_wb ethr_atomic_add_read_wb +#define erts_atomic64_add_wb ethr_atomic_add_wb +#define erts_atomic64_read_bor_wb ethr_atomic_read_bor_wb +#define erts_atomic64_read_band_wb ethr_atomic_read_band_wb +#define erts_atomic64_xchg_wb ethr_atomic_xchg_wb +#define erts_atomic64_cmpxchg_wb ethr_atomic_cmpxchg_wb +#define erts_atomic64_read_bset_wb erts_atomic_read_bset_wb + +#define erts_atomic64_set_dirty erts_atomic_set_dirty +#define erts_atomic64_read_dirty erts_atomic_read_dirty + +#elif defined(ARCH_32) + +#undef ERTS_ATOMIC64_OPS_DECL__ + +#define ERTS_ATOMIC64_OPS_DECL__(BARRIER) \ +ERTS_GLB_INLINE void \ +erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE void \ +erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE void \ +erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE void \ +erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE void \ +erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t new, \ + erts_aint64_t exp); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t mask, \ + erts_aint64_t set) + +ERTS_ATOMIC64_OPS_DECL__(nob); +ERTS_ATOMIC64_OPS_DECL__(mb); +ERTS_ATOMIC64_OPS_DECL__(acqb); +ERTS_ATOMIC64_OPS_DECL__(relb); +ERTS_ATOMIC64_OPS_DECL__(ddrb); +ERTS_ATOMIC64_OPS_DECL__(rb); +ERTS_ATOMIC64_OPS_DECL__(wb); + +#undef ERTS_ATOMIC64_OPS_DECL__ + +ERTS_GLB_INLINE void +erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val); +ERTS_GLB_INLINE erts_aint64_t +erts_atomic64_read_dirty(erts_atomic64_t *var); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +/* + * The ethr_dw_atomic_*_nob() functions below + * are here to make it possible for the + * ERTS_ATOMIC64_OPS_IMPL__() to map erts + * barriers to ethread barriers... + */ +static ERTS_INLINE void +ethr_dw_atomic_init_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *val) +{ + ethr_dw_atomic_init(var, val); +} + +static ERTS_INLINE void +ethr_dw_atomic_set_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *val) +{ + ethr_dw_atomic_set(var, val); +} + +static ERTS_INLINE void +ethr_dw_atomic_read_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *val) +{ + ethr_dw_atomic_read(var, val); +} + +static ERTS_INLINE int +ethr_dw_atomic_cmpxchg_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *new, + ethr_dw_sint_t *xchg) +{ + return ethr_dw_atomic_cmpxchg(var, new, xchg); +} + +#undef ERTS_ATOMIC64_OPS_IMPL__ +#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__ +#undef ERTS_DW_SINT_TO_AINT64__ +#undef ERTS_AINT64_TO_DW_SINT__ + +#ifdef ETHR_SU_DW_NAINT_T__ +#define ERTS_DW_SINT_TO_AINT64__(DW) \ + ((erts_aint64_t) DW.dw_sint) +#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) \ + (DW.dw_sint = (ETHR_SU_DW_NAINT_T__) AINT64) +#else /* !ETHR_SU_DW_NAINT_T__ */ +#define ERTS_DW_SINT_TO_AINT64__(DW) \ + ((((erts_aint64_t) DW.sint[ETHR_DW_SINT_HIGH_WORD]) << 32) \ + | (((erts_aint64_t) DW.sint[ETHR_DW_SINT_LOW_WORD]) \ + & ((erts_aint64_t) 0xffffffff))) +#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) \ + do { \ + DW.sint[ETHR_DW_SINT_LOW_WORD] = \ + (ethr_sint_t) (AINT64 & 0xffffffff); \ + DW.sint[ETHR_DW_SINT_HIGH_WORD] = \ + (ethr_sint_t) ((AINT64 >> 32) & 0xffffffff); \ + } while (0) +#endif /* !ETHR_SU_DW_NAINT_T__ */ + +#define ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(CmpXchgOp, \ + AVarP, XchgVar, NewVar, \ + ModificationCode) \ +do { \ + ethr_dw_sint_t dw_xchg__, dw_new__; \ + ethr_dw_atomic_read(AVarP, &dw_xchg__); \ + do { \ + XchgVar = ERTS_DW_SINT_TO_AINT64__(dw_xchg__); \ + { \ + ModificationCode; \ + } \ + ERTS_AINT64_TO_DW_SINT__(dw_new__, NewVar); \ + } while (!CmpXchgOp((AVarP), &dw_new__, &dw_xchg__)); \ +} while (0) + +#define ERTS_ATOMIC64_OPS_IMPL__(BARRIER) \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + ethr_dw_sint_t dw; \ + ERTS_AINT64_TO_DW_SINT__(dw, val); \ + ethr_dw_atomic_init_ ## BARRIER(var, &dw); \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + ethr_dw_sint_t dw; \ + ERTS_AINT64_TO_DW_SINT__(dw, val); \ + ethr_dw_atomic_set_ ## BARRIER(var, &dw); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + ethr_dw_sint_t dw; \ + ethr_dw_atomic_read_ ## BARRIER(var, &dw); \ + return ERTS_DW_SINT_TO_AINT64__(dw); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + 1); \ + return new; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg - 1); \ + return new; \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + 1); \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg - 1); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + val); \ + return new; \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + val); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg | val); \ + return xchg; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg & val); \ + return xchg; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = val); \ + return xchg; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t new, \ + erts_aint64_t exp) \ +{ \ + ethr_dw_sint_t dw_xchg, dw_new; \ + ERTS_AINT64_TO_DW_SINT__(dw_xchg, exp); \ + ERTS_AINT64_TO_DW_SINT__(dw_new, new); \ + if (ethr_dw_atomic_cmpxchg_ ## BARRIER(var, &dw_new, &dw_xchg)) \ + return exp; \ + return ERTS_DW_SINT_TO_AINT64__(dw_xchg); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t mask, \ + erts_aint64_t set) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + { \ + new = xchg & ~mask; \ + new |= mask & set; \ + }); \ + return xchg; \ +} + +ERTS_ATOMIC64_OPS_IMPL__(nob) +ERTS_ATOMIC64_OPS_IMPL__(mb) +ERTS_ATOMIC64_OPS_IMPL__(acqb) +ERTS_ATOMIC64_OPS_IMPL__(relb) +ERTS_ATOMIC64_OPS_IMPL__(ddrb) +ERTS_ATOMIC64_OPS_IMPL__(rb) +ERTS_ATOMIC64_OPS_IMPL__(wb) + +#undef ERTS_ATOMIC64_OPS_IMPL__ +#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__ + +ERTS_GLB_INLINE void +erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val) +{ + ethr_sint_t *sint = ethr_dw_atomic_addr(var); + ethr_dw_sint_t dw; + ERTS_AINT64_TO_DW_SINT__(dw, val); + sint[0] = dw.sint[0]; + sint[1] = dw.sint[1]; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_atomic64_read_dirty(erts_atomic64_t *var) +{ + ethr_sint_t *sint; + ethr_dw_sint_t dw; + sint = ethr_dw_atomic_addr(var); + dw.sint[0] = sint[0]; + dw.sint[1] = sint[1]; + return ERTS_DW_SINT_TO_AINT64__(dw); +} + +#undef ERTS_DW_SINT_TO_AINT64__ +#undef ERTS_AINT64_TO_DW_SINT__ + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ARCH_32 */ + #else /* !USE_THREADS */ /* Double word size atomics */ @@ -1462,6 +1933,116 @@ erts_atomic32_read_dirty(erts_atomic32_t *var) #define erts_atomic32_set_dirty erts_no_atomic32_set #define erts_atomic32_read_dirty erts_no_atomic32_read +/* 64-bit atomics */ + +#define erts_atomic64_init_nob erts_no_atomic64_set +#define erts_atomic64_set_nob erts_no_atomic64_set +#define erts_atomic64_read_nob erts_no_atomic64_read +#define erts_atomic64_inc_read_nob erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_nob erts_no_atomic64_dec_read +#define erts_atomic64_inc_nob erts_no_atomic64_inc +#define erts_atomic64_dec_nob erts_no_atomic64_dec +#define erts_atomic64_add_read_nob erts_no_atomic64_add_read +#define erts_atomic64_add_nob erts_no_atomic64_add +#define erts_atomic64_read_bor_nob erts_no_atomic64_read_bor +#define erts_atomic64_read_band_nob erts_no_atomic64_read_band +#define erts_atomic64_xchg_nob erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_nob erts_no_atomic64_read_bset + +#define erts_atomic64_init_mb erts_no_atomic64_set +#define erts_atomic64_set_mb erts_no_atomic64_set +#define erts_atomic64_read_mb erts_no_atomic64_read +#define erts_atomic64_inc_read_mb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_mb erts_no_atomic64_dec_read +#define erts_atomic64_inc_mb erts_no_atomic64_inc +#define erts_atomic64_dec_mb erts_no_atomic64_dec +#define erts_atomic64_add_read_mb erts_no_atomic64_add_read +#define erts_atomic64_add_mb erts_no_atomic64_add +#define erts_atomic64_read_bor_mb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_mb erts_no_atomic64_read_band +#define erts_atomic64_xchg_mb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_mb erts_no_atomic64_read_bset + +#define erts_atomic64_init_acqb erts_no_atomic64_set +#define erts_atomic64_set_acqb erts_no_atomic64_set +#define erts_atomic64_read_acqb erts_no_atomic64_read +#define erts_atomic64_inc_read_acqb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_acqb erts_no_atomic64_dec_read +#define erts_atomic64_inc_acqb erts_no_atomic64_inc +#define erts_atomic64_dec_acqb erts_no_atomic64_dec +#define erts_atomic64_add_read_acqb erts_no_atomic64_add_read +#define erts_atomic64_add_acqb erts_no_atomic64_add +#define erts_atomic64_read_bor_acqb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_acqb erts_no_atomic64_read_band +#define erts_atomic64_xchg_acqb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_acqb erts_no_atomic64_read_bset + +#define erts_atomic64_init_relb erts_no_atomic64_set +#define erts_atomic64_set_relb erts_no_atomic64_set +#define erts_atomic64_read_relb erts_no_atomic64_read +#define erts_atomic64_inc_read_relb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_relb erts_no_atomic64_dec_read +#define erts_atomic64_inc_relb erts_no_atomic64_inc +#define erts_atomic64_dec_relb erts_no_atomic64_dec +#define erts_atomic64_add_read_relb erts_no_atomic64_add_read +#define erts_atomic64_add_relb erts_no_atomic64_add +#define erts_atomic64_read_bor_relb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_relb erts_no_atomic64_read_band +#define erts_atomic64_xchg_relb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_relb erts_no_atomic64_read_bset + +#define erts_atomic64_init_ddrb erts_no_atomic64_set +#define erts_atomic64_set_ddrb erts_no_atomic64_set +#define erts_atomic64_read_ddrb erts_no_atomic64_read +#define erts_atomic64_inc_read_ddrb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_ddrb erts_no_atomic64_dec_read +#define erts_atomic64_inc_ddrb erts_no_atomic64_inc +#define erts_atomic64_dec_ddrb erts_no_atomic64_dec +#define erts_atomic64_add_read_ddrb erts_no_atomic64_add_read +#define erts_atomic64_add_ddrb erts_no_atomic64_add +#define erts_atomic64_read_bor_ddrb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_ddrb erts_no_atomic64_read_band +#define erts_atomic64_xchg_ddrb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_ddrb erts_no_atomic64_read_bset + +#define erts_atomic64_init_rb erts_no_atomic64_set +#define erts_atomic64_set_rb erts_no_atomic64_set +#define erts_atomic64_read_rb erts_no_atomic64_read +#define erts_atomic64_inc_read_rb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_rb erts_no_atomic64_dec_read +#define erts_atomic64_inc_rb erts_no_atomic64_inc +#define erts_atomic64_dec_rb erts_no_atomic64_dec +#define erts_atomic64_add_read_rb erts_no_atomic64_add_read +#define erts_atomic64_add_rb erts_no_atomic64_add +#define erts_atomic64_read_bor_rb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_rb erts_no_atomic64_read_band +#define erts_atomic64_xchg_rb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_rb erts_no_atomic64_read_bset + +#define erts_atomic64_init_wb erts_no_atomic64_set +#define erts_atomic64_set_wb erts_no_atomic64_set +#define erts_atomic64_read_wb erts_no_atomic64_read +#define erts_atomic64_inc_read_wb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_wb erts_no_atomic64_dec_read +#define erts_atomic64_inc_wb erts_no_atomic64_inc +#define erts_atomic64_dec_wb erts_no_atomic64_dec +#define erts_atomic64_add_read_wb erts_no_atomic64_add_read +#define erts_atomic64_add_wb erts_no_atomic64_add +#define erts_atomic64_read_bor_wb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_wb erts_no_atomic64_read_band +#define erts_atomic64_xchg_wb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_wb erts_no_atomic64_read_bset + +#define erts_atomic64_set_dirty erts_no_atomic64_set +#define erts_atomic64_read_dirty erts_no_atomic64_read + #endif /* !USE_THREADS */ #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -2383,6 +2964,104 @@ erts_no_atomic32_read_bset(erts_no_atomic32_t *var, return old; } +/* atomic64 */ + +ERTS_GLB_INLINE void +erts_no_atomic64_set(erts_no_atomic64_t *var, erts_aint64_t i) +{ + *var = i; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read(erts_no_atomic64_t *var) +{ + return *var; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_inc_read(erts_no_atomic64_t *incp) +{ + return ++(*incp); +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_dec_read(erts_no_atomic64_t *decp) +{ + return --(*decp); +} + +ERTS_GLB_INLINE void +erts_no_atomic64_inc(erts_no_atomic64_t *incp) +{ + ++(*incp); +} + +ERTS_GLB_INLINE void +erts_no_atomic64_dec(erts_no_atomic64_t *decp) +{ + --(*decp); +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_add_read(erts_no_atomic64_t *addp, erts_aint64_t i) +{ + return *addp += i; +} + +ERTS_GLB_INLINE void +erts_no_atomic64_add(erts_no_atomic64_t *addp, erts_aint64_t i) +{ + *addp += i; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read_bor(erts_no_atomic64_t *var, erts_aint64_t mask) +{ + erts_aint64_t old; + old = *var; + *var |= mask; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read_band(erts_no_atomic64_t *var, erts_aint64_t mask) +{ + erts_aint64_t old; + old = *var; + *var &= mask; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, erts_aint64_t new) +{ + erts_aint64_t old = *xchgp; + *xchgp = new; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp, + erts_aint64_t new, + erts_aint64_t expected) +{ + erts_aint64_t old = *xchgp; + if (old == expected) + *xchgp = new; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read_bset(erts_no_atomic64_t *var, + erts_aint64_t mask, + erts_aint64_t set) +{ + erts_aint64_t old = *var; + *var &= ~mask; + *var |= (mask & set); + return old; +} + /* spinlock */ ERTS_GLB_INLINE void |