Merge branch 'rickard/atomic64/OTP-12351' into maint

* rickard/atomic64/OTP-12351:
  Use the new 64-bit atomic ops API
  Introduce a 64-bit atomic ops API

10 files changed, 930 insertions, 418 deletions

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index f84677dea4..ea63d20dfa 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -716,72 +716,24 @@ sched_wall_time_ts(void)
 
 #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
 
-#ifdef ARCH_64
-
-static ERTS_INLINE Uint64
-aschedtime_read(ErtsAtomicSchedTime *var)
-{
-    return (Uint64) erts_atomic_read_nob((erts_atomic_t *) var);
-}
-
-static ERTS_INLINE void
-aschedtime_set(ErtsAtomicSchedTime *var, Uint64 val)
-{
-    erts_atomic_set_nob((erts_atomic_t *) var, (erts_aint_t) val);
-}
-
-static ERTS_INLINE void
-aschedtime_init(ErtsAtomicSchedTime *var)
-{
-    erts_atomic_init_nob((erts_atomic_t *) var, (erts_aint_t) 0);
-}
-
-#elif defined(ARCH_32)
-
 static ERTS_INLINE Uint64
 aschedtime_read(ErtsAtomicSchedTime *var)
 {
-    erts_dw_aint_t dw;
-    erts_dw_atomic_read_nob((erts_dw_atomic_t *) var, &dw);
-#ifdef ETHR_SU_DW_NAINT_T__
-    return (Uint64) dw.dw_sint;
-#else
-    {
-	Uint64 res;
-	res = (Uint64) ((Uint32) dw.sint[ERTS_DW_AINT_HIGH_WORD]);
-	res <<= 32;
-	res |= (Uint64) ((Uint32) dw.sint[ERTS_DW_AINT_LOW_WORD]);
-	return res;
-    }
-#endif    
+    return (Uint64) erts_atomic64_read_nob((erts_atomic64_t *) var);
 }
 
 static ERTS_INLINE void
 aschedtime_set(ErtsAtomicSchedTime *var, Uint64 val)
 {
-    erts_dw_aint_t dw;
-#ifdef ETHR_SU_DW_NAINT_T__
-    dw.dw_sint = (ETHR_SU_DW_NAINT_T__) val;
-#else
-    dw.sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff);
-    dw.sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff);
-#endif
-    erts_dw_atomic_set_nob((erts_dw_atomic_t *) var, &dw);
+    erts_atomic64_set_nob((erts_atomic64_t *) var, (erts_aint64_t) val);
 }
 
 static ERTS_INLINE void
 aschedtime_init(ErtsAtomicSchedTime *var)
 {
-    erts_dw_aint_t dw;
-    dw.sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) 0;
-    dw.sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) 0;
-    erts_dw_atomic_init_nob((erts_dw_atomic_t *) var, &dw);
+    erts_atomic64_init_nob((erts_atomic64_t *) var, (erts_aint64_t) 0);
 }
 
-#else
-#  error :-/
-#endif
-
 #define ERTS_GET_AVG_MAX_UNLOCKED_TRY 50
 #define ERTS_SCHED_AVG_UTIL_WRITE_MARKER (~((Uint64) 0))
 
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 27a3a3553b..f50b217d4a 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -351,13 +351,7 @@ typedef struct {
 #undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
 #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT
 
-#ifdef ARCH_64
-typedef erts_atomic_t ErtsAtomicSchedTime;
-#elif defined(ARCH_32)
-typedef erts_dw_atomic_t ErtsAtomicSchedTime;
-#else
-# error :-/
-#endif
+typedef erts_atomic64_t ErtsAtomicSchedTime;
 
 #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
 typedef struct {
diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c
index eabf016081..02943ee683 100644
--- a/erts/emulator/beam/erl_ptab.c
+++ b/erts/emulator/beam/erl_ptab.c
@@ -280,124 +280,38 @@ struct ErtsPTabListBifData_ {
 
 };
 
-#ifdef ARCH_32
-
-static ERTS_INLINE Uint64
-dw_aint_to_uint64(erts_dw_aint_t *dw)
-{
-#ifdef ETHR_SU_DW_NAINT_T__
-    return (Uint64) dw->dw_sint;
-#else
-    Uint64 res;
-    res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]);
-    res <<= 32;
-    res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]);
-    return res;
-#endif
-}
-
-static void
-unint64_to_dw_aint(erts_dw_aint_t *dw, Uint64 val)
-{
-#ifdef ETHR_SU_DW_NAINT_T__
-    dw->dw_sint = (ETHR_SU_DW_NAINT_T__) val;
-#else
-    dw->sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff);
-    dw->sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff);
-#endif
-}
-
 static ERTS_INLINE void
 last_data_init_nob(ErtsPTab *ptab, Uint64 val)
 {
-    erts_dw_aint_t dw;
-    unint64_to_dw_aint(&dw, val);
-    erts_smp_dw_atomic_init_nob(&ptab->vola.tile.last_data, &dw);
+    erts_smp_atomic64_init_nob(&ptab->vola.tile.last_data, (erts_aint64_t) val);
 }
 
 static ERTS_INLINE void
 last_data_set_relb(ErtsPTab *ptab, Uint64 val)
 {
-    erts_dw_aint_t dw;
-    unint64_to_dw_aint(&dw, val);
-    erts_smp_dw_atomic_set_relb(&ptab->vola.tile.last_data, &dw);
+    erts_smp_atomic64_set_relb(&ptab->vola.tile.last_data, (erts_aint64_t) val);
 }
 
 static ERTS_INLINE Uint64
 last_data_read_nob(ErtsPTab *ptab)
 {
-    erts_dw_aint_t dw;
-    erts_smp_dw_atomic_read_nob(&ptab->vola.tile.last_data, &dw);
-    return dw_aint_to_uint64(&dw);
+    return (Uint64) erts_smp_atomic64_read_nob(&ptab->vola.tile.last_data);
 }
 
 static ERTS_INLINE Uint64
 last_data_read_acqb(ErtsPTab *ptab)
 {
-    erts_dw_aint_t dw;
-    erts_smp_dw_atomic_read_acqb(&ptab->vola.tile.last_data, &dw);
-    return dw_aint_to_uint64(&dw);
+    return (Uint64) erts_smp_atomic64_read_acqb(&ptab->vola.tile.last_data);
 }
 
 static ERTS_INLINE Uint64
 last_data_cmpxchg_relb(ErtsPTab *ptab, Uint64 new, Uint64 exp)
 {
-    erts_dw_aint_t dw_new, dw_xchg;
-
-    unint64_to_dw_aint(&dw_new, new);
-    unint64_to_dw_aint(&dw_xchg, exp);
-
-    if (erts_smp_dw_atomic_cmpxchg_relb(&ptab->vola.tile.last_data,
-					&dw_new,
-					&dw_xchg))
-	return exp;
-    else
-	return dw_aint_to_uint64(&dw_xchg);
-}
-
-#elif defined(ARCH_64)
-
-union {
-    erts_smp_atomic_t pid_data;
-    char align[ERTS_CACHE_LINE_SIZE];
-} last erts_align_attribute(ERTS_CACHE_LINE_SIZE);
-
-static ERTS_INLINE void
-last_data_init_nob(ErtsPTab *ptab, Uint64 val)
-{
-    erts_smp_atomic_init_nob(&ptab->vola.tile.last_data, (erts_aint_t) val);
+    return (Uint64) erts_smp_atomic64_cmpxchg_relb(&ptab->vola.tile.last_data,
+						   (erts_aint64_t) new,
+						   (erts_aint64_t) exp);
 }
 
-static ERTS_INLINE void
-last_data_set_relb(ErtsPTab *ptab, Uint64 val)
-{
-    erts_smp_atomic_set_relb(&ptab->vola.tile.last_data, (erts_aint_t) val);
-}
-
-static ERTS_INLINE Uint64
-last_data_read_nob(ErtsPTab *ptab)
-{
-    return (Uint64) erts_smp_atomic_read_nob(&ptab->vola.tile.last_data);
-}
-
-static ERTS_INLINE Uint64
-last_data_read_acqb(ErtsPTab *ptab)
-{
-    return (Uint64) erts_smp_atomic_read_acqb(&ptab->vola.tile.last_data);
-}
-
-static ERTS_INLINE Uint64
-last_data_cmpxchg_relb(ErtsPTab *ptab, Uint64 new, Uint64 exp)
-{
-    return (Uint64) erts_smp_atomic_cmpxchg_relb(&ptab->vola.tile.last_data,
-						 (erts_aint_t) new,
-						 (erts_aint_t) exp);
-}
-
-#else
-#  error "Not 64-bit, nor 32-bit architecture..."
-#endif
-
 static ERTS_INLINE int
 last_data_cmp(Uint64 ld1, Uint64 ld2)
 {
diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h
index e3e05f14af..876241159b 100644
--- a/erts/emulator/beam/erl_ptab.h
+++ b/erts/emulator/beam/erl_ptab.h
@@ -88,11 +88,7 @@ typedef struct {
 } ErtsPTabListData;
 
 typedef struct {
-#ifdef ARCH_32
-    erts_smp_dw_atomic_t last_data;
-#else
-    erts_smp_atomic_t last_data;
-#endif
+    erts_smp_atomic64_t last_data;
     erts_smp_atomic32_t count;
     erts_smp_atomic32_t aid_ix;
     erts_smp_atomic32_t fid_ix;
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index c38ef47d87..6c40edeb3e 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -60,6 +60,7 @@ typedef erts_tsd_key_t erts_smp_tsd_key_t;
 #define erts_smp_dw_atomic_t erts_dw_atomic_t
 #define erts_smp_atomic_t erts_atomic_t
 #define erts_smp_atomic32_t erts_atomic32_t
+#define erts_smp_atomic64_t erts_atomic64_t
 typedef erts_spinlock_t erts_smp_spinlock_t;
 typedef erts_rwlock_t erts_smp_rwlock_t;
 void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */
@@ -95,6 +96,7 @@ typedef int erts_smp_tsd_key_t;
 #define erts_smp_dw_atomic_t erts_no_dw_atomic_t
 #define erts_smp_atomic_t erts_no_atomic_t
 #define erts_smp_atomic32_t erts_no_atomic32_t
+#define erts_smp_atomic64_t erts_no_atomic64_t
 #if __GNUC__ > 2
 typedef struct { } erts_smp_spinlock_t;
 typedef struct { } erts_smp_rwlock_t;
@@ -489,6 +491,116 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty
 #define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty
 
+/* 64-bit atomics */
+
+#define erts_smp_atomic64_init_nob erts_atomic64_init_nob
+#define erts_smp_atomic64_set_nob erts_atomic64_set_nob
+#define erts_smp_atomic64_read_nob erts_atomic64_read_nob
+#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob
+#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob
+#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob
+#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob
+#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob
+#define erts_smp_atomic64_add_nob erts_atomic64_add_nob
+#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob
+#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob
+#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob
+#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob
+#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob
+
+#define erts_smp_atomic64_init_mb erts_atomic64_init_mb
+#define erts_smp_atomic64_set_mb erts_atomic64_set_mb
+#define erts_smp_atomic64_read_mb erts_atomic64_read_mb
+#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb
+#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb
+#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb
+#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb
+#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb
+#define erts_smp_atomic64_add_mb erts_atomic64_add_mb
+#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb
+#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb
+#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb
+#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb
+#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb
+
+#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb
+#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb
+#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb
+#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb
+#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb
+#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb
+#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb
+#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb
+#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb
+#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb
+#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb
+#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb
+#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb
+#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb
+
+#define erts_smp_atomic64_init_relb erts_atomic64_init_relb
+#define erts_smp_atomic64_set_relb erts_atomic64_set_relb
+#define erts_smp_atomic64_read_relb erts_atomic64_read_relb
+#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb
+#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb
+#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb
+#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb
+#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb
+#define erts_smp_atomic64_add_relb erts_atomic64_add_relb
+#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb
+#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb
+#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb
+#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb
+#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb
+
+#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb
+#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb
+#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb
+#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb
+#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb
+#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb
+#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb
+#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb
+#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb
+#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb
+#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb
+#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb
+#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb
+#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb
+
+#define erts_smp_atomic64_init_rb erts_atomic64_init_rb
+#define erts_smp_atomic64_set_rb erts_atomic64_set_rb
+#define erts_smp_atomic64_read_rb erts_atomic64_read_rb
+#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb
+#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb
+#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb
+#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb
+#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb
+#define erts_smp_atomic64_add_rb erts_atomic64_add_rb
+#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb
+#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb
+#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb
+#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb
+#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb
+
+#define erts_smp_atomic64_init_wb erts_atomic64_init_wb
+#define erts_smp_atomic64_set_wb erts_atomic64_set_wb
+#define erts_smp_atomic64_read_wb erts_atomic64_read_wb
+#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb
+#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb
+#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb
+#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb
+#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb
+#define erts_smp_atomic64_add_wb erts_atomic64_add_wb
+#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb
+#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb
+#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb
+#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb
+#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb
+
+#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty
+#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty
+
 #else /* !ERTS_SMP */
 
 /* Double word size atomics */
@@ -751,6 +863,116 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic32_set_dirty erts_no_atomic32_set
 #define erts_smp_atomic32_read_dirty erts_no_atomic32_read
 
+/* 64-bit atomics */
+
+#define erts_smp_atomic64_init_nob erts_no_atomic64_set
+#define erts_smp_atomic64_set_nob erts_no_atomic64_set
+#define erts_smp_atomic64_read_nob erts_no_atomic64_read
+#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read
+#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read
+#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc
+#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec
+#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read
+#define erts_smp_atomic64_add_nob erts_no_atomic64_add
+#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor
+#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band
+#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg
+#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
+#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset
+
+#define erts_smp_atomic64_init_mb erts_no_atomic64_set
+#define erts_smp_atomic64_set_mb erts_no_atomic64_set
+#define erts_smp_atomic64_read_mb erts_no_atomic64_read
+#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read
+#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read
+#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc
+#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec
+#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read
+#define erts_smp_atomic64_add_mb erts_no_atomic64_add
+#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor
+#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band
+#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg
+#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
+#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset
+
+#define erts_smp_atomic64_init_acqb erts_no_atomic64_set
+#define erts_smp_atomic64_set_acqb erts_no_atomic64_set
+#define erts_smp_atomic64_read_acqb erts_no_atomic64_read
+#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read
+#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read
+#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc
+#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec
+#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read
+#define erts_smp_atomic64_add_acqb erts_no_atomic64_add
+#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor
+#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band
+#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg
+#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
+#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset
+
+#define erts_smp_atomic64_init_relb erts_no_atomic64_set
+#define erts_smp_atomic64_set_relb erts_no_atomic64_set
+#define erts_smp_atomic64_read_relb erts_no_atomic64_read
+#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read
+#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read
+#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc
+#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec
+#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read
+#define erts_smp_atomic64_add_relb erts_no_atomic64_add
+#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor
+#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band
+#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg
+#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
+#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset
+
+#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set
+#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set
+#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read
+#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
+#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
+#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc
+#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec
+#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read
+#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add
+#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
+#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band
+#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg
+#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
+#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset
+
+#define erts_smp_atomic64_init_rb erts_no_atomic64_set
+#define erts_smp_atomic64_set_rb erts_no_atomic64_set
+#define erts_smp_atomic64_read_rb erts_no_atomic64_read
+#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read
+#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read
+#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc
+#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec
+#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read
+#define erts_smp_atomic64_add_rb erts_no_atomic64_add
+#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor
+#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band
+#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg
+#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
+#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset
+
+#define erts_smp_atomic64_init_wb erts_no_atomic64_set
+#define erts_smp_atomic64_set_wb erts_no_atomic64_set
+#define erts_smp_atomic64_read_wb erts_no_atomic64_read
+#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read
+#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read
+#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc
+#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec
+#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read
+#define erts_smp_atomic64_add_wb erts_no_atomic64_add
+#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor
+#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band
+#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg
+#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
+#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset
+
+#define erts_smp_atomic64_set_dirty erts_no_atomic64_set
+#define erts_smp_atomic64_read_dirty erts_no_atomic64_read
+
 #endif /* !ERTS_SMP */
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 545a0343d0..664c479eb6 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -115,70 +115,24 @@
 #undef read_nob
 #define read_nob erts_thr_prgr_read_nob__
 
-#ifdef ARCH_64
-
 static ERTS_INLINE void
 set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 {
-    erts_atomic_set_mb(atmc, val);
+    erts_atomic64_set_mb(atmc, (erts_aint64_t) val);
 }
 
 static ERTS_INLINE void
 set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 {
-    erts_atomic_set_nob(atmc, val);
+    erts_atomic64_set_nob(atmc, (erts_aint64_t) val);
 }
 
 static ERTS_INLINE void
 init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
 {
-    erts_atomic_init_nob(atmc, val);
-}
-
-#else
-
-#undef dw_aint_to_val
-#define dw_aint_to_val erts_thr_prgr_dw_aint_to_val__
-
-static void
-val_to_dw_aint(erts_dw_aint_t *dw_aint, ErtsThrPrgrVal val)
-{
-#ifdef ETHR_SU_DW_NAINT_T__
-    dw_aint->dw_sint = (ETHR_SU_DW_NAINT_T__) val;
-#else
-    dw_aint->sint[ERTS_DW_AINT_LOW_WORD]
-	= (erts_aint_t) (val & 0xffffffff);
-    dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]
-	= (erts_aint_t) ((val >> 32) & 0xffffffff);
-#endif
+    erts_atomic64_init_nob(atmc, (erts_aint64_t) val);
 }
 
-static ERTS_INLINE void
-set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
-{
-    erts_dw_aint_t dw_aint;
-    val_to_dw_aint(&dw_aint, val);
-    erts_dw_atomic_set_mb(atmc, &dw_aint);
-}
-
-static ERTS_INLINE void
-set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
-{
-    erts_dw_aint_t dw_aint;
-    val_to_dw_aint(&dw_aint, val);
-    erts_dw_atomic_set_nob(atmc, &dw_aint);
-}
-
-static ERTS_INLINE void
-init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
-{
-    erts_dw_aint_t dw_aint;
-    val_to_dw_aint(&dw_aint, val);
-    erts_dw_atomic_init_nob(atmc, &dw_aint);
-}
-
-#endif
-
 /* #define ERTS_THR_PROGRESS_STATE_DEBUG */
 
 #ifdef ERTS_THR_PROGRESS_STATE_DEBUG
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index 5f392944c2..03ddbd467c 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -115,11 +115,7 @@ struct ErtsThrPrgrLaterOp_ {
 
 extern erts_tsd_key_t erts_thr_prgr_data_key__;
 
-#ifdef ARCH_64
-#  define ERTS_THR_PRGR_ATOMIC erts_atomic_t
-#else /* ARCH_32 */
-#  define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t
-#endif
+#define ERTS_THR_PRGR_ATOMIC erts_atomic64_t
 
 typedef struct {
     void *arg;
@@ -158,10 +154,6 @@ void erts_thr_progress_unmanaged_continue__(int umrefc_ix);
 
 void erts_thr_progress_dbg_print_state(void);
 
-#ifdef ARCH_32
-#define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t
-ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint);
-#endif
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc);
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc);
 ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc);
@@ -184,68 +176,24 @@ ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-#ifdef ARCH_64
-
-ERTS_GLB_INLINE ErtsThrPrgrVal
-erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
-{
-    return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc);
-}
-
-ERTS_GLB_INLINE ErtsThrPrgrVal
-erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
-{
-    return (ErtsThrPrgrVal) erts_atomic_read_acqb(atmc);
-}
-
-ERTS_GLB_INLINE ErtsThrPrgrVal
-erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
-{
-    return (ErtsThrPrgrVal) erts_atomic_read_mb(atmc);
-}
-
-#else /* ARCH_32 */
-
-ERTS_GLB_INLINE ErtsThrPrgrVal
-erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint)
-{
-#ifdef ETHR_SU_DW_NAINT_T__
-    return (ErtsThrPrgrVal) dw_aint->dw_sint;
-#else
-    ErtsThrPrgrVal res;
-    res = (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]);
-    res <<= 32;
-    res |= (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_LOW_WORD]);
-    return res;
-#endif
-}
-
 ERTS_GLB_INLINE ErtsThrPrgrVal
 erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
 {
-    erts_dw_aint_t dw_aint;
-    erts_dw_atomic_read_nob(atmc, &dw_aint);
-    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+    return (ErtsThrPrgrVal) erts_atomic64_read_nob(atmc);
 }
 
 ERTS_GLB_INLINE ErtsThrPrgrVal
 erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
 {
-    erts_dw_aint_t dw_aint;
-    erts_dw_atomic_read_acqb(atmc, &dw_aint);
-    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+    return (ErtsThrPrgrVal) erts_atomic64_read_acqb(atmc);
 }
 
 ERTS_GLB_INLINE ErtsThrPrgrVal
 erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
 {
-    erts_dw_aint_t dw_aint;
-    erts_dw_atomic_read_mb(atmc, &dw_aint);
-    return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+    return (ErtsThrPrgrVal) erts_atomic64_read_mb(atmc);
 }
 
-#endif
-
 ERTS_GLB_INLINE int
 erts_thr_progress_is_managed_thread(void)
 {
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 80026104db..7214f3ea33 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -344,6 +344,16 @@ typedef ethr_ts_event erts_tse_t;
 #define erts_aint32_t ethr_sint32_t 
 #define erts_atomic32_t ethr_atomic32_t
 
+#if defined(ARCH_32)
+#  define erts_atomic64_t ethr_dw_atomic_t
+#  define erts_aint64_t ethr_sint64_t
+#elif defined(ARCH_64)
+#  define erts_atomic64_t ethr_atomic_t
+#  define erts_aint64_t ethr_sint_t
+#else
+#  error "Not supported architecture"
+#endif
+
 #define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD
 #define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD
 
@@ -414,10 +424,12 @@ typedef int erts_tse_t;
 typedef struct { SWord sint[2]; } erts_dw_aint_t;
 typedef SWord erts_aint_t;
 typedef Sint32 erts_aint32_t;
+typedef Sint64 erts_aint64_t;
 
 #define erts_dw_atomic_t erts_dw_aint_t
 #define erts_atomic_t erts_aint_t
 #define erts_atomic32_t erts_aint32_t
+#define erts_atomic64_t erts_aint64_t
 
 #if __GNUC__ > 2
 typedef struct { } erts_spinlock_t;
@@ -446,6 +458,7 @@ typedef struct { int gcc_is_buggy; } erts_rwlock_t;
 #define erts_no_dw_atomic_t erts_dw_aint_t
 #define erts_no_atomic_t erts_aint_t
 #define erts_no_atomic32_t erts_aint32_t
+#define erts_no_atomic64_t erts_aint64_t
 
 #define ERTS_AINT_NULL ((erts_aint_t) NULL)
 
@@ -570,6 +583,29 @@ ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp
 ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
 							 erts_aint32_t mask,
 							 erts_aint32_t set);
+ERTS_GLB_INLINE void erts_no_atomic64_set(erts_no_atomic64_t *var,
+					  erts_aint64_t i);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read(erts_no_atomic64_t *var);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_inc_read(erts_no_atomic64_t *incp);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_dec_read(erts_no_atomic64_t *decp);
+ERTS_GLB_INLINE void erts_no_atomic64_inc(erts_no_atomic64_t *incp);
+ERTS_GLB_INLINE void erts_no_atomic64_dec(erts_no_atomic64_t *decp);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_add_read(erts_no_atomic64_t *addp,
+							erts_aint64_t i);
+ERTS_GLB_INLINE void erts_no_atomic64_add(erts_no_atomic64_t *addp,
+					  erts_aint64_t i);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bor(erts_no_atomic64_t *var,
+							erts_aint64_t mask);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_band(erts_no_atomic64_t *var,
+							 erts_aint64_t mask);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp,
+						    erts_aint64_t new);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp,
+						       erts_aint64_t new,
+						       erts_aint64_t expected);
+ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
+							 erts_aint64_t mask,
+							 erts_aint64_t set);
 
 ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock,
 					      char *name,
@@ -1200,6 +1236,441 @@ erts_atomic32_read_dirty(erts_atomic32_t *var)
 
 #endif
 
+/* 64-bit atomics */
+
+#if defined(ARCH_64)
+
+#define erts_atomic64_init_nob ethr_atomic_init
+#define erts_atomic64_set_nob ethr_atomic_set
+#define erts_atomic64_read_nob ethr_atomic_read
+#define erts_atomic64_inc_read_nob ethr_atomic_inc_read
+#define erts_atomic64_dec_read_nob ethr_atomic_dec_read
+#define erts_atomic64_inc_nob ethr_atomic_inc
+#define erts_atomic64_dec_nob ethr_atomic_dec
+#define erts_atomic64_add_read_nob ethr_atomic_add_read
+#define erts_atomic64_add_nob ethr_atomic_add
+#define erts_atomic64_read_bor_nob ethr_atomic_read_bor
+#define erts_atomic64_read_band_nob ethr_atomic_read_band
+#define erts_atomic64_xchg_nob ethr_atomic_xchg
+#define erts_atomic64_cmpxchg_nob ethr_atomic_cmpxchg
+#define erts_atomic64_read_bset_nob erts_atomic_read_bset_nob
+
+#define erts_atomic64_init_mb ethr_atomic_init_mb
+#define erts_atomic64_set_mb ethr_atomic_set_mb
+#define erts_atomic64_read_mb ethr_atomic_read_mb
+#define erts_atomic64_inc_read_mb ethr_atomic_inc_read_mb
+#define erts_atomic64_dec_read_mb ethr_atomic_dec_read_mb
+#define erts_atomic64_inc_mb ethr_atomic_inc_mb
+#define erts_atomic64_dec_mb ethr_atomic_dec_mb
+#define erts_atomic64_add_read_mb ethr_atomic_add_read_mb
+#define erts_atomic64_add_mb ethr_atomic_add_mb
+#define erts_atomic64_read_bor_mb ethr_atomic_read_bor_mb
+#define erts_atomic64_read_band_mb ethr_atomic_read_band_mb
+#define erts_atomic64_xchg_mb ethr_atomic_xchg_mb
+#define erts_atomic64_cmpxchg_mb ethr_atomic_cmpxchg_mb
+#define erts_atomic64_read_bset_mb erts_atomic_read_bset_mb
+
+#define erts_atomic64_init_acqb ethr_atomic_init_acqb
+#define erts_atomic64_set_acqb ethr_atomic_set_acqb
+#define erts_atomic64_read_acqb ethr_atomic_read_acqb
+#define erts_atomic64_inc_read_acqb ethr_atomic_inc_read_acqb
+#define erts_atomic64_dec_read_acqb ethr_atomic_dec_read_acqb
+#define erts_atomic64_inc_acqb ethr_atomic_inc_acqb
+#define erts_atomic64_dec_acqb ethr_atomic_dec_acqb
+#define erts_atomic64_add_read_acqb ethr_atomic_add_read_acqb
+#define erts_atomic64_add_acqb ethr_atomic_add_acqb
+#define erts_atomic64_read_bor_acqb ethr_atomic_read_bor_acqb
+#define erts_atomic64_read_band_acqb ethr_atomic_read_band_acqb
+#define erts_atomic64_xchg_acqb ethr_atomic_xchg_acqb
+#define erts_atomic64_cmpxchg_acqb ethr_atomic_cmpxchg_acqb
+#define erts_atomic64_read_bset_acqb erts_atomic_read_bset_acqb
+
+#define erts_atomic64_init_relb ethr_atomic_init_relb
+#define erts_atomic64_set_relb ethr_atomic_set_relb
+#define erts_atomic64_read_relb ethr_atomic_read_relb
+#define erts_atomic64_inc_read_relb ethr_atomic_inc_read_relb
+#define erts_atomic64_dec_read_relb ethr_atomic_dec_read_relb
+#define erts_atomic64_inc_relb ethr_atomic_inc_relb
+#define erts_atomic64_dec_relb ethr_atomic_dec_relb
+#define erts_atomic64_add_read_relb ethr_atomic_add_read_relb
+#define erts_atomic64_add_relb ethr_atomic_add_relb
+#define erts_atomic64_read_bor_relb ethr_atomic_read_bor_relb
+#define erts_atomic64_read_band_relb ethr_atomic_read_band_relb
+#define erts_atomic64_xchg_relb ethr_atomic_xchg_relb
+#define erts_atomic64_cmpxchg_relb ethr_atomic_cmpxchg_relb
+#define erts_atomic64_read_bset_relb erts_atomic_read_bset_relb
+
+#define erts_atomic64_init_ddrb ethr_atomic_init_ddrb
+#define erts_atomic64_set_ddrb ethr_atomic_set_ddrb
+#define erts_atomic64_read_ddrb ethr_atomic_read_ddrb
+#define erts_atomic64_inc_read_ddrb ethr_atomic_inc_read_ddrb
+#define erts_atomic64_dec_read_ddrb ethr_atomic_dec_read_ddrb
+#define erts_atomic64_inc_ddrb ethr_atomic_inc_ddrb
+#define erts_atomic64_dec_ddrb ethr_atomic_dec_ddrb
+#define erts_atomic64_add_read_ddrb ethr_atomic_add_read_ddrb
+#define erts_atomic64_add_ddrb ethr_atomic_add_ddrb
+#define erts_atomic64_read_bor_ddrb ethr_atomic_read_bor_ddrb
+#define erts_atomic64_read_band_ddrb ethr_atomic_read_band_ddrb
+#define erts_atomic64_xchg_ddrb ethr_atomic_xchg_ddrb
+#define erts_atomic64_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb
+#define erts_atomic64_read_bset_ddrb erts_atomic_read_bset_ddrb
+
+#define erts_atomic64_init_rb ethr_atomic_init_rb
+#define erts_atomic64_set_rb ethr_atomic_set_rb
+#define erts_atomic64_read_rb ethr_atomic_read_rb
+#define erts_atomic64_inc_read_rb ethr_atomic_inc_read_rb
+#define erts_atomic64_dec_read_rb ethr_atomic_dec_read_rb
+#define erts_atomic64_inc_rb ethr_atomic_inc_rb
+#define erts_atomic64_dec_rb ethr_atomic_dec_rb
+#define erts_atomic64_add_read_rb ethr_atomic_add_read_rb
+#define erts_atomic64_add_rb ethr_atomic_add_rb
+#define erts_atomic64_read_bor_rb ethr_atomic_read_bor_rb
+#define erts_atomic64_read_band_rb ethr_atomic_read_band_rb
+#define erts_atomic64_xchg_rb ethr_atomic_xchg_rb
+#define erts_atomic64_cmpxchg_rb ethr_atomic_cmpxchg_rb
+#define erts_atomic64_read_bset_rb erts_atomic_read_bset_rb
+
+#define erts_atomic64_init_wb ethr_atomic_init_wb
+#define erts_atomic64_set_wb ethr_atomic_set_wb
+#define erts_atomic64_read_wb ethr_atomic_read_wb
+#define erts_atomic64_inc_read_wb ethr_atomic_inc_read_wb
+#define erts_atomic64_dec_read_wb ethr_atomic_dec_read_wb
+#define erts_atomic64_inc_wb ethr_atomic_inc_wb
+#define erts_atomic64_dec_wb ethr_atomic_dec_wb
+#define erts_atomic64_add_read_wb ethr_atomic_add_read_wb
+#define erts_atomic64_add_wb ethr_atomic_add_wb
+#define erts_atomic64_read_bor_wb ethr_atomic_read_bor_wb
+#define erts_atomic64_read_band_wb ethr_atomic_read_band_wb
+#define erts_atomic64_xchg_wb ethr_atomic_xchg_wb
+#define erts_atomic64_cmpxchg_wb ethr_atomic_cmpxchg_wb
+#define erts_atomic64_read_bset_wb erts_atomic_read_bset_wb
+
+#define erts_atomic64_set_dirty erts_atomic_set_dirty
+#define erts_atomic64_read_dirty erts_atomic_read_dirty
+
+#elif defined(ARCH_32)
+
+#undef ERTS_ATOMIC64_OPS_DECL__
+
+#define ERTS_ATOMIC64_OPS_DECL__(BARRIER)				\
+ERTS_GLB_INLINE void							\
+erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var,			\
+			       erts_aint64_t val);			\
+ERTS_GLB_INLINE void							\
+erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var,			\
+			      erts_aint64_t val);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var);		\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var);		\
+ERTS_GLB_INLINE void							\
+erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var);			\
+ERTS_GLB_INLINE void							\
+erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var,		\
+				   erts_aint64_t val);			\
+ERTS_GLB_INLINE void							\
+erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var,			\
+			      erts_aint64_t val);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var,		\
+				   erts_aint64_t val);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var,		\
+				    erts_aint64_t val);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var,			\
+			       erts_aint64_t val);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var,			\
+				  erts_aint64_t new,			\
+				  erts_aint64_t exp);			\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var,		\
+				    erts_aint64_t mask,			\
+				    erts_aint64_t set)
+
+ERTS_ATOMIC64_OPS_DECL__(nob);
+ERTS_ATOMIC64_OPS_DECL__(mb);
+ERTS_ATOMIC64_OPS_DECL__(acqb);
+ERTS_ATOMIC64_OPS_DECL__(relb);
+ERTS_ATOMIC64_OPS_DECL__(ddrb);
+ERTS_ATOMIC64_OPS_DECL__(rb);
+ERTS_ATOMIC64_OPS_DECL__(wb);
+
+#undef ERTS_ATOMIC64_OPS_DECL__
+
+ERTS_GLB_INLINE void
+erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val);
+ERTS_GLB_INLINE erts_aint64_t
+erts_atomic64_read_dirty(erts_atomic64_t *var);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+/*
+ * The ethr_dw_atomic_*_nob() functions below
+ * are here to make it possible for the
+ * ERTS_ATOMIC64_OPS_IMPL__() to map erts
+ * barriers to ethread barriers...
+ */
+static ERTS_INLINE void
+ethr_dw_atomic_init_nob(ethr_dw_atomic_t *var,
+			ethr_dw_sint_t *val)
+{
+    ethr_dw_atomic_init(var, val);
+}
+
+static ERTS_INLINE void
+ethr_dw_atomic_set_nob(ethr_dw_atomic_t *var,
+		       ethr_dw_sint_t *val)
+{
+    ethr_dw_atomic_set(var, val);
+}
+
+static ERTS_INLINE void
+ethr_dw_atomic_read_nob(ethr_dw_atomic_t *var,
+			ethr_dw_sint_t *val)
+{
+    ethr_dw_atomic_read(var, val);
+}
+
+static ERTS_INLINE int
+ethr_dw_atomic_cmpxchg_nob(ethr_dw_atomic_t *var,
+			   ethr_dw_sint_t *new,
+			   ethr_dw_sint_t *xchg)
+{
+    return ethr_dw_atomic_cmpxchg(var, new, xchg);
+}
+
+#undef ERTS_ATOMIC64_OPS_IMPL__
+#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__
+#undef ERTS_DW_SINT_TO_AINT64__
+#undef ERTS_AINT64_TO_DW_SINT__
+
+#ifdef ETHR_SU_DW_NAINT_T__
+#define ERTS_DW_SINT_TO_AINT64__(DW)					\
+    ((erts_aint64_t) DW.dw_sint)
+#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) 				\
+    (DW.dw_sint = (ETHR_SU_DW_NAINT_T__) AINT64)
+#else /* !ETHR_SU_DW_NAINT_T__ */
+#define ERTS_DW_SINT_TO_AINT64__(DW)					\
+    ((((erts_aint64_t) DW.sint[ETHR_DW_SINT_HIGH_WORD]) << 32)		\
+     | (((erts_aint64_t) DW.sint[ETHR_DW_SINT_LOW_WORD])		\
+	& ((erts_aint64_t) 0xffffffff)))
+#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) 				\
+    do {								\
+	DW.sint[ETHR_DW_SINT_LOW_WORD] =				\
+	    (ethr_sint_t) (AINT64 & 0xffffffff);			\
+	DW.sint[ETHR_DW_SINT_HIGH_WORD] = 				\
+	    (ethr_sint_t) ((AINT64 >> 32) & 0xffffffff);		\
+    } while (0)
+#endif /* !ETHR_SU_DW_NAINT_T__ */
+
+#define ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(CmpXchgOp, 			\
+					AVarP, XchgVar, NewVar, 	\
+					ModificationCode)		\
+do {									\
+    ethr_dw_sint_t dw_xchg__, dw_new__;					\
+    ethr_dw_atomic_read(AVarP, &dw_xchg__);				\
+    do {								\
+	XchgVar = ERTS_DW_SINT_TO_AINT64__(dw_xchg__);			\
+	{								\
+	    ModificationCode;						\
+	}								\
+	ERTS_AINT64_TO_DW_SINT__(dw_new__, NewVar);			\
+    } while (!CmpXchgOp((AVarP), &dw_new__, &dw_xchg__));		\
+} while (0)
+
+#define ERTS_ATOMIC64_OPS_IMPL__(BARRIER)				\
+    									\
+ERTS_GLB_INLINE void							\
+erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var,			\
+			       erts_aint64_t val)			\
+{									\
+    ethr_dw_sint_t dw;							\
+    ERTS_AINT64_TO_DW_SINT__(dw, val);					\
+    ethr_dw_atomic_init_ ## BARRIER(var, &dw);				\
+}									\
+									\
+ERTS_GLB_INLINE void							\
+erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var,			\
+			      erts_aint64_t val)			\
+{									\
+    ethr_dw_sint_t dw;							\
+    ERTS_AINT64_TO_DW_SINT__(dw, val);					\
+    ethr_dw_atomic_set_ ## BARRIER(var, &dw);				\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var)			\
+{									\
+    ethr_dw_sint_t dw;							\
+    ethr_dw_atomic_read_ ## BARRIER(var, &dw);				\
+    return ERTS_DW_SINT_TO_AINT64__(dw);				\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var)		\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg + 1);			\
+    return new;								\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var)		\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg - 1);			\
+    return new;								\
+}									\
+									\
+ERTS_GLB_INLINE void							\
+erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg + 1);			\
+}									\
+									\
+ERTS_GLB_INLINE void							\
+erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg - 1);			\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var,		\
+				   erts_aint64_t val)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg + val);			\
+    return new;								\
+}									\
+									\
+ERTS_GLB_INLINE void							\
+erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var,			\
+			      erts_aint64_t val)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg + val);			\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var,		\
+				   erts_aint64_t val)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg | val);			\
+    return xchg;							\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var,		\
+				    erts_aint64_t val)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = xchg & val);			\
+    return xchg;							\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var,			\
+			       erts_aint64_t val)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    new = val);				\
+    return xchg;							\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var,			\
+				  erts_aint64_t new,			\
+				  erts_aint64_t exp)			\
+{									\
+    ethr_dw_sint_t dw_xchg, dw_new;					\
+    ERTS_AINT64_TO_DW_SINT__(dw_xchg, exp);				\
+    ERTS_AINT64_TO_DW_SINT__(dw_new, new);				\
+    if (ethr_dw_atomic_cmpxchg_ ## BARRIER(var, &dw_new, &dw_xchg))	\
+	return exp;							\
+    return ERTS_DW_SINT_TO_AINT64__(dw_xchg);				\
+}									\
+									\
+ERTS_GLB_INLINE erts_aint64_t						\
+erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var,		\
+				    erts_aint64_t mask,			\
+				    erts_aint64_t set)			\
+{									\
+    erts_aint64_t xchg, new;						\
+    ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER,	\
+				    var, xchg, new,			\
+				    {					\
+					new = xchg & ~mask;		\
+					new |= mask & set;		\
+				    });					\
+    return xchg;							\
+}
+
+ERTS_ATOMIC64_OPS_IMPL__(nob)
+ERTS_ATOMIC64_OPS_IMPL__(mb)
+ERTS_ATOMIC64_OPS_IMPL__(acqb)
+ERTS_ATOMIC64_OPS_IMPL__(relb)
+ERTS_ATOMIC64_OPS_IMPL__(ddrb)
+ERTS_ATOMIC64_OPS_IMPL__(rb)
+ERTS_ATOMIC64_OPS_IMPL__(wb)
+
+#undef ERTS_ATOMIC64_OPS_IMPL__
+#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__
+
+ERTS_GLB_INLINE void
+erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val)
+{
+    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
+    ethr_dw_sint_t dw;
+    ERTS_AINT64_TO_DW_SINT__(dw, val);
+    sint[0] = dw.sint[0];
+    sint[1] = dw.sint[1];
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_atomic64_read_dirty(erts_atomic64_t *var)
+{
+    ethr_sint_t *sint;
+    ethr_dw_sint_t dw;
+    sint = ethr_dw_atomic_addr(var);
+    dw.sint[0] = sint[0];
+    dw.sint[1] = sint[1];
+    return ERTS_DW_SINT_TO_AINT64__(dw);
+}
+
+#undef ERTS_DW_SINT_TO_AINT64__
+#undef ERTS_AINT64_TO_DW_SINT__
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /* ARCH_32 */
+
 #else /* !USE_THREADS */
 
 /* Double word size atomics */
@@ -1462,6 +1933,116 @@ erts_atomic32_read_dirty(erts_atomic32_t *var)
 #define erts_atomic32_set_dirty erts_no_atomic32_set
 #define erts_atomic32_read_dirty erts_no_atomic32_read
 
+/* 64-bit atomics */
+
+#define erts_atomic64_init_nob erts_no_atomic64_set
+#define erts_atomic64_set_nob erts_no_atomic64_set
+#define erts_atomic64_read_nob erts_no_atomic64_read
+#define erts_atomic64_inc_read_nob erts_no_atomic64_inc_read
+#define erts_atomic64_dec_read_nob erts_no_atomic64_dec_read
+#define erts_atomic64_inc_nob erts_no_atomic64_inc
+#define erts_atomic64_dec_nob erts_no_atomic64_dec
+#define erts_atomic64_add_read_nob erts_no_atomic64_add_read
+#define erts_atomic64_add_nob erts_no_atomic64_add
+#define erts_atomic64_read_bor_nob erts_no_atomic64_read_bor
+#define erts_atomic64_read_band_nob erts_no_atomic64_read_band
+#define erts_atomic64_xchg_nob erts_no_atomic64_xchg
+#define erts_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg
+#define erts_atomic64_read_bset_nob erts_no_atomic64_read_bset
+
+#define erts_atomic64_init_mb erts_no_atomic64_set
+#define erts_atomic64_set_mb erts_no_atomic64_set
+#define erts_atomic64_read_mb erts_no_atomic64_read
+#define erts_atomic64_inc_read_mb erts_no_atomic64_inc_read
+#define erts_atomic64_dec_read_mb erts_no_atomic64_dec_read
+#define erts_atomic64_inc_mb erts_no_atomic64_inc
+#define erts_atomic64_dec_mb erts_no_atomic64_dec
+#define erts_atomic64_add_read_mb erts_no_atomic64_add_read
+#define erts_atomic64_add_mb erts_no_atomic64_add
+#define erts_atomic64_read_bor_mb erts_no_atomic64_read_bor
+#define erts_atomic64_read_band_mb erts_no_atomic64_read_band
+#define erts_atomic64_xchg_mb erts_no_atomic64_xchg
+#define erts_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg
+#define erts_atomic64_read_bset_mb erts_no_atomic64_read_bset
+
+#define erts_atomic64_init_acqb erts_no_atomic64_set
+#define erts_atomic64_set_acqb erts_no_atomic64_set
+#define erts_atomic64_read_acqb erts_no_atomic64_read
+#define erts_atomic64_inc_read_acqb erts_no_atomic64_inc_read
+#define erts_atomic64_dec_read_acqb erts_no_atomic64_dec_read
+#define erts_atomic64_inc_acqb erts_no_atomic64_inc
+#define erts_atomic64_dec_acqb erts_no_atomic64_dec
+#define erts_atomic64_add_read_acqb erts_no_atomic64_add_read
+#define erts_atomic64_add_acqb erts_no_atomic64_add
+#define erts_atomic64_read_bor_acqb erts_no_atomic64_read_bor
+#define erts_atomic64_read_band_acqb erts_no_atomic64_read_band
+#define erts_atomic64_xchg_acqb erts_no_atomic64_xchg
+#define erts_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg
+#define erts_atomic64_read_bset_acqb erts_no_atomic64_read_bset
+
+#define erts_atomic64_init_relb erts_no_atomic64_set
+#define erts_atomic64_set_relb erts_no_atomic64_set
+#define erts_atomic64_read_relb erts_no_atomic64_read
+#define erts_atomic64_inc_read_relb erts_no_atomic64_inc_read
+#define erts_atomic64_dec_read_relb erts_no_atomic64_dec_read
+#define erts_atomic64_inc_relb erts_no_atomic64_inc
+#define erts_atomic64_dec_relb erts_no_atomic64_dec
+#define erts_atomic64_add_read_relb erts_no_atomic64_add_read
+#define erts_atomic64_add_relb erts_no_atomic64_add
+#define erts_atomic64_read_bor_relb erts_no_atomic64_read_bor
+#define erts_atomic64_read_band_relb erts_no_atomic64_read_band
+#define erts_atomic64_xchg_relb erts_no_atomic64_xchg
+#define erts_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg
+#define erts_atomic64_read_bset_relb erts_no_atomic64_read_bset
+
+#define erts_atomic64_init_ddrb erts_no_atomic64_set
+#define erts_atomic64_set_ddrb erts_no_atomic64_set
+#define erts_atomic64_read_ddrb erts_no_atomic64_read
+#define erts_atomic64_inc_read_ddrb erts_no_atomic64_inc_read
+#define erts_atomic64_dec_read_ddrb erts_no_atomic64_dec_read
+#define erts_atomic64_inc_ddrb erts_no_atomic64_inc
+#define erts_atomic64_dec_ddrb erts_no_atomic64_dec
+#define erts_atomic64_add_read_ddrb erts_no_atomic64_add_read
+#define erts_atomic64_add_ddrb erts_no_atomic64_add
+#define erts_atomic64_read_bor_ddrb erts_no_atomic64_read_bor
+#define erts_atomic64_read_band_ddrb erts_no_atomic64_read_band
+#define erts_atomic64_xchg_ddrb erts_no_atomic64_xchg
+#define erts_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg
+#define erts_atomic64_read_bset_ddrb erts_no_atomic64_read_bset
+
+#define erts_atomic64_init_rb erts_no_atomic64_set
+#define erts_atomic64_set_rb erts_no_atomic64_set
+#define erts_atomic64_read_rb erts_no_atomic64_read
+#define erts_atomic64_inc_read_rb erts_no_atomic64_inc_read
+#define erts_atomic64_dec_read_rb erts_no_atomic64_dec_read
+#define erts_atomic64_inc_rb erts_no_atomic64_inc
+#define erts_atomic64_dec_rb erts_no_atomic64_dec
+#define erts_atomic64_add_read_rb erts_no_atomic64_add_read
+#define erts_atomic64_add_rb erts_no_atomic64_add
+#define erts_atomic64_read_bor_rb erts_no_atomic64_read_bor
+#define erts_atomic64_read_band_rb erts_no_atomic64_read_band
+#define erts_atomic64_xchg_rb erts_no_atomic64_xchg
+#define erts_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg
+#define erts_atomic64_read_bset_rb erts_no_atomic64_read_bset
+
+#define erts_atomic64_init_wb erts_no_atomic64_set
+#define erts_atomic64_set_wb erts_no_atomic64_set
+#define erts_atomic64_read_wb erts_no_atomic64_read
+#define erts_atomic64_inc_read_wb erts_no_atomic64_inc_read
+#define erts_atomic64_dec_read_wb erts_no_atomic64_dec_read
+#define erts_atomic64_inc_wb erts_no_atomic64_inc
+#define erts_atomic64_dec_wb erts_no_atomic64_dec
+#define erts_atomic64_add_read_wb erts_no_atomic64_add_read
+#define erts_atomic64_add_wb erts_no_atomic64_add
+#define erts_atomic64_read_bor_wb erts_no_atomic64_read_bor
+#define erts_atomic64_read_band_wb erts_no_atomic64_read_band
+#define erts_atomic64_xchg_wb erts_no_atomic64_xchg
+#define erts_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg
+#define erts_atomic64_read_bset_wb erts_no_atomic64_read_bset
+
+#define erts_atomic64_set_dirty erts_no_atomic64_set
+#define erts_atomic64_read_dirty erts_no_atomic64_read
+
 #endif /* !USE_THREADS */
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -2383,6 +2964,104 @@ erts_no_atomic32_read_bset(erts_no_atomic32_t *var,
     return old;
 }
 
+/* atomic64 */
+
+ERTS_GLB_INLINE void
+erts_no_atomic64_set(erts_no_atomic64_t *var, erts_aint64_t i)
+{
+    *var = i;
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_read(erts_no_atomic64_t *var)
+{
+    return *var;
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_inc_read(erts_no_atomic64_t *incp)
+{
+    return ++(*incp);
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_dec_read(erts_no_atomic64_t *decp)
+{
+    return --(*decp);
+}
+
+ERTS_GLB_INLINE void
+erts_no_atomic64_inc(erts_no_atomic64_t *incp)
+{
+    ++(*incp);
+}
+
+ERTS_GLB_INLINE void
+erts_no_atomic64_dec(erts_no_atomic64_t *decp)
+{
+    --(*decp);
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_add_read(erts_no_atomic64_t *addp, erts_aint64_t i)
+{
+    return *addp += i;
+}
+
+ERTS_GLB_INLINE void
+erts_no_atomic64_add(erts_no_atomic64_t *addp, erts_aint64_t i)
+{
+    *addp += i;
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_read_bor(erts_no_atomic64_t *var, erts_aint64_t mask)
+{
+    erts_aint64_t old;
+    old = *var;
+    *var |= mask;
+    return old;
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_read_band(erts_no_atomic64_t *var, erts_aint64_t mask)
+{
+    erts_aint64_t old;
+    old = *var;
+    *var &= mask;
+    return old;
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, erts_aint64_t new)
+{
+    erts_aint64_t old = *xchgp;
+    *xchgp = new;
+    return old;
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp,
+			 erts_aint64_t new,
+			 erts_aint64_t expected)
+{
+    erts_aint64_t old = *xchgp;
+    if (old == expected)
+        *xchgp = new;
+    return old;
+}
+
+ERTS_GLB_INLINE erts_aint64_t
+erts_no_atomic64_read_bset(erts_no_atomic64_t *var,
+			   erts_aint64_t mask,
+			   erts_aint64_t set)
+{
+    erts_aint64_t old = *var;
+    *var &= ~mask;
+    *var |= (mask & set);
+    return old;
+}
+
 /* spinlock */
 
 ERTS_GLB_INLINE void
diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h
index 0807649ea1..c32f8fd61c 100644
--- a/erts/emulator/beam/erl_utils.h
+++ b/erts/emulator/beam/erl_utils.h
@@ -32,11 +32,7 @@ typedef struct {
 #endif
     union {
 	Uint64 not_atomic;
-#ifdef ARCH_64
-	erts_atomic_t atomic;
-#else
-	erts_dw_atomic_t atomic;
-#endif
+	erts_atomic64_t atomic;
     } counter;
 } erts_interval_t;
 
@@ -50,9 +46,6 @@ Uint64 erts_ensure_later_interval_nob(erts_interval_t *, Uint64);
 Uint64 erts_ensure_later_interval_acqb(erts_interval_t *, Uint64);
 Uint64 erts_smp_ensure_later_interval_nob(erts_interval_t *, Uint64);
 Uint64 erts_smp_ensure_later_interval_acqb(erts_interval_t *, Uint64);
-#ifdef ARCH_32
-ERTS_GLB_INLINE Uint64 erts_interval_dw_aint_to_val__(erts_dw_aint_t *);
-#endif
 ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *);
 ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *);
 ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *);
@@ -62,46 +55,16 @@ ERTS_GLB_INLINE Uint64 erts_smp_current_interval_acqb(erts_interval_t *);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-#ifdef ARCH_32
-
-ERTS_GLB_INLINE Uint64
-erts_interval_dw_aint_to_val__(erts_dw_aint_t *dw)
-{
-#ifdef ETHR_SU_DW_NAINT_T__
-    return (Uint64) dw->dw_sint;
-#else
-    Uint64 res;
-    res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]);
-    res <<= 32;
-    res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]);
-    return res;
-#endif
-}
-
-#endif
-
 ERTS_GLB_INLINE Uint64
 erts_current_interval_nob__(erts_interval_t *icp)
 {
-#ifdef ARCH_64
-    return (Uint64) erts_atomic_read_nob(&icp->counter.atomic);
-#else
-    erts_dw_aint_t dw;
-    erts_dw_atomic_read_nob(&icp->counter.atomic, &dw);
-    return erts_interval_dw_aint_to_val__(&dw);
-#endif
+    return (Uint64) erts_atomic64_read_nob(&icp->counter.atomic);
 }
 
 ERTS_GLB_INLINE Uint64
 erts_current_interval_acqb__(erts_interval_t *icp)
 {
-#ifdef ARCH_64
-    return (Uint64) erts_atomic_read_acqb(&icp->counter.atomic);
-#else
-    erts_dw_aint_t dw;
-    erts_dw_atomic_read_acqb(&icp->counter.atomic, &dw);
-    return erts_interval_dw_aint_to_val__(&dw);
-#endif
+    return (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic);
 }
 
 ERTS_GLB_INLINE Uint64
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index f20e6e5665..c505c44905 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -4216,19 +4216,7 @@ void erts_silence_warn_unused_result(long unused)
 void
 erts_interval_init(erts_interval_t *icp)
 {
-#ifdef ARCH_64
-    erts_atomic_init_nob(&icp->counter.atomic, 0);
-#else
-    erts_dw_aint_t dw;
-#ifdef ETHR_SU_DW_NAINT_T__
-    dw.dw_sint = 0;
-#else
-    dw.sint[ERTS_DW_AINT_HIGH_WORD] = 0;
-    dw.sint[ERTS_DW_AINT_LOW_WORD] = 0;
-#endif
-    erts_dw_atomic_init_nob(&icp->counter.atomic, &dw);
-
-#endif
+    erts_atomic64_init_nob(&icp->counter.atomic, 0);
 #ifdef DEBUG
     icp->smp_api = 0;
 #endif
@@ -4250,55 +4238,13 @@ erts_smp_interval_init(erts_interval_t *icp)
 static ERTS_INLINE Uint64
 step_interval_nob(erts_interval_t *icp)
 {
-#ifdef ARCH_64
-    return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic);
-#else
-    erts_dw_aint_t exp;
-
-    erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
-    while (1) {
-	erts_dw_aint_t new = exp;
-
-#ifdef ETHR_SU_DW_NAINT_T__
-	new.dw_sint++;
-#else
-	new.sint[ERTS_DW_AINT_LOW_WORD]++;
-	if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
-	    new.sint[ERTS_DW_AINT_HIGH_WORD]++;
-#endif
-
-	if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp))
-	    return erts_interval_dw_aint_to_val__(&new);
-
-    }
-#endif
+    return (Uint64) erts_atomic64_inc_read_nob(&icp->counter.atomic);
 }
 
 static ERTS_INLINE Uint64
 step_interval_relb(erts_interval_t *icp)
 {
-#ifdef ARCH_64
-    return (Uint64) erts_atomic_inc_read_relb(&icp->counter.atomic);
-#else
-    erts_dw_aint_t exp;
-
-    erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
-    while (1) {
-	erts_dw_aint_t new = exp;
-
-#ifdef ETHR_SU_DW_NAINT_T__
-	new.dw_sint++;
-#else
-	new.sint[ERTS_DW_AINT_LOW_WORD]++;
-	if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
-	    new.sint[ERTS_DW_AINT_HIGH_WORD]++;
-#endif
-
-	if (erts_dw_atomic_cmpxchg_relb(&icp->counter.atomic, &new, &exp))
-	    return erts_interval_dw_aint_to_val__(&new);
-
-    }
-#endif
+    return (Uint64) erts_atomic64_inc_read_relb(&icp->counter.atomic);
 }
 
 
@@ -4306,38 +4252,10 @@ static ERTS_INLINE Uint64
 ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic)
 {
     Uint64 curr_ic;
-#ifdef ARCH_64
-    curr_ic = (Uint64) erts_atomic_read_nob(&icp->counter.atomic);
+    curr_ic = (Uint64) erts_atomic64_read_nob(&icp->counter.atomic);
     if (curr_ic > ic)
 	return curr_ic;
-    return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic);
-#else
-    erts_dw_aint_t exp;
-
-    erts_dw_atomic_read_nob(&icp->counter.atomic, &exp);
-    curr_ic = erts_interval_dw_aint_to_val__(&exp);
-    if (curr_ic > ic)
-	return curr_ic;
-
-    while (1) {
-	erts_dw_aint_t new = exp;
-
-#ifdef ETHR_SU_DW_NAINT_T__
-	new.dw_sint++;
-#else
-	new.sint[ERTS_DW_AINT_LOW_WORD]++;
-	if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
-	    new.sint[ERTS_DW_AINT_HIGH_WORD]++;
-#endif
-
-	if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp))
-	    return erts_interval_dw_aint_to_val__(&new);
-
-	curr_ic = erts_interval_dw_aint_to_val__(&exp);
-	if (curr_ic > ic)
-	    return curr_ic;
-    }
-#endif
+    return (Uint64) erts_atomic64_inc_read_nob(&icp->counter.atomic);
 }
 
 
@@ -4345,38 +4263,10 @@ static ERTS_INLINE Uint64
 ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic)
 {
     Uint64 curr_ic;
-#ifdef ARCH_64
-    curr_ic = (Uint64) erts_atomic_read_acqb(&icp->counter.atomic);
+    curr_ic = (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic);
     if (curr_ic > ic)
 	return curr_ic;
-    return (Uint64) erts_atomic_inc_read_acqb(&icp->counter.atomic);
-#else
-    erts_dw_aint_t exp;
-
-    erts_dw_atomic_read_acqb(&icp->counter.atomic, &exp);
-    curr_ic = erts_interval_dw_aint_to_val__(&exp);
-    if (curr_ic > ic)
-	return curr_ic;
-
-    while (1) {
-	erts_dw_aint_t new = exp;
-
-#ifdef ETHR_SU_DW_NAINT_T__
-	new.dw_sint++;
-#else
-	new.sint[ERTS_DW_AINT_LOW_WORD]++;
-	if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0)
-	    new.sint[ERTS_DW_AINT_HIGH_WORD]++;
-#endif
-
-	if (erts_dw_atomic_cmpxchg_acqb(&icp->counter.atomic, &new, &exp))
-	    return erts_interval_dw_aint_to_val__(&new);
-
-	curr_ic = erts_interval_dw_aint_to_val__(&exp);
-	if (curr_ic > ic)
-	    return curr_ic;
-    }
-#endif
+    return (Uint64) erts_atomic64_inc_read_acqb(&icp->counter.atomic);
 }
 
 Uint64