Merge branch 'rickard/thread-queue-fix/OTP-10854' into rickard/r16b/thread-queue-fix/OTP-10854

Conflicts:
	erts/emulator/beam/erl_threads.h

4 files changed, 196 insertions, 78 deletions

diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 34c90c0bda..0dd9e29e8e 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -259,6 +259,9 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb
 #define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb
 
+#define erts_smp_dw_atomic_set_dirty erts_dw_atomic_set_dirty
+#define erts_smp_dw_atomic_read_dirty erts_dw_atomic_read_dirty
+
 /* Word size atomics */
 
 #define erts_smp_atomic_init_nob erts_atomic_init_nob
@@ -366,6 +369,9 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb
 #define erts_smp_atomic_read_bset_wb erts_atomic_read_bset_wb
 
+#define erts_smp_atomic_set_dirty erts_atomic_set_dirty
+#define erts_smp_atomic_read_dirty erts_atomic_read_dirty
+
 /* 32-bit atomics */
 
 #define erts_smp_atomic32_init_nob erts_atomic32_init_nob
@@ -473,6 +479,9 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb
 #define erts_smp_atomic32_read_bset_wb erts_atomic32_read_bset_wb
 
+#define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty
+#define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty
+
 #else /* !ERTS_SMP */
 
 /* Double word size atomics */
@@ -512,6 +521,9 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read
 #define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg
 
+#define erts_smp_dw_atomic_set_dirty erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_dirty erts_no_dw_atomic_read
+
 /* Word size atomics */
 
 #define erts_smp_atomic_init_nob erts_no_atomic_set
@@ -619,6 +631,9 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
 #define erts_smp_atomic_read_bset_wb erts_no_atomic_read_bset
 
+#define erts_smp_atomic_set_dirty erts_no_atomic_set
+#define erts_smp_atomic_read_dirty erts_no_atomic_read
+
 /* 32-bit atomics */
 
 #define erts_smp_atomic32_init_nob erts_no_atomic32_set
@@ -726,6 +741,9 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
 #define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
 #define erts_smp_atomic32_read_bset_wb erts_no_atomic32_read_bset
 
+#define erts_smp_atomic32_set_dirty erts_no_atomic32_set
+#define erts_smp_atomic32_read_dirty erts_no_atomic32_read
+
 #endif /* !ERTS_SMP */
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c
index f07964a265..ee2ff765e0 100644
--- a/erts/emulator/beam/erl_thr_queue.c
+++ b/erts/emulator/beam/erl_thr_queue.c
@@ -113,6 +113,11 @@ sl_element_free(ErtsThrQElement_t *p)
 
 #endif
 
+#define ErtsThrQDirtyReadEl(A) \
+    ((ErtsThrQElement_t *) erts_atomic_read_dirty((A)))
+#define ErtsThrQDirtySetEl(A, V) \
+    erts_atomic_set_dirty((A), (erts_aint_t) (V))
+
 typedef union {
     ErtsThrQ_t q;
     char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsThrQ_t))];
@@ -137,7 +142,7 @@ erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi)
     q->last = NULL;
     q->q.blk = NULL;
 #else
-    erts_atomic_init_nob(&q->tail.data.marker.next.atmc, ERTS_AINT_NULL);
+    erts_atomic_init_nob(&q->tail.data.marker.next, ERTS_AINT_NULL);
     q->tail.data.marker.data.ptr = NULL;
     erts_atomic_init_nob(&q->tail.data.last,
 			 (erts_aint_t) &q->tail.data.marker);
@@ -150,7 +155,7 @@ erts_thr_q_initialize(ErtsThrQ_t *q, ErtsThrQInit_t *qi)
     if (!q->tail.data.notify)
 	q->tail.data.notify = noop_callback;
 
-    q->head.head.ptr = &q->tail.data.marker;
+    erts_atomic_init_nob(&q->head.head, (erts_aint_t) &q->tail.data.marker);
     q->head.live = qi->live.objects;
     q->head.first = &q->tail.data.marker;
     q->head.unref_end = &q->tail.data.marker;
@@ -296,17 +301,17 @@ element_free(ErtsThrQ_t *q, ErtsThrQElement_t *el)
 #ifdef USE_THREADS
 
 static ERTS_INLINE ErtsThrQElement_t *
-enqueue_managed(ErtsThrQ_t *q, ErtsThrQElement_t *this, int want_last)
+enqueue_managed(ErtsThrQ_t *q, ErtsThrQElement_t *this)
 {
     erts_aint_t ilast, itmp;
 
-    erts_atomic_init_nob(&this->next.atmc, ERTS_AINT_NULL);
+    erts_atomic_init_nob(&this->next, ERTS_AINT_NULL);
     /* Enqueue at end of list... */
 
     ilast = erts_atomic_read_nob(&q->tail.data.last);
     while (1) {
 	ErtsThrQElement_t *last = (ErtsThrQElement_t *) ilast;
-	itmp = erts_atomic_cmpxchg_mb(&last->next.atmc,
+	itmp = erts_atomic_cmpxchg_mb(&last->next,
 				      (erts_aint_t) this,
 				      ERTS_AINT_NULL);
 	if (itmp == ERTS_AINT_NULL)
@@ -316,31 +321,57 @@ enqueue_managed(ErtsThrQ_t *q, ErtsThrQElement_t *this, int want_last)
 
     /* Move last pointer forward... */
     while (1) {
-	if (want_last) {
-	    if (erts_atomic_read_rb(&this->next.atmc) != ERTS_AINT_NULL) {
-		/* Someone else will move it forward */
-		ilast = erts_atomic_read_rb(&q->tail.data.last);
-		return (ErtsThrQElement_t *) ilast;
-	    }
-	}
-	else {
-	    if (erts_atomic_read_nob(&this->next.atmc) != ERTS_AINT_NULL) {
-		/* Someone else will move it forward */
-		return NULL;
-	    }
+	if (erts_atomic_read_rb(&this->next) != ERTS_AINT_NULL) {
+	    /* Someone else will move it forward */
+	    ilast = erts_atomic_read_rb(&q->tail.data.last);
+	    return (ErtsThrQElement_t *) ilast;
 	}
 	itmp = erts_atomic_cmpxchg_mb(&q->tail.data.last,
 				      (erts_aint_t) this,
 				      ilast);
 	if (ilast == itmp)
-	    return want_last ? this : NULL;
+	    return this;
 	ilast = itmp;
     }
 }
 
+static ERTS_INLINE ErtsThrQElement_t *
+enqueue_marker(ErtsThrQ_t *q, ErtsThrQElement_t **headp)
+{
+    int maybe_notify;
+    erts_aint_t inext;
+    ErtsThrQElement_t *last, *head;
+
+    if (headp)
+	head = *headp;
+    else
+	head = ErtsThrQDirtyReadEl(&q->head.head);
+
+    ASSERT(!q->head.used_marker);
+    q->head.used_marker = 1;
+    last = enqueue_managed(q, &q->tail.data.marker);
+    maybe_notify = &q->tail.data.marker == last;
+    inext = erts_atomic_read_acqb(&head->next);
+    if (inext == (erts_aint_t) &q->tail.data.marker) {
+	ErtsThrQDirtySetEl(&q->head.head, &q->tail.data.marker);
+	if (headp)
+	    *headp = &q->tail.data.marker;
+    }
+    else if (maybe_notify) {
+	/*
+	 * We need to notify; otherwise, we might loose a notification
+	 * for a concurrently inserted element.
+	 */
+	q->head.notify(q->head.arg);
+    }
+    return last;
+}
+
+
 static ErtsThrQCleanState_t
 clean(ErtsThrQ_t *q, int max_ops, int do_notify)
 {
+    ErtsThrQElement_t *head;
     erts_aint_t ilast;
     int um_refc_ix;
     int ops;
@@ -349,7 +380,8 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify)
 	ErtsThrQElement_t *tmp;
     restart:
 	ASSERT(q->head.first);
-	if (q->head.first == q->head.head.ptr) {
+	head = ErtsThrQDirtyReadEl(&q->head.head);
+	if (q->head.first == head) {
 	    q->head.clean_reached_head_count++;
 	    if (q->head.clean_reached_head_count
 		>= ERTS_THR_Q_MAX_CLEAN_REACHED_HEAD_COUNT) {
@@ -362,19 +394,20 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify)
 	    break;
 	if (q->head.first == &q->tail.data.marker) {
 	    q->head.used_marker = 0;
-	    q->head.first = q->head.first->next.ptr;
+	    q->head.first = ErtsThrQDirtyReadEl(&q->head.first->next);
 	    goto restart;
 	}
 	tmp = q->head.first;
-	q->head.first = q->head.first->next.ptr;
+	q->head.first = ErtsThrQDirtyReadEl(&q->head.first->next);
 	if (q->head.deq_fini.automatic)
 	    element_free(q, tmp);
 	else {
 	    tmp->data.ptr = (void *) (UWord) q->head.live;
 	    if (!q->head.deq_fini.start)
 		q->head.deq_fini.start = tmp;
-	    else if (q->head.deq_fini.end->next.ptr == &q->tail.data.marker)
-		q->head.deq_fini.end->next.ptr = tmp;
+	    else if (ErtsThrQDirtyReadEl(&q->head.deq_fini.end->next)
+		     == &q->tail.data.marker)
+		ErtsThrQDirtySetEl(&q->head.deq_fini.end->next, tmp);
 	    q->head.deq_fini.end = tmp;
 	}
     }
@@ -401,21 +434,8 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify)
 	    q->head.unref_end = q->head.next.unref_end;
 
 	    if (!q->head.used_marker
-		&& q->head.unref_end == (ErtsThrQElement_t *) ilast) {
-		q->head.used_marker = 1;
-		ilast = (erts_aint_t) enqueue_managed(q,
-						      &q->tail.data.marker,
-						      1);
-		if (q->head.head.ptr == q->head.unref_end) {
-		    ErtsThrQElement_t *next;
-		    next = ((ErtsThrQElement_t *)
-			    erts_atomic_read_acqb(&q->head.head.ptr->next.atmc));
-		    if (next == &q->tail.data.marker) {
-			q->head.head.ptr->next.ptr = &q->tail.data.marker;
-			q->head.head.ptr = &q->tail.data.marker;
-		    }
-		}
-	    }
+		&& q->head.unref_end == (ErtsThrQElement_t *) ilast)
+		ilast = (erts_aint_t) enqueue_marker(q, NULL);
 
 	    if (q->head.unref_end == (ErtsThrQElement_t *) ilast)
 		ERTS_SMP_MEMORY_BARRIER;
@@ -436,20 +456,16 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify)
     }
 #endif
 
-    if (q->head.first == q->head.head.ptr) {
+    head = ErtsThrQDirtyReadEl(&q->head.head);
+    if (q->head.first == head) {
     inspect_head:
 	if (!q->head.used_marker) {
 	    erts_aint_t inext;
-	    inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc);
+	    inext = erts_atomic_read_acqb(&head->next);
 	    if (inext == ERTS_AINT_NULL) {
-		q->head.used_marker = 1;
-		(void) enqueue_managed(q, &q->tail.data.marker, 0);
-		inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc);
-		if (inext == (erts_aint_t) &q->tail.data.marker) {
-		    q->head.head.ptr->next.ptr = &q->tail.data.marker;
-		    q->head.head.ptr = &q->tail.data.marker;
+		enqueue_marker(q, &head);
+		if (head == &q->tail.data.marker)
 		    goto check_thr_progress;
-		}
 	    }
 	}
 
@@ -506,26 +522,27 @@ erts_thr_q_inspect(ErtsThrQ_t *q, int ensure_empty)
 #ifndef USE_THREADS
     return ERTS_THR_Q_CLEAN;
 #else
+    ErtsThrQElement_t *head = ErtsThrQDirtyReadEl(&q->head.head);
     if (ensure_empty) {
 	erts_aint_t inext;
-	inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc);
+	inext = erts_atomic_read_acqb(&head->next);
 	if (inext != ERTS_AINT_NULL) {
 	    if (&q->tail.data.marker != (ErtsThrQElement_t *) inext)
 		return ERTS_THR_Q_DIRTY;
 	    else {
-		q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext;
-		q->head.head.ptr = (ErtsThrQElement_t *) inext;
-		inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc);
+		head = (ErtsThrQElement_t *) inext;
+		ErtsThrQDirtySetEl(&q->head.head, head);
+		inext = erts_atomic_read_acqb(&head->next);
 		if (inext != ERTS_AINT_NULL)
 		    return ERTS_THR_Q_DIRTY;
 	    }
 	}
     }
 
-    if (q->head.first == q->head.head.ptr) {
+    if (q->head.first == head) {
 	if (!q->head.used_marker) {
 	    erts_aint_t inext;
-	    inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc);
+	    inext = erts_atomic_read_acqb(&head->next);
 	    if (inext == ERTS_AINT_NULL)
 		return ERTS_THR_Q_DIRTY;
 	}
@@ -553,11 +570,11 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this)
 #ifndef USE_THREADS
     ASSERT(data);
 
-    this->next.ptr = NULL;
+    this->next = NULL;
     this->data.ptr = data;
 
     if (q->last)
-	q->last->next.ptr = this;
+	q->last->next = this;
     else {
 	q->first = q->last = this;
 	q->init.notify(q->init.arg);
@@ -595,7 +612,7 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this)
 	}
     }
 
-    notify = this == enqueue_managed(q, this, 1);
+    notify = this == enqueue_managed(q, this);
 	
 
 #ifdef ERTS_SMP
@@ -638,17 +655,17 @@ erts_thr_q_get_finalize_dequeue_data(ErtsThrQ_t *q, ErtsThrQFinDeQ_t *fdp)
 	ErtsThrQElement_t *e = q->head.deq_fini.start;
 	ErtsThrQElement_t *end = q->head.deq_fini.end;
 	while (e != end) {
-	    ASSERT(q->head.head.ptr != e);
+	    ASSERT(ErtsThrQDirtyReadEl(&q->head.head) != e);
 	    ASSERT(q->head.first != e);
 	    ASSERT(q->head.unref_end != e);
-	    e = e->next.ptr;
+	    e = ErtsThrQDirtyReadEl(&e->next);
 	}
     }	
 #endif
     fdp->start = q->head.deq_fini.start;
     fdp->end = q->head.deq_fini.end;
     if (fdp->end)
-	fdp->end->next.ptr = NULL;
+	ErtsThrQDirtySetEl(&fdp->end->next, NULL);
     q->head.deq_fini.start = NULL;
     q->head.deq_fini.end = NULL;
     return fdp->start != NULL;
@@ -662,7 +679,7 @@ erts_thr_q_append_finalize_dequeue_data(ErtsThrQFinDeQ_t *fdp0,
 #ifdef USE_THREADS
     if (fdp1->start) {
 	if (fdp0->end)
-	    fdp0->end->next.ptr = fdp1->start;
+	    ErtsThrQDirtySetEl(&fdp0->end->next, fdp1->start);
 	else
 	    fdp0->start = fdp1->start;
 	fdp0->end = fdp1->end;
@@ -683,7 +700,7 @@ int erts_thr_q_finalize_dequeue(ErtsThrQFinDeQ_t *state)
 	    if (!start)
 		break;
 	    tmp = start;
-	    start = start->next.ptr;
+	    start = ErtsThrQDirtyReadEl(&start->next);
 	    live = (ErtsThrQLive_t) (UWord) tmp->data.ptr;
 	    element_live_free(live, tmp);
 	}
@@ -724,7 +741,7 @@ erts_thr_q_dequeue(ErtsThrQ_t *q)
 	return NULL;
     tmp = q->first;
     res = tmp->data.ptr;
-    q->first = tmp->next.ptr;
+    q->first = tmp->next;
     if (!q->first)
 	q->last = NULL;
 
@@ -732,24 +749,26 @@ erts_thr_q_dequeue(ErtsThrQ_t *q)
 
     return res;
 #else
+    ErtsThrQElement_t *head;
     erts_aint_t inext;
     void *res;
 
-    inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc);
+    head = ErtsThrQDirtyReadEl(&q->head.head);
+    inext = erts_atomic_read_acqb(&head->next);
     if (inext == ERTS_AINT_NULL)
 	return NULL;
-    q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext;
-    q->head.head.ptr = (ErtsThrQElement_t *) inext;
-    if (q->head.head.ptr == &q->tail.data.marker) {
-	inext = erts_atomic_read_acqb(&q->head.head.ptr->next.atmc);
+    head = (ErtsThrQElement_t *) inext;
+    ErtsThrQDirtySetEl(&q->head.head, head);
+    if (head == &q->tail.data.marker) {
+	inext = erts_atomic_read_acqb(&head->next);
 	if (inext == ERTS_AINT_NULL)
 	    return NULL;
-	q->head.head.ptr->next.ptr = (ErtsThrQElement_t *) inext;
-	q->head.head.ptr = (ErtsThrQElement_t *) inext;
+	head = (ErtsThrQElement_t *) inext;
+	ErtsThrQDirtySetEl(&q->head.head, head);
     }
-    res = q->head.head.ptr->data.ptr;
+    res = head->data.ptr;
 #if ERTS_THR_Q_DBG_CHK_DATA
-    q->head.head.ptr->data.ptr = NULL;
+    head->data.ptr = NULL;
     if (!res)
 	erl_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n");
 #endif
diff --git a/erts/emulator/beam/erl_thr_queue.h b/erts/emulator/beam/erl_thr_queue.h
index edcf2c3823..ae8c7fb19a 100644
--- a/erts/emulator/beam/erl_thr_queue.h
+++ b/erts/emulator/beam/erl_thr_queue.h
@@ -76,13 +76,12 @@ typedef struct {
 typedef struct ErtsThrQElement_t_ ErtsThrQElement_t;
 typedef struct ErtsThrQElement_t ErtsThrQPrepEnQ_t;
 
-typedef union {
-    erts_atomic_t atmc;
-    ErtsThrQElement_t *ptr;
-} ErtsThrQPtr_t;
-
 struct ErtsThrQElement_t_ {
-    ErtsThrQPtr_t next;
+#ifdef USE_THREADS
+    erts_atomic_t next;
+#else
+    ErtsThrQElement_t *next;
+#endif
     union {
 	erts_atomic_t atmc;
 	void *ptr;
@@ -130,7 +129,7 @@ struct ErtsThrQ_t_ {
      * thread dequeuing.
      */
     struct {
-	ErtsThrQPtr_t head;
+	erts_atomic_t head;
 	ErtsThrQLive_t live;
 	ErtsThrQElement_t *first;
 	ErtsThrQElement_t *unref_end;
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 1dc3ffeb3c..594d38b2a1 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -690,6 +690,19 @@ do {									\
 } while (0)
 #endif
 
+ERTS_GLB_INLINE void
+erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val);
+ERTS_GLB_INLINE void
+erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val);
+ERTS_GLB_INLINE void
+erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val);
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_dirty(erts_atomic_t *var);
+ERTS_GLB_INLINE void
+erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val);
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_dirty(erts_atomic32_t *var);
+
 /*
  * See "Documentation of atomics and memory barriers" at the top
  * of this file for info on atomics.
@@ -732,6 +745,26 @@ do {									\
 #define erts_dw_atomic_read_wb ethr_dw_atomic_read_wb
 #define erts_dw_atomic_cmpxchg_wb ethr_dw_atomic_cmpxchg_wb
 
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE void
+erts_dw_atomic_set_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val)
+{
+    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
+    sint[0] = val->sint[0];
+    sint[1] = val->sint[1];    
+}
+
+ERTS_GLB_INLINE void
+erts_dw_atomic_read_dirty(erts_dw_atomic_t *var, erts_dw_aint_t *val)
+{
+    ethr_sint_t *sint = ethr_dw_atomic_addr(var);
+    val->sint[0] = sint[0];
+    val->sint[1] = sint[1];
+}
+
+#endif
+
 /* Word size atomics */
 
 #define erts_atomic_init_nob ethr_atomic_init
@@ -911,6 +944,7 @@ erts_atomic_read_bset_rb(erts_atomic_t *var,
 #define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
 ERTS_GLB_INLINE erts_aint_t
 erts_atomic_read_bset_wb(erts_atomic_t *var,
 			 erts_aint_t mask,
@@ -921,6 +955,25 @@ erts_atomic_read_bset_wb(erts_atomic_t *var,
 			    ethr_atomic_cmpxchg_wb,
 			    var, mask, set);
 }
+
+#endif
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE void
+erts_atomic_set_dirty(erts_atomic_t *var, erts_aint_t val)
+{
+    ethr_sint_t *sint = ethr_atomic_addr(var);
+    *sint = val;
+}
+
+ERTS_GLB_INLINE erts_aint_t
+erts_atomic_read_dirty(erts_atomic_t *var)
+{
+    ethr_sint_t *sint = ethr_atomic_addr(var);
+    return *sint;
+}
+
 #endif
 
 /* 32-bit atomics */
@@ -1102,6 +1155,7 @@ erts_atomic32_read_bset_rb(erts_atomic32_t *var,
 #define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
 ERTS_GLB_INLINE erts_aint32_t
 erts_atomic32_read_bset_wb(erts_atomic32_t *var,
 			   erts_aint32_t mask,
@@ -1112,10 +1166,29 @@ erts_atomic32_read_bset_wb(erts_atomic32_t *var,
 			    ethr_atomic32_cmpxchg_wb,
 			    var, mask, set);
 }
+
 #endif
 
 #undef ERTS_ATOMIC_BSET_IMPL__
 
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE void
+erts_atomic32_set_dirty(erts_atomic32_t *var, erts_aint32_t val)
+{
+    ethr_sint32_t *sint = ethr_atomic32_addr(var);
+    *sint = val;
+}
+
+ERTS_GLB_INLINE erts_aint32_t
+erts_atomic32_read_dirty(erts_atomic32_t *var)
+{
+    ethr_sint32_t *sint = ethr_atomic32_addr(var);
+    return *sint;
+}
+
+#endif
+
 #else /* !USE_THREADS */
 
 /* Double word size atomics */
@@ -1155,6 +1228,9 @@ erts_atomic32_read_bset_wb(erts_atomic32_t *var,
 #define erts_dw_atomic_read_wb erts_no_dw_atomic_read
 #define erts_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg
 
+#define erts_dw_atomic_set_dirty erts_no_dw_atomic_set
+#define erts_dw_atomic_read_dirty erts_no_dw_atomic_read
+
 /* Word size atomics */
 
 #define erts_atomic_init_nob erts_no_atomic_set
@@ -1262,6 +1338,9 @@ erts_atomic32_read_bset_wb(erts_atomic32_t *var,
 #define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
 #define erts_atomic_read_bset_wb erts_no_atomic_read_bset
 
+#define erts_atomic_set_dirty erts_no_atomic_set
+#define erts_atomic_read_dirty erts_no_atomic_read
+
 /* 32-bit atomics */
 
 #define erts_atomic32_init_nob erts_no_atomic32_set
@@ -1369,6 +1448,9 @@ erts_atomic32_read_bset_wb(erts_atomic32_t *var,
 #define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
 #define erts_atomic32_read_bset_wb erts_no_atomic32_read_bset
 
+#define erts_atomic32_set_dirty erts_no_atomic32_set
+#define erts_atomic32_read_dirty erts_no_atomic32_read
+
 #endif /* !USE_THREADS */
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF