From 9a88fc02527063be5096bdf4d49a48b144471553 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Fri, 14 Oct 2016 18:48:00 +0200
Subject: erts: Add erts internal secret atom

---
 erts/emulator/beam/atom.c     | 5 ++++-
 erts/emulator/beam/atom.h     | 2 --
 erts/emulator/beam/atom.names | 3 +++
 3 files changed, 7 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c
index b47739059b..2b5ad097a0 100644
--- a/erts/emulator/beam/atom.c
+++ b/erts/emulator/beam/atom.c
@@ -199,7 +199,7 @@ atom_alloc(Atom* tmpl)
 static void
 atom_free(Atom* obj)
 {
-    erts_free(ERTS_ALC_T_ATOM, (void*) obj);
+    ASSERT(obj->slot.index == atom_val(am_ErtsSecretAtom));
 }
 
 static void latin1_to_utf8(byte* conv_buf, const byte** srcp, int* lenp)
@@ -467,6 +467,9 @@ init_atom_table(void)
 	atom_space -= a.len;
 	atom_tab(ix)->name = (byte*)erl_atom_names[i];
     }
+
+    /* Hide am_ErtsSecretAtom */
+    hash_erase(&erts_atom_table.htable, atom_tab(atom_val(am_ErtsSecretAtom)));
 }
 
 void
diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h
index f3b21e1687..abd3b44993 100644
--- a/erts/emulator/beam/atom.h
+++ b/erts/emulator/beam/atom.h
@@ -133,8 +133,6 @@ int atom_table_sz(void);	/* table size in bytes, excluding stored objects */
 
 Eterm am_atom_put(const char*, int); /* ONLY 7-bit ascii! */
 Eterm erts_atom_put(const byte *name, int len, ErtsAtomEncoding enc, int trunc);
-int atom_erase(byte*, int);
-int atom_static_put(byte*, int);
 void init_atom_table(void);
 void atom_info(fmtfn_t, void *);
 void dump_atoms(fmtfn_t, void *);
diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 66af05c1f2..b1aeed7889 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -59,6 +59,9 @@ atom nocatch
 atom undefined_function
 atom undefined_lambda
 
+# Secret internal atom that can never be found by string lookup
+# and should never leak out to be seen by the user.
+atom ErtsSecretAtom='3RT$'
 
 # All other atoms.  Try to keep the order alphabetic.
 #
-- 
cgit v1.2.3


From f7a3650b6827242772f957d624dbb91b5e472744 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 29 Nov 2016 18:21:17 +0100
Subject: erts: Use secret atom as sysname for ETS compressed

---
 erts/emulator/beam/external.c | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c
index a49b242d7c..10e452fa25 100644
--- a/erts/emulator/beam/external.c
+++ b/erts/emulator/beam/external.c
@@ -2163,12 +2163,8 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags)
  * We use this atom as sysname in local pid/port/refs
  * for the ETS compressed format (DFLAG_INTERNAL_TAGS).
  *
- * We used atom '' earlier but that turned out to cause problems
- * for buggy erl_interface/ic usage of c-nodes with empty node names.
- * A long atom reduces risk of nodes actually called this and the length
- * does not matter anyway as it's encoded with atom index (ATOM_INTERNAL_REF2).
  */
-#define INTERNAL_LOCAL_SYSNAME am_await_microstate_accounting_modifications
+#define INTERNAL_LOCAL_SYSNAME am_ErtsSecretAtom
 
 static byte*
 enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags)
-- 
cgit v1.2.3


From 6c76bfa96c12b9f29e66baf3a2df7e84bb9b05ef Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Tue, 29 Nov 2016 20:54:14 +0100
Subject: erts: Refactor BEAM_NIF_MIN_FUNC_SZ

to be declared once in beam_load.h
and get rid of #ifdef kludge.
---
 erts/emulator/beam/beam_load.c | 11 +++--------
 erts/emulator/beam/beam_load.h |  6 ++++++
 erts/emulator/beam/erl_nif.c   |  7 ++-----
 3 files changed, 11 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 3cd395c2c1..8f1faa6719 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -2546,15 +2546,10 @@ load_code(LoaderState* stp)
 
 		if (stp->may_load_nif) {
 		    const int finfo_ix = ci - FUNC_INFO_SZ;
-#ifdef ERTS_DIRTY_SCHEDULERS
-		    enum { MIN_FUNC_SZ = 4 };
-#else
-		    enum { MIN_FUNC_SZ = 3 };
-#endif
-		    if (finfo_ix - last_func_start < MIN_FUNC_SZ && last_func_start) {		   
+		    if (finfo_ix - last_func_start < BEAM_NIF_MIN_FUNC_SZ && last_func_start) {
 			/* Must make room for call_nif op */
-			int pad = MIN_FUNC_SZ - (finfo_ix - last_func_start);
-			ASSERT(pad > 0 && pad < MIN_FUNC_SZ);
+			int pad = BEAM_NIF_MIN_FUNC_SZ - (finfo_ix - last_func_start);
+			ASSERT(pad > 0 && pad < BEAM_NIF_MIN_FUNC_SZ);
 			CodeNeed(pad);
 			sys_memmove(&code[finfo_ix+pad], &code[finfo_ix],
 				    FUNC_INFO_SZ*sizeof(BeamInstr));
diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h
index d420a4346c..659b9c303f 100644
--- a/erts/emulator/beam/beam_load.h
+++ b/erts/emulator/beam/beam_load.h
@@ -111,6 +111,12 @@ typedef struct beam_code_header {
 
 }BeamCodeHeader;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+#  define BEAM_NIF_MIN_FUNC_SZ 4
+#else
+#  define BEAM_NIF_MIN_FUNC_SZ 3
+#endif
+
 void erts_release_literal_area(struct ErtsLiteralArea_* literal_area);
 int erts_is_module_native(BeamCodeHeader* code);
 void erts_beam_bif_load_init(void);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 202f713f67..5499512dd1 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3341,11 +3341,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 				     mod_atom, f->name, f->arity);
 #endif
 	    }
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (4))
-#else
-	    else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (3))
-#endif
+	    else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0])
+                     < BEAM_NIF_MIN_FUNC_SZ)
 	    {
 		ret = load_nif_error(BIF_P,bad_lib,"No explicit call to load_nif"
 				     " in module (%T:%s/%u too small)",
-- 
cgit v1.2.3


From 73e6fc31e8ef1808fb079a4bac2f1ccff44efa3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Wed, 30 Nov 2016 15:16:17 +0100
Subject: erts: Add driver and nif locks to lcnt

---
 erts/emulator/beam/erl_drv_thread.c | 98 +++++++++++++++++++++++++++++++------
 1 file changed, 84 insertions(+), 14 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c
index 92edce5176..0e6aadf568 100644
--- a/erts/emulator/beam/erl_drv_thread.c
+++ b/erts/emulator/beam/erl_drv_thread.c
@@ -54,6 +54,9 @@ fatal_error(int err, char *func)
 
 struct ErlDrvMutex_ {
     ethr_mutex mtx;
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock_t lcnt;
+#endif
     char *name;
 };
 
@@ -64,6 +67,9 @@ struct ErlDrvCond_ {
 
 struct ErlDrvRWLock_ {
     ethr_rwmutex rwmtx;
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock_t lcnt;
+#endif
     char *name;
 };
 
@@ -161,14 +167,17 @@ erl_drv_mutex_create(char *name)
 	opt.posix_compliant = 1;
 	if (ethr_mutex_init_opt(&dmtx->mtx, &opt) != 0) {
 	    erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx);
-	    dmtx = NULL;
+	    return NULL;
 	}
-	else if (!name)
-	    dmtx->name = no_name;
-	else {
+	if (name) {
 	    dmtx->name = ((char *) dmtx) + sizeof(ErlDrvMutex);
 	    sys_strcpy(dmtx->name, name);
+        } else {
+	    dmtx->name = no_name;
 	}
+#ifdef ERTS_ENABLE_LOCK_COUNT
+        erts_lcnt_init_lock(&dmtx->lcnt, dmtx->name, ERTS_LCNT_LT_MUTEX);
+#endif
     }
     return dmtx;
 #else
@@ -180,7 +189,11 @@ void
 erl_drv_mutex_destroy(ErlDrvMutex *dmtx)
 {
 #ifdef USE_THREADS
-    int res = dmtx ? ethr_mutex_destroy(&dmtx->mtx) : EINVAL;
+    int res;
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_destroy_lock(&dmtx->lcnt);
+#endif
+    res = dmtx ? ethr_mutex_destroy(&dmtx->mtx) : EINVAL;
     if (res != 0)
 	fatal_error(res, "erl_drv_mutex_destroy()");
     erts_free(ERTS_ALC_T_DRV_MTX, (void *) dmtx);
@@ -202,9 +215,14 @@ int
 erl_drv_mutex_trylock(ErlDrvMutex *dmtx)
 {
 #ifdef USE_THREADS
+    int res;
     if (!dmtx)
 	fatal_error(EINVAL, "erl_drv_mutex_trylock()");
-    return ethr_mutex_trylock(&dmtx->mtx);
+    res = ethr_mutex_trylock(&dmtx->mtx);
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_trylock(&dmtx->lcnt, res);
+#endif
+    return res;
 #else
     return 0;
 #endif
@@ -216,8 +234,14 @@ erl_drv_mutex_lock(ErlDrvMutex *dmtx)
 #ifdef USE_THREADS
     if (!dmtx)
 	fatal_error(EINVAL, "erl_drv_mutex_lock()");
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock(&dmtx->lcnt);
+#endif
     ethr_mutex_lock(&dmtx->mtx);
 #endif
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock_post(&dmtx->lcnt);
+#endif
 }
 
 void
@@ -226,6 +250,9 @@ erl_drv_mutex_unlock(ErlDrvMutex *dmtx)
 #ifdef USE_THREADS
     if (!dmtx)
 	fatal_error(EINVAL, "erl_drv_mutex_unlock()");
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_unlock(&dmtx->lcnt);
+#endif
     ethr_mutex_unlock(&dmtx->mtx);
 #endif
 }
@@ -306,10 +333,18 @@ erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx)
     if (!dcnd || !dmtx) {
 	fatal_error(EINVAL, "erl_drv_cond_wait()");
     }
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_unlock(&dmtx->lcnt);
+#endif
     while (1) {
 	int res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx);
-	if (res == 0)
+	if (res == 0) {
+#ifdef ERTS_ENABLE_LOCK_COUNT
+            erts_lcnt_lock(&dmtx->lcnt);
+            erts_lcnt_lock_post(&dmtx->lcnt);
+#endif
 	    break;
+        }
     }
 #endif
 }
@@ -324,14 +359,17 @@ erl_drv_rwlock_create(char *name)
     if (drwlck) {
 	if (ethr_rwmutex_init(&drwlck->rwmtx) != 0) {
 	    erts_free(ERTS_ALC_T_DRV_RWLCK, (void *) drwlck);
-	    drwlck = NULL;
+	    return NULL;
 	}
-	else if (!name)
-	    drwlck->name = no_name;
-	else {
+	if (name) {
 	    drwlck->name = ((char *) drwlck) + sizeof(ErlDrvRWLock);
 	    sys_strcpy(drwlck->name, name);
+        } else {
+	    drwlck->name = no_name;
 	}
+#ifdef ERTS_ENABLE_LOCK_COUNT
+        erts_lcnt_init_lock(&drwlck->lcnt, drwlck->name, ERTS_LCNT_LT_RWMUTEX);
+#endif
     }
     return drwlck;
 #else
@@ -343,7 +381,11 @@ void
 erl_drv_rwlock_destroy(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
-    int res = drwlck ? ethr_rwmutex_destroy(&drwlck->rwmtx) : EINVAL;
+    int res;
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_destroy_lock(&drwlck->lcnt);
+#endif
+    res = drwlck ? ethr_rwmutex_destroy(&drwlck->rwmtx) : EINVAL;
     if (res != 0)
 	fatal_error(res, "erl_drv_rwlock_destroy()");
     erts_free(ERTS_ALC_T_DRV_RWLCK, (void *) drwlck);
@@ -364,9 +406,14 @@ int
 erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
+    int res;
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()");
-    return ethr_rwmutex_tryrlock(&drwlck->rwmtx);
+    res = ethr_rwmutex_tryrlock(&drwlck->rwmtx);
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LCNT_LO_READ);
+#endif
+    return res;
 #else
     return 0;
 #endif
@@ -378,7 +425,13 @@ erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck)
 #ifdef USE_THREADS
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rlock()");
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ);
+#endif
     ethr_rwmutex_rlock(&drwlck->rwmtx);
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock_post(&drwlck->lcnt);
+#endif
 #endif
 }
 
@@ -388,6 +441,9 @@ erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck)
 #ifdef USE_THREADS
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_runlock()");
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ);
+#endif
     ethr_rwmutex_runlock(&drwlck->rwmtx);
 #endif
 }
@@ -396,9 +452,14 @@ int
 erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck)
 {
 #ifdef USE_THREADS
+    int res;
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()");
-    return ethr_rwmutex_tryrwlock(&drwlck->rwmtx);
+    res = ethr_rwmutex_tryrwlock(&drwlck->rwmtx);
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_trylock_opt(&drwlck->lcnt, res, ERTS_LCNT_LO_READ_WRITE);
+#endif
+    return res;
 #else
     return 0;
 #endif
@@ -410,7 +471,13 @@ erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck)
 #ifdef USE_THREADS
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rwlock()");
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ_WRITE);
+#endif
     ethr_rwmutex_rwlock(&drwlck->rwmtx);
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_lock_post(&drwlck->lcnt);
+#endif
 #endif
 }
 
@@ -420,6 +487,9 @@ erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck)
 #ifdef USE_THREADS
     if (!drwlck)
 	fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()");
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    erts_lcnt_unlock_opt(&drwlck->lcnt, ERTS_LCNT_LO_READ_WRITE);
+#endif
     ethr_rwmutex_rwunlock(&drwlck->rwmtx);
 #endif
 }
-- 
cgit v1.2.3


From 8e5c762d597b9a3ba4d78bcf57ebe379ad666599 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 2 Dec 2016 16:44:43 +0100
Subject: erts: Fix whitespace errors in lock counter

---
 erts/emulator/beam/erl_lock_count.c | 25 ++++++++++++++-----------
 erts/emulator/beam/erl_lock_count.h |  4 ++--
 2 files changed, 16 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index 481e92b2cd..6354fc8663 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -55,11 +55,11 @@ static erts_lcnt_thread_data_t *lcnt_thread_data[2048];
 /* local functions */
 
 static ERTS_INLINE void lcnt_lock(void) {
-    ethr_mutex_lock(&lcnt_data_lock); 
+    ethr_mutex_lock(&lcnt_data_lock);
 }
 
 static ERTS_INLINE void lcnt_unlock(void) {
-    ethr_mutex_unlock(&lcnt_data_lock); 
+    ethr_mutex_unlock(&lcnt_data_lock);
 }
 
 const int log2_tab64[64] = {
@@ -159,7 +159,7 @@ static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) {
     lcnt_thread_data[eltd->id] = eltd;
 
     return eltd;
-} 
+}
 
 static erts_lcnt_thread_data_t *lcnt_get_thread_data(void) {
     return (erts_lcnt_thread_data_t *)ethr_tsd_get(lcnt_thr_data_key);
@@ -254,9 +254,9 @@ void erts_lcnt_init() {
     /* init lock */
     if (ethr_mutex_init(&lcnt_data_lock) != 0) abort();
 
-    /* init tsd */    
+    /* init tsd */
     lcnt_n_thr = 0;
-    ethr_tsd_key_create(&lcnt_thr_data_key,"lcnt_data");
+    ethr_tsd_key_create(&lcnt_thr_data_key, "lcnt_data");
 
     lcnt_lock();
 
@@ -352,11 +352,11 @@ void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock)
 
 /* interface to erl_threads.h */
 /* only lock on init and destroy, all others should use atomics */
-void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { 
+void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) {
     erts_lcnt_init_lock_x(lock, name, flag, NIL);
 }
 
-void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { 
+void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) {
     int i;
     if (name == NULL) { ERTS_LCNT_CLEAR_FLAG(lock); return; }
     lcnt_lock();
@@ -382,7 +382,10 @@ void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eter
     erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock);
     lcnt_unlock();
 }
-/* init empty, instead of zero struct */
+
+/* init empty, instead of zero struct
+ * used by process locks probes
+ */
 void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock) {
     lock->next = NULL;
     lock->prev = NULL;
@@ -444,7 +447,7 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
     }
 
     /* we cannot acquire w_lock if either w or r are taken */
-    /* we cannot acquire r_lock if w_lock is taken */	
+    /* we cannot acquire r_lock if w_lock is taken */
 
     if ((w_state > 0) || (r_state > 0)) {
         eltd->lock_in_conflict = 1;
@@ -561,7 +564,7 @@ void erts_lcnt_unlock(erts_lcnt_lock_t *lock) {
     if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
 #ifdef DEBUG
     {
-        erts_aint_t w_state;  
+        erts_aint_t w_state;
         erts_aint_t flowstate;
 
         /* flowstate */
@@ -647,7 +650,7 @@ Uint16 erts_lcnt_set_rt_opt(Uint16 opt) {
     return prev;
 }
 
-Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) {			
+Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) {
     Uint16 prev;
     prev = (erts_lcnt_rt_options & opt);
     erts_lcnt_rt_options &= ~opt;
diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h
index 3e8dcefe69..4f838f7faa 100644
--- a/erts/emulator/beam/erl_lock_count.h
+++ b/erts/emulator/beam/erl_lock_count.h
@@ -148,13 +148,13 @@ typedef struct erts_lcnt_lock_stats_s {
 typedef struct erts_lcnt_lock_s {
     char *name;            /* lock name */
     Uint16 flag;           /* lock type */
-    Eterm id;              /* id if possible */ 
+    Eterm id;              /* id if possible */
 
 #ifdef DEBUG
     ethr_atomic_t flowstate;
 #endif
 
-    /* lock states */    
+    /* lock states */
     ethr_atomic_t w_state; /* 0 not taken, otherwise n threads waiting */
     ethr_atomic_t r_state; /* 0 not taken, > 0 -> writes will wait */
 
-- 
cgit v1.2.3


From 7d161f5b475575bd79bd90977b3a79334a8ec658 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn-Egil=20Dahlberg?= <egil@erlang.org>
Date: Fri, 2 Dec 2016 17:20:00 +0100
Subject: erts: Remove unnecessary 'enable_lcnt' option in locks

---
 erts/emulator/beam/erl_alloc_util.c   |  8 ++++----
 erts/emulator/beam/erl_port_task.h    |  8 +++-----
 erts/emulator/beam/erl_process_lock.c | 21 +++++++--------------
 erts/emulator/beam/erl_smp.h          |  4 ++--
 erts/emulator/beam/erl_threads.h      | 33 +++++++++------------------------
 erts/emulator/beam/io.c               | 21 +++++++++------------
 6 files changed, 34 insertions(+), 61 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 61cce224dd..d346f8c8b6 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -6134,18 +6134,18 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
 #ifdef USE_THREADS
     if (init->ts) {
 	allctr->thread_safe = 1;
-	
+
 #ifdef ERTS_ENABLE_LOCK_COUNT
 	erts_mtx_init_x_opt(&allctr->mutex,
 			    "alcu_allocator",
 			    make_small(allctr->alloc_no),
-			    ERTS_LCNT_LT_ALLOC,1);
+			    ERTS_LCNT_LT_ALLOC);
 #else
 	erts_mtx_init_x(&allctr->mutex,
 			"alcu_allocator",
-			make_small(allctr->alloc_no),1);
+			make_small(allctr->alloc_no));
 #endif /*ERTS_ENABLE_LOCK_COUNT*/
-	
+
 #ifdef DEBUG
 	allctr->debug.saved_tid = 0;
 #endif
diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h
index 2a6bd165a3..e3550e878e 100644
--- a/erts/emulator/beam/erl_port_task.h
+++ b/erts/emulator/beam/erl_port_task.h
@@ -188,13 +188,11 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp, Eterm instr_id)
     ptsp->taskq.in.last = NULL;
     erts_smp_atomic32_init_nob(&ptsp->flags, 0);
 #ifdef ERTS_SMP
-    erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id,
 #ifdef ERTS_ENABLE_LOCK_COUNT
-		    (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)
-#else
-		    1
+    if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK))
+        lock_str = NULL;
 #endif
-		    );
+    erts_mtx_init_x(&ptsp->mtx, lock_str, instr_id);
 #endif
 }
 
diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index a69185bc5c..180df229eb 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -123,7 +123,7 @@ erts_init_proc_lock(int cpus)
     for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) {
 #ifdef ERTS_ENABLE_LOCK_COUNT
 	erts_mtx_init_x(&erts_pix_locks[i].u.mtx,
-			"pix_lock", make_small(i), 1);
+			"pix_lock", make_small(i));
 #else
 	erts_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock");
 #endif
@@ -1027,39 +1027,32 @@ erts_proc_lock_init(Process *p)
 #endif
 #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL
 
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    int do_lock_count = 1;
-#else
-    int do_lock_count = 0;
-#endif
-
-    erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id, do_lock_count);
+    erts_mtx_init_x(&p->lock.main, "proc_main", p->common.id);
     ethr_mutex_lock(&p->lock.main.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.main.lc);
 #endif
-    erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id, do_lock_count);
+    erts_mtx_init_x(&p->lock.link, "proc_link", p->common.id);
     ethr_mutex_lock(&p->lock.link.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.link.lc);
 #endif
-    erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id, do_lock_count);
+    erts_mtx_init_x(&p->lock.msgq, "proc_msgq", p->common.id);
     ethr_mutex_lock(&p->lock.msgq.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.msgq.lc);
 #endif
-    erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id, do_lock_count);
+    erts_mtx_init_x(&p->lock.btm, "proc_btm", p->common.id);
     ethr_mutex_lock(&p->lock.btm.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.btm.lc);
 #endif
-    erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id,
-		    do_lock_count);
+    erts_mtx_init_x(&p->lock.status, "proc_status", p->common.id);
     ethr_mutex_lock(&p->lock.status.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.status.lc);
 #endif
-    erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id, do_lock_count);
+    erts_mtx_init_x(&p->lock.trace, "proc_trace", p->common.id);
     ethr_mutex_lock(&p->lock.trace.mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     erts_lc_trylock(1, &p->lock.trace.lc);
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 713ed50b86..14be511f86 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -1065,7 +1065,7 @@ ERTS_GLB_INLINE void
 erts_smp_mtx_init_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
 {
 #ifdef ERTS_SMP
-    erts_mtx_init_x(mtx, name, extra, 1);
+    erts_mtx_init_x(mtx, name, extra);
 #endif
 }
 
@@ -1073,7 +1073,7 @@ ERTS_GLB_INLINE void
 erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, char *name, Eterm extra)
 {
 #ifdef ERTS_SMP
-    erts_mtx_init_locked_x(mtx, name, extra, 1);
+    erts_mtx_init_locked_x(mtx, name, extra);
 #endif
 }
 
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index eccd49f2a9..9e75f6fee5 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -479,14 +479,9 @@ ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void));
 ERTS_GLB_INLINE erts_tid_t erts_thr_self(void);
 ERTS_GLB_INLINE int erts_thr_getname(erts_tid_t tid, char *buf, size_t len);
 ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y);
-ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra,
-				     int enable_lcnt);
-ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra,
-					 Uint16 opt, int enable_lcnt);
-ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx,
-					    char *name,
-					    Eterm extra,
-					    int enable_lcnt);
+ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra);
+ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt);
+ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra);
 ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name);
 ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx);
@@ -2164,7 +2159,7 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y)
 }
 
 ERTS_GLB_INLINE void
-erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
+erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra)
 {
 #ifdef USE_THREADS
     int res = ethr_mutex_init(&mtx->mtx);
@@ -2174,17 +2169,13 @@ erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
     erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    if (enable_lcnt)
-      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
-    else
-      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra);
+    erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
 #endif
 #endif
 }
 
 ERTS_GLB_INLINE void
-erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt,
-		    int enable_lcnt)
+erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt)
 {
 #ifdef USE_THREADS
     int res = ethr_mutex_init(&mtx->mtx);
@@ -2194,17 +2185,14 @@ erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt,
     erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    if (enable_lcnt)
-      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
-    else
-      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX | opt, extra);
+    erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra);
 #endif
 #endif
 }
 
 
 ERTS_GLB_INLINE void
-erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt)
+erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra)
 {
 #ifdef USE_THREADS
     int res = ethr_mutex_init(&mtx->mtx);
@@ -2214,10 +2202,7 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra, int enable_lcnt
     erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra);
 #endif
 #ifdef ERTS_ENABLE_LOCK_COUNT
-    if (enable_lcnt)
-      erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
-    else
-      erts_lcnt_init_lock_x(&mtx->lcnt, NULL, ERTS_LCNT_LT_MUTEX, extra);
+    erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra);
 #endif
     ethr_mutex_lock(&mtx->mtx);
 #ifdef ERTS_ENABLE_LOCK_CHECK
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 4f131c74de..33b74f30b7 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -259,13 +259,11 @@ static ERTS_INLINE void port_init_instr(Port *prt
     ASSERT(prt->drv_ptr && prt->lock);
     if (!prt->drv_ptr->lock) {
 	char *lock_str = "port_lock";
-	erts_mtx_init_locked_x(prt->lock, lock_str, id,
 #ifdef ERTS_ENABLE_LOCK_COUNT
-			       (erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK)
-#else
-			       0
+        if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PORTLOCK))
+            lock_str = NULL;
 #endif
-			       );
+	erts_mtx_init_locked_x(prt->lock, lock_str, id);
     }
 #endif
     erts_port_task_init_sched(&prt->sched, id);
@@ -7107,7 +7105,7 @@ driver_pdl_create(ErlDrvPort dp)
 	return NULL;
     pdl = erts_alloc(ERTS_ALC_T_PORT_DATA_LOCK,
 		     sizeof(struct erl_drv_port_data_lock));
-    erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id, 1);
+    erts_mtx_init_x(&pdl->mtx, "port_data_lock", pp->common.id);
     pdl_init_refc(pdl);
     erts_port_inc_refc(pp);
     pdl->prt = pp;
@@ -8290,14 +8288,13 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
 	erts_mtx_init_x(drv->lock,
 			"driver_lock",
 #if defined(ERTS_ENABLE_LOCK_CHECK) || defined(ERTS_ENABLE_LOCK_COUNT)
-			    erts_atom_put((byte *) drv->name,
-					  sys_strlen(drv->name),
-					  ERTS_ATOM_ENC_LATIN1,
-					  1),
+                        erts_atom_put((byte *) drv->name,
+				       sys_strlen(drv->name),
+                                       ERTS_ATOM_ENC_LATIN1,
+                                       1)
 #else
-			NIL,
+			NIL
 #endif
-			1
 	    );
     }
 #endif
-- 
cgit v1.2.3


From 3eddb0f762de248d3230b38bc9d478bfbc8e7331 Mon Sep 17 00:00:00 2001
From: Erlang/OTP <otp@erlang.org>
Date: Wed, 7 Dec 2016 13:15:31 +0100
Subject: Update copyright-year

---
 erts/emulator/beam/erl_hl_timer.h | 2 +-
 erts/emulator/beam/erl_msacc.c    | 2 +-
 erts/emulator/beam/erl_msacc.h    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hl_timer.h b/erts/emulator/beam/erl_hl_timer.h
index 705be94532..9cdcd581a0 100644
--- a/erts/emulator/beam/erl_hl_timer.h
+++ b/erts/emulator/beam/erl_hl_timer.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  * 
- * Copyright Ericsson AB 2015. All Rights Reserved.
+ * Copyright Ericsson AB 2015-2016. All Rights Reserved.
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index 421445fbad..7ddf49937f 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014-2015. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
index ad7c8c5eee..4c8e1c8e22 100644
--- a/erts/emulator/beam/erl_msacc.h
+++ b/erts/emulator/beam/erl_msacc.h
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2014-2015. All Rights Reserved.
+ * Copyright Ericsson AB 2014-2016. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
-- 
cgit v1.2.3


From 2f8d59aa9e8a96d094172db339fd94aae45a90b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?H=C3=A5kan=20Mattsson?= <hm@tail-f.com>
Date: Wed, 2 Nov 2016 12:56:39 +0100
Subject: erts: Make depth of current_stacktrace configurable

The BIF process_info(Pid, current_stacktrace) truncates the
stacktrace. The old behavior was to truncate long stacktraces to max
8 items. And this was hard coded. Now it is truncated to the value of
system_flag(backtrace_depth) instead. The backtrace_depth defaults to
8, but is configurable.
---
 erts/emulator/beam/erl_bif_info.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 735aabbee3..88a052cad7 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -1672,11 +1672,11 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp)
     Eterm mfa;
     Eterm res = NIL;
 
-    depth = 8;
+    depth = erts_backtrace_depth;
     sz = offsetof(struct StackTrace, trace) + sizeof(BeamInstr *)*depth;
     s = (struct StackTrace *) erts_alloc(ERTS_ALC_T_TMP, sz);
     s->depth = 0;
-    if (rp->i) {
+    if (depth > 0 && rp->i) {
 	s->trace[s->depth++] = rp->i;
 	depth--;
     }
-- 
cgit v1.2.3


From 064a2630129caadbca58aaebe83f78735c1e374d Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 19 Dec 2016 14:06:12 +0100
Subject: Fix printout of timer data in crash dump

---
 erts/emulator/beam/erl_hl_timer.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index d29d079fc5..cc61f9abf1 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -2865,7 +2865,8 @@ btm_print(ErtsHLTimer *tmr, void *vbtmp)
 
     if (tmr->timeout <= btmp->now)
 	left = 0;
-    left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now);
+    else
+        left = ERTS_CLKTCKS_TO_MSEC(tmr->timeout - btmp->now);
 
     receiver = ((tmr->head.roflgs & ERTS_TMR_ROFLG_REG_NAME)
 		? tmr->receiver.name
-- 
cgit v1.2.3


From d44f2e85c058e10082427a354a7d65e9af607501 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 25 Nov 2016 15:03:33 +0100
Subject: Fix stack-trace generated by a traced process

---
 erts/emulator/beam/beam_emu.c | 45 ++++++++++++++++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 11 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 59a9ea1417..8c9bc1fa65 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5844,12 +5844,13 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
     s->depth = 0;
 
     /*
-     * If the failure was in a BIF other than 'error', 'exit' or
-     * 'throw', find the bif-table index and save the argument
-     * registers by consing up an arglist.
+     * If the failure was in a BIF other than 'error/1', 'error/2',
+     * 'exit/1' or 'throw/1', find the bif-table index and save the
+     * argument registers by consing up an arglist.
      */
-    if (bf != NULL && bf != error_1 && bf != error_2 &&
-	bf != exit_1 && bf != throw_1) {
+    if (bf != NULL && bf != error_1 && bf != error_2 && bf != exit_1
+	&& bf != throw_1 && bf != wrap_error_1 && bf != wrap_error_2
+	&& bf != wrap_exit_1 && bf != wrap_throw_1) {
         int i;
 	int a = 0;
 	for (i = 0; i < BIF_SIZE; i++) {
@@ -5945,12 +5946,30 @@ erts_save_stacktrace(Process* p, struct StackTrace* s, int depth)
 	    p->cp) {
 	    /* Cannot follow cp here - code may be unloaded */
 	    BeamInstr *cpp = p->cp;
+	    int trace_cp;
 	    if (cpp == beam_exception_trace || cpp == beam_return_trace) {
 		/* Skip return_trace parameters */
 		ptr += 2;
+		trace_cp = 1;
 	    } else if (cpp == beam_return_to_trace) {
 		/* Skip return_to_trace parameters */
 		ptr += 1;
+		trace_cp = 1;
+	    }
+	    else {
+		trace_cp = 0;
+	    }
+	    if (trace_cp && s->pc == cpp) {
+		/*
+		 * If process 'cp' points to a return/exception trace
+		 * instruction and 'cp' has been saved as 'pc' in
+		 * stacktrace, we need to update 'pc' in stacktrace
+		 * with the actual 'cp' located on the top of the
+		 * stack; otherwise, we will lose the top stackframe
+		 * when building the stack trace.
+		 */
+		ASSERT(is_CP(p->stop[0]));
+		s->pc = cp_val(p->stop[0]);
 	    }
 	}
 	while (ptr < STACK_START(p) && depth > 0) {
@@ -6070,12 +6089,15 @@ build_stacktrace(Process* c_p, Eterm exc) {
 	erts_set_current_function(&fi, s->current);
     }
 
+    depth = s->depth;
     /*
-     * If fi.current is still NULL, default to the initial function
+     * If fi.current is still NULL, and we have no
+     * stack at all, default to the initial function
      * (e.g. spawn_link(erlang, abs, [1])).
      */
     if (fi.current == NULL) {
-	erts_set_current_function(&fi, c_p->u.initial);
+	if (depth <= 0)
+	    erts_set_current_function(&fi, c_p->u.initial);
 	args = am_true; /* Just in case */
     } else {
 	args = get_args_from_exc(exc);
@@ -6085,10 +6107,9 @@ build_stacktrace(Process* c_p, Eterm exc) {
      * Look up all saved continuation pointers and calculate
      * needed heap space.
      */
-    depth = s->depth;
     stk = stkp = (FunctionInfo *) erts_alloc(ERTS_ALC_T_TMP,
 				      depth*sizeof(FunctionInfo));
-    heap_size = fi.needed + 2;
+    heap_size = fi.current ? fi.needed + 2 : 0;
     for (i = 0; i < depth; i++) {
 	erts_lookup_function_info(stkp, s->trace[i], 1);
 	if (stkp->current) {
@@ -6107,8 +6128,10 @@ build_stacktrace(Process* c_p, Eterm exc) {
 	res = CONS(hp, mfa, res);
 	hp += 2;
     }
-    hp = erts_build_mfa_item(&fi, hp, args, &mfa);
-    res = CONS(hp, mfa, res);
+    if (fi.current) {
+	hp = erts_build_mfa_item(&fi, hp, args, &mfa);
+	res = CONS(hp, mfa, res);
+    }
 
     erts_free(ERTS_ALC_T_TMP, (void *) stk);
     return res;
-- 
cgit v1.2.3


From 0612bcedf520388d58a9441e9e6490bf86bf5a4b Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 28 Nov 2016 18:13:38 +0100
Subject: Fix stactrace for apply on error/[1,2], exit/1, or throw/1

---
 erts/emulator/beam/beam_emu.c | 185 +++++++++++++++++++++++++++++++++++++-----
 1 file changed, 163 insertions(+), 22 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 8c9bc1fa65..66bccedd94 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1047,9 +1047,11 @@ static BeamInstr* handle_error(Process* c_p, BeamInstr* pc,
 			       Eterm* reg, BifFunction bf) NOINLINE;
 static BeamInstr* call_error_handler(Process* p, BeamInstr* ip,
 				     Eterm* reg, Eterm func) NOINLINE;
-static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE;
+static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity,
+			      BeamInstr *I, Uint offs) NOINLINE;
 static BeamInstr* apply(Process* p, Eterm module, Eterm function,
-			Eterm args, Eterm* reg) NOINLINE;
+			Eterm args, Eterm* reg,
+			BeamInstr *I, Uint offs) NOINLINE;
 static BeamInstr* call_fun(Process* p, int arity,
 			   Eterm* reg, Eterm args) NOINLINE;
 static BeamInstr* apply_fun(Process* p, Eterm fun,
@@ -3194,7 +3196,7 @@ do {						\
  OpCase(i_apply): {
      BeamInstr *next;
      HEAVY_SWAPOUT;
-     next = apply(c_p, r(0), x(1), x(2), reg);
+     next = apply(c_p, r(0), x(1), x(2), reg, NULL, 0);
      HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, I+1);
@@ -3208,7 +3210,7 @@ do {						\
  OpCase(i_apply_last_P): {
      BeamInstr *next;
      HEAVY_SWAPOUT;
-     next = apply(c_p, r(0), x(1), x(2), reg);
+     next = apply(c_p, r(0), x(1), x(2), reg, I, Arg(0));
      HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, (BeamInstr *) E[0]);
@@ -3223,7 +3225,7 @@ do {						\
  OpCase(i_apply_only): {
      BeamInstr *next;
      HEAVY_SWAPOUT;
-     next = apply(c_p, r(0), x(1), x(2), reg);
+     next = apply(c_p, r(0), x(1), x(2), reg, I, 0);
      HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_I(next);
@@ -3237,7 +3239,7 @@ do {						\
      BeamInstr *next;
 
      HEAVY_SWAPOUT;
-     next = fixed_apply(c_p, reg, Arg(0));
+     next = fixed_apply(c_p, reg, Arg(0), NULL, 0);
      HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, I+2);
@@ -3252,7 +3254,7 @@ do {						\
      BeamInstr *next;
 
      HEAVY_SWAPOUT;
-     next = fixed_apply(c_p, reg, Arg(0));
+     next = fixed_apply(c_p, reg, Arg(0), I, Arg(1));
      HEAVY_SWAPIN;
      if (next != NULL) {
 	 SET_CP(c_p, (BeamInstr *) E[0]);
@@ -6228,8 +6230,107 @@ apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity,
     return ep;
 }
 
+static ERTS_INLINE void
+apply_bif_error_adjustment(Process *p, Export *ep,
+			   Eterm *reg, Uint arity,
+			   BeamInstr *I, Uint stack_offset)
+{
+    /*
+     * I is only set when the apply is a tail call, i.e.,
+     * from the instructions i_apply_only, i_apply_last_P,
+     * and apply_last_IP.
+     */
+    if (I
+	&& ep->code[3] == (BeamInstr) em_apply_bif
+	&& (ep == bif_export[BIF_error_1]
+	    || ep == bif_export[BIF_error_2]
+	    || ep == bif_export[BIF_exit_1]
+	    || ep == bif_export[BIF_throw_1])) {
+	/*
+	 * We are about to tail apply one of the BIFs
+	 * erlang:error/1, erlang:error/2, erlang:exit/1,
+	 * or erlang:throw/1. Error handling of these BIFs is
+	 * special!
+	 *
+	 * We need 'p->cp' to point into the calling
+	 * function when handling the error after the BIF has
+	 * been applied. This in order to get the topmost
+	 * stackframe correct. Without the following adjustment,
+	 * 'p->cp' will point into the function that called
+	 * current function when handling the error. We add a
+	 * dummy stackframe in order to achive this.
+	 *
+	 * Note that these BIFs unconditionally will cause
+	 * an exception to be raised. That is, our modifications
+	 * of 'p->cp' as well as the stack will be corrected by
+	 * the error handling code.
+	 *
+	 * If we find an exception/return-to trace continuation
+	 * pointer as the topmost continuation pointer, we do not
+	 * need to do anything since the information already will
+	 * be available for generation of the stacktrace.
+	 */
+	int apply_only = stack_offset == 0;
+	BeamInstr *cpp;
+
+	if (apply_only) {
+	    ASSERT(p->cp != NULL);
+	    cpp = p->cp;
+	}
+	else {
+	    ASSERT(is_CP(p->stop[0]));
+	    cpp = cp_val(p->stop[0]);
+	}
+
+	if (cpp != beam_exception_trace
+	    && cpp != beam_return_trace
+	    && cpp != beam_return_to_trace) {
+	    Uint need = stack_offset /* bytes */ / sizeof(Eterm);
+	    if (need == 0)
+		need = 1; /* i_apply_only */
+	    if (p->stop - p->htop < need)
+		erts_garbage_collect(p, (int) need, reg, arity+1);
+	    p->stop -= need;
+
+	    if (apply_only) {
+		/*
+		 * Called from the i_apply_only instruction.
+		 *
+		 * 'p->cp' contains continuation pointer pointing
+		 * into the function that called current function.
+		 * We push that continuation pointer onto the stack,
+		 * and set 'p->cp' to point into current function.
+		 */
+
+		p->stop[0] = make_cp(p->cp);
+		p->cp = I;
+	    }
+	    else {
+		/*
+		 * Called from an i_apply_last_p, or apply_last_IP,
+		 * instruction.
+		 *
+		 * Calling instruction will after we return read
+		 * a continuation pointer from the stack and write
+		 * it to 'p->cp', and then remove the topmost
+		 * stackframe of size 'stack_offset'.
+		 *
+		 * We have sized the dummy-stackframe so that it
+		 * will be removed by the instruction we currently
+		 * are executing, and leave the stackframe that
+		 * normally would have been removed intact.
+		 *
+		 */
+		p->stop[0] = make_cp(I);
+	    }
+	}
+    }
+}
+
 static BeamInstr*
-apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
+apply(
+Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg,
+BeamInstr *I, Uint stack_offset)
 {
     int arity;
     Export* ep;
@@ -6254,21 +6355,54 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
 	return 0;
     }
 
-    /* The module argument may be either an atom or an abstract module
-     * (currently implemented using tuples, but this might change).
-     */
-    this = THE_NON_VALUE;
-    if (is_not_atom(module)) {
-	Eterm* tp;
+    while (1) {
+	Eterm m, f, a;
+	/* The module argument may be either an atom or an abstract module
+	 * (currently implemented using tuples, but this might change).
+	 */
+	this = THE_NON_VALUE;
+	if (is_not_atom(module)) {
+	    Eterm* tp;
+
+	    if (is_not_tuple(module)) goto error;
+	    tp = tuple_val(module);
+	    if (arityval(tp[0]) < 1) goto error;
+	    this = module;
+	    module = tp[1];
+	    if (is_not_atom(module)) goto error;
+	}
 
-        if (is_not_tuple(module)) goto error;
-        tp = tuple_val(module);
-        if (arityval(tp[0]) < 1) goto error;
-        this = module;
-        module = tp[1];
-        if (is_not_atom(module)) goto error;
+	if (module != am_erlang || function != am_apply)
+	    break;
+
+	/* Adjust for multiple apply of apply/3... */
+
+	a = args;
+	if (is_list(a)) {
+	    Eterm *consp = list_val(a);
+	    m = CAR(consp);
+	    a = CDR(consp);
+	    if (is_list(a)) {
+		consp = list_val(a);
+		f = CAR(consp);
+		a = CDR(consp);
+		if (is_list(a)) {
+		    consp = list_val(a);
+		    a = CAR(consp);
+		    if (is_nil(CDR(consp))) {
+			/* erlang:apply/3 */
+			module = m;
+			function = f;
+			args = a;
+			if (is_not_atom(f))
+			    goto error;
+			continue;
+		    }
+		}
+	    }
+	}
+	break; /* != erlang:apply/3 */
     }
-    
     /*
      * Walk down the 3rd parameter of apply (the argument list) and copy
      * the parameters to the x registers (reg[]). If the module argument
@@ -6306,12 +6440,14 @@ apply(Process* p, Eterm module, Eterm function, Eterm args, Eterm* reg)
     } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) {
 	save_calls(p, ep);
     }
+    apply_bif_error_adjustment(p, ep, reg, arity, I, stack_offset);
     DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep);
     return ep->addressv[erts_active_code_ix()];
 }
 
 static BeamInstr*
-fixed_apply(Process* p, Eterm* reg, Uint arity)
+fixed_apply(Process* p, Eterm* reg, Uint arity,
+	    BeamInstr *I, Uint stack_offset)
 {
     Export* ep;
     Eterm module;
@@ -6341,6 +6477,10 @@ fixed_apply(Process* p, Eterm* reg, Uint arity)
         if (is_not_atom(module)) goto error;
         ++arity;
     }
+
+    /* Handle apply of apply/3... */
+    if (module == am_erlang && function == am_apply && arity == 3)
+	return apply(p, reg[0], reg[1], reg[2], reg, I, stack_offset);
     
     /*
      * Get the index into the export table, or failing that the export
@@ -6355,6 +6495,7 @@ fixed_apply(Process* p, Eterm* reg, Uint arity)
     } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(p)) {
 	save_calls(p, ep);
     }
+    apply_bif_error_adjustment(p, ep, reg, arity, I, stack_offset);
     DTRACE_GLOBAL_CALL_FROM_EXPORT(p, ep);
     return ep->addressv[erts_active_code_ix()];
 }
-- 
cgit v1.2.3


From ca7e946af9c2fdc86c1c74259ee7b6881c5aec1e Mon Sep 17 00:00:00 2001
From: Mikael Pettersson <mikpelinux@gmail.com>
Date: Mon, 19 Dec 2016 11:17:05 +0100
Subject: erts: add erlang:system_info(atom_count)

---
 erts/emulator/beam/erl_bif_info.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 0e0842e139..9a3b78ae8d 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2863,6 +2863,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
     else if (ERTS_IS_ATOM_STR("atom_limit",BIF_ARG_1)) {
         BIF_RET(make_small(erts_get_atom_limit()));
     }
+    else if (ERTS_IS_ATOM_STR("atom_count",BIF_ARG_1)) {
+        BIF_RET(make_small(atom_table_size()));
+    }
     else if (ERTS_IS_ATOM_STR("tolerant_timeofday",BIF_ARG_1)) {
 	if (erts_has_time_correction()
 	    && erts_time_offset_state() == ERTS_TIME_OFFSET_FINAL) {
-- 
cgit v1.2.3


From 3eef3cd38c6beb48bfe4c2e26ede2ab2d107c146 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 28 Dec 2016 18:21:44 +0100
Subject: Fix crash due to GC of node entry on dirty scheduler

The dirty scheduler failed to set a timer. Dispatch this job
to an ordinary scheduler.
---
 erts/emulator/beam/erl_hl_timer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_hl_timer.c b/erts/emulator/beam/erl_hl_timer.c
index d29d079fc5..38bebc7576 100644
--- a/erts/emulator/beam/erl_hl_timer.c
+++ b/erts/emulator/beam/erl_hl_timer.c
@@ -2666,7 +2666,7 @@ erts_start_timer_callback(ErtsMonotonicTime tmo,
 				  tmo);
     twt = tmo < ERTS_TIMER_WHEEL_MSEC;
 
-    if (esdp)
+    if (esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp))
 	start_callback_timer(esdp,
 			     twt,
 			     timeout_pos,
-- 
cgit v1.2.3


From 32c71f85bd2d17a6c58f6d1fa450dfb8cf3de6cb Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 28 Dec 2016 18:27:54 +0100
Subject: Fix premature removal of process struct

Refc on process struct could reach zero while it was still
referred by dirty scheduler
---
 erts/emulator/beam/erl_process.c | 119 +++++++++++++++++++++++++++------------
 1 file changed, 82 insertions(+), 37 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b345c35a7e..6810210e4d 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -2060,6 +2060,7 @@ handle_thr_prgr_later_op(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int wait
 #endif
     for (lops = 0; lops < ERTS_MAX_THR_PRGR_LATER_OPS; lops++) {
 	ErtsThrPrgrLaterOp *lop = awdp->later_op.first;
+
 	if (!erts_thr_progress_has_reached_this(current, lop->later))
 	    return aux_work & ~ERTS_SSI_AUX_WORK_THR_PRGR_LATER_OP;
 	awdp->later_op.first = lop->next;
@@ -6234,6 +6235,12 @@ check_dirty_enqueue_in_prio_queue(Process *c_p,
     int queue;
     erts_aint32_t dact, max_qbit;
 
+    /* Do not enqueue free process... */
+    if (actual & ERTS_PSFLG_FREE) {
+	*newp &= ~ERTS_PSFLGS_DIRTY_WORK;
+        return ERTS_ENQUEUE_NOT;
+    }
+
     /* Termination should be done on an ordinary scheduler */
     if ((*newp) & ERTS_PSFLG_EXITING) {
 	*newp &= ~ERTS_PSFLGS_DIRTY_WORK;
@@ -6426,6 +6433,9 @@ select_enqueue_run_queue(int enqueue, int enq_prio, Process *p, erts_aint32_t st
 
 /*
  * schedule_out_process() return with c_rq locked.
+ *
+ * Return non-zero value if caller should decrease
+ * reference count on the process when done with it...
  */
 static ERTS_INLINE int
 schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
@@ -6480,12 +6490,18 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 
 	erts_smp_runq_lock(c_rq);
 
-	return 0;
-
+#if !defined(ERTS_SMP)
+        /* Decrement refc if process struct is free... */
+        return !!(n & ERTS_PSFLG_FREE);
+#else
+        /* Decrement refc if scheduled out from dirty scheduler... */
+	return !is_normal_sched;
+#endif
     }
     else {
 	Process* sched_p;
 
+        ASSERT(!(n & ERTS_PSFLG_FREE));
 	ASSERT(!(n & ERTS_PSFLG_SUSPENDED) || (n & (ERTS_PSFLG_ACTIVE_SYS
 						    | ERTS_PSFLG_DIRTY_ACTIVE_SYS)));
 
@@ -6501,11 +6517,14 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 
 	erts_smp_runq_lock(runq);
 
+        if (is_normal_sched && sched_p == p && ERTS_RUNQ_IX_IS_DIRTY(runq->ix))
+            erts_proc_inc_refc(p); /* Needs to be done before enqueue_process() */
+
 	/* Enqueue the process */
 	enqueue_process(runq, (int) enq_prio, sched_p);
 
 	if (runq == c_rq)
-	    return 1;
+	    return 0;
 
 	erts_smp_runq_unlock(runq);
 
@@ -6513,9 +6532,15 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
 
 	erts_smp_runq_lock(c_rq);
 
-	return 1;
+        /*
+         * Decrement refc if process is scheduled out by a
+         * dirty scheduler, and we have not just scheduled
+         * the process using the ordinary process struct
+         * on a dirty run-queue again...
+         */
+        return !is_normal_sched && (sched_p != p
+                                    || !ERTS_RUNQ_IX_IS_DIRTY(runq->ix));
     }
-
 }
 
 static ERTS_INLINE void
@@ -6530,8 +6555,17 @@ add2runq(int enqueue, erts_aint32_t prio,
     if (runq) {
 	Process *sched_p;
 
-	if (enqueue > 0)
+	if (enqueue > 0) {
 	    sched_p = proc;
+	    /*
+	     * Refc on process struct (i.e. true struct,
+	     * not proxy-struct) increased while in a
+	     * dirty run-queue or executing on a dirty
+	     * scheduler.
+	     */
+            if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix))
+                erts_proc_inc_refc(proc);
+        }
 	else {
 	    Process *pxy;
 
@@ -9587,40 +9621,45 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	esdp->reductions += reds;
 
-	/* schedule_out_process() returns with rq locked! */
-	schedule_out_process(rq, state, p, proxy_p, is_normal_sched);
-	proxy_p = NULL;
+        {
+            int dec_refc;
 
-	ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq,
-				      (int) ERTS_PSFLGS_GET_USR_PRIO(state),
-				      reds,
-				      actual_reds);
+            /* schedule_out_process() returns with rq locked! */
+            dec_refc = schedule_out_process(rq, state, p,
+                                            proxy_p, is_normal_sched);
+            proxy_p = NULL;
 
-	esdp->current_process = NULL;
+            ERTS_PROC_REDUCTIONS_EXECUTED(esdp, rq,
+                                          (int) ERTS_PSFLGS_GET_USR_PRIO(state),
+                                          reds,
+                                          actual_reds);
+
+            esdp->current_process = NULL;
 #ifdef ERTS_SMP
-	p->scheduler_data = NULL;
+            p->scheduler_data = NULL;
 #endif
 
-	erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN
-				 | ERTS_PROC_LOCK_STATUS
-				 | ERTS_PROC_LOCK_TRACE));
+            erts_smp_proc_unlock(p, (ERTS_PROC_LOCK_MAIN
+                                     | ERTS_PROC_LOCK_STATUS
+                                     | ERTS_PROC_LOCK_TRACE));
 
-        ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER);
+            ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_OTHER);
 
-	if (state & ERTS_PSFLG_FREE) {
-	    if (!is_normal_sched) {
-		ASSERT(p->flags & F_DELAYED_DEL_PROC);
-		erts_proc_dec_refc(p);
-	    }
-	    else {
 #ifdef ERTS_SMP
-		ASSERT(esdp->free_process == p);
-		esdp->free_process = NULL;
-#else
-		erts_proc_dec_refc(p);
+            if (state & ERTS_PSFLG_FREE) {
+                if (!is_normal_sched) {
+                    ASSERT(p->flags & F_DELAYED_DEL_PROC);
+                }
+                else {
+                    ASSERT(esdp->free_process == p);
+                    esdp->free_process = NULL;
+                }
+            }
 #endif
-	    }
-	}
+
+            if (dec_refc)
+                erts_proc_dec_refc(p);
+        }
 
 #ifdef ERTS_SMP
 	ASSERT(!esdp->free_process);
@@ -9896,8 +9935,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    if (!(state & ERTS_PSFLG_PROXY))
 		psflg_band_mask &= ~ERTS_PSFLG_IN_RUNQ;
 	    else {
+                Eterm pid = p->common.id;
 		proxy_p = p;
-		p = erts_proc_lookup_raw(proxy_p->common.id);
+		p = (is_normal_sched
+                     ? erts_proc_lookup_raw(pid)
+                     : erts_pid2proc_opt(NULL, 0, pid, 0,
+                                         ERTS_P2P_FLG_INC_REFC));
 		if (!p) {
 		    free_proxy_proc(proxy_p);
 		    proxy_p = NULL;
@@ -9929,6 +9972,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 					    | ERTS_PSFLG_FREE)))
 #ifdef ERTS_DIRTY_SCHEDULERS
 				| (((state & (ERTS_PSFLG_RUNNING
+
 					      | ERTS_PSFLG_FREE
 					      | ERTS_PSFLG_RUNNING_SYS
 					      | ERTS_PSFLG_EXITING))
@@ -9961,6 +10005,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 			    /* free and not queued by proxy */
 			    erts_proc_dec_refc(p);
 			}
+                        if (!is_normal_sched)
+                            erts_proc_dec_refc(p);
 			goto pick_next_process;
 		    }
 		    state = new;
@@ -13091,15 +13137,14 @@ erts_continue_exit_process(Process *p)
 	}
 
 #ifdef ERTS_DIRTY_SCHEDULERS
-	if (a & (ERTS_PSFLG_DIRTY_RUNNING
-		 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
+        if (a & (ERTS_PSFLG_DIRTY_RUNNING
+                 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
 	    p->flags |= F_DELAYED_DEL_PROC;
 	    delay_del_proc = 1;
 	    /*
-	     * The dirty scheduler will also decrease
-	     * refc when done...
+	     * The dirty scheduler decrease refc
+             * when done with the process...
 	     */
-	    erts_proc_inc_refc(p);
 	}
 #endif
 
-- 
cgit v1.2.3


From 88ae8fd6c952cd1f1edba1321de0dc97d8eafaf2 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 28 Dec 2016 18:56:18 +0100
Subject: Leave dirty work in dirty run-queues on multi scheduling block

---
 erts/emulator/beam/erl_process.c | 30 ++++++------------------------
 1 file changed, 6 insertions(+), 24 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 6810210e4d..fba81db1cd 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -7165,16 +7165,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
     ASSERT(sched_type != ERTS_SCHED_NORMAL || no != 1);
 
-    if (sched_type != ERTS_SCHED_NORMAL) {
-	if (erts_smp_mtx_trylock(&schdlr_sspnd.mtx) == EBUSY) {
-	    erts_smp_runq_unlock(esdp->run_queue);
-	    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-	    erts_smp_runq_lock(esdp->run_queue);
-	}
-	if (schdlr_sspnd.msb.ongoing)
-	    evacuate_run_queue(esdp->run_queue, &sbp);
+    if (sched_type != ERTS_SCHED_NORMAL)
 	erts_smp_runq_unlock(esdp->run_queue);
-    }
     else {
 	evacuate_run_queue(esdp->run_queue, &sbp);
 
@@ -7187,8 +7179,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 	sched_wall_time_change(esdp, 0);
 
-	erts_smp_mtx_lock(&schdlr_sspnd.mtx);
     }
+    erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
     flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
     if (flgs & ERTS_SSI_FLG_SUSPENDED) {
@@ -7304,24 +7296,14 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 	    while (1) {
 		ErtsMonotonicTime current_time;
-		erts_aint32_t qmask;
 		erts_aint32_t flgs;
 
-		qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue)
-			 & ERTS_RUNQ_FLGS_QMASK);
-
-		if (sched_type != ERTS_SCHED_NORMAL) {
-		    if (qmask) {
-			erts_smp_mtx_lock(&schdlr_sspnd.mtx);
-			erts_smp_runq_lock(esdp->run_queue);
-			if (schdlr_sspnd.msb.ongoing)
-			    evacuate_run_queue(esdp->run_queue, &sbp);
-			erts_smp_runq_unlock(esdp->run_queue);
-			erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
-		    }
+		if (sched_type != ERTS_SCHED_NORMAL)
 		    aux_work = 0;
-		}
 		else {
+		    erts_aint32_t qmask;
+		    qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue)
+			     & ERTS_RUNQ_FLGS_QMASK);
 
 		    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 
-- 
cgit v1.2.3


From efc331a04191a124ee7428c25f2ec05ded748faf Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 28 Dec 2016 19:20:29 +0100
Subject: Fix VM global GC info for dirty schedulers

---
 erts/emulator/beam/erl_gc.c         | 39 +++++++++++++++++++++++++++++++++++--
 erts/emulator/beam/erl_lock_check.c |  1 +
 2 files changed, 38 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index af799d09da..cb48b31b7e 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -176,6 +176,13 @@ typedef struct {
     erts_smp_atomic32_t refc;
 } ErtsGCInfoReq;
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+static struct {
+    erts_mtx_t mtx;
+    ErtsGCInfo info;
+} dirty_gc;
+#endif
+
 static ERTS_INLINE int
 gc_cost(Uint gc_moved_live_words, Uint resize_moved_words)
 {
@@ -259,6 +266,11 @@ erts_init_gc(void)
       init_gc_info(&esdp->gc_info);
     }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_smp_mtx_init(&dirty_gc.mtx, "dirty_gc_info");
+    init_gc_info(&dirty_gc.info);
+#endif
+
     init_gcireq_alloc();
 }
 
@@ -735,8 +747,19 @@ do_major_collection:
 	    monitor_large_heap(p);
     }
 
-    esdp->gc_info.garbage_cols++;
-    esdp->gc_info.reclaimed += reclaimed_now;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	erts_mtx_lock(&dirty_gc.mtx);
+	dirty_gc.info.garbage_cols++;
+	dirty_gc.info.reclaimed += reclaimed_now;
+	erts_mtx_unlock(&dirty_gc.mtx);
+    }
+    else
+#endif
+    {
+	esdp->gc_info.garbage_cols++;
+	esdp->gc_info.reclaimed += reclaimed_now;
+    }
     
     FLAGS(p) &= ~F_FORCE_GC;
     p->live_hf_end = ERTS_INVALID_HFRAG_PTR;
@@ -3017,6 +3040,18 @@ reply_gc_info(void *vgcirp)
 
     reclaimed = esdp->gc_info.reclaimed;
     garbage_cols = esdp->gc_info.garbage_cols;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    /*
+     * Add dirty schedulers info on requesting
+     * schedulers info
+     */
+    if (gcirp->req_sched == esdp->no) {
+	erts_mtx_lock(&dirty_gc.mtx);
+	reclaimed += dirty_gc.info.reclaimed;
+	garbage_cols += dirty_gc.info.garbage_cols;
+	erts_mtx_unlock(&dirty_gc.mtx);
+    }
+#endif
 
     sz = 0;
     hpp = NULL;
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 06266363b5..13a4b2cd93 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -129,6 +129,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
     {	"run_queue",				"address"		},
 #ifdef ERTS_DIRTY_SCHEDULERS
     {   "dirty_run_queue_sleep_list",		"address"		},
+    {	"dirty_gc_info",			NULL			},
 #endif
     {	"process_table",			NULL			},
     {	"cpu_info",				NULL			},
-- 
cgit v1.2.3


From 765e63dbffc4ec7f8cc3a7a1463151d6d863746f Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 30 Dec 2016 17:15:53 +0100
Subject: Multi scheduling block bug-fixes

---
 erts/emulator/beam/erl_process.c | 80 +++++++++++++++++++++++++---------------
 erts/emulator/beam/erl_process.h |  2 +-
 2 files changed, 52 insertions(+), 30 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b345c35a7e..58097ad42f 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -7055,12 +7055,18 @@ typedef struct {
 } ErtsSchdlrSspndResume;
 
 static void
-schdlr_sspnd_resume_proc(Eterm pid)
+schdlr_sspnd_resume_proc(ErtsSchedType sched_type, Eterm pid)
 {
-    Process *p = erts_pid2proc(NULL, 0, pid, ERTS_PROC_LOCK_STATUS);
+    Process *p;
+    p = erts_pid2proc_opt(NULL, 0, pid, ERTS_PROC_LOCK_STATUS,
+                          (sched_type != ERTS_SCHED_NORMAL
+                           ? ERTS_P2P_FLG_INC_REFC
+                           : 0));
     if (p) {
 	resume_process(p, ERTS_PROC_LOCK_STATUS);
 	erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS);
+        if (sched_type != ERTS_SCHED_NORMAL)
+            erts_proc_dec_refc(p);
     }
 }
 
@@ -7069,17 +7075,20 @@ schdlr_sspnd_resume_procs(ErtsSchedType sched_type,
 			  ErtsSchdlrSspndResume *resume)
 {
     if (is_internal_pid(resume->onln.chngr)) {
-	schdlr_sspnd_resume_proc(resume->onln.chngr);
+	schdlr_sspnd_resume_proc(sched_type,
+                                 resume->onln.chngr);
 	resume->onln.chngr = NIL;
     }
     if (is_internal_pid(resume->onln.nxt)) {
-	schdlr_sspnd_resume_proc(resume->onln.nxt);
+	schdlr_sspnd_resume_proc(sched_type,
+                                 resume->onln.nxt);
 	resume->onln.nxt = NIL;
     }
     while (resume->msb.chngrs) {
 	ErtsProcList *plp = resume->msb.chngrs;
 	resume->msb.chngrs = plp->next;
-	schdlr_sspnd_resume_proc(plp->pid);
+	schdlr_sspnd_resume_proc(sched_type,
+                                 plp->pid);
 	proclist_destroy(plp);
     }
 }
@@ -7198,15 +7207,18 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 			(void) erts_proclist_fetch(&msb[i]->chngq, &end_plp);
 			/* resume processes that initiated the multi scheduling block... */
 			plp = msb[i]->chngq;
-			while (plp) {
-			    erts_proclist_store_last(&msb[i]->blckrs,
-						     proclist_copy(plp));
-			    plp = plp->next;
-			}
-			if (end_plp)
+			if (plp) {
+                            ASSERT(end_plp);
+                            ASSERT(msb[i]->ongoing);
+                            do {
+                                erts_proclist_store_last(&msb[i]->blckrs,
+                                                         proclist_copy(plp));
+                                plp = plp->next;
+                            } while (plp);
 			    end_plp->next = resume.msb.chngrs;
-			resume.msb.chngrs = msb[i]->chngq;
-			msb[i]->chngq = NULL;
+                            resume.msb.chngrs = msb[i]->chngq;
+                            msb[i]->chngq = NULL;
+                        }
 		    }
 		}
 	    }
@@ -7415,12 +7427,23 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 	schdlr_sspnd_inc_nscheds(&schdlr_sspnd.active, sched_type);
 	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
-	if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
-	    && schdlr_sspnd.online == schdlr_sspnd.active) {
-	    erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
-					    ~ERTS_SCHDLR_SSPND_CHNG_MSB);
-	}
-
+        if (changing) {
+            if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
+                && !schdlr_sspnd.msb.ongoing
+                && schdlr_sspnd.online == schdlr_sspnd.active) {
+                erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+                                                ~ERTS_SCHDLR_SSPND_CHNG_MSB);
+            }
+            if ((changing & ERTS_SCHDLR_SSPND_CHNG_NMSB)
+                && !schdlr_sspnd.nmsb.ongoing
+                && (schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+                                             ERTS_SCHED_NORMAL)
+                    == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+                                                ERTS_SCHED_NORMAL))) {
+                erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
+                                                ~ERTS_SCHDLR_SSPND_CHNG_NMSB);
+            }
+        }
 	ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type));
 	ASSERT((sched_type == ERTS_SCHED_NORMAL
 		? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)
@@ -7553,7 +7576,7 @@ abort_sched_onln_chng_waitq(Process *p)
     erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 
     if (is_internal_pid(resume))
-	schdlr_sspnd_resume_proc(resume);
+	schdlr_sspnd_resume_proc(ERTS_SCHED_NORMAL, resume);
 }
 
 ErtsSchedSuspendResult
@@ -7951,21 +7974,20 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
     }
     else {  /* ------ UNBLOCK ------ */
 	if (p->flags & have_blckd_flg) {
-	    ErtsProcList *plps[2];
+	    ErtsProcList **plpps[3] = {0};
 	    ErtsProcList *plp;
-	    int limit = 0;
 
-	    plps[limit++] = erts_proclist_peek_first(msbp->blckrs);
-	    if (all)
-		plps[limit++] = erts_proclist_peek_first(msbp->chngq);
+            plpps[0] = &msbp->blckrs;
+            if (all)
+                plpps[1] = &msbp->chngq;
 
-	    for (ix = 0; ix < limit; ix++) {
-		plp = plps[ix];
+	    for (ix = 0; plpps[ix]; ix++) {
+                plp = erts_proclist_peek_first(*plpps[ix]);
 		while (plp) {
 		    ErtsProcList *tmp_plp = plp;
-		    plp = erts_proclist_peek_next(msbp->blckrs, plp);
+		    plp = erts_proclist_peek_next(*plpps[ix], plp);
 		    if (erts_proclist_same(tmp_plp, p)) {
-			erts_proclist_remove(&msbp->blckrs, tmp_plp);
+			erts_proclist_remove(plpps[ix], tmp_plp);
 			proclist_destroy(tmp_plp);
 			if (!all)
 			    break;
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 9f7084c127..68fbb10602 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1684,7 +1684,7 @@ ERTS_GLB_INLINE ErtsProcList *erts_proclist_fetch_first(ErtsProcList **list)
 	return NULL;
     else {
 	ErtsProcList *res = *list;
-	if (res == *list)
+	if (res->next == *list)
 	    *list = NULL;
 	else
 	    *list = res->next;
-- 
cgit v1.2.3


From 67355d1f2f4efb61ef9345751c15c0263cdc97a0 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 2 Jan 2017 13:09:47 +0100
Subject: Fix faulty assertion

---
 erts/emulator/beam/erl_process.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b345c35a7e..d15430b902 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -8604,8 +8604,15 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 	     * from being selected for normal execution regardless
 	     * of locks held or not held on it...
 	     */
-	    ASSERT(!((ERTS_PSFLG_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS)
-		     & erts_smp_atomic32_read_nob(&rp->state)));
+#ifdef DEBUG
+	    {
+		erts_aint32_t state;
+		state = erts_smp_atomic32_read_nob(&rp->state);
+		ASSERT((state & ERTS_PSFLG_PENDING_EXIT)
+		       || !(state & (ERTS_PSFLG_RUNNING
+				     | ERTS_PSFLG_DIRTY_RUNNING_SYS)));
+	    }
+#endif
 
 	    if (!suspend)
 		resume_process(rp, pid_locks|ERTS_PROC_LOCK_STATUS);
-- 
cgit v1.2.3


From 5f4d538b2abfd1ab3f495879996edaa004309623 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 5 Jan 2017 20:06:04 +0100
Subject: erts: Assert sufficient space need after GC

---
 erts/emulator/beam/erl_gc.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index af799d09da..50f09277fa 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -765,6 +765,7 @@ do_major_collection:
 
     ASSERT(!p->mbuf);
     ASSERT(!ERTS_IS_GC_DESIRED(p));
+    ASSERT(need <= HEAP_LIMIT(p) - HEAP_TOP(p));
 
     return reds;
 }
-- 
cgit v1.2.3


From 287edce0090a14fdd88a9e10d13327e74e52831a Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 5 Jan 2017 20:43:49 +0100
Subject: erts: Fix GC tracing to use temp heap

Can't use HAlloc as it might consume part of callers 'need'.
---
 erts/emulator/beam/erl_gc.c    | 3 ++-
 erts/emulator/beam/erl_nif.c   | 4 ++--
 erts/emulator/beam/erl_trace.c | 5 ++++-
 3 files changed, 8 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 50f09277fa..b05209402d 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -602,7 +602,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     Uint reclaimed_now = 0;
     Eterm gc_trace_end_tag;
     int reds;
-    ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */
+    ErtsMonotonicTime start_time;
     ErtsSchedulerData *esdp;
     erts_aint32_t state;
     ERTS_MSACC_PUSH_STATE_M();
@@ -610,6 +610,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
     DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
 #endif
 
+    ERTS_UNDEF(start_time, 0);
     ERTS_CHK_MBUF_SZ(p);
 
     ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls)
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 6b265a8b80..19ce0f6965 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -3561,8 +3561,8 @@ Eterm erts_nif_call_function(Process *p, Process *tracee,
 #endif
     if (p) {
         /* This is almost a normal nif call like in beam_emu,
-           except that any heap fragment created in the nif will be
-           discarded without checking if anything in it is live.
+           except that any heap consumed by the nif will be
+           released without checking if anything in it is live.
            This is because we cannot do a GC here as we don't know
            the number of live registers that have to be preserved.
            This means that any heap part of the returned term may
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 8c84303997..ac9e91e31f 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -1436,6 +1436,7 @@ void
 trace_gc(Process *p, Eterm what, Uint size, Eterm msg)
 {
     ErtsTracerNif *tnif = NULL;
+    Eterm* o_hp = NULL;
     Eterm* hp;
     Uint sz = 0;
     Eterm tup;
@@ -1446,7 +1447,7 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg)
         if (is_non_value(msg)) {
 
             (void) erts_process_gc_info(p, &sz, NULL, 0, 0);
-            hp = HAlloc(p, sz + 3 + 2);
+            o_hp = hp = erts_alloc(ERTS_ALC_T_TMP, (sz + 3 + 2) * sizeof(Eterm));
 
             msg = erts_process_gc_info(p, NULL, &hp, 0, 0);
             tup = TUPLE2(hp, am_wordsize, make_small(size)); hp += 3;
@@ -1455,6 +1456,8 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg)
 
         send_to_tracer_nif(p, &p->common, p->common.id, tnif, TRACE_FUN_T_GC,
                            what, msg, THE_NON_VALUE, am_true);
+        if (o_hp)
+            erts_free(ERTS_ALC_T_TMP, o_hp);
     }
 }
 
-- 
cgit v1.2.3


From 5bf38ca930b75189cc83ed01e4a34eeb6a0892d2 Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Mon, 9 Jan 2017 13:57:34 +0100
Subject: erts: Fix error report message for erlang:delete/1

---
 erts/emulator/beam/beam_bif_load.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index ea1323d651..644fd9fbf9 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -658,7 +658,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1)
 	}
 	else if (modp->old.code_hdr) {
 	    erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf();
-	    erts_dsprintf(dsbufp, "Module %T must be purged before loading\n",
+	    erts_dsprintf(dsbufp, "Module %T must be purged before deleting\n",
 			  BIF_ARG_1);
 	    erts_send_error_to_logger(BIF_P->group_leader, dsbufp);
 	    ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG);
-- 
cgit v1.2.3


From 8df00f51a4a2c0c6788b3dcf3c6be56c50e44461 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 4 Jan 2017 17:51:54 +0100
Subject: Fix issues with abandoned heap

---
 erts/emulator/beam/beam_bif_load.c    |  11 +++
 erts/emulator/beam/erl_gc.c           | 158 +++++++++++++++++++---------------
 erts/emulator/beam/erl_message.c      |   2 +-
 erts/emulator/beam/erl_process_dump.c |   3 +
 4 files changed, 103 insertions(+), 71 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index ea1323d651..1307fe22b5 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -995,6 +995,12 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed
     if (any_heap_refs(c_p->heap, c_p->htop, literals, lit_bsize))
 	goto literal_gc;
     *redsp += 1;
+    if (c_p->abandoned_heap) {
+	if (any_heap_refs(c_p->abandoned_heap, c_p->abandoned_heap + c_p->heap_sz,
+			  literals, lit_bsize))
+	    goto literal_gc;
+	*redsp += 1;
+    }
     if (any_heap_refs(c_p->old_heap, c_p->old_htop, literals, lit_bsize))
 	goto literal_gc;
 
@@ -1295,6 +1301,11 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls
 #endif
 	if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize))
 	    goto try_literal_gc;
+	if (rp->abandoned_heap) {
+	    if (any_heap_refs(rp->abandoned_heap, rp->abandoned_heap + rp->heap_sz,
+			      literals, lit_bsize))
+		goto try_literal_gc;
+	}
 	if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize))
 	    goto try_literal_gc;
 
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index af799d09da..561e7fbdee 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -110,7 +110,7 @@ typedef struct {
 
 static Uint setup_rootset(Process*, Eterm*, int, Rootset*);
 static void cleanup_rootset(Rootset *rootset);
-static void remove_message_buffers(Process* p);
+static void deallocate_previous_young_generation(Process *c_p);
 static Eterm *full_sweep_heaps(Process *p,
 			       int hibernate,
 			       Eterm *n_heap, Eterm* n_htop,
@@ -153,7 +153,7 @@ static void init_gc_info(ErtsGCInfo *gcip);
 static Uint64 next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz);
 
 #ifdef HARDDEBUG
-static void disallow_heap_frag_ref_in_heap(Process* p);
+static void disallow_heap_frag_ref_in_heap(Process *p, Eterm *heap, Eterm *htop);
 static void disallow_heap_frag_ref_in_old_heap(Process* p);
 #endif
 
@@ -477,24 +477,26 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need, int
     hsz = ssz + need + ERTS_DELAY_GC_EXTRA_FREE;
 
     hfrag = new_message_buffer(hsz);
-    hfrag->next = p->mbuf;
-    p->mbuf = hfrag;
-    p->mbuf_sz += hsz;
     p->heap = p->htop = &hfrag->mem[0];
     p->hend = hend = &hfrag->mem[hsz];
     p->stop = stop = hend - ssz;
     sys_memcpy((void *) stop, (void *) orig_stop, ssz * sizeof(Eterm));
 
     if (p->abandoned_heap) {
-	/* Active heap already in a fragment; adjust it... */
-	ErlHeapFragment *hfrag = ((ErlHeapFragment *)
-				  (((char *) orig_heap)
-				   - offsetof(ErlHeapFragment, mem)));
-	Uint unused = orig_hend - orig_htop;
-	ASSERT(hfrag->used_size == hfrag->alloc_size);
-	ASSERT(hfrag->used_size >= unused);
-	hfrag->used_size -= unused;
-	p->mbuf_sz -= unused;
+	/*
+         * Active heap already in a fragment; adjust it and
+         * save it into mbuf list...
+         */
+        ErlHeapFragment *hfrag = ((ErlHeapFragment *)
+                                  (((char *) orig_heap)
+                                   - offsetof(ErlHeapFragment, mem)));
+        Uint used = orig_htop - orig_heap;
+        hfrag->used_size = used;
+        p->mbuf_sz += used;
+        ASSERT(hfrag->used_size <= hfrag->alloc_size);
+        ASSERT(!hfrag->off_heap.first && !hfrag->off_heap.overhead);
+        hfrag->next = p->mbuf;
+        p->mbuf = hfrag;
     }
     else {
 	/* Do not leave a hole in the abandoned heap... */
@@ -556,17 +558,14 @@ young_gen_usage(Process *p)
 	}
     }
 
+    hsz += p->htop - p->heap;
     aheap = p->abandoned_heap;
-    if (!aheap)
-	hsz += p->htop - p->heap;
-    else {
+    if (aheap) {
 	/* used in orig heap */
 	if (p->flags & F_ABANDONED_HEAP_USE)
 	    hsz += aheap[p->heap_sz-1];
 	else
 	    hsz += p->heap_sz;
-	/* Remove unused part in latest fragment */
-	hsz -= p->hend - p->htop;
     }
     return hsz;
 }
@@ -789,6 +788,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
 	   >= erts_proc_sched_data(p)->virtual_reds);
 }
 
+
 /*
  * Place all living data on a the new heap; deallocate any old heap.
  * Meant to be used by hibernate/3.
@@ -835,11 +835,11 @@ erts_garbage_collect_hibernate(Process* p)
 			    p->arg_reg,
 			    p->arity);
 
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
-		   (p->abandoned_heap
-		    ? p->abandoned_heap
-		    : p->heap),
-		   p->heap_sz * sizeof(Eterm));
+#ifdef HARDDEBUG
+    disallow_heap_frag_ref_in_heap(p, heap, htop);
+#endif
+
+    deallocate_previous_young_generation(p);
 
     p->heap = heap;
     p->high_water = htop;
@@ -874,8 +874,6 @@ erts_garbage_collect_hibernate(Process* p)
     sys_memcpy((void *) heap, (void *) p->heap, actual_size*sizeof(Eterm));
     ERTS_HEAP_FREE(ERTS_ALC_T_TMP_HEAP, p->heap, p->heap_sz*sizeof(Eterm));
 
-    remove_message_buffers(p);
-
     p->stop = p->hend = heap + heap_size;
 
     offs = heap - p->heap;
@@ -1494,22 +1492,16 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
     }
 #endif
 
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
-		   (p->abandoned_heap
-		    ? p->abandoned_heap
-		    : HEAP_START(p)),
-		   HEAP_SIZE(p) * sizeof(Eterm));
-    p->abandoned_heap = NULL;
-    p->flags &= ~F_ABANDONED_HEAP_USE;
+#ifdef HARDDEBUG
+    disallow_heap_frag_ref_in_heap(p, n_heap, n_htop);
+#endif
+
+    deallocate_previous_young_generation(p);
+
     HEAP_START(p) = n_heap;
     HEAP_TOP(p) = n_htop;
     HEAP_SIZE(p) = new_sz;
     HEAP_END(p) = n_heap + new_sz;
-
-#ifdef HARDDEBUG
-    disallow_heap_frag_ref_in_heap(p);
-#endif
-    remove_message_buffers(p);
 }
 
 /*
@@ -1598,13 +1590,12 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
     }
 #endif
 
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
-		   (p->abandoned_heap
-		    ? p->abandoned_heap
-		    : HEAP_START(p)),
-		   p->heap_sz * sizeof(Eterm));
-    p->abandoned_heap = NULL;
-    p->flags &= ~F_ABANDONED_HEAP_USE;
+#ifdef HARDDEBUG
+    disallow_heap_frag_ref_in_heap(p, n_heap, n_htop);
+#endif
+
+    deallocate_previous_young_generation(p);
+
     HEAP_START(p) = n_heap;
     HEAP_TOP(p) = n_htop;
     HEAP_SIZE(p) = new_sz;
@@ -1613,11 +1604,6 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
 
     HIGH_WATER(p) = HEAP_TOP(p);
 
-#ifdef HARDDEBUG
-    disallow_heap_frag_ref_in_heap(p);
-#endif
-    remove_message_buffers(p);
-
     if (p->flags & F_ON_HEAP_MSGQ)
 	move_msgq_to_heap(p);
 
@@ -1770,22 +1756,56 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
     return adjusted;
 }
 
-/*
- * Remove all message buffers.
- */
 static void
-remove_message_buffers(Process* p)
+deallocate_previous_young_generation(Process *c_p)
 {
-    if (MBUF(p) != NULL) {
-	free_message_buffer(MBUF(p));
-	MBUF(p) = NULL;
+    Eterm *orig_heap;
+
+    if (!c_p->abandoned_heap) {
+	orig_heap = c_p->heap;
+	ASSERT(!(c_p->flags & F_ABANDONED_HEAP_USE));
+    }
+    else {
+	ErlHeapFragment *hfrag;
+
+	orig_heap = c_p->abandoned_heap;
+	c_p->abandoned_heap = NULL;
+	c_p->flags &= ~F_ABANDONED_HEAP_USE;
+
+	/*
+	 * Temporary heap located in heap fragment
+	 * only referred to by 'c_p->heap'. Add it to
+	 * 'c_p->mbuf' list and deallocate it as any
+	 * other heap fragment...
+	 */
+	hfrag = ((ErlHeapFragment *)
+		 (((char *) c_p->heap)
+		  - offsetof(ErlHeapFragment, mem)));
+
+	ASSERT(!hfrag->off_heap.first);
+	ASSERT(!hfrag->off_heap.overhead);
+	ASSERT(!hfrag->next);
+	ASSERT(c_p->htop - c_p->heap <= hfrag->alloc_size);
+
+	hfrag->next = c_p->mbuf;
+	c_p->mbuf = hfrag;
+    }
+
+    if (c_p->mbuf) {
+	free_message_buffer(c_p->mbuf);
+	c_p->mbuf = NULL;
     }    
-    if (p->msg_frag) {
-	erts_cleanup_messages(p->msg_frag);
-	p->msg_frag = NULL;
+    if (c_p->msg_frag) {
+	erts_cleanup_messages(c_p->msg_frag);
+	c_p->msg_frag = NULL;
     }
-    MBUF_SIZE(p) = 0;    
+    c_p->mbuf_sz = 0;  
+
+    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
+		   orig_heap,
+		   c_p->heap_sz * sizeof(Eterm));
 }
+
 #ifdef HARDDEBUG
 
 /*
@@ -1797,19 +1817,15 @@ remove_message_buffers(Process* p)
  */
 
 static void
-disallow_heap_frag_ref_in_heap(Process* p)
+disallow_heap_frag_ref_in_heap(Process *p, Eterm *heap, Eterm *htop)
 {
     Eterm* hp;
-    Eterm* htop;
-    Eterm* heap;
     Uint heap_size;
 
     if (p->mbuf == 0) {
 	return;
     }
 
-    htop = p->htop;
-    heap = p->heap;
     heap_size = (htop - heap)*sizeof(Eterm);
 
     hp = heap;
@@ -3297,13 +3313,15 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop)
     ErtsMessage* mp;
     Eterm *htop, *heap;
 
-    if (p->abandoned_heap)
+    if (p->abandoned_heap) {
 	ERTS_GET_ORIG_HEAP(p, heap, htop);
-    else {
-	heap = p->heap;
-	htop = real_htop ? real_htop : HEAP_TOP(p);
+	if (heap <= ptr && ptr < htop)
+	    return 1;
     }
 
+    heap = p->heap;
+    htop = real_htop ? real_htop : HEAP_TOP(p);
+
     if (OLD_HEAP(p) && (OLD_HEAP(p) <= ptr && ptr < OLD_HEND(p))) {
         return 1;
     }
diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c
index e4c696ae3b..f45e6974cd 100644
--- a/erts/emulator/beam/erl_message.c
+++ b/erts/emulator/beam/erl_message.c
@@ -193,7 +193,7 @@ free_message_buffer(ErlHeapFragment* bp)
 
 	erts_cleanup_offheap(&bp->off_heap);
 	ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp,
-		       ERTS_HEAP_FRAG_SIZE(bp->size));	
+		       ERTS_HEAP_FRAG_SIZE(bp->alloc_size));	
 	bp = next_bp;
     }while (bp != NULL);
 }
diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c
index d8bb00e8c6..a19db74763 100644
--- a/erts/emulator/beam/erl_process_dump.c
+++ b/erts/emulator/beam/erl_process_dump.c
@@ -90,9 +90,12 @@ Uint erts_process_memory(Process *p, int incl_msg_inq) {
   erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size);
   erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size);
   size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm);
+  if (p->abandoned_heap)
+      size += (p->hend - p->heap) * sizeof(Eterm);
   if (p->old_hend && p->old_heap)
     size += (p->old_hend - p->old_heap) * sizeof(Eterm);
 
+
   size += p->msg.len * sizeof(ErtsMessage);
 
   for (mp = p->msg.first; mp; mp = mp->next)
-- 
cgit v1.2.3


From a0d79ee6eeca9d99fbb644717ee2df57b824a527 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 10 Jan 2017 20:40:56 +0100
Subject: Fix call_time trace for NIFs

---
 erts/emulator/beam/beam_emu.c | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 59a9ea1417..6ac156a15d 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -3550,18 +3550,11 @@ do {						\
 	    BifFunction vbf;
 	    ErlHeapFragment *live_hf_end;
 
-            if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) {
-                /* If we have run out of reductions, we do a context
-                   switch before calling the nif */
-                goto context_switch;
-            }
-
 	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
 
 	    DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]);
 	    c_p->current = I-3; /* current and vbf set to please handle_error */ 
-	    SWAPOUT;
-	    c_p->fcalls = FCALLS - 1;
+	    HEAVY_SWAPOUT;
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
 	    bif_nif_arity = I[-1];
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
-- 
cgit v1.2.3


From c057b468ee7535f199aa01301ff93e059fad4b07 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 11 Jan 2017 18:18:11 +0100
Subject: Fix call time tracing with dirty schedulers

---
 erts/emulator/beam/beam_bp.c        | 60 +++++++++++++++++++++++++++++++++----
 erts/emulator/beam/beam_bp.h        |  6 ++--
 erts/emulator/beam/erl_bif_trace.c  |  7 +++++
 erts/emulator/beam/erl_lock_check.c |  1 +
 4 files changed, 64 insertions(+), 10 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index bbb2e4f34f..0df2df0eaa 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -74,6 +74,9 @@ extern BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */
 
 erts_smp_atomic32_t erts_active_bp_index;
 erts_smp_atomic32_t erts_staging_bp_index;
+#ifdef ERTS_DIRTY_SCHEDULERS
+erts_smp_mtx_t erts_dirty_bp_ix_mtx;
+#endif
 
 /*
  * Inlined helpers
@@ -85,6 +88,31 @@ get_mtime(Process *c_p)
     return erts_get_monotonic_time(erts_proc_sched_data(c_p));
 }
 
+static ERTS_INLINE Uint32
+acquire_bp_sched_ix(Process *c_p)
+{
+    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
+    ASSERT(esdp);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx);
+        return (Uint32) erts_no_schedulers;
+    }
+#endif
+    return (Uint32) esdp->no - 1;
+}
+
+static ERTS_INLINE void
+release_bp_sched_ix(Uint32 ix)
+{
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ix == (Uint32) erts_no_schedulers)
+        erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx);
+#endif
+}
+
+
+
 /* *************************************************************************
 ** Local prototypes
 */
@@ -135,6 +163,9 @@ void
 erts_bp_init(void) {
     erts_smp_atomic32_init_nob(&erts_active_bp_index, 0);
     erts_smp_atomic32_init_nob(&erts_staging_bp_index, 1);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_smp_mtx_init(&erts_dirty_bp_ix_mtx, "dirty_break_point_index");
+#endif
 }
 
 
@@ -973,6 +1004,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
     bp_data_time_item_t sitem, *item = NULL;
     bp_time_hash_t *h = NULL;
     BpDataTime *pbdt = NULL;
+    Uint32 six = acquire_bp_sched_ix(c_p);
 
     ASSERT(c_p);
     ASSERT(erts_smp_atomic32_read_acqb(&c_p->state) & (ERTS_PSFLG_RUNNING
@@ -980,7 +1012,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
 
     /* get previous timestamp and breakpoint
      * from the process psd  */
-
+    
     pbt = ERTS_PROC_GET_CALL_TIME(c_p);
     time = get_mtime(c_p);
 
@@ -1006,7 +1038,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
 
 	/* if null then the breakpoint was removed */
 	if (pbdt) {
-	    h = &(pbdt->hash[bp_sched2ix_proc(c_p)]);
+	    h = &(pbdt->hash[six]);
 
 	    ASSERT(h);
 	    ASSERT(h->item);
@@ -1027,7 +1059,7 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
 
     /* this breakpoint */
     ASSERT(bdt);
-    h = &(bdt->hash[bp_sched2ix_proc(c_p)]);
+    h = &(bdt->hash[six]);
 
     ASSERT(h);
     ASSERT(h->item);
@@ -1041,6 +1073,8 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt)
 
     pbt->pc = I;
     pbt->time = time;
+
+    release_bp_sched_ix(six);
 }
 
 void
@@ -1051,6 +1085,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc)
     bp_data_time_item_t sitem, *item = NULL;
     bp_time_hash_t *h = NULL;
     BpDataTime *pbdt = NULL;
+    Uint32 six = acquire_bp_sched_ix(p);
 
     ASSERT(p);
     ASSERT(erts_smp_atomic32_read_acqb(&p->state) & (ERTS_PSFLG_RUNNING
@@ -1071,6 +1106,7 @@ erts_trace_time_return(Process *p, BeamInstr *pc)
      */
 
     if (pbt) {
+
 	/* might have been removed due to
 	 * trace_pattern(false)
 	 */
@@ -1085,7 +1121,8 @@ erts_trace_time_return(Process *p, BeamInstr *pc)
 
 	/* beware, the trace_pattern might have been removed */
 	if (pbdt) {
-	    h = &(pbdt->hash[bp_sched2ix_proc(p)]);
+
+	    h = &(pbdt->hash[six]);
 
 	    ASSERT(h);
 	    ASSERT(h->item);
@@ -1096,11 +1133,15 @@ erts_trace_time_return(Process *p, BeamInstr *pc)
 	    } else {
 		BP_TIME_ADD(item, &sitem);
 	    }
+
 	}
 
 	pbt->pc = pc;
 	pbt->time = time;
+
     }
+
+    release_bp_sched_ix(six);
 }
 
 int 
@@ -1353,6 +1394,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) {
     bp_data_time_item_t sitem, *item = NULL;
     bp_time_hash_t *h = NULL;
     BpDataTime *pbdt = NULL;
+    Uint32 six = acquire_bp_sched_ix(p);
 
     ASSERT(p);
 
@@ -1375,7 +1417,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) {
 		sitem.pid   = p->common.id;
 		sitem.count = 0;
 
-		h = &(pbdt->hash[bp_sched2ix_proc(p)]);
+		h = &(pbdt->hash[six]);
 
 		ASSERT(h);
 		ASSERT(h->item);
@@ -1401,6 +1443,8 @@ void erts_schedule_time_break(Process *p, Uint schedule) {
 	    break;
 	}
     } /* pbt */
+
+    release_bp_sched_ix(six);
 }
 
 /* *************************************************************************
@@ -1517,7 +1561,11 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags,
 	ASSERT((bp->flags & ERTS_BPF_TIME_TRACE) == 0);
 	bdt = Alloc(sizeof(BpDataTime));
 	erts_refc_init(&bdt->refc, 1);
-	bdt->n = erts_no_total_schedulers;
+#ifdef ERTS_DIRTY_SCHEDULERS
+	bdt->n = erts_no_schedulers + 1;
+#else
+	bdt->n = erts_no_schedulers;
+#endif
 	bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n));
 	for (i = 0; i < bdt->n; i++) {
 	    bp_hash_init(&(bdt->hash[i]), 32);
diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h
index 7206ef471a..4743e4fc2f 100644
--- a/erts/emulator/beam/beam_bp.h
+++ b/erts/emulator/beam/beam_bp.h
@@ -79,10 +79,8 @@ typedef struct generic_bp {
 #define ERTS_BP_CALL_TIME_SCHEDULE_OUT     (1)
 #define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2)
 
-#ifdef ERTS_SMP
-#define bp_sched2ix_proc(p) (erts_proc_sched_data(p)->thr_id - 1)
-#else
-#define bp_sched2ix_proc(p) (0)
+#ifdef ERTS_DIRTY_SCHEDULERS
+extern erts_smp_mtx_t erts_dirty_bp_ix_mtx;
 #endif
 
 enum erts_break_op{
diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c
index 96275eb228..0627526d7e 100644
--- a/erts/emulator/beam/erl_bif_trace.c
+++ b/erts/emulator/beam/erl_bif_trace.c
@@ -1062,9 +1062,16 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key)
 	erts_smp_thr_progress_block();
     }
 #endif
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_smp_mtx_lock(&erts_dirty_bp_ix_mtx);
+#endif
+
 
     r = function_is_traced(p, mfa, &ms, &ms_meta, &meta, &count, &call_time);
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    erts_smp_mtx_unlock(&erts_dirty_bp_ix_mtx);
+#endif
 #ifdef ERTS_SMP
     if ( (key == am_call_time) || (key == am_all)) {
 	erts_smp_thr_progress_unblock();
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 13a4b2cd93..5d4823c077 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -130,6 +130,7 @@ static erts_lc_lock_order_t erts_lock_order[] = {
 #ifdef ERTS_DIRTY_SCHEDULERS
     {   "dirty_run_queue_sleep_list",		"address"		},
     {	"dirty_gc_info",			NULL			},
+    {	"dirty_break_point_index",		NULL			},
 #endif
     {	"process_table",			NULL			},
     {	"cpu_info",				NULL			},
-- 
cgit v1.2.3


From af5169d85fcd545e3c857a219db081a62f33404d Mon Sep 17 00:00:00 2001
From: Sverker Eriksson <sverker@erlang.org>
Date: Thu, 12 Jan 2017 13:58:26 +0100
Subject: erts: Fix race bug between export fun creation and code loading

Symptom: SEGV crash on ARM in delete_code() -> export_list().
Could probably happen on other machines as well.

Problem: Staging export table was iterated in an unsafe way
while an entry was added for a new export fun.

Solution: Correct write order and some memory barriers.
---
 erts/emulator/beam/beam_bif_load.c | 14 ++++++++------
 erts/emulator/beam/beam_load.c     | 11 ++++++-----
 erts/emulator/beam/export.c        |  2 +-
 erts/emulator/beam/index.c         |  9 ++++++++-
 erts/emulator/beam/index.h         | 14 ++++++++++++++
 5 files changed, 37 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 5969197168..93f5ed500b 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -786,7 +786,7 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
     }
 
     if (BIF_ARG_2 == am_true) {
-	int i;
+	int i, num_exps;
 
 	/*
 	 * Make the code with the on_load function current.
@@ -802,7 +802,8 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	/*
 	 * The on_load function succeded. Fix up export entries.
 	 */
-	for (i = 0; i < export_list_size(code_ix); i++) {
+	num_exps = export_list_size(code_ix);
+	for (i = 0; i < num_exps; i++) {
 	    Export *ep = export_list(i,code_ix);
 	    if (ep == NULL || ep->code[0] != BIF_ARG_1) {
 		continue;
@@ -822,14 +823,15 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2)
 	modp->curr.code_hdr->on_load_function_ptr = NULL;
 	set_default_trace_pattern(BIF_ARG_1);
     } else if (BIF_ARG_2 == am_false) {
-	int i;
+	int i, num_exps;
 
 	/*
 	 * The on_load function failed. Remove references to the
 	 * code that is about to be purged from the export entries.
 	 */
 
-	for (i = 0; i < export_list_size(code_ix); i++) {
+	num_exps = export_list_size(code_ix);
+	for (i = 0; i < num_exps; i++) {
 	    Export *ep = export_list(i,code_ix);
 	    if (ep == NULL || ep->code[0] != BIF_ARG_1) {
 		continue;
@@ -2011,9 +2013,9 @@ delete_code(Module* modp)
 {
     ErtsCodeIndex code_ix = erts_staging_code_ix();
     Eterm module = make_atom(modp->module);
-    int i;
+    int i, num_exps = export_list_size(code_ix);
 
-    for (i = 0; i < export_list_size(code_ix); i++) {
+    for (i = 0; i < num_exps; i++) {
 	Export *ep = export_list(i, code_ix);
         if (ep != NULL && (ep->code[0] == module)) {
 	    if (ep->addressv[code_ix] == ep->code+3) {
diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c
index 0afdedf6c2..3f2bdf3f9d 100644
--- a/erts/emulator/beam/beam_load.c
+++ b/erts/emulator/beam/beam_load.c
@@ -797,14 +797,14 @@ erts_finish_loading(Binary* magic, Process* c_p,
     } else {
 	ErtsCodeIndex code_ix = erts_staging_code_ix();
 	Eterm module = stp->module;
-	int i;
+	int i, num_exps;
 
 	/*
 	 * There is an -on_load() function. We will keep the current
 	 * code, but we must turn off any tracing.
 	 */
-
-	for (i = 0; i < export_list_size(code_ix); i++) {
+        num_exps = export_list_size(code_ix);
+	for (i = 0; i < num_exps; i++) {
 	    Export *ep = export_list(i, code_ix);
 	    if (ep == NULL || ep->code[0] != module) {
 		continue;
@@ -5754,12 +5754,13 @@ exported_from_module(Process* p, /* Process whose heap to use. */
                      ErtsCodeIndex code_ix,
 		     Eterm mod) /* Tagged atom for module. */
 {
-    int i;
+    int i, num_exps;
     Eterm* hp = NULL;
     Eterm* hend = NULL;
     Eterm result = NIL;
 
-    for (i = 0; i < export_list_size(code_ix); i++) {
+    num_exps = export_list_size(code_ix);
+    for (i = 0; i < num_exps; i++) {
 	Export* ep = export_list(i,code_ix);
 	
 	if (ep->code[0] == mod) {
diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c
index 2a19211987..2a007ce860 100644
--- a/erts/emulator/beam/export.c
+++ b/erts/emulator/beam/export.c
@@ -348,7 +348,7 @@ Export *export_list(int i, ErtsCodeIndex code_ix)
 
 int export_list_size(ErtsCodeIndex code_ix)
 {
-    return export_tables[code_ix].entries;
+    return erts_index_num_entries(&export_tables[code_ix]);
 }
 
 int export_table_sz(void)
diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c
index 26d6c04ea0..4cebb4d8fa 100644
--- a/erts/emulator/beam/index.c
+++ b/erts/emulator/beam/index.c
@@ -91,9 +91,16 @@ index_put_entry(IndexTable* t, void* tmpl)
 	t->seg_table[ix>>INDEX_PAGE_SHIFT] = erts_alloc(t->type, sz);
 	t->size += INDEX_PAGE_SIZE;
     }
-    t->entries++;
     p->index = ix;
     t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK] = p;
+
+    /*
+     * Do a write barrier here to allow readers to do lock free iteration.
+     * erts_index_num_entries() does matching read barrier.
+     */
+    ERTS_SMP_WRITE_MEMORY_BARRIER;
+    t->entries++;
+
     return p;
 }
 
diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h
index 0a109d8699..532dec0168 100644
--- a/erts/emulator/beam/index.h
+++ b/erts/emulator/beam/index.h
@@ -65,6 +65,7 @@ void index_erase_latest_from(IndexTable*, Uint ix);
 
 ERTS_GLB_INLINE int index_put(IndexTable*, void*);
 ERTS_GLB_INLINE IndexSlot* erts_index_lookup(IndexTable*, Uint);
+ERTS_GLB_INLINE int erts_index_num_entries(IndexTable* t);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
@@ -78,6 +79,19 @@ erts_index_lookup(IndexTable* t, Uint ix)
 {
     return t->seg_table[ix>>INDEX_PAGE_SHIFT][ix&INDEX_PAGE_MASK];
 }
+
+ERTS_GLB_INLINE int erts_index_num_entries(IndexTable* t)
+{
+    int ret = t->entries;
+    /*
+     * Do a read barrier here to allow lock free iteration
+     * on tables where entries are never erased.
+     * index_put_entry() does matching write barrier.
+     */
+    ERTS_SMP_READ_MEMORY_BARRIER;
+    return ret;
+}
+
 #endif
 
 #endif
-- 
cgit v1.2.3


From 6bcdd45abd97134fddfb5b0307b1d256337b0c67 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 28 Dec 2016 18:23:06 +0100
Subject: Reduction counting on non-tail return

---
 erts/emulator/beam/beam_emu.c | 21 +++++++++++++++++----
 erts/emulator/beam/erl_vm.h   |  2 +-
 2 files changed, 18 insertions(+), 5 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index c554bd73b6..bb77dbd955 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -633,21 +633,34 @@ void** beam_ops;
       y[4] = xt4;				\
  } while (0)
 
+#define DispatchReturn                          \
+do {                                            \
+    if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
+        FCALLS--;				\
+        Goto(*I);                               \
+    }                                           \
+    else {					\
+        c_p->current = NULL;                    \
+        c_p->arity = 1;                         \
+        goto context_switch3;			\
+    }						\
+} while (0)
+
 #define MoveReturn(Src)				\
     x(0) = (Src);				\
     I = c_p->cp;				\
     ASSERT(VALID_INSTR(*c_p->cp));		\
     c_p->cp = 0;				\
     CHECK_TERM(r(0));				\
-    Goto(*I)
+    DispatchReturn
 
 #define DeallocateReturn(Deallocate)       \
   do {                                     \
     int words_to_pop = (Deallocate);       \
-    SET_I((BeamInstr *) cp_val(*E));                     \
+    SET_I((BeamInstr *) cp_val(*E));       \
     E = ADD_BYTE_OFFSET(E, words_to_pop);  \
     CHECK_TERM(r(0));                      \
-    Goto(*I);                              \
+    DispatchReturn;                        \
   } while (0)
 
 #define MoveDeallocateReturn(Src, Deallocate)	\
@@ -1681,7 +1694,7 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
     c_p->cp = 0;
     CHECK_TERM(r(0));
     HEAP_SPACE_VERIFIED(0);
-    Goto(*I);
+    DispatchReturn;
  }
 
     /*
diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h
index 93cfe08105..d88cafc5bf 100644
--- a/erts/emulator/beam/erl_vm.h
+++ b/erts/emulator/beam/erl_vm.h
@@ -36,7 +36,7 @@
 #define EMULATOR "BEAM"
 #define SEQ_TRACE 1
 
-#define CONTEXT_REDS 2000	/* Swap process out after this number */
+#define CONTEXT_REDS 4000	/* Swap process out after this number */
 #define MAX_ARG 255	        /* Max number of arguments allowed */
 #define MAX_REG 1024            /* Max number of x(N) registers used */
 
-- 
cgit v1.2.3


From 5aff60d96efac96a41b514ed167f13eb787a415f Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Mon, 26 Sep 2016 17:05:50 +0200
Subject: Support for dirty BIFs

---
 erts/emulator/beam/atom.names        |   9 +
 erts/emulator/beam/beam_bif_load.c   |   6 +
 erts/emulator/beam/beam_debug.c      | 347 +++++++++++++++++
 erts/emulator/beam/beam_emu.c        | 117 +++---
 erts/emulator/beam/bif.c             | 317 +++++++++++++++-
 erts/emulator/beam/bif.h             |  87 ++++-
 erts/emulator/beam/bif.tab           |   3 +
 erts/emulator/beam/erl_bif_info.c    |   3 +
 erts/emulator/beam/erl_dirty_bif.tab |  82 ++++
 erts/emulator/beam/erl_gc.c          |  24 +-
 erts/emulator/beam/erl_msacc.c       |   4 +-
 erts/emulator/beam/erl_msacc.h       |   2 +-
 erts/emulator/beam/erl_nfunc_sched.c | 144 +++++++
 erts/emulator/beam/erl_nfunc_sched.h | 289 ++++++++++++++
 erts/emulator/beam/erl_nif.c         | 707 +++++++++++++----------------------
 erts/emulator/beam/erl_process.c     |  18 +-
 erts/emulator/beam/erl_process.h     |   8 +-
 erts/emulator/beam/error.h           |  88 +++--
 erts/emulator/beam/global.h          |  14 +-
 erts/emulator/beam/utils.c           |   2 +
 20 files changed, 1663 insertions(+), 608 deletions(-)
 create mode 100644 erts/emulator/beam/erl_dirty_bif.tab
 create mode 100644 erts/emulator/beam/erl_nfunc_sched.c
 create mode 100644 erts/emulator/beam/erl_nfunc_sched.h

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index b1aeed7889..7df350116a 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -76,6 +76,7 @@ atom ac
 atom accessor
 atom active
 atom active_tasks
+atom alive
 atom all
 atom all_but_first
 atom all_names
@@ -199,10 +200,15 @@ atom dexit
 atom depth
 atom dgroup_leader
 atom dictionary
+atom dirty_bif_exception
+atom dirty_bif_result
+atom dirty_bif_trap
 atom dirty_cpu
 atom dirty_cpu_schedulers_online
 atom dirty_execution
 atom dirty_io
+atom dirty_nif_exception
+atom dirty_nif_finalizer
 atom disable_trace
 atom disabled
 atom discard
@@ -243,6 +249,7 @@ atom ERROR='ERROR'
 atom error_handler
 atom error_logger
 atom erts_code_purger
+atom erts_debug
 atom erts_internal
 atom ets
 atom ETS_TRANSFER='ETS-TRANSFER'
@@ -573,6 +580,7 @@ atom scientific
 atom scope
 atom second
 atom seconds
+atom send
 atom send_to_non_existing_process
 atom sensitive
 atom sequential_tracer
@@ -665,6 +673,7 @@ atom value
 atom values
 atom version
 atom visible
+atom wait
 atom waiting
 atom wall_clock
 atom warning
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 09331eb8ad..36bbe81ed5 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -36,6 +36,7 @@
 #include "erl_nif.h"
 #include "erl_bits.h"
 #include "erl_thr_progress.h"
+#include "erl_nfunc_sched.h"
 #ifdef HIPE
 #  include "hipe_bif0.h"
 #  define IF_HIPE(X) (X)
@@ -1102,6 +1103,11 @@ check_process_code(Process* rp, Module* modp, int *redsp, int fcalls)
  
     *redsp += 1;
 
+    if (erts_check_nif_export_in_area(rp, mod_start, mod_size))
+	return am_true;
+
+    *redsp += 1;
+
     if (erts_check_nif_export_in_area(rp, mod_start, mod_size))
 	return am_true;
 
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index e72d7f8de4..3ffbb0364c 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -39,6 +39,7 @@
 #include "beam_bp.h"
 #include "erl_binary.h"
 #include "erl_thr_progress.h"
+#include "erl_nfunc_sched.h"
 
 #ifdef ARCH_64
 # define HEXF "%016bpX"
@@ -764,3 +765,349 @@ static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif)
 	erts_print(to, to_arg, "%T/%u", name, arity);
     }
 }
+
+/*
+ * Dirty BIF testing.
+ *
+ * The erts_debug:dirty_cpu/2, erts_debug:dirty_io/1, and
+ * erts_debug:dirty/3 BIFs are used by the dirty_bif_SUITE
+ * test suite.
+ */
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+static int ms_wait(Process *c_p, Eterm etimeout, int busy);
+static int dirty_send_message(Process *c_p, Eterm to, Eterm tag);
+#endif
+static BIF_RETTYPE dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I);
+
+/*
+ * erts_debug:dirty_cpu/2 is statically determined to execute on
+ * a dirty CPU scheduler (see erts_dirty_bif.tab).
+ */
+BIF_RETTYPE
+erts_debug_dirty_cpu_2(BIF_ALIST_2)
+{
+    return dirty_test(BIF_P, am_dirty_cpu, BIF_ARG_1, BIF_ARG_2, BIF_I);
+}
+
+/*
+ * erts_debug:dirty_io/2 is statically determined to execute on
+ * a dirty I/O scheduler (see erts_dirty_bif.tab).
+ */
+BIF_RETTYPE
+erts_debug_dirty_io_2(BIF_ALIST_2)
+{
+    return dirty_test(BIF_P, am_dirty_io, BIF_ARG_1, BIF_ARG_2, BIF_I);
+}
+
+/*
+ * erts_debug:dirty/3 executes on a normal scheduler.
+ */
+BIF_RETTYPE
+erts_debug_dirty_3(BIF_ALIST_3)
+{
+#ifdef ERTS_DIRTY_SCHEDULERS
+    Eterm argv[2];
+    switch (BIF_ARG_1) {
+    case am_normal:
+	return dirty_test(BIF_P, am_normal, BIF_ARG_2, BIF_ARG_3, BIF_I);
+    case am_dirty_cpu:
+	argv[0] = BIF_ARG_2;
+	argv[1] = BIF_ARG_3;
+	return erts_schedule_bif(BIF_P,
+				 argv,
+				 BIF_I,
+				 erts_debug_dirty_cpu_2,
+				 ERTS_SCHED_DIRTY_CPU,
+				 am_erts_debug,
+				 am_dirty_cpu,
+				 2);
+    case am_dirty_io:
+	argv[0] = BIF_ARG_2;
+	argv[1] = BIF_ARG_3;
+	return erts_schedule_bif(BIF_P,
+				 argv,
+				 BIF_I,
+				 erts_debug_dirty_io_2,
+				 ERTS_SCHED_DIRTY_IO,
+				 am_erts_debug,
+				 am_dirty_io,
+				 2);
+    default:
+	BIF_ERROR(BIF_P, EXC_BADARG);
+    }
+#else
+	BIF_ERROR(BIF_P, EXC_UNDEF);
+#endif
+}
+
+
+static BIF_RETTYPE
+dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I)
+{
+    BIF_RETTYPE ret;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (am_scheduler == arg1) {
+	ErtsSchedulerData *esdp;
+	if (arg2 != am_type) 
+	    goto badarg;
+	esdp = erts_proc_sched_data(c_p);
+	if (!esdp)
+	    ERTS_BIF_PREP_RET(ret, am_error);
+	else if (!ERTS_SCHEDULER_IS_DIRTY(esdp))
+	    ERTS_BIF_PREP_RET(ret, am_normal);
+	else if (ERTS_SCHEDULER_IS_DIRTY_CPU(esdp))
+	    ERTS_BIF_PREP_RET(ret, am_dirty_cpu);
+	else if (ERTS_SCHEDULER_IS_DIRTY_IO(esdp))
+	    ERTS_BIF_PREP_RET(ret, am_dirty_io);
+	else
+	    ERTS_BIF_PREP_RET(ret, am_error);
+    }
+    else if (am_error == arg1) {
+	switch (arg2) {
+	case am_notsup:
+	    ERTS_BIF_PREP_ERROR(ret, c_p, EXC_NOTSUP);
+	    break;
+	case am_undef:
+	    ERTS_BIF_PREP_ERROR(ret, c_p, EXC_UNDEF);
+	    break;
+	case am_badarith:
+	    ERTS_BIF_PREP_ERROR(ret, c_p, EXC_BADARITH);
+	    break;
+	case am_noproc:
+	    ERTS_BIF_PREP_ERROR(ret, c_p, EXC_NOPROC);
+	    break;
+	case am_system_limit:
+	    ERTS_BIF_PREP_ERROR(ret, c_p, SYSTEM_LIMIT);
+	    break;
+	case am_badarg:
+	default:
+	    goto badarg;
+	}
+    }
+    else if (am_copy == arg1) {
+	int i;
+	Eterm res;
+
+	for (res = NIL, i = 0; i < 1000; i++) {
+	    Eterm *hp, sz;
+	    Eterm cpy;
+	    /* We do not want this to be optimized,
+	       but rather the oposite... */
+	    sz = size_object(arg2);
+	    hp = HAlloc(c_p, sz);
+	    cpy = copy_struct(arg2, sz, &hp, &c_p->off_heap);
+	    hp = HAlloc(c_p, 2);
+	    res = CONS(hp, cpy, res);
+	}
+
+	ERTS_BIF_PREP_RET(ret, res);
+    }
+    else if (am_send == arg1) {
+	dirty_send_message(c_p, arg2, am_ok);
+	ERTS_BIF_PREP_RET(ret, am_ok);
+    }
+    else if (ERTS_IS_ATOM_STR("wait", arg1)) {
+	if (!ms_wait(c_p, arg2, type == am_dirty_cpu))
+	    goto badarg;
+	ERTS_BIF_PREP_RET(ret, am_ok);
+    }
+    else if (ERTS_IS_ATOM_STR("reschedule", arg1)) {
+	/*
+	 * Reschedule operation after decrement of two until we reach
+	 * zero. Switch between dirty scheduler types when 'n' is
+	 * evenly divided by 4. If the initial value wasn't evenly
+	 * dividable by 2, throw badarg exception.
+	 */
+	Eterm next_type;
+	Sint n;
+	if (!term_to_Sint(arg2, &n) || n < 0)
+	    goto badarg;
+	if (n == 0)
+	    ERTS_BIF_PREP_RET(ret, am_ok);
+	else {
+	    Eterm argv[3];
+	    Eterm eint = erts_make_integer((Uint) (n - 2), c_p);
+	    if (n % 4 != 0)
+		next_type = type;
+	    else {
+		switch (type) {
+		case am_dirty_cpu: next_type = am_dirty_io; break;
+		case am_dirty_io: next_type = am_normal; break;
+		case am_normal: next_type = am_dirty_cpu; break;
+		default: goto badarg;
+		}
+	    }
+	    switch (next_type) {
+	    case am_dirty_io:
+		argv[0] = arg1;
+		argv[1] = eint;
+		ret = erts_schedule_bif(c_p,
+					argv,
+					I,
+					erts_debug_dirty_io_2,
+					ERTS_SCHED_DIRTY_IO,
+					am_erts_debug,
+					am_dirty_io,
+					2);
+		break;
+	    case am_dirty_cpu:
+		argv[0] = arg1;
+		argv[1] = eint;
+		ret = erts_schedule_bif(c_p,
+					argv,
+					I,
+					erts_debug_dirty_cpu_2,
+					ERTS_SCHED_DIRTY_CPU,
+					am_erts_debug,
+					am_dirty_cpu,
+					2);
+		break;
+	    case am_normal:
+		argv[0] = am_normal;
+		argv[1] = arg1;
+		argv[2] = eint;
+		ret = erts_schedule_bif(c_p,
+					argv,
+					I,
+					erts_debug_dirty_3,
+					ERTS_SCHED_NORMAL,
+					am_erts_debug,
+					am_dirty,
+					3);
+		break;
+	    default:
+		goto badarg;
+	    }
+	}
+    }
+    else if (ERTS_IS_ATOM_STR("ready_wait6_done", arg1)) {
+	ERTS_DECL_AM(ready);
+	ERTS_DECL_AM(done);
+	dirty_send_message(c_p, arg2, AM_ready);
+	ms_wait(c_p, make_small(6000), 0);
+	dirty_send_message(c_p, arg2, AM_done);
+	ERTS_BIF_PREP_RET(ret, am_ok);
+    }
+    else if (ERTS_IS_ATOM_STR("alive_waitexiting", arg1)) {
+	Process *real_c_p = erts_proc_shadow2real(c_p);
+	Eterm *hp, *hp2;
+	Uint sz;
+	int i;
+	if (ERTS_PROC_IS_EXITING(real_c_p))
+	    goto badarg;
+	dirty_send_message(c_p, arg2, am_alive);
+	/* Wait until dead */
+
+	while (!ERTS_PROC_IS_EXITING(real_c_p))
+	    erts_thr_yield();
+
+	ms_wait(c_p, make_small(1000), 0);
+
+	/* Should still be able to allocate memory */
+	hp = HAlloc(c_p, 3); /* Likely on heap */
+	sz = 10000;
+	hp2 = HAlloc(c_p, sz); /* Likely in heap fragment */
+	*hp2 = make_pos_bignum_header(sz);
+	for (i = 1; i < sz; i++)
+	    hp2[i] = (Eterm) 4711;
+	ERTS_BIF_PREP_RET(ret, TUPLE2(hp, am_ok, make_big(hp2)));
+    }
+    else {
+    badarg:
+	ERTS_BIF_PREP_ERROR(ret, c_p, BADARG);
+    }
+#else
+    ERTS_BIF_PREP_ERROR(ret, c_p, EXC_UNDEF);
+#endif
+    return ret;
+}
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+static int
+dirty_send_message(Process *c_p, Eterm to, Eterm tag)
+{
+    ErtsProcLocks c_p_locks, rp_locks;
+    Process *rp, *real_c_p;
+    Eterm msg, *hp;
+    ErlOffHeap *ohp;
+    ErtsMessage *mp;
+
+    ASSERT(is_immed(tag));
+
+    real_c_p = erts_proc_shadow2real(c_p);
+    if (real_c_p != c_p)
+	c_p_locks = 0;
+    else
+	c_p_locks = ERTS_PROC_LOCK_MAIN;
+
+    ASSERT(real_c_p->common.id == c_p->common.id);
+
+    rp = erts_pid2proc_opt(real_c_p, c_p_locks,
+			   to, 0,
+			   ERTS_P2P_FLG_INC_REFC);
+
+    if (!rp)
+	return 0;
+
+    rp_locks = 0;
+    mp = erts_alloc_message_heap(rp, &rp_locks, 3, &hp, &ohp);
+
+    msg = TUPLE2(hp, tag, c_p->common.id);
+    erts_queue_message(rp, rp_locks, mp, msg, c_p->common.id);
+
+    if (rp == real_c_p)
+	rp_locks &= ~c_p_locks;
+    if (rp_locks)
+	erts_smp_proc_unlock(rp, rp_locks);
+
+    erts_proc_dec_refc(rp);
+
+    return 1;
+}
+
+static int
+ms_wait(Process *c_p, Eterm etimeout, int busy)
+{
+    ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
+    ErtsMonotonicTime time, timeout_time;
+    Sint64 ms;
+
+    if (!term_to_Sint64(etimeout, &ms))
+	return 0;
+
+    time = erts_get_monotonic_time(esdp);
+
+    if (ms < 0)
+	timeout_time = time;
+    else
+	timeout_time = time + ERTS_MSEC_TO_MONOTONIC(ms);
+
+    while (time < timeout_time) {
+	if (busy)
+	    erts_thr_yield();
+	else {
+	    ErtsMonotonicTime timeout = timeout_time - time;
+
+#ifdef __WIN32__
+	    Sleep((DWORD) ERTS_MONOTONIC_TO_MSEC(timeout));
+#else
+	    {
+		ErtsMonotonicTime to = ERTS_MONOTONIC_TO_USEC(timeout);
+		struct timeval tv;
+
+		tv.tv_sec = (long) to / (1000*1000);
+		tv.tv_usec = (long) to % (1000*1000);
+
+		select(0, NULL, NULL, NULL, &tv);
+	    }
+#endif
+	}
+
+	time = erts_get_monotonic_time(esdp);
+    }
+    return 1;
+}
+
+#endif /* ERTS_DIRTY_SCHEDULERS */
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index bb77dbd955..befd2989f9 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -38,6 +38,7 @@
 #include "beam_bp.h"
 #include "beam_catches.h"
 #include "erl_thr_progress.h"
+#include "erl_nfunc_sched.h"
 #ifdef HIPE
 #include "hipe_mode_switch.h"
 #include "hipe_bif1.h"
@@ -214,6 +215,7 @@ BeamInstr beam_continue_exit[1];
 BeamInstr* em_call_error_handler;
 BeamInstr* em_apply_bif;
 BeamInstr* em_call_nif;
+BeamInstr* em_call_bif_e;
 
 
 /* NOTE These should be the only variables containing trace instructions.
@@ -2581,7 +2583,7 @@ do {						\
 
  OpCase(bif1_fbsd):
     {
-	Eterm (*bf)(Process*, Eterm*);
+	ErtsBifFunc bf;
 	Eterm tmp_reg[1];
 	Eterm result;
 
@@ -2592,7 +2594,7 @@ do {						\
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, tmp_reg);
+	result = (*bf)(c_p, tmp_reg, I);
 	ERTS_CHK_MBUF_SZ(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
@@ -2613,19 +2615,19 @@ do {						\
 
  OpCase(bif1_body_bsd):
     {
-	Eterm (*bf)(Process*, Eterm*);
+	ErtsBifFunc bf;
 
 	Eterm tmp_reg[1];
 	Eterm result;
 
 	GetArg1(1, tmp_reg[0]);
-	bf = (BifFunction) Arg(0);
+	bf = (ErtsBifFunc) Arg(0);
 	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, tmp_reg);
+	result = (*bf)(c_p, tmp_reg, I);
 	ERTS_CHK_MBUF_SZ(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
@@ -2775,17 +2777,17 @@ do {						\
  OpCase(i_bif2_fbssd):
     {
 	Eterm tmp_reg[2];
-	Eterm (*bf)(Process*, Eterm*);
+	ErtsBifFunc bf;
 	Eterm result;
 
 	GetArg2(2, tmp_reg[0], tmp_reg[1]);
-	bf = (BifFunction) Arg(1);
+	bf = (ErtsBifFunc) Arg(1);
 	ERTS_DBG_CHK_REDS(c_p, FCALLS);
 	c_p->fcalls = FCALLS;
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, tmp_reg);
+	result = (*bf)(c_p, tmp_reg, I);
 	ERTS_CHK_MBUF_SZ(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
@@ -2806,15 +2808,15 @@ do {						\
  OpCase(i_bif2_body_bssd):
     {
 	Eterm tmp_reg[2];
-	Eterm (*bf)(Process*, Eterm*);
+	ErtsBifFunc bf;
 	Eterm result;
 
 	GetArg2(1, tmp_reg[0], tmp_reg[1]);
-	bf = (BifFunction) Arg(0);
+	bf = (ErtsBifFunc) Arg(0);
 	PROCESS_MAIN_CHK_LOCKS(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 	ERTS_CHK_MBUF_SZ(c_p);
-	result = (*bf)(c_p, tmp_reg);
+	result = (*bf)(c_p, tmp_reg, I);
 	ERTS_CHK_MBUF_SZ(c_p);
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result));
 	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
@@ -2837,7 +2839,7 @@ do {						\
      */
  OpCase(call_bif_e):
     {
-        Eterm (*bf)(Process*, Eterm*, BeamInstr*);
+	ErtsBifFunc bf;
 	Eterm result;
 	BeamInstr *next;
 	ErlHeapFragment *live_hf_end;
@@ -3644,7 +3646,7 @@ do {						\
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
 	    {
-		Eterm (*bf)(Process*, Eterm*, BeamInstr*) = vbf;
+		ErtsBifFunc bf = vbf;
 		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 		live_hf_end = c_p->mbuf;
 		ERTS_CHK_MBUF_SZ(c_p);
@@ -5137,6 +5139,7 @@ do {						\
      em_call_error_handler = OpCode(call_error_handler);
      em_apply_bif = OpCode(apply_bif);
      em_call_nif = OpCode(call_nif);
+     em_call_bif_e = OpCode(call_bif_e);
 
      beam_apply[0]             = (BeamInstr) OpCode(i_apply);
      beam_apply[1]             = (BeamInstr) OpCode(normal_exit);
@@ -5342,8 +5345,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 
 	I = c_p->i;
 
-	ASSERT(em_call_nif == (BeamInstr *) *I);
-
 	/*
 	 * Set fcalls even though we ignore it, so we don't
 	 * confuse code accessing it...
@@ -5379,10 +5380,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
     }
 
     {
-#ifdef DEBUG
-	Eterm result;
-#endif
-	Eterm arity;
+	int exiting;
 
 	{
 	    /*
@@ -5398,7 +5396,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
              *
              * This layout is determined by the NifExport struct
 	     */
-	    BifFunction vbf;
             ErtsCodeMFA *codemfa;
 
 	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
@@ -5406,44 +5403,29 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
             codemfa = erts_code_to_codemfa(I);
 
 	    DTRACE_NIF_ENTRY(c_p, codemfa);
-            /* current and vbf set to please handle_error */
 	    c_p->current = codemfa;
 	    SWAPOUT;
 	    PROCESS_MAIN_CHK_LOCKS(c_p);
-	    arity = codemfa->arity;
 	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 
 	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	    {
-		typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]);
-		NifF* fp = vbf = (NifF*) I[1];
-		struct enif_environment_t env;
-		ASSERT(!c_p->scheduler_data);
-
-		erts_pre_dirty_nif(esdp, &env, c_p,
-				   (struct erl_module_nif*)I[2]);
-
-#ifdef DEBUG
-		result =
-#else
-		(void)
-#endif
-		    (*fp)(&env, arity, reg);
-
-		erts_post_dirty_nif(&env);
+	    if (em_apply_bif == (BeamInstr *) *I) {
+		exiting = erts_call_dirty_bif(esdp, c_p, I, reg);
+	    }
+	    else {
+		ASSERT(em_call_nif == (BeamInstr *) *I);
+		exiting = erts_call_dirty_nif(esdp, c_p, I, reg);
+	    }
 
-		ASSERT(!is_value(result));
-		ASSERT(c_p->freason == TRAP);
-		ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
+	    ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
 
-		PROCESS_MAIN_CHK_LOCKS(c_p);
-		ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
-		ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-		ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
-		if (env.exiting)
-		    goto do_dirty_schedule;
-		ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	    }
+	    PROCESS_MAIN_CHK_LOCKS(c_p);
+	    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+	    if (exiting)
+		goto do_dirty_schedule;
+	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 
 	    DTRACE_NIF_RETURN(c_p, codemfa);
 	    ERTS_HOLE_CHECK(c_p);
@@ -5529,10 +5511,15 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
     Eterm* hp;
     Eterm Value = c_p->fvalue;
     Eterm Args = am_true;
-    c_p->i = pc;    /* In case we call erts_exit(). */
 
     ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */
 
+    if (c_p->freason & EXF_RESTORE_NIF) {
+	erts_nif_export_restore_error(c_p, &pc, reg, &bf);
+    }
+
+    c_p->i = pc;    /* In case we call erts_exit(). */
+
     /*
      * Check if we have an arglist for the top level call. If so, this
      * is encoded in Value, so we have to dig out the real Value as well
@@ -5829,6 +5816,18 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) {
  * error terms.)
  */
 
+static void
+save_stacktrace_current(struct StackTrace *s, Process *c_p, ErtsCodeMFA *mfa)
+{
+    NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
+    if (!nep || &nep->exp.info.mfa != mfa)
+	s->current = mfa;
+    else {
+	s->mfa = *mfa;
+	s->current = &s->mfa;
+    }
+}
+
 static void
 save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 		Eterm args) {
@@ -5858,12 +5857,10 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 	&& bf != throw_1 && bf != wrap_error_1 && bf != wrap_error_2
 	&& bf != wrap_exit_1 && bf != wrap_throw_1) {
         int i;
-	int a = 0;
+	int a;
 	for (i = 0; i < BIF_SIZE; i++) {
 	    if (bf == bif_table[i].f || bf == bif_table[i].traced) {
-		Export *ep = bif_export[i];
-		s->current = &ep->info.mfa;
-	        a = bif_table[i].arity;
+		save_stacktrace_current(s, c_p, &bif_export[i]->info.mfa);
 		break;
 	    }
 	}
@@ -5875,9 +5872,11 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 	     * OR it is a NIF called by call_nif where current is also set.
 	     */
 	    ASSERT(c_p->current);
-	    s->current = c_p->current;
-	    a = s->current->arity;
+	    save_stacktrace_current(s, c_p, c_p->current);
 	}
+
+	a = s->current->arity;
+
 	/* Save first stack entry */
 	ASSERT(pc);
 	if (depth > 0) {
@@ -5892,7 +5891,9 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 	s->pc = NULL;
 	args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */
     } else {
-	s->current = c_p->current;
+
+	save_stacktrace_current(s, c_p, c_p->current);
+
         /* 
 	 * For a function_clause error, the arguments are in the beam
 	 * registers, c_p->cp is valid, and c_p->current is set.
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index d886c2985e..7bcb7c196d 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4958,7 +4958,7 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 Export bif_return_trap_export;
 
 void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
-			   Eterm (*bif)(BIF_ALIST_0))
+			   Eterm (*bif)(BIF_ALIST))
 {
     int i;
     sys_memset((void *) ep, 0, sizeof(Export));
@@ -5019,6 +5019,321 @@ void erts_init_bif(void)
     erts_smp_atomic32_init_nob(&msacc, ERTS_MSACC_IS_ENABLED());
 }
 
+/*
+ * Scheduling of BIFs via NifExport...
+ */
+#define ERTS_WANT_NFUNC_SCHED_INTERNALS__
+#include "erl_nfunc_sched.h"
+
+#define ERTS_SCHED_BIF_TRAP_MARKER ((void *) (UWord) 1)
+
+static void
+schedule(Process *c_p, Process *dirty_shadow_proc,
+	 ErtsCodeMFA *mfa, BifFunction *nif, BeamInstr *pc,
+	 ErtsBifFunc dfunc, void *ifunc,
+	 Eterm module, Eterm function,
+	 int argc, Eterm *argv)
+{
+    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
+    (void) erts_nif_export_schedule(c_p, dirty_shadow_proc,
+				    mfa, nif, pc,
+				    (BeamInstr) em_apply_bif,
+				    dfunc, ifunc,
+				    module, function,
+				    argc, argv);
+}
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+static BIF_RETTYPE dirty_bif_result(BIF_ALIST_1)
+{
+    NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P);
+    erts_nif_export_restore(BIF_P, nep);
+    BIF_RET(BIF_ARG_1);
+}
+
+static BIF_RETTYPE dirty_bif_trap(BIF_ALIST)
+{
+    NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P);
+
+    /*
+     * Arity and argument registers already set
+     * correct by call to dirty_bif_trap()...
+     */
+
+    ASSERT(BIF_P->arity == nep->exp.info.mfa.arity);
+
+    erts_nif_export_restore(BIF_P, nep);
+
+    BIF_P->i = (BeamInstr *) nep->func;
+    BIF_P->freason = TRAP;
+    return THE_NON_VALUE;
+}
+
+static BIF_RETTYPE dirty_bif_exception(BIF_ALIST_2)
+{
+    Eterm freason;
+
+    ASSERT(is_small(BIF_ARG_1));
+
+    freason = signed_val(BIF_ARG_1);
+
+    /* Restore orig info for error and clear nif export in handle_error() */
+    freason |= EXF_RESTORE_NIF;
+
+    BIF_P->fvalue = BIF_ARG_2;
+
+    BIF_ERROR(BIF_P, freason);
+}
+
+#endif /* ERTS_DIRTY_SCHEDULERS */
+
+extern BeamInstr* em_call_bif_e;
+static BIF_RETTYPE call_bif(Process *c_p, Eterm *reg, BeamInstr *I);
+
+BIF_RETTYPE
+erts_schedule_bif(Process *proc,
+		  Eterm *argv,
+		  BeamInstr *i,
+		  ErtsBifFunc bif,
+		  ErtsSchedType sched_type,
+		  Eterm mod,
+		  Eterm func,
+		  int argc)
+{
+    Process *c_p, *dirty_shadow_proc;
+    ErtsCodeMFA *mfa;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
+	dirty_shadow_proc = proc;
+	c_p = proc->next;
+	ASSERT(c_p->common.id == dirty_shadow_proc->common.id);
+	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+    }
+    else
+#endif
+    {
+	dirty_shadow_proc = NULL;
+	c_p = proc;
+    }
+
+    if (!ERTS_PROC_IS_EXITING(c_p)) {
+	Export *exp;
+	BifFunction obif, dbif, ibif;
+	BeamInstr *pc;
+
+	/*
+	 * obif - original bif
+	 * dbif - direct bif
+	 * ibif - indirect bif
+	 */
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+	erts_aint32_t set, mask;
+	mask = (ERTS_PSFLG_DIRTY_CPU_PROC
+		| ERTS_PSFLG_DIRTY_IO_PROC);
+	switch (sched_type) {
+	case ERTS_SCHED_DIRTY_CPU:
+	    set = ERTS_PSFLG_DIRTY_CPU_PROC;
+	    dbif = bif;
+	    ibif = NULL;
+	    break;
+	case ERTS_SCHED_DIRTY_IO:
+	    set = ERTS_PSFLG_DIRTY_IO_PROC;
+	    dbif = bif;
+	    ibif = NULL;
+	    break;
+	case ERTS_SCHED_NORMAL:
+	default:
+	    set = 0;
+	    dbif = call_bif;
+	    ibif = bif;
+	    break;
+	}
+
+	(void) erts_smp_atomic32_read_bset_nob(&c_p->state, mask, set);
+#else
+	dbif = call_bif;
+	ibif = bif;
+#endif
+
+	if (i == NULL) {
+	    ERTS_INTERNAL_ERROR("Missing instruction pointer");
+	    obif = NULL;
+	}
+#ifdef HIPE
+	else if (proc->flags & F_HIPE_MODE) {
+	    /* Pointer to bif export in i */
+	    exp = (Export *) i;
+	    pc = c_p->cp;
+	    obif = (BifFunction) exp->beam[1];
+	    mfa = &exp->info.mfa;
+	}
+#endif
+	else if (em_call_bif_e == (BeamInstr *) *i) {
+	    /* Pointer to bif export in i+1 */
+	    exp = (Export *) i[1];
+	    pc = i;
+	    obif = (BifFunction) exp->beam[1];
+	    mfa = &exp->info.mfa;
+	}
+	else if (em_apply_bif == (BeamInstr *) *i) {
+	    /* Pointer to bif in i+1, and mfa in i-3 */	    
+	    obif = (BifFunction) i[1];
+	    pc = c_p->cp;
+	    mfa = erts_code_to_codemfa(i);
+	}
+	else {
+	    ERTS_INTERNAL_ERROR("erts_schedule_bif() called "
+				"from unexpected instruction");
+	    obif = NULL;
+	}
+	ASSERT(bif);
+
+	if (argc < 0) { /* reschedule original call */
+	    mod = mfa->module;
+	    func = mfa->function;
+	    argc = (int) mfa->arity;
+	}
+
+	schedule(c_p, dirty_shadow_proc, mfa, obif,
+		 pc, dbif, ibif, mod, func, argc, argv);
+    }
+
+    if (dirty_shadow_proc)
+	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    return THE_NON_VALUE;
+}
+
+static BIF_RETTYPE
+call_bif(Process *c_p, Eterm *reg, BeamInstr *I)
+{
+    NifExport *nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I);
+    ErtsBifFunc bif = (ErtsBifFunc) nep->func;
+    BIF_RETTYPE ret;
+
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
+
+    nep->func = ERTS_SCHED_BIF_TRAP_MARKER;
+
+    ASSERT(bif);
+
+    ret = (*bif)(c_p, reg, I);
+
+    if (is_value(ret))
+	erts_nif_export_restore(c_p, nep);
+    else if (c_p->freason != TRAP)
+	c_p->freason |= EXF_RESTORE_NIF; /* restore in handle_error() */
+    else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) {
+	/* BIF did an ordinary trap... */
+	erts_nif_export_restore(c_p, nep);
+    }
+    /* else:
+     *   BIF rescheduled itself using erts_schedule_bif().
+     */
+
+    return ret;
+}
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+int
+erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg)
+{
+    BIF_RETTYPE result;
+    int exiting;
+    Process *dirty_shadow_proc;
+    ErtsBifFunc bf;
+    NifExport *nep;
+#ifdef DEBUG
+    Eterm *c_p_htop;
+    erts_aint32_t state;
+
+    ASSERT(!c_p->scheduler_data);
+    state = erts_smp_atomic32_read_nob(&c_p->state);
+    ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING)
+	   && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)));
+    ASSERT(esdp);
+
+#endif
+
+    nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I);
+    ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p));
+
+    nep->func = ERTS_SCHED_BIF_TRAP_MARKER;
+
+    bf = (ErtsBifFunc) I[1];
+
+    erts_smp_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
+						  | ERTS_PSFLG_DIRTY_IO_PROC));
+
+    dirty_shadow_proc = erts_make_dirty_shadow_proc(esdp, c_p);
+
+    dirty_shadow_proc->freason = c_p->freason;
+    dirty_shadow_proc->fvalue = c_p->fvalue;
+    dirty_shadow_proc->ftrace = c_p->ftrace;
+    dirty_shadow_proc->cp = c_p->cp;
+    dirty_shadow_proc->i = c_p->i;
+
+#ifdef DEBUG
+    c_p_htop = c_p->htop;
+#endif
+
+    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    result = (*bf)(dirty_shadow_proc, reg, I);
+
+    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    ASSERT(c_p_htop == c_p->htop);
+    ASSERT(dirty_shadow_proc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
+    ASSERT(dirty_shadow_proc->next == c_p);
+
+    exiting = ERTS_PROC_IS_EXITING(c_p);
+
+    if (!exiting) {
+	if (is_value(result))
+	    schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL, dirty_bif_result,
+		     NULL, am_erts_internal, am_dirty_bif_result, 1, &result);
+	else if (dirty_shadow_proc->freason != TRAP) {
+	    Eterm argv[2];
+	    ASSERT(dirty_shadow_proc->freason <= MAX_SMALL);
+	    argv[0] = make_small(dirty_shadow_proc->freason);
+	    argv[1] = dirty_shadow_proc->fvalue;
+	    schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL,
+		     dirty_bif_exception, NULL, am_erts_internal,
+		     am_dirty_bif_exception, 2, argv);
+	}
+	else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) {
+	    /* Dirty BIF did an ordinary trap... */
+	    ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state)
+		     & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)));
+	    schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL,
+		     dirty_bif_trap, (void *) dirty_shadow_proc->i,
+		     am_erts_internal, am_dirty_bif_trap,
+		     dirty_shadow_proc->arity, reg);
+	}
+	/* else:
+	 *   BIF rescheduled itself using erts_schedule_bif().
+	 */
+	c_p->freason = dirty_shadow_proc->freason;
+	c_p->fvalue = dirty_shadow_proc->fvalue;
+	c_p->ftrace = dirty_shadow_proc->ftrace;
+	c_p->cp = dirty_shadow_proc->cp;
+	c_p->i = dirty_shadow_proc->i;
+	c_p->arity = dirty_shadow_proc->arity;
+    }
+
+    erts_flush_dirty_shadow_proc(dirty_shadow_proc);
+
+    return exiting;
+}
+
+#endif /* ERTS_DIRTY_SCHEDULERS */
+
+
 #ifdef HARDDEBUG
 /*
 You'll need this line in bif.tab to be able to use this debug bif
diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index 0c85e19ef0..ad55b19d0a 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -29,17 +29,35 @@ extern Export *erts_convert_time_unit_trap;
 
 #define BIF_P A__p
 
-#define BIF_ALIST_0 Process* A__p, Eterm* BIF__ARGS
-#define BIF_ALIST_1 Process* A__p, Eterm* BIF__ARGS
-#define BIF_ALIST_2 Process* A__p, Eterm* BIF__ARGS
-#define BIF_ALIST_3 Process* A__p, Eterm* BIF__ARGS
-#define BIF_ALIST_4 Process* A__p, Eterm* BIF__ARGS
+#define BIF_ALIST Process* A__p, Eterm* BIF__ARGS, BeamInstr *A__I
+#define BIF_CALL_ARGS A__p, BIF__ARGS, A__I
+
+#define BIF_ALIST_0 BIF_ALIST
+#define BIF_ALIST_1 BIF_ALIST
+#define BIF_ALIST_2 BIF_ALIST
+#define BIF_ALIST_3 BIF_ALIST
+#define BIF_ALIST_4 BIF_ALIST
 
 #define BIF_ARG_1  (BIF__ARGS[0])
 #define BIF_ARG_2  (BIF__ARGS[1])
 #define BIF_ARG_3  (BIF__ARGS[2])
 #define BIF_ARG_4  (BIF__ARGS[3])
 
+#define BIF_I A__I
+
+/* NBIF_* is for bif calls from native code... */
+
+#define NBIF_ALIST Process* A__p, Eterm* BIF__ARGS
+#define NBIF_CALL_ARGS A__p, BIF__ARGS
+
+#define NBIF_ALIST_0 NBIF_ALIST
+#define NBIF_ALIST_1 NBIF_ALIST
+#define NBIF_ALIST_2 NBIF_ALIST
+#define NBIF_ALIST_3 NBIF_ALIST
+#define NBIF_ALIST_4 NBIF_ALIST
+
+typedef BIF_RETTYPE (*ErtsBifFunc)(BIF_ALIST);
+
 #define ERTS_IS_PROC_OUT_OF_REDS(p)		\
     ((p)->fcalls > 0				\
      ? 0					\
@@ -480,6 +498,49 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 					 Eterm args[],
 					 int nargs);
 
+typedef enum {
+    ERTS_SCHED_NORMAL,
+    ERTS_SCHED_DIRTY_CPU,
+    ERTS_SCHED_DIRTY_IO
+} ErtsSchedType;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p,
+			BeamInstr *I, Eterm *reg);
+#endif
+
+BIF_RETTYPE
+erts_schedule_bif(Process *proc,
+		  Eterm *argv,
+		  BeamInstr *i,
+		  ErtsBifFunc dbf,
+		  ErtsSchedType sched_type,
+		  Eterm mod,
+		  Eterm func,
+		  int argc);
+
+ERTS_GLB_INLINE BIF_RETTYPE
+erts_reschedule_bif(Process *proc,
+		    Eterm *argv,
+		    BeamInstr *i,
+		    ErtsBifFunc dbf,
+		    ErtsSchedType sched_type);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE BIF_RETTYPE
+erts_reschedule_bif(Process *proc,
+		    Eterm *argv,
+		    BeamInstr *i,
+		    ErtsBifFunc dbf,
+		    ErtsSchedType sched_type)
+{
+    return erts_schedule_bif(proc, argv, i, dbf, sched_type,
+			     THE_NON_VALUE, THE_NON_VALUE, -1);
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
 #ifdef ERL_WANT_HIPE_BIF_WRAPPER__
 
 #ifndef HIPE
@@ -510,16 +571,16 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 
 
 #define HIPE_WRAPPER_BIF_DISABLE_GC(BIF_NAME, ARITY)			\
-BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p,	\
-						     Eterm* args);	\
-BIF_RETTYPE hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (Process* c_p,	\
-						     Eterm* args)	\
+BIF_RETTYPE								\
+nbif_impl_hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (NBIF_ALIST);		\
+BIF_RETTYPE								\
+nbif_impl_hipe_wrapper_ ## BIF_NAME ## _ ## ARITY (NBIF_ALIST)		\
 {									\
     BIF_RETTYPE  res;							\
-    hipe_reserve_beam_trap_frame(c_p, args, ARITY);			\
-    res =  BIF_NAME ## _ ## ARITY (c_p, args);				\
-    if (is_value(res) || c_p->freason != TRAP) {			\
-	hipe_unreserve_beam_trap_frame(c_p);				\
+    hipe_reserve_beam_trap_frame(BIF_P, BIF__ARGS, ARITY);		\
+    res = nbif_impl_ ## BIF_NAME ## _ ## ARITY (NBIF_CALL_ARGS);	\
+    if (is_value(res) || BIF_P->freason != TRAP) {			\
+	hipe_unreserve_beam_trap_frame(BIF_P);				\
     }									\
     return res;								\
 }
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 32600f4338..f461da913e 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -420,6 +420,9 @@ bif erts_debug:set_internal_state/2
 bif erts_debug:display/1
 bif erts_debug:dist_ext_to_term/2
 bif erts_debug:instructions/0
+bif erts_debug:dirty_cpu/2
+bif erts_debug:dirty_io/2
+bif erts_debug:dirty/3
 
 #
 # Monitor testing bif's...
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 009d2b72d3..4659cb5a7b 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -93,6 +93,9 @@ static char erts_system_version[] = ("Erlang/OTP " ERLANG_OTP_RELEASE
 #ifdef USE_THREADS
 #if defined(ERTS_DIRTY_SCHEDULERS) && defined(ERTS_SMP)
 				     " [ds:%beu:%beu:%beu]"
+#endif
+#if defined(ERTS_DIRTY_SCHEDULERS_TEST)
+				     " [dirty-schedulers-TEST]"
 #endif
 				     " [async-threads:%d]"
 #endif
diff --git a/erts/emulator/beam/erl_dirty_bif.tab b/erts/emulator/beam/erl_dirty_bif.tab
new file mode 100644
index 0000000000..69421dcfcc
--- /dev/null
+++ b/erts/emulator/beam/erl_dirty_bif.tab
@@ -0,0 +1,82 @@
+#
+# %CopyrightBegin%
+#
+# Copyright Ericsson AB 2016. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# %CopyrightEnd%
+#
+
+#
+# Static declaration of BIFs that should execute on dirty schedulers.
+#
+# <dirty-bif-decl> ::= <type> <bif>
+# <bif> ::= <module> ":" <name> "/" <arity>
+# <type> ::= dirty-cpu | dirty-io | dirty-cpu-test | dirty-io-test
+#
+# When dirty scheduler support is available, a BIF declared with the
+# 'dirty-cpu' type will unconditionally execute on a dirty CPU scheduler,
+# and a BIF declared with the type 'dirty-io' will unconditionally execute
+# on a dirty IO scheduler. When dirty scheduler support is not available
+# all BIFs will of course execute on normal schedulers.
+#
+# When the emulator has been configured with the debug option
+# '--enable-dirty-schedulers-test', BIFs with the types 'dirty-cpu-test',
+# and 'dirty-io-test' will unconditionally execute on dirty schedulers.
+# When this debug option has not been enabled, these BIFs will be executed
+# on normal schedulers.
+#
+# BIFs marked as 'ubif' in ./bif.tab will be ignored, i.e., will always
+# execute on normal schedulers.
+#
+
+# --- Dirty BIFs ---
+
+dirty-cpu erts_debug:dirty_cpu/2
+dirty-io erts_debug:dirty_io/2
+
+# --- TEST of Dirty BIF functionality ---
+#  Functions below will execute on dirty schedulers when emulator has
+#  been configured for testing dirty schedulers. This is used for test
+#  and debug purposes only. We really do *not* want to execute these
+#  on dirty schedulers on a real system.
+
+dirty-cpu-test erlang:'++'/2
+dirty-cpu-test erlang:append/2
+dirty-cpu-test erlang:'--'/2
+dirty-cpu-test erlang:subtract/2
+dirty-cpu-test erlang:iolist_size/1
+dirty-cpu-test erlang:make_tuple/2
+dirty-cpu-test erlang:make_tuple/3
+dirty-cpu-test erlang:append_element/2
+dirty-cpu-test erlang:insert_element/3
+dirty-cpu-test erlang:delete_element/2
+dirty-cpu-test erlang:atom_to_list/1
+dirty-cpu-test erlang:list_to_atom/1
+dirty-cpu-test erlang:list_to_existing_atom/1
+dirty-cpu-test erlang:integer_to_list/1
+dirty-cpu-test erlang:string_to_integer/1
+dirty-cpu-test erlang:list_to_integer/1
+dirty-cpu-test erlang:list_to_integer/2
+dirty-cpu-test erlang:float_to_list/1
+dirty-cpu-test erlang:float_to_list/2
+dirty-cpu-test erlang:float_to_binary/1
+dirty-cpu-test erlang:float_to_binary/2
+dirty-cpu-test erlang:string_to_float/1
+dirty-cpu-test erlang:list_to_float/1
+dirty-cpu-test erlang:binary_to_float/1
+dirty-cpu-test erlang:tuple_to_list/1
+dirty-cpu-test erlang:list_to_tuple/1
+dirty-cpu-test erlang:display/1
+dirty-cpu-test erlang:display_string/1
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 643b46c861..6093b0cf39 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -42,6 +42,7 @@
 #include "dtrace-wrapper.h"
 #include "erl_bif_unique.h"
 #include "dist.h"
+#include "erl_nfunc_sched.h"
 
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20
@@ -2423,17 +2424,10 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
     }
 
     /*
-     * If a NIF has saved arguments, they need to be added
+     * If a NIF or BIF has saved arguments, they need to be added
      */
-    if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) {
-	Eterm* argv;
-	int argc;
-	if (erts_setup_nif_gc(p, &argv, &argc)) {
-	    roots[n].v = argv;
-	    roots[n].sz = argc;
-	    n++;
-	}
-    }
+    if (erts_setup_nif_export_rootset(p, &roots[n].v, &roots[n].sz))
+	n++;
 
     ASSERT(n <= rootset->size);
 
@@ -2985,6 +2979,8 @@ static void ERTS_INLINE
 offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size,
 	       Eterm* objv, int nobj)
 {
+    Eterm *v;
+    Uint sz;
     if (p->dictionary)  {
 	offset_heap(ERTS_PD_START(p->dictionary),
 		    ERTS_PD_SIZE(p->dictionary),
@@ -3005,12 +3001,8 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size,
 	offset_heap_ptr(objv, nobj, offs, area, area_size);
     }
     offset_off_heap(p, offs, area, area_size);
-    if (ERTS_PROC_GET_NIF_TRAP_EXPORT(p)) {
-	Eterm* argv;
-	int argc;
-	if (erts_setup_nif_gc(p, &argv, &argc))
-	    offset_heap_ptr(argv, argc, offs, area, area_size);
-    }
+    if (erts_setup_nif_export_rootset(p, &v, &sz))
+	offset_heap_ptr(v, sz, offs, area, area_size);
 }
 
 static void
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index 7ddf49937f..66bb55e6c8 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -137,8 +137,8 @@ void erts_msacc_init_thread(char *type, int id, int managed) {
 void erts_msacc_set_bif_state(ErtsMsAcc *__erts_msacc_cache, Eterm mod, void *fn) {
 
 #ifdef ERTS_MSACC_EXTENDED_BIFS
-#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num)                           \
-    if (fn == &FuncAddr) {                                             \
+#define BIF_LIST(Mod,Func,Arity,BifFuncAddr,FuncAddr,Num)	       \
+    if (fn == &BifFuncAddr) {                                             \
         ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATIC_STATE_COUNT + Num); \
     } else
 #include "erl_bif_list.h"
diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
index 9ef86a1293..d64ef8c8b9 100644
--- a/erts/emulator/beam/erl_msacc.h
+++ b/erts/emulator/beam/erl_msacc.h
@@ -122,7 +122,7 @@ static char *erts_msacc_states[] = {
     "sleep",
     "timers"
 #ifdef ERTS_MSACC_EXTENDED_BIFS
-#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num)   \
+#define BIF_LIST(Mod,Func,Arity,BifFuncAddr,FuncAddr,Num)	\
         ,"bif_" #Mod "_" #Func "_" #Arity
 #include "erl_bif_list.h"
 #undef BIF_LIST
diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c
new file mode 100644
index 0000000000..0333545255
--- /dev/null
+++ b/erts/emulator/beam/erl_nfunc_sched.c
@@ -0,0 +1,144 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#define ERTS_WANT_NFUNC_SCHED_INTERNALS__
+
+#include "global.h"
+#include "erl_process.h"
+#include "bif.h"
+#include "erl_nfunc_sched.h"
+
+NifExport *
+erts_new_proc_nif_export(Process *c_p, int argc)
+{
+    size_t size;
+    int i;
+    NifExport *nep, *old_nep;
+
+    size = sizeof(NifExport) + (argc-1)*sizeof(Eterm);
+    nep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, size);
+
+    for (i = 0; i < ERTS_NUM_CODE_IX; i++)
+	nep->exp.addressv[i] = &nep->exp.beam[0];
+
+    nep->argc = -1; /* unused marker */
+    nep->argv_size = argc;
+    old_nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(c_p, nep);
+    if (old_nep)
+	erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, old_nep);
+    return nep;
+}
+
+void
+erts_destroy_nif_export(Process *p)
+{
+    NifExport *nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL);
+    if (nep) {
+	if (nep->m)
+	    erts_nif_export_cleanup_nif_mod(nep);
+	erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, nep);
+    }
+}
+
+NifExport *
+erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
+			 ErtsCodeMFA *mfa, void *nif, BeamInstr *pc,
+			 BeamInstr instr,
+			 void *dfunc, void *ifunc,
+			 Eterm mod, Eterm func,
+			 int argc, const Eterm *argv)
+{
+    Process *used_proc;
+    ErtsSchedulerData *esdp;
+    Eterm* reg;
+    NifExport* nep;
+    int i;
+
+    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
+		       & ERTS_PROC_LOCK_MAIN);
+
+    if (dirty_shadow_proc) {
+	esdp = erts_get_scheduler_data();
+	ASSERT(esdp && ERTS_SCHEDULER_IS_DIRTY(esdp));
+
+	used_proc = dirty_shadow_proc;
+    }
+    else {
+	esdp = erts_proc_sched_data(c_p);
+	ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp));
+
+	used_proc = c_p;
+	ERTS_VBUMP_ALL_REDS(c_p);
+    }
+
+    reg = esdp->x_reg_array;
+
+    if (mfa)
+	nep = erts_get_proc_nif_export(c_p, (int) mfa->arity);
+    else {
+	/* If no mfa, this is not the first schedule... */
+	nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
+	ASSERT(nep && nep->argc >= 0);
+    }
+
+    if (nep->argc < 0) {
+	/*
+	 * First schedule; save things that might
+	 * need to be restored...
+	 */
+	for (i = 0; i < (int) mfa->arity; i++)
+	    nep->argv[i] = reg[i];
+	nep->pc = pc;
+	nep->cp = c_p->cp;
+	ASSERT(nif);
+	nep->nif = nif;
+	nep->mfa = mfa;
+	nep->current = c_p->current;
+	ASSERT(argc >= 0);
+	nep->argc = (int) mfa->arity;
+	nep->m = NULL;
+
+	ASSERT(!erts_check_nif_export_in_area(c_p,
+					      (char *) nep,
+					      (sizeof(NifExport)
+					       + (sizeof(Eterm)
+						  *(nep->argc-1)))));
+    }
+    /* Copy new arguments into register array if necessary... */
+    if (reg != argv) {
+	for (i = 0; i < argc; i++)
+	    reg[i] = argv[i];
+    }
+    ASSERT(is_atom(mod) && is_atom(func));
+    nep->exp.info.mfa.module = mod;
+    nep->exp.info.mfa.function = func;
+    nep->exp.info.mfa.arity = (Uint) argc;
+    nep->exp.beam[0] = (BeamInstr) instr; /* call_nif || apply_bif */
+    nep->exp.beam[1] = (BeamInstr) dfunc;
+    nep->func = ifunc;
+    used_proc->arity = argc;
+    used_proc->freason = TRAP;
+    used_proc->i = (BeamInstr*) nep->exp.addressv[0];
+    return nep;
+}
diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h
new file mode 100644
index 0000000000..72de7f0eb5
--- /dev/null
+++ b/erts/emulator/beam/erl_nfunc_sched.h
@@ -0,0 +1,289 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2016. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifndef ERL_NFUNC_SCHED_H__
+#define ERL_NFUNC_SCHED_H__
+
+#include "erl_process.h"
+#include "bif.h"
+
+/*
+ * NIF exports need a few more items than the Export struct provides,
+ * including the erl_module_nif* and a NIF function pointer, so the
+ * NifExport below adds those. The Export member must be first in the
+ * struct. A number of values are stored for error handling purposes
+ * only.
+ *
+ * 'argc' is >= 0 when NifExport is in use, and < 0 when not.
+ */
+
+typedef struct {
+    Export exp;
+    struct erl_module_nif* m; /* NIF module, or NULL if BIF */
+    void *func;		/* Indirect NIF or BIF to execute (may be unused) */
+    ErtsCodeMFA *current;/* Current as set when originally called */
+    /* --- The following is only used on error --- */
+    BeamInstr *pc;	/* Program counter */
+    BeamInstr *cp;	/* Continuation pointer */
+    void *nif;		/* Original NIF/BIF call */
+    ErtsCodeMFA *mfa;	/* MFA of original call */
+    int argc;		/* Number of arguments in original call */
+    int argv_size;	/* Allocated size of argv */
+    Eterm argv[1];	/* Saved arguments from the original call */
+} NifExport;
+
+NifExport *erts_new_proc_nif_export(Process *c_p, int argc);
+void erts_destroy_nif_export(Process *p);
+NifExport *erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
+				    ErtsCodeMFA *mfa, void *nif, BeamInstr *pc,
+				    BeamInstr instr,
+				    void *dfunc, void *ifunc,
+				    Eterm mod, Eterm func,
+				    int argc, const Eterm *argv);
+void erts_nif_export_cleanup_nif_mod(NifExport *ep); /* erl_nif.c */
+ERTS_GLB_INLINE NifExport *erts_get_proc_nif_export(Process *c_p, int extra);
+ERTS_GLB_INLINE int erts_setup_nif_export_rootset(Process* proc, Eterm** objv,
+						  Uint* nobj);
+ERTS_GLB_INLINE int erts_check_nif_export_in_area(Process *p,
+						  char *start, Uint size);
+ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep);
+ERTS_GLB_INLINE void erts_nif_export_restore_error(Process* c_p, BeamInstr **pc,
+						   Eterm *reg, void **nif);
+ERTS_GLB_INLINE Process *erts_proc_shadow2real(Process *c_p);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE NifExport *
+erts_get_proc_nif_export(Process *c_p, int argc)
+{
+    NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
+    if (!nep || (nep->argc < 0 && nep->argv_size < argc))
+	return erts_new_proc_nif_export(c_p, argc);
+    return nep;
+}
+
+/*
+ * If a process has saved arguments, they need to be part of the GC
+ * rootset. The function below is called from setup_rootset() in
+ * erl_gc.c. Any exception term saved in the NifExport is also made
+ * part of the GC rootset here; it always resides in rootset[0].
+ */
+ERTS_GLB_INLINE int
+erts_setup_nif_export_rootset(Process* proc, Eterm** objv, Uint* nobj)
+{
+    NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+
+    if (!ep || ep->argc <= 0)
+	return 0;
+
+    *objv = ep->argv;
+    *nobj = ep->argc;
+    return 1;
+}
+
+/*
+ * Check if nif export points into code area...
+ */
+ERTS_GLB_INLINE int
+erts_check_nif_export_in_area(Process *p, char *start, Uint size)
+{
+    NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p);
+    if (!nep || nep->argc < 0)
+	return 0;
+    if (ErtsInArea(nep->pc, start, size))
+	return 1;
+    if (ErtsInArea(nep->cp, start, size))
+	return 1;
+    if (ErtsInArea(nep->mfa, start, size))
+	return 1;
+    if (ErtsInArea(nep->current, start, size))
+	return 1;
+    return 0;
+}
+
+ERTS_GLB_INLINE void
+erts_nif_export_restore(Process *c_p, NifExport *ep)
+{
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
+    ERTS_SMP_LC_ASSERT(!(c_p->static_flags
+			 & ERTS_STC_FLG_SHADOW_PROC));
+    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
+		       & ERTS_PROC_LOCK_MAIN);
+
+    c_p->current = ep->current;
+    ep->argc = -1; /* Unused nif-export marker... */
+}
+
+ERTS_GLB_INLINE void
+erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, Eterm *reg,
+			      void **nif)
+{
+    NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
+    int ix;
+
+    ASSERT(nep);
+    *pc = nep->pc;
+    c_p->cp = nep->cp;
+    *nif = nep->nif;
+    for (ix = 0; ix < nep->argc; ix++)
+	reg[ix] = nep->argv[ix];
+    erts_nif_export_restore(c_p, nep);
+}
+
+ERTS_GLB_INLINE Process *
+erts_proc_shadow2real(Process *c_p)
+{
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (c_p->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
+	Process *real_c_p = c_p->next;
+	ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
+	ASSERT(real_c_p->common.id == c_p->common.id);
+	return real_c_p;
+    }
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
+#endif
+    return c_p;
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /* ERL_NFUNC_SCHED_H__ */
+
+#if defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__)
+#define ERTS_NFUNC_SCHED_INTERNALS__
+
+#define ERTS_I_BEAM_OP_TO_NIF_EXPORT(I)					\
+    (ASSERT(BeamOp(op_apply_bif) == (BeamInstr *) (*(I))		\
+	    || BeamOp(op_call_nif) == (BeamInstr *) (*(I))),		\
+     ((NifExport *) (((char *) (I)) - offsetof(NifExport, exp.beam[0]))))
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+#include "erl_message.h"
+#include <stddef.h>
+
+ERTS_GLB_INLINE void erts_flush_dirty_shadow_proc(Process *sproc);
+ERTS_GLB_INLINE void erts_cache_dirty_shadow_proc(Process *sproc);
+ERTS_GLB_INLINE Process *erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp,
+						     Process *c_p);
+
+#if ERTS_GLB_INLINE_INCL_FUNC_DEF
+
+ERTS_GLB_INLINE void
+erts_flush_dirty_shadow_proc(Process *sproc)
+{
+    Process *c_p = sproc->next;
+
+    ASSERT(sproc->common.id == c_p->common.id);
+    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
+		       & ERTS_PROC_LOCK_MAIN);
+
+    ASSERT(c_p->stop == sproc->stop);
+    ASSERT(c_p->hend == sproc->hend);
+    ASSERT(c_p->heap == sproc->heap);
+    ASSERT(c_p->abandoned_heap == sproc->abandoned_heap);
+    ASSERT(c_p->heap_sz == sproc->heap_sz);
+    ASSERT(c_p->high_water == sproc->high_water);
+    ASSERT(c_p->old_heap == sproc->old_heap);
+    ASSERT(c_p->old_htop == sproc->old_htop);
+    ASSERT(c_p->old_hend == sproc->old_hend);
+
+    ASSERT(c_p->htop <= sproc->htop && sproc->htop <= c_p->stop);
+
+    c_p->htop = sproc->htop;
+
+    if (!c_p->mbuf)
+	c_p->mbuf = sproc->mbuf;
+    else if (sproc->mbuf) {
+	ErlHeapFragment *bp;
+	for (bp = sproc->mbuf; bp->next; bp = bp->next)
+	    ASSERT(!bp->off_heap.first);
+	bp->next = c_p->mbuf;
+	c_p->mbuf = sproc->mbuf;
+    }
+
+    c_p->mbuf_sz += sproc->mbuf_sz;
+
+    if (!c_p->off_heap.first)
+	c_p->off_heap.first = sproc->off_heap.first;
+    else if (sproc->off_heap.first) {
+	struct erl_off_heap_header *ohhp;
+	for (ohhp = sproc->off_heap.first; ohhp->next; ohhp = ohhp->next)
+	    ;
+	ohhp->next = c_p->off_heap.first;
+	c_p->off_heap.first = sproc->off_heap.first;
+    }
+
+    c_p->off_heap.overhead += sproc->off_heap.overhead;
+}
+
+ERTS_GLB_INLINE void
+erts_cache_dirty_shadow_proc(Process *sproc)
+{
+    Process *c_p = sproc->next;
+    ASSERT(c_p);
+    ASSERT(sproc->common.id == c_p->common.id);
+    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
+		       & ERTS_PROC_LOCK_MAIN);
+
+    sproc->htop = c_p->htop;
+    sproc->stop = c_p->stop;
+    sproc->hend = c_p->hend;
+    sproc->heap = c_p->heap;
+    sproc->abandoned_heap = c_p->abandoned_heap;
+    sproc->heap_sz = c_p->heap_sz;
+    sproc->high_water = c_p->high_water;
+    sproc->old_hend = c_p->old_hend;
+    sproc->old_htop = c_p->old_htop;
+    sproc->old_heap = c_p->old_heap;
+    sproc->mbuf = NULL;
+    sproc->mbuf_sz = 0;
+    ERTS_INIT_OFF_HEAP(&sproc->off_heap);
+}
+
+ERTS_GLB_INLINE Process *
+erts_make_dirty_shadow_proc(ErtsSchedulerData *esdp, Process *c_p)
+{
+    Process *sproc;
+
+    ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp));
+
+    sproc = esdp->dirty_shadow_process;
+    ASSERT(sproc);
+    ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
+    ASSERT(erts_smp_atomic32_read_nob(&sproc->state)
+	   == (ERTS_PSFLG_ACTIVE
+	       | ERTS_PSFLG_DIRTY_RUNNING
+	       | ERTS_PSFLG_PROXY));
+
+    sproc->next = c_p;
+    sproc->common.id = c_p->common.id;
+
+    erts_cache_dirty_shadow_proc(sproc);
+
+    return sproc;
+}
+
+#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */
+
+#endif /* ERTS_DIRTY_SCHEDULERS */
+
+#endif /* defined(ERTS_WANT_NFUNC_SCHED_INTERNALS__) && !defined(ERTS_NFUNC_SCHED_INTERNALS__) */
+
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 5499512dd1..ee4b22f7ca 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -55,6 +55,9 @@
 #include "dtrace-wrapper.h"
 #include "erl_process.h"
 #include "erl_bif_unique.h"
+#undef ERTS_WANT_NFUNC_SCHED_INTERNALS__
+#define ERTS_WANT_NFUNC_SCHED_INTERNALS__
+#include "erl_nfunc_sched.h"
 #if defined(USE_DYNAMIC_TRACE) && (defined(USE_DTRACE) || defined(USE_SYSTEMTAP))
 #define HAVE_USE_DTRACE 1
 #endif
@@ -79,8 +82,11 @@ struct erl_module_nif {
     ErlNifFunc _funcs_copy_[1];  /* only used for old libs */
 };
 
+typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]);
+
 #ifdef DEBUG
 #  define READONLY_CHECK
+#  define ERTS_DBG_NIF_NOT_SCHED_MARKER ((void *) (UWord) 1)
 #endif
 #ifdef READONLY_CHECK
 #  define ADD_READONLY_CHECK(ENV,PTR,SIZE) add_readonly_check(ENV,PTR,SIZE)
@@ -219,38 +225,6 @@ static void cache_env(ErlNifEnv* env);
 static void full_flush_env(ErlNifEnv *env);
 static void flush_env(ErlNifEnv* env);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-void erts_pre_dirty_nif(ErtsSchedulerData *esdp,
-			ErlNifEnv* env, Process* p,
-			struct erl_module_nif* mod_nif)
-{
-    Process *sproc;
-#ifdef DEBUG
-    erts_aint32_t state = erts_smp_atomic32_read_nob(&p->state);
-
-    ASSERT(!p->scheduler_data);
-    ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING)
-	   && !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)));
-    ASSERT(esdp);
-#endif
-
-    erts_pre_nif(env, p, mod_nif, NULL);
-
-    sproc = esdp->dirty_shadow_process;
-    ASSERT(sproc);
-    ASSERT(sproc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
-    ASSERT(erts_smp_atomic32_read_nob(&sproc->state)
-	   == (ERTS_PSFLG_ACTIVE
-	       | ERTS_PSFLG_DIRTY_RUNNING
-	       | ERTS_PSFLG_PROXY));
-
-    sproc->next = p;
-    sproc->common.id = p->common.id;
-    env->proc = sproc;
-    full_cache_env(env);
-}
-#endif
-
 /* Temporary object header, auto-deallocated when NIF returns
  * or when independent environment is cleared.
  */
@@ -278,115 +252,158 @@ void erts_post_nif(ErlNifEnv* env)
     env->exiting = ERTS_PROC_IS_EXITING(env->proc);
 }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-void erts_post_dirty_nif(ErlNifEnv* env)
+
+/*
+ * Initialize a NifExport struct. Create it if needed and store it in the
+ * proc. The direct_fp function is what will be invoked by op_call_nif, and
+ * the indirect_fp function, if not NULL, is what the direct_fp function
+ * will call. If the allocated NifExport isn't enough to hold all of argv,
+ * allocate a larger one. Save 'current' and registers if first time this
+ * call is scheduled.
+ */
+
+static ERTS_INLINE ERL_NIF_TERM
+schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
+	 Eterm mod, Eterm func_name, int argc, const ERL_NIF_TERM argv[])
 {
-    Process *c_p;
-    ASSERT(env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
-    ASSERT(env->proc->next);
-    erts_unblock_fpe(env->fpe_was_unmasked);
-    full_flush_env(env);
-    free_tmp_objs(env);
-    c_p = env->proc->next;
-    env->exiting = ERTS_PROC_IS_EXITING(c_p);
-    ERTS_VBUMP_ALL_REDS(c_p);
+    Export *exp;
+    NifExport *ep;
+    Process *c_p, *dirty_shadow_proc;
+
+    execution_state(env, &c_p, NULL);
+    if (c_p == env->proc)
+	dirty_shadow_proc = NULL;
+    else
+	dirty_shadow_proc = env->proc;
+
+    ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
+
+    exp = ErtsContainerStruct(c_p->current, Export, info.mfa);
+
+    ep = erts_nif_export_schedule(c_p, dirty_shadow_proc,
+				  c_p->current,
+				  (BifFunction) exp->beam[1],
+				  c_p->cp,
+				  (BeamInstr) em_call_nif,
+				  direct_fp, indirect_fp,
+				  mod, func_name,
+				  argc, (const Eterm *) argv);
+    if (!ep->m) {
+	/* First time this call is scheduled... */
+	erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1);
+	ep->m = env->mod_nif;
+    }
+    return (ERL_NIF_TERM) THE_NON_VALUE;
 }
-#endif
 
-static void full_flush_env(ErlNifEnv* env)
-{
 #ifdef ERTS_DIRTY_SCHEDULERS
-    if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
-	/* Dirty nif call using shadow process struct */
-	Process *c_p = env->proc->next;
-
-	ASSERT(is_scheduler() < 0);
-	ASSERT(env->proc->common.id == c_p->common.id);
-	ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
-			   & ERTS_PROC_LOCK_MAIN);
-
-	if (!env->heap_frag) {
-	    ASSERT(env->hp_end == HEAP_LIMIT(c_p));
-	    ASSERT(env->hp >= HEAP_TOP(c_p));
-	    ASSERT(env->hp <= HEAP_LIMIT(c_p));
-	    HEAP_TOP(c_p) = env->hp;
+
+static ERL_NIF_TERM dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+static ERL_NIF_TERM dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]);
+
+int
+erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *reg)
+{
+    int exiting;
+    ERL_NIF_TERM *argv = (ERL_NIF_TERM *) reg;
+    NifExport *nep = ERTS_I_BEAM_OP_TO_NIF_EXPORT(I);
+    ErtsCodeMFA *codemfa = erts_code_to_codemfa(I);
+    NativeFunPtr dirty_nif = (NativeFunPtr) I[1];
+    ErlNifEnv env;
+    ERL_NIF_TERM result;
+#ifdef DEBUG
+    erts_aint32_t state = erts_smp_atomic32_read_nob(&c_p->state);
+
+    ASSERT(nep == ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p));
+
+    ASSERT(!c_p->scheduler_data);
+    ASSERT((state & ERTS_PSFLG_DIRTY_RUNNING)
+	&& !(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS)));
+    ASSERT(esdp);
+
+    nep->func = ERTS_DBG_NIF_NOT_SCHED_MARKER;
+#endif
+
+    erts_pre_nif(&env, c_p, nep->m, NULL);
+
+    env.proc = erts_make_dirty_shadow_proc(esdp, c_p);
+
+    env.proc->freason = EXC_NULL;
+    env.proc->fvalue = NIL;
+    env.proc->ftrace = NIL;
+    env.proc->i = c_p->i;
+
+    ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p)));
+
+    erts_smp_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
+						   | ERTS_PSFLG_DIRTY_IO_PROC));
+
+    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */
+
+    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+
+    ASSERT(env.proc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
+    ASSERT(env.proc->next == c_p);
+
+    exiting = ERTS_PROC_IS_EXITING(c_p);
+
+    if (!exiting) {
+	if (env.exception_thrown) {
+	schedule_exception:
+	    schedule(&env, dirty_nif_exception, NULL,
+		     am_erts_internal, am_dirty_nif_exception,
+		     1, &env.proc->fvalue);
+	}
+	else if (is_value(result)) {
+	    schedule(&env, dirty_nif_finalizer, NULL,
+		     am_erts_internal, am_dirty_nif_finalizer,
+		     1, &result);
+	}
+	else if (env.proc->freason != TRAP) { /* user returned garbage... */
+	    ERTS_DECL_AM(badreturn);
+	    (void) enif_raise_exception(&env, AM_badreturn);
+	    goto schedule_exception;
 	}
 	else {
-	    Uint usz;
-	    ASSERT(env->hp_end != HEAP_LIMIT(c_p));
-	    ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size);
-
-	    HEAP_TOP(c_p) = HEAP_TOP(env->proc);
-	    usz = env->hp - env->heap_frag->mem;
-	    env->proc->mbuf_sz += usz - env->heap_frag->used_size;
-	    env->heap_frag->used_size = usz;
-
-	    ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size);
-
-	    if (c_p->mbuf) {
-		ErlHeapFragment *bp;
-		for (bp = env->proc->mbuf; bp->next; bp = bp->next)
-		    ;
-		bp->next = c_p->mbuf;
-	    }
+	    /* Rescheduled by dirty NIF call... */
+	    ASSERT(nep->func != ERTS_DBG_NIF_NOT_SCHED_MARKER);
+	}
+	c_p->i = env.proc->i;
+	c_p->arity = env.proc->arity;
+    }
 
-	    c_p->mbuf = env->proc->mbuf;
-	    c_p->mbuf_sz += env->proc->mbuf_sz;
+#ifdef DEBUG
+    if (nep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER)
+	nep->func = NULL;
+#endif
 
-	}
+    erts_unblock_fpe(env.fpe_was_unmasked);
+    full_flush_env(&env);
+    free_tmp_objs(&env);
 
-	if (!c_p->off_heap.first)
-	    c_p->off_heap.first = env->proc->off_heap.first;
-	else if (env->proc->off_heap.first) {
-	    struct erl_off_heap_header *ohhp;
-	    for (ohhp = env->proc->off_heap.first; ohhp->next; ohhp = ohhp->next)
-		;
-	    ohhp->next = c_p->off_heap.first;
-	    c_p->off_heap.first = env->proc->off_heap.first;
-	}
-	c_p->off_heap.overhead += env->proc->off_heap.overhead;
+    return exiting;
+}
 
-	return;
-    }
 #endif
 
+static void full_flush_env(ErlNifEnv* env)
+{
     flush_env(env);
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)
+	/* Dirty nif call using shadow process struct */
+	erts_flush_dirty_shadow_proc(env->proc);
+#endif
 }
 
 static void full_cache_env(ErlNifEnv* env)
 {    
 #ifdef ERTS_DIRTY_SCHEDULERS
-    if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) {
-	/* Dirty nif call using shadow process struct */
-	Process *sproc = env->proc;
-	Process *c_p = sproc->next;
-	ASSERT(c_p);
-	ASSERT(is_scheduler() < 0);
-	ASSERT(env->proc->common.id == c_p->common.id);
-	ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p)
-			   & ERTS_PROC_LOCK_MAIN);
-
-	sproc->htop = c_p->htop;
-	sproc->stop = c_p->stop;
-	sproc->hend = c_p->hend;
-	sproc->heap = c_p->heap;
-	sproc->abandoned_heap = c_p->abandoned_heap;
-	sproc->heap_sz = c_p->heap_sz;
-	sproc->high_water = c_p->high_water;
-	sproc->old_hend = c_p->old_hend;
-	sproc->old_htop = c_p->old_htop;
-	sproc->old_heap = c_p->old_heap;
-	sproc->mbuf = NULL;
-	sproc->mbuf_sz = 0;
-	ERTS_INIT_OFF_HEAP(&sproc->off_heap);
-
-	env->hp_end = HEAP_LIMIT(c_p);
-	env->hp = HEAP_TOP(c_p);
-	env->heap_frag = NULL;
-	return;
-    }
+    if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC)
+	erts_cache_dirty_shadow_proc(env->proc);
 #endif
-
     cache_env(env);
 }
 
@@ -1303,22 +1320,15 @@ Eterm enif_make_badarg(ErlNifEnv* env)
 
 Eterm enif_raise_exception(ErlNifEnv* env, ERL_NIF_TERM reason)
 {
-    Process *c_p;
-
-    execution_state(env, &c_p, NULL);
-
     env->exception_thrown = 1;
-    c_p->fvalue = reason;
-    BIF_ERROR(c_p, EXC_ERROR);
+    env->proc->fvalue = reason;
+    BIF_ERROR(env->proc, EXC_ERROR);
 }
 
 int enif_has_pending_exception(ErlNifEnv* env, ERL_NIF_TERM* reason)
 {
-    if (env->exception_thrown && reason != NULL) {
-	Process *c_p;
-	execution_state(env, &c_p, NULL);
-	*reason = c_p->fvalue;
-    }
+    if (env->exception_thrown && reason != NULL)
+	*reason = env->proc->fvalue;
     return env->exception_thrown;
 }
 
@@ -2269,188 +2279,28 @@ int enif_consume_timeslice(ErlNifEnv* env, int percent)
     return ERTS_BIF_REDS_LEFT(proc) == 0;
 }
 
-/*
- * NIF exports need a few more items than the Export struct provides,
- * including the erl_module_nif* and a NIF function pointer, so the
- * NifExport below adds those. The Export member must be first in the
- * struct. The saved_current, exception_thrown, saved_argc, rootset_extra, and
- * rootset members are used to track the MFA, any pending exception, and
- * arguments of the top NIF in case a chain of one or more
- * enif_schedule_nif() calls results in an exception, since in that case
- * the original MFA and registers have to be restored before returning to
- * Erlang to ensure stacktrace information associated with the exception is
- * correct.
- */
-typedef ERL_NIF_TERM (*NativeFunPtr)(ErlNifEnv*, int, const ERL_NIF_TERM[]);
-
-typedef struct {
-    Export exp;
-    struct erl_module_nif* m;
-    NativeFunPtr fp;
-    ErtsCodeMFA *saved_current;
-    int exception_thrown;
-    int saved_argc;
-    int rootset_extra;
-    Eterm rootset[1];
-} NifExport;
-
-/*
- * If a process has saved arguments, they need to be part of the GC
- * rootset. The function below is called from setup_rootset() in
- * erl_gc.c. This function is declared in erl_process.h. Any exception term
- * saved in the NifExport is also made part of the GC rootset here; it
- * always resides in rootset[0].
- */
-int
-erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj)
-{
-    NifExport* ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    int gc = ep && (ep->saved_argc > 0 || ep->rootset[0] != NIL);
-
-    if (gc) {
-	*objv = ep->rootset;
-	*nobj = 1 + ep->saved_argc;
-    }
-    return gc;
-}
-
-int
-erts_check_nif_export_in_area(Process *p, char *start, Uint size)
-{
-    NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(p);
-    if (!nep || !nep->saved_current)
-	return 0;
-    if (ErtsInArea(nep->saved_current, start, size))
-	return 1;
-    return 0;
-}
-
-/*
- * Allocate a NifExport and set it in proc specific data
- */
-static NifExport*
-allocate_nif_sched_data(Process* proc, int argc)
-{
-    NifExport* ep;
-    size_t total;
-    int i;
-
-    total = sizeof(NifExport) + argc*sizeof(Eterm);
-    ep = erts_alloc(ERTS_ALC_T_NIF_TRAP_EXPORT, total);
-    sys_memset((void*) ep, 0, total);
-    ep->rootset_extra = argc;
-    ep->rootset[0] = NIL;
-    for (i=0; i<ERTS_NUM_CODE_IX; i++) {
-	ep->exp.addressv[i] = &ep->exp.beam[0];
-    }
-    ep->exp.beam[0] = (BeamInstr) em_call_nif;
-    (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep);
-    return ep;
-}
-
 static ERTS_INLINE void
-destroy_nif_export(NifExport *nif_export)
+nif_export_cleanup_nif_mod(NifExport *ep)
 {
-    erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, (void *) nif_export);
+    if (erts_refc_dectest(&ep->m->rt_dtor_cnt, 0) == 0 && ep->m->mod == NULL)
+	close_lib(ep->m);
+    ep->m = NULL;
 }
 
 void
-erts_destroy_nif_export(void *nif_export)
+erts_nif_export_cleanup_nif_mod(NifExport *ep)
 {
-    destroy_nif_export((NifExport *) nif_export);
+    nif_export_cleanup_nif_mod(ep);
 }
 
-/*
- * Initialize a NifExport struct. Create it if needed and store it in the
- * proc. The direct_fp function is what will be invoked by op_call_nif, and
- * the indirect_fp function, if not NULL, is what the direct_fp function
- * will call. If the allocated NifExport isn't enough to hold all of argv,
- * allocate a larger one. Save MFA and registers only if the need_save
- * parameter is true.
- */
-static ERL_NIF_TERM
-init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
-		    int need_save, int argc, const ERL_NIF_TERM argv[])
+static ERTS_INLINE void
+nif_export_restore(Process *c_p, NifExport *ep)
 {
-    Process* proc;
-    Eterm* reg;
-    NifExport* ep;
-    int i, scheduler;
-    int orig_argc;
-
-    execution_state(env, &proc, &scheduler);
-
-    ASSERT(scheduler);
-
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc)
-		       & ERTS_PROC_LOCK_MAIN);
-
-    reg = erts_proc_sched_data(proc)->x_reg_array;
-
-    ASSERT(!need_save || proc->current);
-    orig_argc = need_save ? (int) proc->current->arity : 0;
-
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    if (!ep)
-	ep = allocate_nif_sched_data(proc, orig_argc);
-    else if (need_save && ep->rootset_extra < orig_argc) {
-	NifExport* new_ep = allocate_nif_sched_data(proc, orig_argc);
-	destroy_nif_export(ep);
-	ep = new_ep;
-    }
-    if (env->exception_thrown) {
-	ep->exception_thrown = 1;
-	ep->rootset[0] = proc->fvalue;
-    } else {
-	ep->exception_thrown = 0;
-	ep->rootset[0] = NIL;
-    }
-    if (scheduler > 0)
-	ERTS_VBUMP_ALL_REDS(proc);
-    if (need_save) {
-	ep->saved_current = proc->current;
-	ep->saved_argc = orig_argc;
-	for (i = 0; i < orig_argc; i++)
-	    ep->rootset[i+1] = reg[i];
-    }
-    for (i = 0; i < argc; i++)
-	reg[i] = (Eterm) argv[i];
-    proc->i = (BeamInstr*) ep->exp.addressv[0];
-    ep->exp.info.mfa.module = proc->current->module;
-    ep->exp.info.mfa.function = proc->current->function;
-    ep->exp.info.mfa.arity = argc;
-    ep->exp.beam[1] = (BeamInstr) direct_fp;
-    ep->m = env->mod_nif;
-    ep->fp = indirect_fp;
-    proc->freason = TRAP;
-    proc->arity = argc;
-    return THE_NON_VALUE;
+    erts_nif_export_restore(c_p, ep);
+    ASSERT(ep->m);
+    nif_export_cleanup_nif_mod(ep);
 }
 
-/*
- * Restore saved MFA and registers. Registers are restored only when the
- * exception flag is true.
- */
-static void
-restore_nif_mfa(Process* proc, NifExport* ep, int exception)
-{
-    int i;
-
-    ERTS_SMP_LC_ASSERT(!(proc->static_flags
-			 & ERTS_STC_FLG_SHADOW_PROC));
-    ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc)
-		       & ERTS_PROC_LOCK_MAIN);
-
-    ASSERT(ep->saved_current != &ep->exp.info.mfa);
-    proc->current = ep->saved_current;
-    ep->saved_current = NULL;
-    if (exception) {
-	Eterm* reg = erts_proc_sched_data(proc)->x_reg_array;
-	for (i = 0; i < ep->saved_argc; i++)
-	    reg[i] = ep->rootset[i+1];
-    }
-    ep->saved_argc = 0;
-}
 
 #ifdef ERTS_DIRTY_SCHEDULERS
 
@@ -2459,7 +2309,7 @@ restore_nif_mfa(Process* proc, NifExport* ep, int exception)
  * switch the process off a dirty scheduler thread and back onto a regular
  * scheduler thread, and then return the result from the dirty NIF. It also
  * restores the original NIF MFA when necessary based on the value of
- * ep->fp set by execute_dirty_nif via init_nif_sched_data -- non-NULL
+ * ep->func set by execute_dirty_nif via init_nif_sched_data -- non-NULL
  * means restore, NULL means do not restore.
  */
 static ERL_NIF_TERM
@@ -2474,9 +2324,7 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc)));
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
-    ASSERT(!ep->exception_thrown);
-    if (ep->fp)
-	restore_nif_mfa(proc, ep, 0);
+    nif_export_restore(proc, ep);
     return argv[0];
 }
 
@@ -2486,148 +2334,100 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 static ERL_NIF_TERM
 dirty_nif_exception(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
+    ERL_NIF_TERM ret;
     Process* proc;
     NifExport* ep;
+    Eterm exception;
 
     execution_state(env, &proc, NULL);
 
+    ASSERT(argc == 1);
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc)));
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
-    ASSERT(ep->exception_thrown);
-    if (ep->fp)
-	restore_nif_mfa(proc, ep, 1);
-    return enif_raise_exception(env, ep->rootset[0]);
+    exception = argv[0]; /* argv overwritten by restore below... */
+    nif_export_cleanup_nif_mod(ep);
+    ret = enif_raise_exception(env, exception);
+
+    /* Restore orig info for error and clear nif export in handle_error() */
+    proc->freason |= EXF_RESTORE_NIF;
+    return ret;
 }
 
 /*
- * Dirty NIF execution wrapper function. Invoke an application's dirty NIF,
- * then check the result and schedule the appropriate finalizer function
- * where needed. Also restore the original NIF MFA when appropriate.
+ * Dirty NIF scheduling wrapper function. Schedule a dirty NIF to execute.
+ * The dirty scheduler thread type (CPU or I/O) is indicated in flags
+ * parameter.
  */
-static ERL_NIF_TERM
-execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+static ERTS_INLINE ERL_NIF_TERM
+schedule_dirty_nif(ErlNifEnv* env, int flags, NativeFunPtr fp,
+		   Eterm func_name, int argc, const ERL_NIF_TERM argv[])
 {
     Process* proc;
-    NativeFunPtr fp;
-    NifExport* ep;
-    ERL_NIF_TERM result;
-
-    execution_state(env, &proc, NULL);
 
-    ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa);
-    fp = ep->fp;
-
-    ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc)));
-
-    /*
-     * Set ep->fp to NULL before the native call so we know later whether it scheduled another NIF for execution
-     */
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    ASSERT(ep && fp);
-
-    ep->fp = NULL;
-    erts_smp_atomic32_read_band_mb(&proc->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
-						   | ERTS_PSFLG_DIRTY_IO_PROC));
+    ASSERT(is_atom(func_name));
+    ASSERT(fp);
 
-    erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
+    ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND);
 
-    result = (*fp)(env, argc, argv);
+    execution_state(env, &proc, NULL);
 
-    erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
+    (void) erts_smp_atomic32_read_bset_nob(&proc->state,
+					   (ERTS_PSFLG_DIRTY_CPU_PROC
+					    | ERTS_PSFLG_DIRTY_IO_PROC),
+					   (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND
+					    ? ERTS_PSFLG_DIRTY_CPU_PROC
+					    : ERTS_PSFLG_DIRTY_IO_PROC));
 
-    if (erts_refc_dectest(&env->mod_nif->rt_dtor_cnt, 0) == 0 && env->mod_nif->mod == NULL)
-	close_lib(env->mod_nif);
-    /*
-     * If no more NIFs were scheduled by the native call via
-     * enif_schedule_nif(), then ep->fp will still be NULL as set above, in
-     * which case we need to restore the original NIF calling
-     * context. Reuse fp essentially as a boolean for this, passing it to
-     * init_nif_sched_data below. Both dirty_nif_exception and
-     * dirty_nif_finalizer then check ep->fp to decide whether or not to
-     * restore the original calling context.
-     */
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    ASSERT(ep);
-    if (ep->fp)
-	fp = NULL;
-    if (is_non_value(result) || env->exception_thrown) {
-	if (proc->freason != TRAP) {
-	    return init_nif_sched_data(env, dirty_nif_exception, fp, 0, argc, argv);
-	} else {
-	    if (ep->fp == NULL)
-		restore_nif_mfa(proc, ep, 1);
-	    return THE_NON_VALUE;
-	}
-    }
-    else
-	return init_nif_sched_data(env, dirty_nif_finalizer, fp, 0, 1, &result);
+    return schedule(env, fp, NULL, proc->current->module, func_name, argc, argv);
 }
 
-/*
- * Dirty NIF scheduling wrapper function. Schedule a dirty NIF to execute
- * via the execute_dirty_nif() wrapper function. The dirty scheduler thread
- * type (CPU or I/O) is indicated in flags parameter.
- */
 static ERTS_INLINE ERL_NIF_TERM
-schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[])
+static_schedule_dirty_nif(ErlNifEnv* env, erts_aint32_t dirty_psflg,
+			     int argc, const ERL_NIF_TERM argv[])
 {
-    ERL_NIF_TERM result;
-    erts_aint32_t act, dirty_flag;
-    Process* proc;
+    Process *proc;
+    NifExport *ep;
+    Eterm mod, func;
     NativeFunPtr fp;
-    NifExport* ep;
-    int need_save, scheduler;
 
-    execution_state(env, &proc, &scheduler);
-    if (scheduler <= 0) {
-	ASSERT(scheduler < 0);
-	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
-    }
+    execution_state(env, &proc, NULL);
+
+    /*
+     * Called in order to schedule statically determined
+     * dirty NIF calls...
+     *
+     * Note that 'current' does not point into a NifExport
+     * structure; only a structure with similar
+     * parts (located in code).
+     */
 
     ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa);
-    fp = ep->fp;
+    mod = proc->current->module;
+    func = proc->current->function;
+    fp = (NativeFunPtr) ep->func;
 
+    ASSERT(is_atom(mod) && is_atom(func));
     ASSERT(fp);
 
-    ASSERT(flags==ERL_NIF_DIRTY_JOB_IO_BOUND || flags==ERL_NIF_DIRTY_JOB_CPU_BOUND);
-
-    if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND)
-	dirty_flag = ERTS_PSFLG_DIRTY_CPU_PROC;
-    else
-	dirty_flag = ERTS_PSFLG_DIRTY_IO_PROC;
-
-    act = erts_smp_atomic32_read_bor_nob(&proc->state, dirty_flag);
-    if (!(act & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)))
-	erts_refc_inc(&env->mod_nif->rt_dtor_cnt, 1);
-    else if ((act & (ERTS_PSFLG_DIRTY_CPU_PROC
-		     | ERTS_PSFLG_DIRTY_IO_PROC)) & ~dirty_flag) {
-	/* clear other flag... */
-	if (flags == ERL_NIF_DIRTY_JOB_CPU_BOUND)
-	    dirty_flag = ERTS_PSFLG_DIRTY_IO_PROC;
-	else
-	    dirty_flag = ERTS_PSFLG_DIRTY_CPU_PROC;
-	erts_smp_atomic32_read_band_nob(&proc->state, ~dirty_flag);
-    }
+    (void) erts_smp_atomic32_read_bset_nob(&proc->state,
+					   (ERTS_PSFLG_DIRTY_CPU_PROC
+					    | ERTS_PSFLG_DIRTY_IO_PROC),
+					   dirty_psflg);
 
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    need_save = (ep == NULL || !ep->saved_current);
-    result = init_nif_sched_data(env, execute_dirty_nif, fp, need_save, argc, argv);
-    if (scheduler <= 0)
-	erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
-    return result;
+    return schedule(env, fp, NULL, mod, func, argc, argv);
 }
 
 static ERL_NIF_TERM
-schedule_dirty_io_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+static_schedule_dirty_io_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_IO_BOUND, argc, argv);
+    return static_schedule_dirty_nif(env, ERTS_PSFLG_DIRTY_IO_PROC, argc, argv);
 }
 
 static ERL_NIF_TERM
-schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
+static_schedule_dirty_cpu_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 {
-    return schedule_dirty_nif(env, ERL_NIF_DIRTY_JOB_CPU_BOUND, argc, argv);
+    return static_schedule_dirty_nif(env, ERTS_PSFLG_DIRTY_CPU_PROC, argc, argv);
 }
 
 #endif /* ERTS_DIRTY_SCHEDULERS */
@@ -2646,23 +2446,42 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     ERL_NIF_TERM result;
 
     execution_state(env, &proc, NULL);
-    ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa);
-    fp = ep->fp;
 
-    ASSERT(!env->exception_thrown);
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
+    ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa);
+    fp = ep->func;
     ASSERT(ep);
-    ep->fp = NULL;
+    ASSERT(!env->exception_thrown);
+
+    fp = (NativeFunPtr) ep->func;
+
+#ifdef DEBUG
+    ep->func = ERTS_DBG_NIF_NOT_SCHED_MARKER;
+#endif
+
     result = (*fp)(env, argc, argv);
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    ASSERT(ep);
-    /*
-     * If no NIFs were scheduled by the native call via
-     * enif_schedule_nif(), then ep->fp will still be NULL as set above, in
-     * which case we need to restore the original NIF MFA.
-     */
-    if (ep->fp == NULL)
-	restore_nif_mfa(proc, ep, env->exception_thrown);
+
+    ASSERT(ep == ERTS_PROC_GET_NIF_TRAP_EXPORT(proc));
+
+    if (is_value(result) || proc->freason != TRAP) {
+	/* Done (not rescheduled)... */
+	ASSERT(ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER);
+	if (!env->exception_thrown)
+	    nif_export_restore(proc, ep);
+	else {
+	    nif_export_cleanup_nif_mod(ep);
+	    /*
+	     * Restore orig info for error and clear nif
+	     * export in handle_error()
+	     */
+	    proc->freason |= EXF_RESTORE_NIF;
+	}
+    }
+
+#ifdef DEBUG
+    if (ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER)
+	ep->func = NULL;
+#endif
+
     return result;
 }
 
@@ -2672,9 +2491,8 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags,
 		  int argc, const ERL_NIF_TERM argv[])
 {
     Process* proc;
-    NifExport* ep;
     ERL_NIF_TERM fun_name_atom, result;
-    int need_save, scheduler;
+    int scheduler;
 
     if (argc > MAX_ARG)
 	return enif_make_badarg(env);
@@ -2689,35 +2507,16 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags,
 	erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN);
     }
 
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    need_save = (ep == NULL || !ep->saved_current);
-
-    if (flags) {
+    if (flags == 0)
+	result = schedule(env, execute_nif, fp, proc->current->module,
+			  fun_name_atom, argc, argv);
 #ifdef ERTS_DIRTY_SCHEDULERS
-	NativeFunPtr sched_fun;
-	int chkflgs = (flags & (ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND));
-	if (chkflgs == ERL_NIF_DIRTY_JOB_IO_BOUND)
-	    sched_fun = schedule_dirty_io_nif;
-	else if (chkflgs == ERL_NIF_DIRTY_JOB_CPU_BOUND)
-	    sched_fun = schedule_dirty_cpu_nif;
-	else {
-	    result = enif_make_badarg(env);
-	    goto done;
-	}
-	result = init_nif_sched_data(env, sched_fun, fp, need_save, argc, argv);
-#else
-	result = enif_make_badarg(env);
+    else if (!(flags & ~(ERL_NIF_DIRTY_JOB_IO_BOUND|ERL_NIF_DIRTY_JOB_CPU_BOUND)))
+	result = schedule_dirty_nif(env, flags, fp, fun_name_atom, argc, argv);
 #endif
-	goto done;
-    }
     else
-	result = init_nif_sched_data(env, execute_nif, fp, need_save, argc, argv);
-
-    ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
-    ASSERT(ep);
-    ep->exp.info.mfa.function = (BeamInstr) fun_name_atom;
+	result = enif_make_badarg(env);
 
-done:
     if (scheduler < 0)
 	erts_smp_proc_unlock(proc, ERTS_PROC_LOCK_MAIN);
 
@@ -3416,8 +3215,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2)
 	    if (f->flags) {
 		code_ptr[3] = (BeamInstr) f->fptr;
 		code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ?
-		    (BeamInstr) schedule_dirty_io_nif :
-		    (BeamInstr) schedule_dirty_cpu_nif;
+		    (BeamInstr) static_schedule_dirty_io_nif :
+		    (BeamInstr) static_schedule_dirty_cpu_nif;
 	    }
 	    else
 #endif
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 6928b282ee..955b98b35b 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -48,6 +48,7 @@
 #include "erl_bif_unique.h"
 #define ERTS_WANT_TIMER_WHEEL_API
 #include "erl_time.h"
+#include "erl_nfunc_sched.h"
 
 #define ERTS_CHECK_TIME_REDS CONTEXT_REDS
 #define ERTS_DELAYED_WAKEUP_INFINITY (~(Uint64) 0)
@@ -193,12 +194,6 @@ static ErtsAuxWorkData *aux_thread_aux_work_data;
 #define ERTS_SCHDLR_SSPND_CHNG_ONLN		(((erts_aint32_t) 1) << 2)
 #define ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN	(((erts_aint32_t) 1) << 3)
 
-typedef enum {
-    ERTS_SCHED_NORMAL,
-    ERTS_SCHED_DIRTY_CPU,
-    ERTS_SCHED_DIRTY_IO
-} ErtsSchedType;
-
 typedef struct {
     int ongoing;
     ErtsProcList *blckrs;
@@ -10102,9 +10097,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		goto sunlock_sched_out_proc;
 	    }
 
-	    ASSERT((state & ERTS_PSFLG_DIRTY_ACTIVE_SYS)
-		   || *p->i == (BeamInstr) em_call_nif);
-
 	    ASSERT(rq == ERTS_DIRTY_CPU_RUNQ
 		   ? (state & (ERTS_PSFLG_DIRTY_CPU_PROC
 			       | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
@@ -11976,7 +11968,6 @@ delete_process(Process* p)
     ErtsPSD *psd;
     struct saved_calls *scb;
     process_breakpoint_time_t *pbt;
-    void *nif_export;
 
     VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->common.id));
     VERBOSE(DEBUG_SHCOPY, ("[pid=%T] delete process: %p %p %p %p\n", p->common.id,
@@ -11993,9 +11984,7 @@ delete_process(Process* p)
     if (pbt)
         erts_free(ERTS_ALC_T_BPD, (void *) pbt);
 
-    nif_export = ERTS_PROC_SET_NIF_TRAP_EXPORT(p, NULL);
-    if (nif_export)
-	erts_destroy_nif_export(nif_export);
+    erts_destroy_nif_export(p);
 
     /* Cleanup psd */
 
@@ -13564,6 +13553,9 @@ erts_dbg_check_halloc_lock(Process *p)
     ErtsSchedulerData *esdp;
     if (ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(p))
 	return 1;
+    if ((p->static_flags & ERTS_STC_FLG_SHADOW_PROC)
+	&& ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()))
+	return 1;
     if (p->common.id == ERTS_INVALID_PID)
 	return 1;
     esdp = erts_proc_sched_data(p);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 7193b7d1db..58a50f8cf4 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -838,8 +838,8 @@ typedef struct {
 #define ERTS_PSD_DELAYED_GC_TASK_QS_GET_LOCKS ERTS_PROC_LOCK_MAIN
 #define ERTS_PSD_DELAYED_GC_TASK_QS_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
-#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ((ErtsProcLocks) 0)
-#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ((ErtsProcLocks) 0)
+#define ERTS_PSD_NIF_TRAP_EXPORT_GET_LOCKS ERTS_PROC_LOCK_MAIN
+#define ERTS_PSD_NIF_TRAP_EXPORT_SET_LOCKS ERTS_PROC_LOCK_MAIN
 
 typedef struct {
     ErtsProcLocks get_locks;
@@ -1582,10 +1582,6 @@ Uint64 erts_get_proc_interval(void);
 Uint64 erts_ensure_later_proc_interval(Uint64);
 Uint64 erts_step_proc_interval(void);
 
-int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c */
-void erts_destroy_nif_export(void *); /* see erl_nif.c */
-int erts_check_nif_export_in_area(Process *p, char *start, Uint size);
-
 ErtsProcList *erts_proclist_create(Process *);
 ErtsProcList *erts_proclist_copy(ErtsProcList *);
 void erts_proclist_destroy(ErtsProcList *);
diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h
index e431c3051b..99a8ff6bad 100644
--- a/erts/emulator/beam/error.h
+++ b/erts/emulator/beam/error.h
@@ -38,15 +38,12 @@
  * Some of these have convenient aliases, like BADARG and BADARITH.
  */
 
-/*
- * Bits 0-1 index the 'exception class tag' table.
- */
-#define EXC_CLASSBITS 3
-#define GET_EXC_CLASS(x) ((x) & EXC_CLASSBITS)
-
 /*
  * Exception class tags (indices into the 'exception_tag' array)
  */
+#define EXTAG_OFFSET	0
+#define EXTAG_BITS	2
+
 #define EXTAG_ERROR	0
 #define EXTAG_EXIT	1
 #define EXTAG_THROWN	2
@@ -54,20 +51,31 @@
 #define NUMBER_EXC_TAGS 3	/* The number of exception class tags */
 
 /*
- * Exit code flags (bits 2-7)
+ * Index to the 'exception class tag' table.
+ */
+#define EXC_CLASSBITS	((1<<EXTAG_BITS)-1)
+#define GET_EXC_CLASS(x) ((x) & EXC_CLASSBITS)
+
+/*
+ * Exit code flags
  *
  * These flags make is easier and quicker to decide what to do with the
  * exception in the early stages, before a handler is found, and also
  * maintains some separation between the class tag and the actions.
  */
-#define EXF_PANIC	(1<<2)	/* ignore catches */
-#define EXF_THROWN	(1<<3)	/* nonlocal return */
-#define EXF_LOG		(1<<4)	/* write to logger on termination */
-#define EXF_NATIVE	(1<<5)	/* occurred in native code */
-#define EXF_SAVETRACE	(1<<6)	/* save stack trace in internal form */
-#define EXF_ARGLIST	(1<<7)	/* has arglist for top of trace */
+#define EXF_OFFSET	EXTAG_BITS
+#define EXF_BITS	7
 
-#define EXC_FLAGBITS 0x00fc
+#define EXF_PANIC	(1<<(0+EXF_OFFSET))	/* ignore catches */
+#define EXF_THROWN	(1<<(1+EXF_OFFSET))	/* nonlocal return */
+#define EXF_LOG		(1<<(2+EXF_OFFSET))	/* write to logger on termination */
+#define EXF_NATIVE	(1<<(3+EXF_OFFSET))	/* occurred in native code */
+#define EXF_SAVETRACE	(1<<(4+EXF_OFFSET))	/* save stack trace in internal form */
+#define EXF_ARGLIST	(1<<(5+EXF_OFFSET))	/* has arglist for top of trace */
+#define EXF_RESTORE_NIF	(1<<(6+EXF_OFFSET))	/* restore original bif/nif */
+
+#define EXC_FLAGBITS	(((1<<(EXF_BITS+EXF_OFFSET))-1) \
+			 & ~((1<<(EXF_OFFSET))-1))
 
 /*
  * The primary fields of an exception code
@@ -77,11 +85,16 @@
 #define NATIVE_EXCEPTION(x) ((x) | EXF_NATIVE)
 
 /*
- * Bits 8-12 of the error code are used for indexing into
+ * Error code used for indexing into
  * the short-hand error descriptor table.
  */
-#define EXC_INDEXBITS 0x1f00
-#define GET_EXC_INDEX(x) (((x) & EXC_INDEXBITS) >> 8)
+#define EXC_OFFSET	(EXF_OFFSET+EXF_BITS)
+#define EXC_BITS	5
+
+#define EXC_INDEXBITS	(((1<<(EXC_BITS+EXC_OFFSET))-1) \
+			 & ~((1<<(EXC_OFFSET))-1))
+
+#define GET_EXC_INDEX(x) (((x) & EXC_INDEXBITS) >> EXC_OFFSET)
 
 /*
  * Exit codes used for raising a fresh exception. The primary exceptions
@@ -107,46 +120,46 @@
 					/* Error with given arglist term
 					 * (exit reason in p->fvalue) */
 
-#define EXC_NORMAL		((1 << 8) | EXC_EXIT)
+#define EXC_NORMAL		((1 << EXC_OFFSET) | EXC_EXIT)
 					/* Normal exit (reason 'normal') */
-#define EXC_INTERNAL_ERROR	((2 << 8) | EXC_ERROR | EXF_PANIC)
+#define EXC_INTERNAL_ERROR	((2 << EXC_OFFSET) | EXC_ERROR | EXF_PANIC)
 					/* Things that shouldn't happen */
-#define EXC_BADARG		((3 << 8) | EXC_ERROR)
+#define EXC_BADARG		((3 << EXC_OFFSET) | EXC_ERROR)
 					/* Bad argument to a BIF */
-#define EXC_BADARITH		((4 << 8) | EXC_ERROR)
+#define EXC_BADARITH		((4 << EXC_OFFSET) | EXC_ERROR)
 					/* Bad arithmetic */
-#define EXC_BADMATCH		((5 << 8) | EXC_ERROR)
+#define EXC_BADMATCH		((5 << EXC_OFFSET) | EXC_ERROR)
 					/* Bad match in function body */
-#define EXC_FUNCTION_CLAUSE	((6 << 8) | EXC_ERROR)
+#define EXC_FUNCTION_CLAUSE	((6 << EXC_OFFSET) | EXC_ERROR)
 					 /* No matching function head */
-#define EXC_CASE_CLAUSE		((7 << 8) | EXC_ERROR)
+#define EXC_CASE_CLAUSE		((7 << EXC_OFFSET) | EXC_ERROR)
 					/* No matching case clause */
-#define EXC_IF_CLAUSE		((8 << 8) | EXC_ERROR)
+#define EXC_IF_CLAUSE		((8 << EXC_OFFSET) | EXC_ERROR)
 					/* No matching if clause */
-#define EXC_UNDEF		((9 << 8) | EXC_ERROR)
+#define EXC_UNDEF		((9 << EXC_OFFSET) | EXC_ERROR)
 				 	/* No farity that matches */
-#define EXC_BADFUN		((10 << 8) | EXC_ERROR)
+#define EXC_BADFUN		((10 << EXC_OFFSET) | EXC_ERROR)
 					/* Not an existing fun */
-#define EXC_BADARITY		((11 << 8) | EXC_ERROR)
+#define EXC_BADARITY		((11 << EXC_OFFSET) | EXC_ERROR)
 					/* Attempt to call fun with
 					 * wrong number of arguments. */
-#define EXC_TIMEOUT_VALUE	((12 << 8) | EXC_ERROR)
+#define EXC_TIMEOUT_VALUE	((12 << EXC_OFFSET) | EXC_ERROR)
 					/* Bad time out value */
-#define EXC_NOPROC		((13 << 8) | EXC_ERROR)
+#define EXC_NOPROC		((13 << EXC_OFFSET) | EXC_ERROR)
 					/* No process or port */
-#define EXC_NOTALIVE		((14 << 8) | EXC_ERROR)
+#define EXC_NOTALIVE		((14 << EXC_OFFSET) | EXC_ERROR)
 					/* Not distributed */
-#define EXC_SYSTEM_LIMIT	((15 << 8) | EXC_ERROR)
+#define EXC_SYSTEM_LIMIT	((15 << EXC_OFFSET) | EXC_ERROR)
 					/* Ran out of something */
-#define EXC_TRY_CLAUSE		((16 << 8) | EXC_ERROR)
+#define EXC_TRY_CLAUSE		((16 << EXC_OFFSET) | EXC_ERROR)
 					/* No matching try clause */
-#define EXC_NOTSUP		((17 << 8) | EXC_ERROR)
+#define EXC_NOTSUP		((17 << EXC_OFFSET) | EXC_ERROR)
 					/* Not supported */
 
-#define EXC_BADMAP		((18 << 8) | EXC_ERROR)
+#define EXC_BADMAP		((18 << EXC_OFFSET) | EXC_ERROR)
 					/* Bad map */
 
-#define EXC_BADKEY		((19 << 8) | EXC_ERROR)
+#define EXC_BADKEY		((19 << EXC_OFFSET) | EXC_ERROR)
 					/* Bad key in map */
 
 #define NUMBER_EXIT_CODES 20	/* The number of exit code indices */
@@ -154,7 +167,7 @@
 /*
  * Internal pseudo-error codes.
  */
-#define TRAP		(1 << 8)	/* BIF Trap to erlang code */
+#define TRAP		(1 << EXC_OFFSET)	/* BIF Trap to erlang code */
 
 /*
  * Aliases for some common exit codes.
@@ -201,6 +214,7 @@ struct StackTrace {
     BeamInstr* pc;
     ErtsCodeMFA* current;
     int depth;	/* number of saved pointers in trace[] */
+    ErtsCodeMFA mfa; /* in case we need to make a copy of mfa */
     BeamInstr *trace[1];  /* varying size - must be last in struct */
 };
 
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 2b2f3c5cdc..d5ca3b04eb 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -61,12 +61,6 @@ struct enif_environment_t /* ErlNifEnv */
 extern void erts_pre_nif(struct enif_environment_t*, Process*,
 			 struct erl_module_nif*, Process* tracee);
 extern void erts_post_nif(struct enif_environment_t* env);
-#ifdef ERTS_DIRTY_SCHEDULERS
-extern void erts_pre_dirty_nif(ErtsSchedulerData *,
-			       struct enif_environment_t*, Process*,
-			       struct erl_module_nif*);
-extern void erts_post_dirty_nif(struct enif_environment_t* env);
-#endif
 extern Eterm erts_nif_taints(Process* p);
 extern void erts_print_nif_taints(fmtfn_t to, void* to_arg);
 void erts_unload_nif(struct erl_module_nif* nif);
@@ -78,6 +72,12 @@ extern Eterm erts_nif_call_function(Process *p, Process *tracee,
                                     struct enif_func_t *,
                                     int argc, Eterm *argv);
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+int erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p,
+			BeamInstr *I, Eterm *reg);
+#endif /* ERTS_DIRTY_SCHEDULERS */
+
+
 /* Driver handle (wrapper for old plain handle) */
 #define ERL_DE_OK      0
 #define ERL_DE_UNLOAD  1
@@ -993,7 +993,7 @@ void erts_queue_monitor_message(Process *,
 				Eterm,
 				Eterm);
 void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a,
-			   Eterm (*bif)(Process*,Eterm*));
+			   Eterm (*bif)(Process*, Eterm*, BeamInstr*));
 void erts_init_bif(void);
 Eterm erl_send(Process *p, Eterm to, Eterm msg);
 
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index ec502d5a78..3fa48da1ec 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -56,6 +56,8 @@
 #ifdef HIPE
 #  include "hipe_mode_switch.h"
 #endif
+#define ERTS_WANT_NFUNC_SCHED_INTERNALS__
+#include "erl_nfunc_sched.h"
 
 #undef M_TRIM_THRESHOLD
 #undef M_TOP_PAD
-- 
cgit v1.2.3


From 95ec5d385cfba23c770d946871c0197bf374ff3c Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 28 Oct 2016 18:34:14 +0200
Subject: Optimize handling of BIF errors

---
 erts/emulator/beam/beam_emu.c        | 139 ++++++++++++++++++-----------------
 erts/emulator/beam/bif.c             |  25 +++----
 erts/emulator/beam/erl_nfunc_sched.c |   4 +-
 erts/emulator/beam/erl_nfunc_sched.h |  10 +--
 erts/emulator/beam/erl_nif.c         |   4 -
 erts/emulator/beam/error.h           |   1 -
 6 files changed, 87 insertions(+), 96 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index befd2989f9..cfdf61eed0 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -1057,9 +1057,10 @@ do {                                            \
  * The following functions are called directly by process_main().
  * Don't inline them.
  */
-static BifFunction translate_gc_bif(void* gcf) NOINLINE;
+static ErtsCodeMFA *ubif2mfa(void* uf) NOINLINE;
+static ErtsCodeMFA *gcbif2mfa(void* gcf) NOINLINE;
 static BeamInstr* handle_error(Process* c_p, BeamInstr* pc,
-			       Eterm* reg, BifFunction bf) NOINLINE;
+			       Eterm* reg, ErtsCodeMFA* bif_mfa) NOINLINE;
 static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa,
 				     Eterm* reg, Eterm func) NOINLINE;
 static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity,
@@ -1088,7 +1089,7 @@ static BeamInstr* next_catch(Process* c_p, Eterm *reg);
 static void terminate_proc(Process* c_p, Eterm Value);
 static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc);
 static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg,
-			     BifFunction bf, Eterm args);
+			    ErtsCodeMFA *bif_mfa, Eterm args);
 static struct StackTrace * get_trace_from_exc(Eterm exc);
 static Eterm make_arglist(Process* c_p, Eterm* reg, int a);
 
@@ -2640,7 +2641,7 @@ do {						\
 	}
 	reg[0] = tmp_reg[0];
 	SWAPOUT;
-	I = handle_error(c_p, I, reg, bf);
+	I = handle_error(c_p, I, reg, ubif2mfa((void *) bf));
 	goto post_error_handling;
     }
 
@@ -2676,7 +2677,7 @@ do {						\
 	    Goto(*I);
 	}
 	x(0) = x(live);
-	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
+	I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf));
 	goto post_error_handling;
     }
 
@@ -2721,7 +2722,7 @@ do {						\
 	live--;
 	x(0) = x(live);
 	x(1) = x(live+1);
-	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
+	I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf));
 	goto post_error_handling;
     }
 
@@ -2767,7 +2768,7 @@ do {						\
 	x(0) = x(live);
 	x(1) = x(live+1);
 	x(2) = x(live+2);
-	I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf));
+	I = handle_error(c_p, I, reg, gcbif2mfa((void *) bf));
 	goto post_error_handling;
     }
 
@@ -2829,7 +2830,7 @@ do {						\
 	reg[0] = tmp_reg[0];
 	reg[1] = tmp_reg[1];
 	SWAPOUT;
-	I = handle_error(c_p, I, reg, bf);
+	I = handle_error(c_p, I, reg, ubif2mfa((void *) bf));
 	goto post_error_handling;
     }
 
@@ -2906,7 +2907,7 @@ do {						\
 	 * Error handling.  SWAPOUT is not needed because it was done above.
 	 */
 	ASSERT(c_p->stop == E);
-	I = handle_error(c_p, I, reg, bf);
+	I = handle_error(c_p, I, reg, &export->info.mfa);
 	goto post_error_handling;
     }
 
@@ -3210,7 +3211,7 @@ do {						\
 	 SET_I(next);
 	 Dispatch();
      }
-     I = handle_error(c_p, I, reg, apply_3);
+     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
      goto post_error_handling;
  }
 
@@ -3225,7 +3226,7 @@ do {						\
 	 SET_I(next);
 	 Dispatch();
      }
-     I = handle_error(c_p, I, reg, apply_3);
+     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
      goto post_error_handling;
  }
 
@@ -3238,7 +3239,7 @@ do {						\
 	 SET_I(next);
 	 Dispatch();
      }
-     I = handle_error(c_p, I, reg, apply_3);
+     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
      goto post_error_handling;
  }
 
@@ -3253,7 +3254,7 @@ do {						\
 	 SET_I(next);
 	 Dispatch();
      }
-     I = handle_error(c_p, I, reg, apply_3);
+     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
      goto post_error_handling;
  }
 
@@ -3269,7 +3270,7 @@ do {						\
 	 SET_I(next);
 	 Dispatch();
      }
-     I = handle_error(c_p, I, reg, apply_3);
+     I = handle_error(c_p, I, reg, &bif_export[BIF_apply_3]->info.mfa);
      goto post_error_handling;
  }
 
@@ -3690,7 +3691,7 @@ do {						\
 		}
 		Dispatch();
 	    }
-	    I = handle_error(c_p, c_p->cp, reg, vbf);
+	    I = handle_error(c_p, c_p->cp, reg, c_p->current);
 	    goto post_error_handling;
 	}
     }
@@ -5045,7 +5046,7 @@ do {						\
 	 goto do_schedule;
      } else {
 	 HEAVY_SWAPIN;
-	 I = handle_error(c_p, I, reg, hibernate_3);
+	 I = handle_error(c_p, I, reg, &bif_export[BIF_hibernate_3]->info.mfa);
 	 goto post_error_handling;
      }
  }
@@ -5437,17 +5438,28 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 #endif /* ERTS_DIRTY_SCHEDULERS */
 }
 
-static BifFunction
-translate_gc_bif(void* gcf)
+static ErtsCodeMFA *
+gcbif2mfa(void* gcf)
 {
-    const ErtsGcBif* p;
-
-    for (p = erts_gc_bifs; p->bif != 0; p++) {
-	if (p->gc_bif == gcf) {
-	    return p->bif;
-	}
+    int i;
+    for (i = 0; erts_gc_bifs[i].bif; i++) {
+	if (erts_gc_bifs[i].gc_bif == gcf)
+	    return &bif_export[erts_gc_bifs[i].exp_ix]->info.mfa;
     }
     erts_exit(ERTS_ERROR_EXIT, "bad gc bif");
+    return NULL;
+}
+
+static ErtsCodeMFA *
+ubif2mfa(void* uf)
+{
+    int i;
+    for (i = 0; erts_u_bifs[i].bif; i++) {
+	if (erts_u_bifs[i].bif == uf)
+	    return &bif_export[erts_u_bifs[i].exp_ix]->info.mfa;
+    }
+    erts_exit(ERTS_ERROR_EXIT, "bad u bif");
+    return NULL;
 }
 
 /*
@@ -5506,7 +5518,7 @@ Eterm error_atom[NUMBER_EXIT_CODES] = {
  */
 
 static BeamInstr*
-handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
+handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa)
 {
     Eterm* hp;
     Eterm Value = c_p->fvalue;
@@ -5514,9 +5526,16 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
 
     ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */
 
-    if (c_p->freason & EXF_RESTORE_NIF) {
-	erts_nif_export_restore_error(c_p, &pc, reg, &bf);
+    if (c_p->freason & EXF_RESTORE_NIF)
+	erts_nif_export_restore_error(c_p, &pc, reg, &bif_mfa);
+
+#ifdef DEBUG
+    if (bif_mfa) {
+	/* Verify that bif_mfa does not point into our nif export */
+	NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
+	ASSERT(!nep || !ErtsInArea(bif_mfa, (char *)nep, sizeof(NifExport)));
     }
+#endif
 
     c_p->i = pc;    /* In case we call erts_exit(). */
 
@@ -5542,7 +5561,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf)
      * more modular.
      */
     if (c_p->freason & EXF_SAVETRACE) {
-        save_stacktrace(c_p, pc, reg, bf, Args);
+        save_stacktrace(c_p, pc, reg, bif_mfa, Args);
     }
 
     /*
@@ -5817,23 +5836,12 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) {
  */
 
 static void
-save_stacktrace_current(struct StackTrace *s, Process *c_p, ErtsCodeMFA *mfa)
-{
-    NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
-    if (!nep || &nep->exp.info.mfa != mfa)
-	s->current = mfa;
-    else {
-	s->mfa = *mfa;
-	s->current = &s->mfa;
-    }
-}
-
-static void
-save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
-		Eterm args) {
+save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg,
+		ErtsCodeMFA *bif_mfa, Eterm args) {
     struct StackTrace* s;
     int sz;
     int depth = erts_backtrace_depth;    /* max depth (never negative) */
+
     if (depth > 0) {
 	/* There will always be a current function */
 	depth --;
@@ -5850,33 +5858,29 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 
     /*
      * If the failure was in a BIF other than 'error/1', 'error/2',
-     * 'exit/1' or 'throw/1', find the bif-table index and save the
-     * argument registers by consing up an arglist.
+     * 'exit/1' or 'throw/1', save BIF-MFA and save the argument
+     * registers by consing up an arglist.
      */
-    if (bf != NULL && bf != error_1 && bf != error_2 && bf != exit_1
-	&& bf != throw_1 && bf != wrap_error_1 && bf != wrap_error_2
-	&& bf != wrap_exit_1 && bf != wrap_throw_1) {
-        int i;
-	int a;
-	for (i = 0; i < BIF_SIZE; i++) {
-	    if (bf == bif_table[i].f || bf == bif_table[i].traced) {
-		save_stacktrace_current(s, c_p, &bif_export[i]->info.mfa);
+    if (bif_mfa) {
+	if (bif_mfa->module == am_erlang) {
+	    switch (bif_mfa->function) {
+	    case am_error:
+		if (bif_mfa->arity == 1 || bif_mfa->arity == 2)
+		    goto non_bif_stacktrace;
+		break;
+	    case am_exit:
+		if (bif_mfa->arity == 1)
+		    goto non_bif_stacktrace;
+		break;
+	    case am_throw:
+		if (bif_mfa->arity == 1)
+		    goto non_bif_stacktrace;
+		break;
+	    default:
 		break;
 	    }
 	}
-	if (i >= BIF_SIZE) {
-	    /* 
-	     * The Bif does not really exist (no BIF entry).  It is a
-	     * TRAP and traps are called through apply_bif, which also
-	     * sets c_p->current (luckily).
-	     * OR it is a NIF called by call_nif where current is also set.
-	     */
-	    ASSERT(c_p->current);
-	    save_stacktrace_current(s, c_p, c_p->current);
-	}
-
-	a = s->current->arity;
-
+	s->current = bif_mfa;
 	/* Save first stack entry */
 	ASSERT(pc);
 	if (depth > 0) {
@@ -5889,11 +5893,12 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf,
 	    depth--;
 	}
 	s->pc = NULL;
-	args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */
+	args = make_arglist(c_p, reg, bif_mfa->arity); /* Overwrite CAR(c_p->ftrace) */
     } else {
 
-	save_stacktrace_current(s, c_p, c_p->current);
+    non_bif_stacktrace:
 
+	s->current = c_p->current;
         /* 
 	 * For a function_clause error, the arguments are in the beam
 	 * registers, c_p->cp is valid, and c_p->current is set.
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 7bcb7c196d..5f0564474f 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -5027,17 +5027,16 @@ void erts_init_bif(void)
 
 #define ERTS_SCHED_BIF_TRAP_MARKER ((void *) (UWord) 1)
 
-static void
+static ERTS_INLINE void
 schedule(Process *c_p, Process *dirty_shadow_proc,
-	 ErtsCodeMFA *mfa, BifFunction *nif, BeamInstr *pc,
+	 ErtsCodeMFA *mfa, BeamInstr *pc,
 	 ErtsBifFunc dfunc, void *ifunc,
 	 Eterm module, Eterm function,
 	 int argc, Eterm *argv)
 {
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
     (void) erts_nif_export_schedule(c_p, dirty_shadow_proc,
-				    mfa, nif, pc,
-				    (BeamInstr) em_apply_bif,
+				    mfa, pc, (BeamInstr) em_apply_bif,
 				    dfunc, ifunc,
 				    module, function,
 				    argc, argv);
@@ -5120,11 +5119,10 @@ erts_schedule_bif(Process *proc,
 
     if (!ERTS_PROC_IS_EXITING(c_p)) {
 	Export *exp;
-	BifFunction obif, dbif, ibif;
+	BifFunction dbif, ibif;
 	BeamInstr *pc;
 
 	/*
-	 * obif - original bif
 	 * dbif - direct bif
 	 * ibif - indirect bif
 	 */
@@ -5160,14 +5158,12 @@ erts_schedule_bif(Process *proc,
 
 	if (i == NULL) {
 	    ERTS_INTERNAL_ERROR("Missing instruction pointer");
-	    obif = NULL;
 	}
 #ifdef HIPE
 	else if (proc->flags & F_HIPE_MODE) {
 	    /* Pointer to bif export in i */
 	    exp = (Export *) i;
 	    pc = c_p->cp;
-	    obif = (BifFunction) exp->beam[1];
 	    mfa = &exp->info.mfa;
 	}
 #endif
@@ -5175,19 +5171,16 @@ erts_schedule_bif(Process *proc,
 	    /* Pointer to bif export in i+1 */
 	    exp = (Export *) i[1];
 	    pc = i;
-	    obif = (BifFunction) exp->beam[1];
 	    mfa = &exp->info.mfa;
 	}
 	else if (em_apply_bif == (BeamInstr *) *i) {
 	    /* Pointer to bif in i+1, and mfa in i-3 */	    
-	    obif = (BifFunction) i[1];
 	    pc = c_p->cp;
 	    mfa = erts_code_to_codemfa(i);
 	}
 	else {
 	    ERTS_INTERNAL_ERROR("erts_schedule_bif() called "
 				"from unexpected instruction");
-	    obif = NULL;
 	}
 	ASSERT(bif);
 
@@ -5197,8 +5190,8 @@ erts_schedule_bif(Process *proc,
 	    argc = (int) mfa->arity;
 	}
 
-	schedule(c_p, dirty_shadow_proc, mfa, obif,
-		 pc, dbif, ibif, mod, func, argc, argv);
+	schedule(c_p, dirty_shadow_proc, mfa, pc, dbif, ibif,
+		 mod, func, argc, argv);
     }
 
     if (dirty_shadow_proc)
@@ -5295,14 +5288,14 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *
 
     if (!exiting) {
 	if (is_value(result))
-	    schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL, dirty_bif_result,
+	    schedule(c_p, dirty_shadow_proc, NULL, NULL, dirty_bif_result,
 		     NULL, am_erts_internal, am_dirty_bif_result, 1, &result);
 	else if (dirty_shadow_proc->freason != TRAP) {
 	    Eterm argv[2];
 	    ASSERT(dirty_shadow_proc->freason <= MAX_SMALL);
 	    argv[0] = make_small(dirty_shadow_proc->freason);
 	    argv[1] = dirty_shadow_proc->fvalue;
-	    schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL,
+	    schedule(c_p, dirty_shadow_proc, NULL, NULL,
 		     dirty_bif_exception, NULL, am_erts_internal,
 		     am_dirty_bif_exception, 2, argv);
 	}
@@ -5310,7 +5303,7 @@ erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *
 	    /* Dirty BIF did an ordinary trap... */
 	    ASSERT(!(erts_smp_atomic32_read_nob(&c_p->state)
 		     & (ERTS_PSFLG_DIRTY_CPU_PROC|ERTS_PSFLG_DIRTY_IO_PROC)));
-	    schedule(c_p, dirty_shadow_proc, NULL, NULL, NULL,
+	    schedule(c_p, dirty_shadow_proc, NULL, NULL,
 		     dirty_bif_trap, (void *) dirty_shadow_proc->i,
 		     am_erts_internal, am_dirty_bif_trap,
 		     dirty_shadow_proc->arity, reg);
diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c
index 0333545255..bc3fd83d7a 100644
--- a/erts/emulator/beam/erl_nfunc_sched.c
+++ b/erts/emulator/beam/erl_nfunc_sched.c
@@ -63,7 +63,7 @@ erts_destroy_nif_export(Process *p)
 
 NifExport *
 erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
-			 ErtsCodeMFA *mfa, void *nif, BeamInstr *pc,
+			 ErtsCodeMFA *mfa, BeamInstr *pc,
 			 BeamInstr instr,
 			 void *dfunc, void *ifunc,
 			 Eterm mod, Eterm func,
@@ -111,8 +111,6 @@ erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
 	    nep->argv[i] = reg[i];
 	nep->pc = pc;
 	nep->cp = c_p->cp;
-	ASSERT(nif);
-	nep->nif = nif;
 	nep->mfa = mfa;
 	nep->current = c_p->current;
 	ASSERT(argc >= 0);
diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h
index 72de7f0eb5..d7eccb28ba 100644
--- a/erts/emulator/beam/erl_nfunc_sched.h
+++ b/erts/emulator/beam/erl_nfunc_sched.h
@@ -52,7 +52,7 @@ typedef struct {
 NifExport *erts_new_proc_nif_export(Process *c_p, int argc);
 void erts_destroy_nif_export(Process *p);
 NifExport *erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
-				    ErtsCodeMFA *mfa, void *nif, BeamInstr *pc,
+				    ErtsCodeMFA *mfa, BeamInstr *pc,
 				    BeamInstr instr,
 				    void *dfunc, void *ifunc,
 				    Eterm mod, Eterm func,
@@ -65,7 +65,7 @@ ERTS_GLB_INLINE int erts_check_nif_export_in_area(Process *p,
 						  char *start, Uint size);
 ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep);
 ERTS_GLB_INLINE void erts_nif_export_restore_error(Process* c_p, BeamInstr **pc,
-						   Eterm *reg, void **nif);
+						   Eterm *reg, ErtsCodeMFA **nif_mfa);
 ERTS_GLB_INLINE Process *erts_proc_shadow2real(Process *c_p);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -132,8 +132,8 @@ erts_nif_export_restore(Process *c_p, NifExport *ep)
 }
 
 ERTS_GLB_INLINE void
-erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, Eterm *reg,
-			      void **nif)
+erts_nif_export_restore_error(Process* c_p, BeamInstr **pc,
+			      Eterm *reg, ErtsCodeMFA **nif_mfa)
 {
     NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
     int ix;
@@ -141,7 +141,7 @@ erts_nif_export_restore_error(Process* c_p, BeamInstr **pc, Eterm *reg,
     ASSERT(nep);
     *pc = nep->pc;
     c_p->cp = nep->cp;
-    *nif = nep->nif;
+    *nif_mfa = nep->mfa;
     for (ix = 0; ix < nep->argc; ix++)
 	reg[ix] = nep->argv[ix];
     erts_nif_export_restore(c_p, nep);
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index ee4b22f7ca..fd756692f9 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -266,7 +266,6 @@ static ERTS_INLINE ERL_NIF_TERM
 schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
 	 Eterm mod, Eterm func_name, int argc, const ERL_NIF_TERM argv[])
 {
-    Export *exp;
     NifExport *ep;
     Process *c_p, *dirty_shadow_proc;
 
@@ -278,11 +277,8 @@ schedule(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirect_fp,
 
     ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks(c_p));
 
-    exp = ErtsContainerStruct(c_p->current, Export, info.mfa);
-
     ep = erts_nif_export_schedule(c_p, dirty_shadow_proc,
 				  c_p->current,
-				  (BifFunction) exp->beam[1],
 				  c_p->cp,
 				  (BeamInstr) em_call_nif,
 				  direct_fp, indirect_fp,
diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h
index 99a8ff6bad..64c08b1570 100644
--- a/erts/emulator/beam/error.h
+++ b/erts/emulator/beam/error.h
@@ -214,7 +214,6 @@ struct StackTrace {
     BeamInstr* pc;
     ErtsCodeMFA* current;
     int depth;	/* number of saved pointers in trace[] */
-    ErtsCodeMFA mfa; /* in case we need to make a copy of mfa */
     BeamInstr *trace[1];  /* varying size - must be last in struct */
 };
 
-- 
cgit v1.2.3


From 04e119e22a68d686b9e8df17c0a4836c4a5b91ea Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Tue, 8 Nov 2016 09:51:03 +0100
Subject: Return and exception trace for nif-export scheduled BIFs

The support is somewhat primitive, since it is determined at
call time if trace on return or exception should be sent.
---
 erts/emulator/beam/beam_bp.c         | 25 ++++++++++++++++++
 erts/emulator/beam/bif.c             |  8 +++---
 erts/emulator/beam/erl_alloc.c       |  7 +++++
 erts/emulator/beam/erl_alloc.types   |  3 ++-
 erts/emulator/beam/erl_nfunc_sched.c | 40 ++++++++++++++++++++++++++++-
 erts/emulator/beam/erl_nfunc_sched.h | 50 +++++++++++++++++++++++++++++++++---
 erts/emulator/beam/erl_nif.c         |  8 +++---
 erts/emulator/beam/erl_trace.h       |  5 ++++
 8 files changed, 133 insertions(+), 13 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c
index 455362f5ae..27329a339e 100644
--- a/erts/emulator/beam/beam_bp.c
+++ b/erts/emulator/beam/beam_bp.c
@@ -32,6 +32,7 @@
 #include "erl_binary.h"
 #include "beam_bp.h"
 #include "erl_term.h"
+#include "erl_nfunc_sched.h"
 
 /* *************************************************************************
 ** Macros
@@ -805,6 +806,30 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I)
 
     result = func(p, args, I);
 
+    if (erts_nif_export_check_save_trace(p, result,
+					 applying, ep,
+					 cp, flags,
+					 flags_meta, I,
+					 meta_tracer)) {
+	/*
+	 * erts_bif_trace_epilogue() will be called
+	 * later when appropriate via the NIF export
+	 * scheduling functionality...
+	 */
+	return result;
+    }
+
+    return erts_bif_trace_epilogue(p, result, applying, ep, cp,
+				   flags, flags_meta, I,
+				   meta_tracer);
+}
+
+Eterm
+erts_bif_trace_epilogue(Process *p, Eterm result, int applying,
+			Export* ep, BeamInstr *cp, Uint32 flags,
+			Uint32 flags_meta, BeamInstr* I,
+			ErtsTracer meta_tracer)
+{
     if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) {
 	BeamInstr i_return_trace      = beam_return_trace[0];
 	BeamInstr i_return_to_trace   = beam_return_to_trace[0];
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 5f0564474f..cae346267c 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -5047,7 +5047,7 @@ schedule(Process *c_p, Process *dirty_shadow_proc,
 static BIF_RETTYPE dirty_bif_result(BIF_ALIST_1)
 {
     NifExport *nep = (NifExport *) ERTS_PROC_GET_NIF_TRAP_EXPORT(BIF_P);
-    erts_nif_export_restore(BIF_P, nep);
+    erts_nif_export_restore(BIF_P, nep, BIF_ARG_1);
     BIF_RET(BIF_ARG_1);
 }
 
@@ -5062,7 +5062,7 @@ static BIF_RETTYPE dirty_bif_trap(BIF_ALIST)
 
     ASSERT(BIF_P->arity == nep->exp.info.mfa.arity);
 
-    erts_nif_export_restore(BIF_P, nep);
+    erts_nif_export_restore(BIF_P, nep, THE_NON_VALUE);
 
     BIF_P->i = (BeamInstr *) nep->func;
     BIF_P->freason = TRAP;
@@ -5216,12 +5216,12 @@ call_bif(Process *c_p, Eterm *reg, BeamInstr *I)
     ret = (*bif)(c_p, reg, I);
 
     if (is_value(ret))
-	erts_nif_export_restore(c_p, nep);
+	erts_nif_export_restore(c_p, nep, ret);
     else if (c_p->freason != TRAP)
 	c_p->freason |= EXF_RESTORE_NIF; /* restore in handle_error() */
     else if (nep->func == ERTS_SCHED_BIF_TRAP_MARKER) {
 	/* BIF did an ordinary trap... */
-	erts_nif_export_restore(c_p, nep);
+	erts_nif_export_restore(c_p, nep, ret);
     }
     /* else:
      *   BIF rescheduled itself using erts_schedule_bif().
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 40a45c961f..4d990a9c56 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -44,6 +44,7 @@
 #include "erl_hl_timer.h"
 #include "erl_cpu_topology.h"
 #include "erl_thr_queue.h"
+#include "erl_nfunc_sched.h"
 #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE)
 #include "erl_check_io.h"
 #endif
@@ -679,6 +680,8 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
     fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_ABIF_TIMER)]
 	= erts_timer_type_size(ERTS_ALC_T_ABIF_TIMER);
 #endif
+    fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NIF_EXP_TRACE)]
+	= sizeof(NifExportTrace);
 
 #ifdef HARD_DEBUG
     hdbg_init();
@@ -2437,6 +2440,10 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
 		       fi,
 		       ERTS_ALC_T_ABIF_TIMER);
 #endif
+	add_fix_values(&size.processes,
+		       &size.processes_used,
+		       fi,
+		       ERTS_ALC_T_NIF_EXP_TRACE);
     }
 
     if (want.atom || want.atom_used) {
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index 6e8710eb8a..70eca5b49c 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -376,7 +376,8 @@ type	DB_MS_PSDO_PROC	LONG_LIVED	ETS		db_match_pseudo_proc
 type	SCHDLR_DATA	LONG_LIVED	SYSTEM		scheduler_data
 type  	LL_TEMP_TERM 	LONG_LIVED	SYSTEM		ll_temp_term
 
-type	NIF_TRAP_EXPORT	STANDARD	CODE		nif_trap_export_entry
+type	NIF_TRAP_EXPORT	STANDARD	PROCESSES	nif_trap_export_entry
+type	NIF_EXP_TRACE	FIXED_SIZE	PROCESSES	nif_export_trace
 type	EXPORT		LONG_LIVED	CODE		export_entry
 type	MONITOR_SH	FIXED_SIZE	PROCESSES	monitor_sh
 type	NLINK_SH	FIXED_SIZE	PROCESSES	nlink_sh
diff --git a/erts/emulator/beam/erl_nfunc_sched.c b/erts/emulator/beam/erl_nfunc_sched.c
index bc3fd83d7a..1bebc1eda4 100644
--- a/erts/emulator/beam/erl_nfunc_sched.c
+++ b/erts/emulator/beam/erl_nfunc_sched.c
@@ -28,6 +28,7 @@
 #include "erl_process.h"
 #include "bif.h"
 #include "erl_nfunc_sched.h"
+#include "erl_trace.h"
 
 NifExport *
 erts_new_proc_nif_export(Process *c_p, int argc)
@@ -44,9 +45,12 @@ erts_new_proc_nif_export(Process *c_p, int argc)
 
     nep->argc = -1; /* unused marker */
     nep->argv_size = argc;
+    nep->trace = NULL;
     old_nep = ERTS_PROC_SET_NIF_TRAP_EXPORT(c_p, nep);
-    if (old_nep)
+    if (old_nep) {
+	ASSERT(!nep->trace);
 	erts_free(ERTS_ALC_T_NIF_TRAP_EXPORT, old_nep);
+    }
     return nep;
 }
 
@@ -61,6 +65,40 @@ erts_destroy_nif_export(Process *p)
     }
 }
 
+void
+erts_nif_export_save_trace(Process *c_p, NifExport *nep, int applying,
+			   Export* ep, BeamInstr *cp, Uint32 flags,
+			   Uint32 flags_meta, BeamInstr* I,
+			   ErtsTracer meta_tracer)
+{
+    NifExportTrace *netp;
+    ASSERT(nep && nep->argc >= 0);
+    ASSERT(!nep->trace);
+    netp = erts_alloc(ERTS_ALC_T_NIF_EXP_TRACE,
+		      sizeof(NifExportTrace));
+    netp->applying = applying;
+    netp->ep = ep;
+    netp->cp = cp;
+    netp->flags = flags;
+    netp->flags_meta = flags_meta;
+    netp->I = I;
+    netp->meta_tracer = NIL;
+    erts_tracer_update(&netp->meta_tracer, meta_tracer);
+    nep->trace = netp;
+}
+
+void
+erts_nif_export_restore_trace(Process *c_p, Eterm result, NifExport *nep)
+{
+    NifExportTrace *netp = nep->trace;
+    nep->trace = NULL;
+    erts_bif_trace_epilogue(c_p, result, netp->applying, netp->ep,
+			    netp->cp, netp->flags, netp->flags_meta,
+			    netp->I, netp->meta_tracer);
+    erts_tracer_update(&netp->meta_tracer, NIL);
+    erts_free(ERTS_ALC_T_NIF_EXP_TRACE, netp);
+}
+
 NifExport *
 erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
 			 ErtsCodeMFA *mfa, BeamInstr *pc,
diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h
index d7eccb28ba..5c98957cb7 100644
--- a/erts/emulator/beam/erl_nfunc_sched.h
+++ b/erts/emulator/beam/erl_nfunc_sched.h
@@ -23,6 +23,17 @@
 
 #include "erl_process.h"
 #include "bif.h"
+#include "error.h"
+
+typedef struct {
+    int applying;
+    Export* ep;
+    BeamInstr *cp;
+    Uint32 flags;
+    Uint32 flags_meta;
+    BeamInstr* I;
+    ErtsTracer meta_tracer;
+} NifExportTrace;
 
 /*
  * NIF exports need a few more items than the Export struct provides,
@@ -39,6 +50,7 @@ typedef struct {
     struct erl_module_nif* m; /* NIF module, or NULL if BIF */
     void *func;		/* Indirect NIF or BIF to execute (may be unused) */
     ErtsCodeMFA *current;/* Current as set when originally called */
+    NifExportTrace *trace;
     /* --- The following is only used on error --- */
     BeamInstr *pc;	/* Program counter */
     BeamInstr *cp;	/* Continuation pointer */
@@ -50,6 +62,11 @@ typedef struct {
 } NifExport;
 
 NifExport *erts_new_proc_nif_export(Process *c_p, int argc);
+void erts_nif_export_save_trace(Process *c_p, NifExport *nep, int applying,
+				Export* ep, BeamInstr *cp, Uint32 flags,
+				Uint32 flags_meta, BeamInstr* I,
+				ErtsTracer meta_tracer);
+void erts_nif_export_restore_trace(Process *c_p, Eterm result, NifExport *nep);
 void erts_destroy_nif_export(Process *p);
 NifExport *erts_nif_export_schedule(Process *c_p, Process *dirty_shadow_proc,
 				    ErtsCodeMFA *mfa, BeamInstr *pc,
@@ -63,9 +80,15 @@ ERTS_GLB_INLINE int erts_setup_nif_export_rootset(Process* proc, Eterm** objv,
 						  Uint* nobj);
 ERTS_GLB_INLINE int erts_check_nif_export_in_area(Process *p,
 						  char *start, Uint size);
-ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep);
+ERTS_GLB_INLINE void erts_nif_export_restore(Process *c_p, NifExport *ep,
+					     Eterm result);
 ERTS_GLB_INLINE void erts_nif_export_restore_error(Process* c_p, BeamInstr **pc,
 						   Eterm *reg, ErtsCodeMFA **nif_mfa);
+ERTS_GLB_INLINE int erts_nif_export_check_save_trace(Process *c_p, Eterm result,
+						     int applying, Export* ep,
+						     BeamInstr *cp, Uint32 flags,
+						     Uint32 flags_meta, BeamInstr* I,
+						     ErtsTracer meta_tracer);
 ERTS_GLB_INLINE Process *erts_proc_shadow2real(Process *c_p);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -119,7 +142,7 @@ erts_check_nif_export_in_area(Process *p, char *start, Uint size)
 }
 
 ERTS_GLB_INLINE void
-erts_nif_export_restore(Process *c_p, NifExport *ep)
+erts_nif_export_restore(Process *c_p, NifExport *ep, Eterm result)
 {
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_get_scheduler_data()));
     ERTS_SMP_LC_ASSERT(!(c_p->static_flags
@@ -129,6 +152,8 @@ erts_nif_export_restore(Process *c_p, NifExport *ep)
 
     c_p->current = ep->current;
     ep->argc = -1; /* Unused nif-export marker... */
+    if (ep->trace)
+	erts_nif_export_restore_trace(c_p, result, ep);
 }
 
 ERTS_GLB_INLINE void
@@ -144,7 +169,26 @@ erts_nif_export_restore_error(Process* c_p, BeamInstr **pc,
     *nif_mfa = nep->mfa;
     for (ix = 0; ix < nep->argc; ix++)
 	reg[ix] = nep->argv[ix];
-    erts_nif_export_restore(c_p, nep);
+    erts_nif_export_restore(c_p, nep, THE_NON_VALUE);
+}
+
+ERTS_GLB_INLINE int
+erts_nif_export_check_save_trace(Process *c_p, Eterm result,
+				 int applying, Export* ep,
+				 BeamInstr *cp, Uint32 flags,
+				 Uint32 flags_meta, BeamInstr* I,
+				 ErtsTracer meta_tracer)
+{
+    if (is_non_value(result) && c_p->freason == TRAP) {
+	NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
+	if (nep && nep->argc >= 0) {
+	    erts_nif_export_save_trace(c_p, nep, applying, ep,
+				       cp, flags, flags_meta,
+				       I, meta_tracer);
+	    return 1;
+	}
+    }
+    return 0;
 }
 
 ERTS_GLB_INLINE Process *
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index fd756692f9..d946844f15 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -2290,9 +2290,9 @@ erts_nif_export_cleanup_nif_mod(NifExport *ep)
 }
 
 static ERTS_INLINE void
-nif_export_restore(Process *c_p, NifExport *ep)
+nif_export_restore(Process *c_p, NifExport *ep, Eterm res)
 {
-    erts_nif_export_restore(c_p, ep);
+    erts_nif_export_restore(c_p, ep, res);
     ASSERT(ep->m);
     nif_export_cleanup_nif_mod(ep);
 }
@@ -2320,7 +2320,7 @@ dirty_nif_finalizer(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
     ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc)));
     ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc);
     ASSERT(ep);
-    nif_export_restore(proc, ep);
+    nif_export_restore(proc, ep, argv[0]);
     return argv[0];
 }
 
@@ -2462,7 +2462,7 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[])
 	/* Done (not rescheduled)... */
 	ASSERT(ep->func == ERTS_DBG_NIF_NOT_SCHED_MARKER);
 	if (!env->exception_thrown)
-	    nif_export_restore(proc, ep);
+	    nif_export_restore(proc, ep, result);
 	else {
 	    nif_export_cleanup_nif_mod(ep);
 	    /*
diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h
index 378b6de49c..01fe1e5e23 100644
--- a/erts/emulator/beam/erl_trace.h
+++ b/erts/emulator/beam/erl_trace.h
@@ -143,6 +143,11 @@ Uint erts_trace_flag2bit(Eterm flag);
 int erts_trace_flags(Eterm List, 
 		 Uint *pMask, ErtsTracer *pTracer, int *pCpuTimestamp);
 Eterm erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr *I);
+Eterm
+erts_bif_trace_epilogue(Process *p, Eterm result, int applying,
+			Export* ep, BeamInstr *cp, Uint32 flags,
+			Uint32 flags_meta, BeamInstr* I,
+			ErtsTracer meta_tracer);
 
 #ifdef ERTS_SMP
 void erts_send_pending_trace_msgs(ErtsSchedulerData *esdp);
-- 
cgit v1.2.3


From d0e88c0c69f94625daf9cafa192bac97115e9072 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 16 Dec 2016 16:35:57 +0100
Subject: Perform potentially long GC on dirty schedulers if available

---
 erts/emulator/beam/beam_bif_load.c   |  26 ++-
 erts/emulator/beam/beam_emu.c        | 119 ++++++-----
 erts/emulator/beam/bif.c             |   9 +-
 erts/emulator/beam/bif.tab           |   1 -
 erts/emulator/beam/erl_gc.c          | 169 +++++++++++++---
 erts/emulator/beam/erl_gc.h          |   9 +-
 erts/emulator/beam/erl_nfunc_sched.h |   1 -
 erts/emulator/beam/erl_process.c     | 375 +++++++++++++++++++++++++++--------
 erts/emulator/beam/erl_process.h     |  45 ++++-
 erts/emulator/beam/ops.tab           |   7 +-
 10 files changed, 562 insertions(+), 199 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 36bbe81ed5..22576503ef 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -913,7 +913,7 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed
 
     la = ERTS_COPY_LITERAL_AREA();
     if (!la)
-	return am_ok;
+        goto return_ok;
 
     oh = la->off_heap;
     literals = (char *) &la->start[0];
@@ -977,6 +977,11 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed
 	 * this is not completely certain). We go for
 	 * the GC directly instead of scanning everything
 	 * one more time...
+	 *
+	 * Also note that calling functions expect a
+	 * major GC to be performed if gc_allowed is set
+	 * to true. If you change this, you need to fix
+	 * callers...
 	 */
 	goto literal_gc;
     }
@@ -1051,6 +1056,13 @@ erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed
 	}
     }
 
+return_ok:
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p)))
+	c_p->flags &= ~F_DIRTY_CLA;
+#endif
+
     return am_ok;
 
 literal_gc:
@@ -1061,13 +1073,13 @@ literal_gc:
     if (c_p->flags & F_DISABLE_GC)
 	return THE_NON_VALUE;
 
-    FLAGS(c_p) |= F_NEED_FULLSWEEP;
-
-    *redsp += erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg, c_p->arity, fcalls);
+    *redsp += erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize,
+					    oh, fcalls);
 
-    erts_garbage_collect_literals(c_p, (Eterm *) literals, lit_bsize, oh);
-
-    *redsp += lit_bsize / 64; /* Need, better value... */
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (c_p->flags & F_DIRTY_CLA)
+	return THE_NON_VALUE;
+#endif
 
     return am_ok;
 }
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index cfdf61eed0..85d92321b8 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -5327,10 +5327,25 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
     ASSERT(!(c_p->flags & F_HIPE_MODE));
     ERTS_MSACC_UPDATE_CACHE_X();
 
-    reg = esdp->x_reg_array;
-    {
+    /*
+     * Set fcalls even though we ignore it, so we don't
+     * confuse code accessing it...
+     */
+    if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
+	c_p->fcalls = 0;
+    else
+	c_p->fcalls = CONTEXT_REDS;
+
+    if (erts_smp_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_DIRTY_RUNNING_SYS) {
+	erts_execute_dirty_system_task(c_p);
+	goto do_dirty_schedule;
+    }
+    else {
+	ErtsCodeMFA *codemfa;
 	Eterm* argp;
-	int i;
+	int i, exiting;
+
+	reg = esdp->x_reg_array;
 
 	argp = c_p->arg_reg;
 	for (i = c_p->arity - 1; i >= 0; i--) {
@@ -5346,15 +5361,6 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 
 	I = c_p->i;
 
-	/*
-	 * Set fcalls even though we ignore it, so we don't
-	 * confuse code accessing it...
-	 */
-	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
-	    c_p->fcalls = 0;
-	else
-	    c_p->fcalls = CONTEXT_REDS;
-
 	SWAPIN;
 
 #ifdef USE_VM_PROBES
@@ -5378,62 +5384,55 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
             DTRACE2(process_scheduled, process_buf, fun_buf);
         }
 #endif
-    }
 
-    {
-	int exiting;
-
-	{
-	    /*
-	     * call_nif is always first instruction in function:
-	     *
-	     * I[-3]: Module
-	     * I[-2]: Function
-	     * I[-1]: Arity
-	     * I[0]: &&call_nif
-	     * I[1]: Function pointer to NIF function
-	     * I[2]: Pointer to erl_module_nif
-	     * I[3]: Function pointer to dirty NIF
-             *
-             * This layout is determined by the NifExport struct
-	     */
-            ErtsCodeMFA *codemfa;
+	/*
+	 * call_nif is always first instruction in function:
+	 *
+	 * I[-3]: Module
+	 * I[-2]: Function
+	 * I[-1]: Arity
+	 * I[0]: &&call_nif
+	 * I[1]: Function pointer to NIF function
+	 * I[2]: Pointer to erl_module_nif
+	 * I[3]: Function pointer to dirty NIF
+	 *
+	 * This layout is determined by the NifExport struct
+	 */
 
-	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
+	ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
 
-            codemfa = erts_code_to_codemfa(I);
+	codemfa = erts_code_to_codemfa(I);
 
-	    DTRACE_NIF_ENTRY(c_p, codemfa);
-	    c_p->current = codemfa;
-	    SWAPOUT;
-	    PROCESS_MAIN_CHK_LOCKS(c_p);
-	    ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
+	DTRACE_NIF_ENTRY(c_p, codemfa);
+	c_p->current = codemfa;
+	SWAPOUT;
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p);
 
-	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	    if (em_apply_bif == (BeamInstr *) *I) {
-		exiting = erts_call_dirty_bif(esdp, c_p, I, reg);
-	    }
-	    else {
-		ASSERT(em_call_nif == (BeamInstr *) *I);
-		exiting = erts_call_dirty_nif(esdp, c_p, I, reg);
-	    }
+	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+	if (em_apply_bif == (BeamInstr *) *I) {
+	    exiting = erts_call_dirty_bif(esdp, c_p, I, reg);
+	}
+	else {
+	    ASSERT(em_call_nif == (BeamInstr *) *I);
+	    exiting = erts_call_dirty_nif(esdp, c_p, I, reg);
+	}
 
-	    ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
+	ASSERT(!(c_p->flags & F_HIBERNATE_SCHED));
 
-	    PROCESS_MAIN_CHK_LOCKS(c_p);
-	    ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
-	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
-	    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
-	    if (exiting)
-		goto do_dirty_schedule;
-	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+	PROCESS_MAIN_CHK_LOCKS(c_p);
+	ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p);
+	ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR);
+	if (exiting)
+	    goto do_dirty_schedule;
+	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
 
-	    DTRACE_NIF_RETURN(c_p, codemfa);
-	    ERTS_HOLE_CHECK(c_p);
-	    SWAPIN;
-	    I = c_p->i;
-	    goto context_switch;
-	}
+	DTRACE_NIF_RETURN(c_p, codemfa);
+	ERTS_HOLE_CHECK(c_p);
+	SWAPIN;
+	I = c_p->i;
+	goto context_switch;
     }
 #endif /* ERTS_DIRTY_SCHEDULERS */
 }
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index cae346267c..a5b66db35c 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3865,13 +3865,6 @@ BIF_RETTYPE now_0(BIF_ALIST_0)
 
 /**********************************************************************/
 
-BIF_RETTYPE garbage_collect_0(BIF_ALIST_0)
-{
-    FLAGS(BIF_P) |= F_NEED_FULLSWEEP;
-    erts_garbage_collect(BIF_P, 0, NULL, 0);
-    return am_true;
-}
-
 /*
  * Pass atom 'minor' for relaxed generational GC run. This is only
  * recommendation, major run may still be chosen by VM.
@@ -4324,7 +4317,7 @@ BIF_RETTYPE group_leader_2(BIF_ALIST_2)
 		erts_smp_proc_unlock(new_member, ERTS_PROC_LOCK_STATUS);
 		if (new_member == BIF_P
 		    || !(erts_smp_atomic32_read_nob(&new_member->state)
-			 & (ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS))) {
+			 & ERTS_PSFLG_DIRTY_RUNNING)) {
 		    new_member->group_leader = STORE_NC_IN_PROC(new_member,
 								BIF_ARG_1);
 		}
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index f461da913e..47fdcfa7a4 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -68,7 +68,6 @@ gcbif erlang:float/1
 bif erlang:float_to_list/1
 bif erlang:float_to_list/2
 bif erlang:fun_info/2
-bif erlang:garbage_collect/0
 bif erts_internal:garbage_collect/1
 bif erlang:get/0
 bif erlang:get/1
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 6093b0cf39..5b299d9acf 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -118,11 +118,14 @@ static Eterm *full_sweep_heaps(Process *p,
 			       char *oh, Uint oh_size,
 			       Eterm *objv, int nobj);
 static int garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
-			   int need, Eterm* objv, int nobj, int fcalls);
+			   int need, Eterm* objv, int nobj, int fcalls,
+			   Uint max_young_gen_usage);
 static int major_collection(Process* p, ErlHeapFragment *live_hf_end,
-			    int need, Eterm* objv, int nobj, Uint *recl);
+			    int need, Eterm* objv, int nobj,
+			    Uint ygen_usage, Uint *recl);
 static int minor_collection(Process* p, ErlHeapFragment *live_hf_end,
-			    int need, Eterm* objv, int nobj, Uint *recl);
+			    int need, Eterm* objv, int nobj,
+			    Uint ygen_usage, Uint *recl);
 static void do_minor(Process *p, ErlHeapFragment *live_hf_end,
 		     char *mature, Uint mature_size,
 		     Uint new_sz, Eterm* objv, int nobj);
@@ -413,15 +416,15 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
 		regs = erts_proc_sched_data(p)->x_reg_array;
 	    }
 #endif
-	    cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls);
+	    cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls, 0);
 	} else {
-	    cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls);
+	    cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls, 0);
 	}
     } else {
 	Eterm val[1];
 
 	val[0] = result;
-	cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls);
+	cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls, 0);
 	result = val[0];
     }
     BUMP_REDS(p, cost);
@@ -599,6 +602,32 @@ young_gen_usage(Process *p)
 	}							\
     } while (0)
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+static ERTS_INLINE void
+check_for_possibly_long_gc(Process *p, Uint ygen_usage)
+{
+    int major;
+    Uint sz;
+
+    major = (p->flags & F_NEED_FULLSWEEP) || GEN_GCS(p) >= MAX_GEN_GCS(p);
+
+    sz = ygen_usage;
+    sz += p->hend - p->stop;
+    if (p->flags & F_ON_HEAP_MSGQ)
+        sz += p->msg.len;
+    if (major)
+	sz += p->old_htop - p->old_heap;
+
+    if (sz >= ERTS_POTENTIALLY_LONG_GC_HSIZE) {
+        ASSERT(!(p->flags & (F_DISABLE_GC|F_DELAY_GC)));
+	p->flags |= major ? F_DIRTY_MAJOR_GC : F_DIRTY_MINOR_GC;
+        erts_schedule_dirty_sys_execution(p);
+    }
+}
+
+#endif
+
 /*
  * Garbage collect a process.
  *
@@ -609,13 +638,15 @@ young_gen_usage(Process *p)
  */
 static int
 garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
-		int need, Eterm* objv, int nobj, int fcalls)
+		int need, Eterm* objv, int nobj, int fcalls,
+		Uint max_young_gen_usage)
 {
     Uint reclaimed_now = 0;
+    Uint ygen_usage;
     Eterm gc_trace_end_tag;
     int reds;
     ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */
-    ErtsSchedulerData *esdp;
+    ErtsSchedulerData *esdp = erts_proc_sched_data(p);
     erts_aint32_t state;
     ERTS_MSACC_PUSH_STATE_M();
 #ifdef USE_VM_PROBES
@@ -624,13 +655,26 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
 
     ERTS_CHK_MBUF_SZ(p);
 
-    ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls)
-	   >= erts_proc_sched_data(p)->virtual_reds);
+    ASSERT(CONTEXT_REDS - ERTS_REDS_LEFT(p, fcalls) >= esdp->virtual_reds);
 
     state = erts_smp_atomic32_read_nob(&p->state);
 
-    if (p->flags & (F_DISABLE_GC|F_DELAY_GC) || state & ERTS_PSFLG_EXITING)
+    if ((p->flags & (F_DISABLE_GC|F_DELAY_GC)) || state & ERTS_PSFLG_EXITING) {
+#ifdef ERTS_DIRTY_SCHEDULERS
+    delay_gc_before_start:
+#endif
 	return delay_garbage_collection(p, live_hf_end, need, fcalls);
+    }
+
+    ygen_usage = max_young_gen_usage ? max_young_gen_usage : young_gen_usage(p);
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+	check_for_possibly_long_gc(p, ygen_usage);
+	if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC))
+	    goto delay_gc_before_start;
+    }
+#endif
 
     if (p->abandoned_heap)
 	live_hf_end = ERTS_INVALID_HFRAG_PTR;
@@ -639,8 +683,6 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
 
     ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_GC);
 
-    esdp = erts_get_scheduler_data();
-
     erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
     if (erts_system_monitor_long_gc != 0)
 	start_time = erts_get_monotonic_time(esdp);
@@ -667,14 +709,25 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
             trace_gc(p, am_gc_minor_start, need, THE_NON_VALUE);
         }
         DTRACE2(gc_minor_start, pidbuf, need);
-        reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
+        reds = minor_collection(p, live_hf_end, need, objv, nobj,
+				ygen_usage, &reclaimed_now);
         DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
         if (reds == -1) {
             if (IS_TRACED_FL(p, F_TRACE_GC)) {
                 trace_gc(p, am_gc_minor_end, reclaimed_now, THE_NON_VALUE);
             }
+#ifdef ERTS_DIRTY_SCHEDULERS
+	    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+		p->flags |= F_NEED_FULLSWEEP;
+		check_for_possibly_long_gc(p, ygen_usage);
+		if (p->flags & F_DIRTY_MAJOR_GC)
+		    goto delay_gc_after_start;
+	    }
+#endif
             goto do_major_collection;
         }
+        if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+            p->flags &= ~F_DIRTY_MINOR_GC;
         gc_trace_end_tag = am_gc_minor_end;
     } else {
 do_major_collection:
@@ -683,7 +736,10 @@ do_major_collection:
             trace_gc(p, am_gc_major_start, need, THE_NON_VALUE);
         }
         DTRACE2(gc_major_start, pidbuf, need);
-        reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
+        reds = major_collection(p, live_hf_end, need, objv, nobj,
+				ygen_usage, &reclaimed_now);
+        if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+            p->flags &= ~(F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC);
         DTRACE2(gc_major_end, pidbuf, reclaimed_now);
         gc_trace_end_tag = am_gc_major_end;
         ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_GC);
@@ -713,6 +769,9 @@ do_major_collection:
                               am_kill, NIL, NULL, 0);
         erts_smp_proc_unlock(p, locks & ERTS_PROC_LOCKS_ALL_MINOR);
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    delay_gc_after_start:
+#endif
         /* erts_send_exit_signal looks for ERTS_PSFLG_GC, so
            we have to remove it after the signal is sent */
         erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
@@ -795,7 +854,7 @@ do_major_collection:
 int
 erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fcalls)
 {
-    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls);
+    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, fcalls, 0);
     int reds_left = ERTS_REDS_LEFT(p, fcalls);
     if (reds > reds_left)
 	reds = reds_left;
@@ -806,7 +865,7 @@ erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj, int fca
 void
 erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
 {
-    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls);
+    int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj, p->fcalls, 0);
     BUMP_REDS(p, reds);
     ASSERT(CONTEXT_REDS - ERTS_BIF_REDS_LEFT(p)
 	   >= erts_proc_sched_data(p)->virtual_reds);
@@ -832,6 +891,20 @@ erts_garbage_collect_hibernate(Process* p)
     if (p->flags & F_DISABLE_GC)
 	ERTS_INTERNAL_ERROR("GC disabled");
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p)))
+        p->flags &= ~(F_DIRTY_GC_HIBERNATE|F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC);
+    else {
+	Uint flags = p->flags;
+	p->flags |= F_NEED_FULLSWEEP;
+	check_for_possibly_long_gc(p, (p->htop - p->heap) + p->mbuf_sz);
+	if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) {
+	    p->flags = flags|F_DIRTY_GC_HIBERNATE;
+	    return;
+	}
+	p->flags = flags;
+    }
+#endif
     /*
      * Preliminaries.
      */
@@ -843,7 +916,6 @@ erts_garbage_collect_hibernate(Process* p)
      * Do it.
      */
 
-
     heap_size = p->heap_sz + (p->old_htop - p->old_heap) + p->mbuf_sz;
 
     heap = (Eterm*) ERTS_HEAP_ALLOC(ERTS_ALC_T_TMP_HEAP,
@@ -984,10 +1056,11 @@ static ERTS_INLINE void offset_nstack(Process* p, Sint offs,
 
 #endif /* HIPE */
 
-void
+int
 erts_garbage_collect_literals(Process* p, Eterm* literals,
 			      Uint byte_lit_size,
-			      struct erl_off_heap_header* oh)
+			      struct erl_off_heap_header* oh,
+			      int fcalls)
 {
     Uint lit_size = byte_lit_size / sizeof(Eterm);
     Uint old_heap_size;
@@ -999,20 +1072,49 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
     Uint area_size;
     Eterm* old_htop;
     Uint n;
+    Uint ygen_usage = 0;
     struct erl_off_heap_header** prev = NULL;
+    Sint64 reds;
+
+    if (p->flags & (F_DISABLE_GC|F_DELAY_GC))
+	ERTS_INTERNAL_ERROR("GC disabled");
+
+    /*
+     * First an ordinary major collection...
+     */
+
+    p->flags |= F_NEED_FULLSWEEP;
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(p)))
+	p->flags &= ~F_DIRTY_CLA;
+    else {
+	ygen_usage = young_gen_usage(p);
+	check_for_possibly_long_gc(p,
+				   (byte_lit_size/sizeof(Uint)
+				    + 2*ygen_usage));
+	if (p->flags & F_DIRTY_MAJOR_GC) {
+	    p->flags |= F_DIRTY_CLA;
+	    return 10;
+	}
+    }
+#endif
+
+    reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, 0,
+				    p->arg_reg, p->arity, fcalls,
+				    ygen_usage);
+
+    ASSERT(!(p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)));
 
-    if (p->flags & F_DISABLE_GC)
-	return;
     /*
      * Set GC state.
      */
     erts_smp_atomic32_read_bor_nob(&p->state, ERTS_PSFLG_GC);
 
     /*
-     * We assume that the caller has already done a major collection
-     * (which has discarded the old heap), so that we don't have to cope
-     * with pointer to literals on the old heap. We will now allocate
-     * an old heap to contain the literals.
+     * Just did a major collection (which has discarded the old heap),
+     * so that we don't have to cope with pointer to literals on the
+     * old heap. We will now allocate an old heap to contain the literals.
      */
     
     ASSERT(p->old_heap == 0);	/* Must NOT have an old heap yet. */
@@ -1155,15 +1257,21 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
      * Restore status.
      */
     erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
+
+    reds += (Sint64) gc_cost((p->htop - p->heap) + byte_lit_size/sizeof(Uint), 0);
+    if (reds > INT_MAX)
+	return INT_MAX;
+    return (int) reds;
 }
 
 static int
 minor_collection(Process* p, ErlHeapFragment *live_hf_end,
-		 int need, Eterm* objv, int nobj, Uint *recl)
+		 int need, Eterm* objv, int nobj,
+		 Uint ygen_usage, Uint *recl)
 {
     Eterm *mature = p->abandoned_heap ? p->abandoned_heap : p->heap;
     Uint mature_size = p->high_water - mature;
-    Uint size_before = young_gen_usage(p);
+    Uint size_before = ygen_usage;
 
     /*
      * Check if we have gone past the max heap size limit
@@ -1534,7 +1642,8 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
 
 static int
 major_collection(Process* p, ErlHeapFragment *live_hf_end,
-		 int need, Eterm* objv, int nobj, Uint *recl)
+		 int need, Eterm* objv, int nobj,
+		 Uint ygen_usage, Uint *recl)
 {
     Uint size_before, size_after, stack_size;
     Eterm* n_heap;
@@ -1552,7 +1661,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
      * to receive all live data.
      */
 
-    size_before = young_gen_usage(p);
+    size_before = ygen_usage;
     size_before += p->old_htop - p->old_heap;
     stack_size = p->hend - p->stop;
 
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 54ea9ca3c0..0dfdf52028 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -25,6 +25,8 @@
 
 /* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */
 
+#define ERTS_POTENTIALLY_LONG_GC_HSIZE (128*1024) /* Words */
+
 #include "erl_map.h"
 
 #if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -145,9 +147,10 @@ void erts_garbage_collect_hibernate(struct process* p);
 Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end,
 				 Eterm result, Eterm* regs, Uint arity);
 Eterm erts_gc_after_bif_call(struct process* p, Eterm result, Eterm* regs, Uint arity);
-void erts_garbage_collect_literals(struct process* p, Eterm* literals,
-				   Uint lit_size,
-				   struct erl_off_heap_header* oh);
+int erts_garbage_collect_literals(struct process* p, Eterm* literals,
+				  Uint lit_size,
+				  struct erl_off_heap_header* oh,
+				  int fcalls);
 Uint erts_next_heap_size(Uint, Uint);
 Eterm erts_heap_sizes(struct process* p);
 
diff --git a/erts/emulator/beam/erl_nfunc_sched.h b/erts/emulator/beam/erl_nfunc_sched.h
index 5c98957cb7..55a3a6dbf6 100644
--- a/erts/emulator/beam/erl_nfunc_sched.h
+++ b/erts/emulator/beam/erl_nfunc_sched.h
@@ -54,7 +54,6 @@ typedef struct {
     /* --- The following is only used on error --- */
     BeamInstr *pc;	/* Program counter */
     BeamInstr *cp;	/* Continuation pointer */
-    void *nif;		/* Original NIF/BIF call */
     ErtsCodeMFA *mfa;	/* MFA of original call */
     int argc;		/* Number of arguments in original call */
     int argv_size;	/* Allocated size of argv */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 955b98b35b..da4b468d66 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -6439,10 +6439,30 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p,
     int enqueue; /* < 0 -> use proxy */
     ErtsRunQueue* runq;
 
-    if (is_normal_sched)
-	running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS;
-    else
+    if (!is_normal_sched)
 	running_flgs = ERTS_PSFLG_DIRTY_RUNNING|ERTS_PSFLG_DIRTY_RUNNING_SYS;
+    else {
+	running_flgs = ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS;
+#ifdef ERTS_DIRTY_SCHEDULERS
+        if (state & ERTS_PSFLG_DIRTY_ACTIVE_SYS
+            && (p->flags & (F_DELAY_GC|F_DISABLE_GC))) {
+            /*
+             * Delay dirty GC; will be enabled automatically
+             * again by next GC...
+             */
+
+            /*
+             * No normal execution until dirty CLA or hibernat has
+             * been handled...
+             */
+            ASSERT(!(p->flags & (F_DIRTY_CLA | F_DIRTY_GC_HIBERNATE)));
+
+            state = erts_smp_atomic32_read_band_nob(&p->state,
+                                                    ~ERTS_PSFLG_DIRTY_ACTIVE_SYS);
+            state &= ~ERTS_PSFLG_DIRTY_ACTIVE_SYS;
+        }
+#endif
+    }
 
     a = state;
 
@@ -8642,8 +8662,7 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks,
 		erts_aint32_t state;
 		state = erts_smp_atomic32_read_nob(&rp->state);
 		ASSERT((state & ERTS_PSFLG_PENDING_EXIT)
-		       || !(state & (ERTS_PSFLG_RUNNING
-				     | ERTS_PSFLG_DIRTY_RUNNING_SYS)));
+		       || !(state & ERTS_PSFLG_RUNNING));
 	    }
 #endif
 
@@ -9981,6 +10000,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 					      | ERTS_PSFLG_FREE
 					      | ERTS_PSFLG_RUNNING_SYS
+					      | ERTS_PSFLG_DIRTY_RUNNING_SYS
 					      | ERTS_PSFLG_EXITING))
 				    == ERTS_PSFLG_EXITING)
 				   & (!!is_normal_sched))
@@ -9992,7 +10012,14 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 					    | ERTS_PSFLG_PENDING_EXIT
 					    | ERTS_PSFLG_ACTIVE_SYS
 					    | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
-				  != ERTS_PSFLG_SUSPENDED));
+				  != ERTS_PSFLG_SUSPENDED)
+#ifdef ERTS_DIRTY_SCHEDULERS
+                               & (!(state & (ERTS_PSFLG_EXITING
+                                             | ERTS_PSFLG_PENDING_EXIT))
+                                  | (!!is_normal_sched))
+#endif
+                    );
+
 		if (run_process) {
 		    if (state & (ERTS_PSFLG_ACTIVE_SYS
 				 | ERTS_PSFLG_DIRTY_ACTIVE_SYS))
@@ -10133,65 +10160,70 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    }
 	}
 
-	if (state & (ERTS_PSFLG_RUNNING_SYS
-		     | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
-	    /*
-	     * GC is normally never delayed when a process
-	     * is scheduled out, but might be when executing
-	     * hand written beam assembly in
-	     * prim_eval:'receive'. If GC is delayed we are
-	     * not allowed to execute system tasks.
-	     */
-	    if (!(p->flags & F_DELAY_GC)) {
-		int cost = execute_sys_tasks(p, &state, reds);
-		calls += cost;
-		reds -= cost;
-		if (reds <= 0
+        if (is_normal_sched) {
+
+            if (state & ERTS_PSFLG_RUNNING_SYS) {
+                /*
+                 * GC is normally never delayed when a process
+                 * is scheduled out, but might be when executing
+                 * hand written beam assembly in
+                 * prim_eval:'receive'. If GC is delayed we are
+                 * not allowed to execute system tasks.
+                 */
+                if (!(p->flags & F_DELAY_GC)) {
+                    int cost = execute_sys_tasks(p, &state, reds);
+                    calls += cost;
+                    reds -= cost;
+                    if (reds <= 0)
+                        goto sched_out_proc;
 #ifdef ERTS_DIRTY_SCHEDULERS
-		    || !is_normal_sched
-		    || (state & ERTS_PSFLGS_DIRTY_WORK)
+                    if (state & ERTS_PSFLGS_DIRTY_WORK)
+                        goto sched_out_proc;
 #endif
-		    ) {
-		    goto sched_out_proc;
-		}
-	    }
+                }
 
-	    ASSERT(state & psflg_running_sys);
-	    ASSERT(!(state & psflg_running));
+                ASSERT(state & psflg_running_sys);
+                ASSERT(!(state & psflg_running));
 
-	    while (1) {
-		erts_aint32_t n, e;
+                while (1) {
+                    erts_aint32_t n, e;
 
-		if (((state & (ERTS_PSFLG_SUSPENDED
-			       | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE)
-		    && !(state & ERTS_PSFLG_EXITING)) {
-		    goto sched_out_proc;
-		}
+                    if (((state & (ERTS_PSFLG_SUSPENDED
+                                   | ERTS_PSFLG_ACTIVE)) != ERTS_PSFLG_ACTIVE)
+                        && !(state & ERTS_PSFLG_EXITING)) {
+                        goto sched_out_proc;
+                    }
 
-		n = e = state;
-		n &= ~psflg_running_sys;
-		n |= psflg_running;
+                    n = e = state;
+                    n &= ~psflg_running_sys;
+                    n |= psflg_running;
 
-		state = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
-		if (state == e) {
-		    state = n;
-		    break;
-		}
+                    state = erts_smp_atomic32_cmpxchg_mb(&p->state, n, e);
+                    if (state == e) {
+                        state = n;
+                        break;
+                    }
 
-		ASSERT(state & psflg_running_sys);
-		ASSERT(!(state & psflg_running));
-	    }
-	}
+                    ASSERT(state & psflg_running_sys);
+                    ASSERT(!(state & psflg_running));
+                }
+            }
 
-	if (ERTS_IS_GC_DESIRED(p) && !ERTS_SCHEDULER_IS_DIRTY_IO(esdp)) {
-	    if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) {
-		int cost = scheduler_gc_proc(p, reds);
-		calls += cost;
-		reds -= cost;
-		if (reds <= 0)
-		    goto sched_out_proc;
-	    }
-	}
+            if (ERTS_IS_GC_DESIRED(p)) {
+                if (!(state & ERTS_PSFLG_EXITING)
+                    && !(p->flags & (F_DELAY_GC|F_DISABLE_GC))) {
+                    int cost = scheduler_gc_proc(p, reds);
+                    calls += cost;
+                    reds -= cost;
+                    if (reds <= 0)
+                        goto sched_out_proc;
+#ifdef ERTS_DIRTY_SCHEDULERS
+                    if (p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC))
+                        goto sched_out_proc;
+#endif
+                }
+            }
+        }
 
 	if (proxy_p) {
 	    free_proxy_proc(proxy_p);
@@ -10203,7 +10235,13 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p);
 
 	/* Never run a suspended process */
-	ASSERT(!(ERTS_PSFLG_SUSPENDED & erts_smp_atomic32_read_nob(&p->state)));
+#ifdef DEBUG
+        {
+            erts_aint32_t dstate = erts_smp_atomic32_read_nob(&p->state);
+            ASSERT(!(ERTS_PSFLG_SUSPENDED & dstate)
+                   || (ERTS_PSFLG_DIRTY_RUNNING_SYS & dstate));
+        }
+#endif
 
 	ASSERT(erts_proc_read_refc(p) > 0);
 
@@ -10224,9 +10262,18 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 }
 
 static int
-notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result)
+notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st,
+			 Eterm st_result, int normal_sched)
 {
-    Process *rp = erts_proc_lookup(st->requester);
+    Process *rp;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (!normal_sched)
+	rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN,
+			       st->requester, 0,
+			       ERTS_P2P_FLG_INC_REFC);
+    else
+#endif
+	rp = erts_proc_lookup(st->requester);
     if (rp) {
 	ErtsProcLocks rp_locks;
 	ErlOffHeap *ohp;
@@ -10274,6 +10321,11 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, Eterm st_result)
 
 	if (rp_locks)
 	    erts_smp_proc_unlock(rp, rp_locks);
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (!normal_sched)
+	    erts_proc_dec_refc(rp);
+#endif
     }
 
     erts_cleanup_offheap(&st->off_heap);
@@ -10429,18 +10481,23 @@ done:
 }
 
 static void save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio);
+#ifdef ERTS_DIRTY_SCHEDULERS
+static void save_dirty_task(Process *c_p, ErtsProcSysTask *st);
+#endif
 
 static int
 execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 {
-    int garbage_collected = 0;
+    int minor_gc = 0, major_gc = 0;
     erts_aint32_t state = *statep;
     int reds = in_reds;
     int qmask = 0;
 
+    ASSERT(!ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(c_p)));
     ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(c_p) == ERTS_PROC_LOCK_MAIN);
 
     do {
+	ErtsProcSysTaskType type;
 	ErtsProcSysTask *st;
 	int st_prio;
 	Eterm st_res;
@@ -10458,7 +10515,9 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	if (!st)
 	    break;
 
-	switch (st->type) {
+	type = st->type;
+
+	switch (type) {
         case ERTS_PSTT_GC_MAJOR:
         case ERTS_PSTT_GC_MINOR:
 	    if (c_p->flags & F_DISABLE_GC) {
@@ -10467,12 +10526,23 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 		reds--;
 	    }
 	    else {
-		if (!garbage_collected) {
-                    if (st->type == ERTS_PSTT_GC_MAJOR) {
+		if (!minor_gc
+		    || (!major_gc && type == ERTS_PSTT_GC_MAJOR)) {
+                    if (type == ERTS_PSTT_GC_MAJOR) {
                         FLAGS(c_p) |= F_NEED_FULLSWEEP;
                     }
 		    reds -= scheduler_gc_proc(c_p, reds);
-		    garbage_collected = 1;
+#ifdef ERTS_DIRTY_SCHEDULERS
+		    if (c_p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) {
+			save_dirty_task(c_p, st);
+			st = NULL;
+			break;
+		    }
+#endif
+		    if (type == ERTS_PSTT_GC_MAJOR)
+			minor_gc = major_gc = 1;
+		    else
+			minor_gc = 1;
 		}
 		st_res = am_true;
 	    }
@@ -10499,20 +10569,31 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	case ERTS_PSTT_CLA: {
 	    int fcalls;
             int cla_reds = 0;
+	    int do_gc;
+
 	    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
 		fcalls = reds;
 	    else
 		fcalls = reds - CONTEXT_REDS;
-	    st_res = erts_proc_copy_literal_area(c_p,
-						 &cla_reds,
-						 fcalls,
-						 st->arg[0] == am_true);
+	    do_gc = st->arg[0] == am_true;
+	    st_res = erts_proc_copy_literal_area(c_p, &cla_reds,
+						 fcalls, do_gc);
             reds -= cla_reds;
 	    if (is_non_value(st_res)) {
+#ifdef ERTS_DIRTY_SCHEDULERS
+		if (c_p->flags & F_DIRTY_CLA) {
+		    save_dirty_task(c_p, st);
+		    st = NULL;
+		    break;
+		}
+#endif
 		/* Needed gc, but gc was disabled */
 		save_gc_task(c_p, st, st_prio);
 		st = NULL;
+		break;
 	    }
+	    if (do_gc) /* We did a major gc */
+		minor_gc = major_gc = 1;
 	    break;
         }
 	case ERTS_PSTT_COHMQ:
@@ -10531,7 +10612,7 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
 	}
 
 	if (st)
-	    reds += notify_sys_task_executed(c_p, st, st_res);
+	    reds += notify_sys_task_executed(c_p, st, st_res, 1);
 
 	state = erts_smp_atomic32_read_acqb(&c_p->state);
     } while (qmask && reds > 0);
@@ -10559,9 +10640,18 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 	Eterm st_res;
 	int st_prio;
 
-	st = fetch_sys_task(c_p, state, &qmask, &st_prio);
-	if (!st)
-	    break;
+#ifdef ERTS_DIRTY_SCHEDULERS
+	if (c_p->dirty_sys_tasks) {
+	    st = c_p->dirty_sys_tasks;
+	    c_p->dirty_sys_tasks = st->next;
+	}
+	else
+#endif
+	{
+	    st = fetch_sys_task(c_p, state, &qmask, &st_prio);
+	    if (!st)
+		break;
+	}
 
 	switch (st->type) {
         case ERTS_PSTT_GC_MAJOR:
@@ -10585,7 +10675,7 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 	    break;
 	}
 
-	reds += notify_sys_task_executed(c_p, st, st_res);
+	reds += notify_sys_task_executed(c_p, st, st_res, 1);
 
 	state = erts_smp_atomic32_read_acqb(&c_p->state);
     } while (qmask && reds < max_reds);
@@ -10595,6 +10685,92 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds)
 
 #ifdef ERTS_DIRTY_SCHEDULERS
 
+void
+erts_execute_dirty_system_task(Process *c_p)
+{
+    Eterm cla_res = THE_NON_VALUE;
+    ErtsProcSysTask *stasks;
+
+    /*
+     * If multiple operations, perform them in the following
+     * order (in order to avoid unnecessary GC):
+     *  1. Copy Literal Area (implies major GC).
+     *  2. GC Hibernate (implies major GC if not woken).
+     *  3. Major GC (implies minor GC).
+     *  4. Minor GC.
+     *
+     * System task requests are handled after the actual
+     * operations have been performed...
+     */
+
+    ASSERT(!(c_p->flags & (F_DELAY_GC|F_DISABLE_GC)));
+
+    if (c_p->flags & F_DIRTY_CLA) {
+	int cla_reds = 0;
+	cla_res = erts_proc_copy_literal_area(c_p, &cla_reds, c_p->fcalls, 1);
+	ASSERT(is_value(cla_res));
+    }
+
+    if (c_p->flags & F_DIRTY_GC_HIBERNATE) {
+	erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+	ERTS_SMP_MSGQ_MV_INQ2PRIVQ(c_p);
+	if (c_p->msg.len)
+            c_p->flags &= ~F_DIRTY_GC_HIBERNATE; /* operation aborted... */
+        else {
+	    erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+	    c_p->fvalue = NIL;
+	    erts_garbage_collect_hibernate(c_p);
+	    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
+	    erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+	    ASSERT(!ERTS_PROC_IS_EXITING(c_p));
+	}
+	erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS);
+    }
+
+    if (c_p->flags & (F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC)) {
+        if (c_p->flags & F_DIRTY_MAJOR_GC)
+            c_p->flags |= F_NEED_FULLSWEEP;
+	(void) erts_garbage_collect_nobump(c_p, 0, c_p->arg_reg,
+					   c_p->arity, c_p->fcalls);
+    }
+
+    ASSERT(!(c_p->flags & (F_DIRTY_CLA
+			   | F_DIRTY_GC_HIBERNATE
+			   | F_DIRTY_MAJOR_GC
+			   | F_DIRTY_MINOR_GC)));
+
+    stasks = c_p->dirty_sys_tasks;
+    c_p->dirty_sys_tasks = NULL;
+    
+    while (stasks) {
+	Eterm st_res;
+	ErtsProcSysTask *st = stasks;
+	stasks = st->next;
+
+	switch (st->type) {
+	case ERTS_PSTT_CLA:
+	    ASSERT(is_value(st_res));
+	    st_res = cla_res;
+	    break;
+	case ERTS_PSTT_GC_MAJOR:
+	    st_res = am_true;
+	    break;
+	case ERTS_PSTT_GC_MINOR:
+	    st_res = am_true;
+	    break;
+
+	default:
+	    ERTS_INTERNAL_ERROR("Not supported dirty system task");
+	    break;
+	}
+
+	(void) notify_sys_task_executed(c_p, st, st_res, 0);
+
+    }
+
+    erts_smp_atomic32_read_band_relb(&c_p->state, ~ERTS_PSFLG_DIRTY_ACTIVE_SYS);
+}
+
 static BIF_RETTYPE
 dispatch_system_task(Process *c_p, erts_aint_t fail_state,
 		     ErtsProcSysTask *st, Eterm target,
@@ -10797,7 +10973,7 @@ request_system_task(Process *c_p, Eterm requester, Eterm target,
 	    ERTS_INTERNAL_ERROR("Unknown failure schedule_process_sys_task()");
 	    failure = am_internal_error;
 	}
-	notify_sys_task_executed(c_p, st, failure);
+	notify_sys_task_executed(c_p, st, failure, 1);
     }
 
     ERTS_BIF_PREP_RET(ret, am_ok);
@@ -11012,6 +11188,15 @@ save_gc_task(Process *c_p, ErtsProcSysTask *st, int prio)
     }
 }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+static void
+save_dirty_task(Process *c_p, ErtsProcSysTask *st)
+{
+    st->next = c_p->dirty_sys_tasks;
+    c_p->dirty_sys_tasks = st;
+}
+#endif
+
 int
 erts_set_gc_state(Process *c_p, int enable)
 {
@@ -11348,6 +11533,7 @@ static void early_init_process_struct(void *varg, Eterm data)
     proc->common.id = make_internal_pid(data);
 #ifdef ERTS_DIRTY_SCHEDULERS
     erts_smp_atomic32_init_nob(&proc->dirty_state, 0);
+    proc->dirty_sys_tasks = NULL;
 #endif
     erts_smp_atomic32_init_relb(&proc->state, arg->state);
 
@@ -11856,6 +12042,7 @@ void erts_init_empty_process(Process *p)
 
 #ifdef ERTS_DIRTY_SCHEDULERS
     erts_smp_atomic32_init_nob(&p->dirty_state, 0);
+    p->dirty_sys_tasks = NULL;
 #endif
     erts_smp_atomic32_init_nob(&p->state, (erts_aint32_t) PRIORITY_NORMAL);
 
@@ -12443,7 +12630,9 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		}
 		set_proc_exiting(c_p, state, rsn, NULL);
 	    }
-	    else if (!(state & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS))) {
+	    else if (!(state & (ERTS_PSFLG_RUNNING
+                                | ERTS_PSFLG_RUNNING_SYS
+                                | ERTS_PSFLG_DIRTY_RUNNING_SYS))) {
 		/* Process not running ... */
 		ErtsProcLocks need_locks = ~(*rp_locks) & ERTS_PROC_LOCKS_ALL;
 		ErlHeapFragment *bp = NULL;
@@ -12470,8 +12659,7 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		    ErlOffHeap *ohp;
 		    Uint rsn_sz = size_object(rsn);
 #ifdef ERTS_DIRTY_SCHEDULERS
-		    if (state & (ERTS_PSFLG_DIRTY_RUNNING
-				 | ERTS_PSFLG_DIRTY_RUNNING_SYS)) {
+		    if (state & ERTS_PSFLG_DIRTY_RUNNING) {
 			bp = new_message_buffer(rsn_sz);
 			ohp = &bp->off_heap;
 			hp = &bp->mem[0];
@@ -12488,7 +12676,7 @@ send_exit_signal(Process *c_p,		/* current process if and only
 		set_proc_exiting(rp, state, rsn_cpy, bp);
 	    }
 	    else { /* Process running... */
-
+ 
 		/*
 		 * The pending exit will be discovered when the process
 		 * is scheduled out if not discovered earlier.
@@ -12510,8 +12698,35 @@ send_exit_signal(Process *c_p,		/* current process if and only
 							  &bp->off_heap);
 		    rp->pending_exit.bp = bp;
 		}
-		erts_smp_atomic32_read_bor_relb(&rp->state,
-						ERTS_PSFLG_PENDING_EXIT);
+
+		/*
+		 * If no dirty work has been scheduled, pending exit will
+                 * be discovered when the process is scheduled. If dirty work
+                 * has been scheduled, we may need to add it to a normal run
+                 * queue...
+		 */
+#ifndef ERTS_DIRTY_SCHEDULERS
+                (void) erts_smp_atomic32_read_bor_relb(&rp->state,
+                                                       ERTS_PSFLG_PENDING_EXIT);
+#else
+                {
+                    erts_aint32_t a = erts_smp_atomic32_read_nob(&rp->state);
+                    while (1) {
+                        erts_aint32_t n, e;
+                        int dwork;
+                        n = e = a;
+                        n |= ERTS_PSFLG_PENDING_EXIT;
+                        dwork = !!(n & ERTS_PSFLGS_DIRTY_WORK);
+                        n &= ~ERTS_PSFLGS_DIRTY_WORK;
+                        a = erts_smp_atomic32_cmpxchg_mb(&rp->state, n, e);
+                        if (a == e) {
+                            if (dwork)
+                                erts_schedule_process(rp, n, *rp_locks);
+                            break;
+                        }
+                    }
+                }
+#endif
 	    }
 	}
 	/* else:
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 58a50f8cf4..3938c4c913 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1060,6 +1060,9 @@ struct process {
     Uint64 bin_old_vheap;	/* Virtual old heap size for binaries */
 
     ErtsProcSysTaskQs *sys_task_qs;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ErtsProcSysTask *dirty_sys_tasks;
+#endif
 
     erts_smp_atomic32_t state;  /* Process state flags (see ERTS_PSFLG_*) */
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -1384,14 +1387,18 @@ extern int erts_system_profile_ts_type;
 #define F_FORCE_GC           (1 << 10) /* Force gc at process in-scheduling */
 #define F_DISABLE_GC         (1 << 11) /* Disable GC (see below) */
 #define F_OFF_HEAP_MSGQ      (1 << 12) /* Off heap msg queue */
-#define F_ON_HEAP_MSGQ       (1 << 13) /* Off heap msg queue */
+#define F_ON_HEAP_MSGQ       (1 << 13) /* On heap msg queue */
 #define F_OFF_HEAP_MSGQ_CHNG (1 << 14) /* Off heap msg queue changing */
 #define F_ABANDONED_HEAP_USE (1 << 15) /* Have usage of abandoned heap */
 #define F_DELAY_GC           (1 << 16) /* Similar to disable GC (see below) */
 #define F_SCHDLR_ONLN_WAITQ  (1 << 17) /* Process enqueued waiting to change schedulers online */
 #define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */
-#define F_HIPE_MODE          (1 << 19)
+#define F_HIPE_MODE          (1 << 19) /* Process is executing in HiPE mode */
 #define F_DELAYED_DEL_PROC   (1 << 20) /* Delay delete process (dirty proc exit case) */
+#define F_DIRTY_CLA          (1 << 21) /* Dirty copy literal area scheduled */
+#define F_DIRTY_GC_HIBERNATE (1 << 22) /* Dirty GC hibernate scheduled */
+#define F_DIRTY_MAJOR_GC     (1 << 23) /* Dirty major GC scheduled */
+#define F_DIRTY_MINOR_GC     (1 << 24) /* Dirty minor GC scheduled */
 
 /*
  * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent
@@ -1573,7 +1580,9 @@ void erts_init_scheduling(int, int
 			  , int, int, int
 #endif
 			  );
-
+#ifdef ERTS_DIRTY_SCHEDULERS
+void erts_execute_dirty_system_task(Process *c_p);
+#endif
 int erts_set_gc_state(Process *c_p, int enable);
 Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
 Eterm erts_system_check_request(Process *c_p);
@@ -1918,6 +1927,8 @@ ErtsSchedulerData *erts_get_scheduler_data(void)
 void erts_schedule_process(Process *, erts_aint32_t, ErtsProcLocks);
 
 ERTS_GLB_INLINE void erts_proc_notify_new_message(Process *p, ErtsProcLocks locks);
+ERTS_GLB_INLINE void erts_schedule_dirty_sys_execution(Process *c_p);
+
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 ERTS_GLB_INLINE void
 erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
@@ -1927,6 +1938,34 @@ erts_proc_notify_new_message(Process *p, ErtsProcLocks locks)
     if (!(state & ERTS_PSFLG_ACTIVE))
 	erts_schedule_process(p, state, locks);
 }
+
+ERTS_GLB_INLINE void
+erts_schedule_dirty_sys_execution(Process *c_p)
+{
+    erts_aint32_t a, n, e;
+
+    a = erts_smp_atomic32_read_nob(&c_p->state);
+
+    /*
+     * Only a currently executing process schedules
+     * itself for dirty-sys execution...
+     */
+
+    ASSERT(a & (ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS));
+
+    /* Don't set dirty-active-sys if we are about to exit... */
+
+    while (!(a & (ERTS_PSFLG_DIRTY_ACTIVE_SYS
+                  | ERTS_PSFLG_EXITING
+                  | ERTS_PSFLG_PENDING_EXIT))) {
+        e = a;
+        n = a | ERTS_PSFLG_DIRTY_ACTIVE_SYS;
+        a = erts_smp_atomic32_cmpxchg_mb(&c_p->state, n, e);
+        if (a == e)
+            break; /* dirty-active-sys set */
+    }
+}
+
 #endif
 
 #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK)
diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab
index 4ef04d020a..ec36b23059 100644
--- a/erts/emulator/beam/ops.tab
+++ b/erts/emulator/beam/ops.tab
@@ -802,15 +802,10 @@ call_ext_last u==1 Bif=u$bif:erts_internal:check_process_code/1 D => i_call_ext_
 call_ext_only u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext_only Bif
 
 #
-# The BIFs erlang:garbage_collect/0 must be called like a function,
+# The BIFs erts_internal:garbage_collect/1 must be called like a function,
 # to allow them to invoke the garbage collector. (The stack pointer must
 # be saved and p->arity must be zeroed, which is not done on ordinary BIF calls.)
 #
-
-call_ext u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext Bif
-call_ext_last u==0 Bif=u$bif:erlang:garbage_collect/0 D => i_call_ext_last Bif D
-call_ext_only u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext_only Bif
-
 call_ext u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext Bif
 call_ext_last u==1 Bif=u$bif:erts_internal:garbage_collect/1 D => i_call_ext_last Bif D
 call_ext_only u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext_only Bif
-- 
cgit v1.2.3


From 5de2d73abd5d369676e43acd17a9d9db9795ccd5 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 13 Jan 2017 14:22:31 +0100
Subject: Fix memory leak of temporary heap

This bug was introduced in previous commit, and has never been
released in an official OTP version.
---
 erts/emulator/beam/erl_gc.c      | 11 +++++------
 erts/emulator/beam/erl_gc.h      |  1 +
 erts/emulator/beam/erl_process.c | 18 +-----------------
 3 files changed, 7 insertions(+), 23 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 561e7fbdee..ace1524326 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -110,7 +110,6 @@ typedef struct {
 
 static Uint setup_rootset(Process*, Eterm*, int, Rootset*);
 static void cleanup_rootset(Rootset *rootset);
-static void deallocate_previous_young_generation(Process *c_p);
 static Eterm *full_sweep_heaps(Process *p,
 			       int hibernate,
 			       Eterm *n_heap, Eterm* n_htop,
@@ -839,7 +838,7 @@ erts_garbage_collect_hibernate(Process* p)
     disallow_heap_frag_ref_in_heap(p, heap, htop);
 #endif
 
-    deallocate_previous_young_generation(p);
+    erts_deallocate_young_generation(p);
 
     p->heap = heap;
     p->high_water = htop;
@@ -1496,7 +1495,7 @@ do_minor(Process *p, ErlHeapFragment *live_hf_end,
     disallow_heap_frag_ref_in_heap(p, n_heap, n_htop);
 #endif
 
-    deallocate_previous_young_generation(p);
+    erts_deallocate_young_generation(p);
 
     HEAP_START(p) = n_heap;
     HEAP_TOP(p) = n_htop;
@@ -1594,7 +1593,7 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
     disallow_heap_frag_ref_in_heap(p, n_heap, n_htop);
 #endif
 
-    deallocate_previous_young_generation(p);
+    erts_deallocate_young_generation(p);
 
     HEAP_START(p) = n_heap;
     HEAP_TOP(p) = n_htop;
@@ -1756,8 +1755,8 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
     return adjusted;
 }
 
-static void
-deallocate_previous_young_generation(Process *c_p)
+void
+erts_deallocate_young_generation(Process *c_p)
 {
     Eterm *orig_heap;
 
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 54ea9ca3c0..9a177b7c59 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -157,5 +157,6 @@ void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*);
 void erts_free_heap_frags(struct process* p);
 Eterm erts_max_heap_size_map(Sint, Uint, Eterm **, Uint *);
 int erts_max_heap_size(Eterm, Uint *, Uint *);
+void erts_deallocate_young_generation(Process *c_p);
 
 #endif /* __ERL_GC_H__ */
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index b345c35a7e..8d13723188 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -11912,7 +11912,6 @@ erts_cleanup_empty_process(Process* p)
 static void
 delete_process(Process* p)
 {
-    Eterm *heap;
     ErtsPSD *psd;
     struct saved_calls *scb;
     process_breakpoint_time_t *pbt;
@@ -11967,13 +11966,8 @@ delete_process(Process* p)
     hipe_delete_process(&p->hipe);
 #endif
 
-    heap = p->abandoned_heap ? p->abandoned_heap : p->heap;
+    erts_deallocate_young_generation(p);
 
-#ifdef DEBUG
-    sys_memset(heap, DEBUG_BAD_BYTE, p->heap_sz*sizeof(Eterm));
-#endif
-
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP, (void*) heap, p->heap_sz*sizeof(Eterm));
     if (p->old_heap != NULL) {
 
 #ifdef DEBUG
@@ -11985,16 +11979,6 @@ delete_process(Process* p)
 		       (p->old_hend-p->old_heap)*sizeof(Eterm));
     }
 
-    /*
-     * Free all pending message buffers.
-     */
-    if (p->mbuf != NULL) {	
-	free_message_buffer(p->mbuf);
-    }
-
-    if (p->msg_frag)
-	erts_cleanup_messages(p->msg_frag);
-
     erts_erase_dicts(p);
 
     /* free all pending messages */
-- 
cgit v1.2.3


From 7350e237d66f5098b53de7fa58e725753c04d530 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Fri, 13 Jan 2017 17:46:58 +0100
Subject: Do not automatically define HARDDEBUG when DEBUG is defined

---
 erts/emulator/beam/erl_gc.c | 4 ++++
 erts/emulator/beam/erl_gc.h | 4 ----
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index af799d09da..46be91c5d5 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -59,6 +59,10 @@
 #  define ERTS_GC_ASSERT(B) ((void) 1)
 #endif
 
+#if defined(DEBUG) && 0
+#  define HARDDEBUG 1
+#endif
+
 /*
  * Returns number of elements in an array.
  */
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index 54ea9ca3c0..49cc43a48f 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -27,10 +27,6 @@
 
 #include "erl_map.h"
 
-#if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF
-#  define HARDDEBUG 1
-#endif
-
 #define IS_MOVED_BOXED(x)	(!is_header((x)))
 #define IS_MOVED_CONS(x)	(is_non_value((x)))
 
-- 
cgit v1.2.3


From 43eebdd25a0d4a17f076f017cb3ea7da5cfd1808 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 22 Dec 2016 17:48:51 +0100
Subject: Switch between scheduler types when multi-scheduling is blocked

---
 erts/emulator/beam/bif.h           |   6 -
 erts/emulator/beam/erl_port_task.c |   2 +-
 erts/emulator/beam/erl_process.c   | 996 ++++++++++++++++++++++++-------------
 erts/emulator/beam/erl_process.h   |  22 +-
 4 files changed, 674 insertions(+), 352 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h
index ad55b19d0a..01cca90a7a 100644
--- a/erts/emulator/beam/bif.h
+++ b/erts/emulator/beam/bif.h
@@ -498,12 +498,6 @@ erts_bif_prep_await_proc_exit_apply_trap(Process *c_p,
 					 Eterm args[],
 					 int nargs);
 
-typedef enum {
-    ERTS_SCHED_NORMAL,
-    ERTS_SCHED_DIRTY_CPU,
-    ERTS_SCHED_DIRTY_IO
-} ErtsSchedType;
-
 #ifdef ERTS_DIRTY_SCHEDULERS
 int erts_call_dirty_bif(ErtsSchedulerData *esdp, Process *c_p,
 			BeamInstr *I, Eterm *reg);
diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c
index 3102e44c11..4836b9e2d3 100644
--- a/erts/emulator/beam/erl_port_task.c
+++ b/erts/emulator/beam/erl_port_task.c
@@ -936,7 +936,7 @@ enqueue_port(ErtsRunQueue *runq, Port *pp)
     erts_smp_inc_runq_len(runq, &runq->ports.info, ERTS_PORT_PRIO_LEVEL);
 
 #ifdef ERTS_SMP
-    if (runq->halt_in_progress)
+    if (ERTS_RUNQ_FLGS_GET_NOB(runq) & ERTS_RUNQ_FLG_HALTING)
 	erts_non_empty_runq(runq);
 #endif
 }
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index da4b468d66..084ecb14ec 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -107,9 +107,31 @@
 #define LOW_BIT       (1 << PRIORITY_LOW)
 #define PORT_BIT      (1 << ERTS_PORT_PRIO_LEVEL)
 
-#define ERTS_EMPTY_RUNQ(RQ)					\
-    ((ERTS_RUNQ_FLGS_GET_NOB((RQ)) & ERTS_RUNQ_FLGS_QMASK) == 0	\
-     && (RQ)->misc.start == NULL)
+#define ERTS_IS_RUNQ_EMPTY_FLGS(FLGS) \
+    (!((FLGS) & (ERTS_RUNQ_FLGS_QMASK|ERTS_RUNQ_FLG_MISC_OP)))
+
+#define ERTS_IS_RUNQ_EMPTY_PORTS_FLGS(FLGS) \
+    (!((FLGS) & (PORT_BIT|ERTS_RUNQ_FLG_MISC_OP)))
+
+#define ERTS_EMPTY_RUNQ(RQ) \
+    ERTS_IS_RUNQ_EMPTY_FLGS(ERTS_RUNQ_FLGS_GET_NOB((RQ)))
+
+#define ERTS_EMPTY_RUNQ_PORTS(RQ) \
+    ERTS_IS_RUNQ_EMPTY_FLGS(ERTS_RUNQ_FLGS_GET_NOB((RQ)))
+
+static ERTS_INLINE int
+runq_got_work_to_execute_flags(Uint32 flags)
+{
+    if (flags & ERTS_RUNQ_FLG_HALTING)
+        return !ERTS_IS_RUNQ_EMPTY_PORTS_FLGS(flags);
+    return !ERTS_IS_RUNQ_EMPTY_FLGS(flags);
+}
+
+static ERTS_INLINE int
+runq_got_work_to_execute(ErtsRunQueue *rq)
+{
+    return runq_got_work_to_execute_flags(ERTS_RUNQ_FLGS_GET_NOB(rq));
+}
 
 #undef RUNQ_READ_RQ
 #undef RUNQ_SET_RQ
@@ -141,9 +163,6 @@ do { \
 #  define ERTS_DBG_VERIFY_VALID_RUNQP(RQP)
 #endif
 
-#define ERTS_EMPTY_RUNQ_PORTS(RQ) \
-    (RUNQ_READ_LEN(&(RQ)->ports.info.len) == 0 && (RQ)->misc.start == NULL)
-
 const Process erts_invalid_process = {{ERTS_INVALID_PID}};
 
 extern BeamInstr beam_apply[];
@@ -200,158 +219,139 @@ typedef struct {
     ErtsProcList *chngq;
 } ErtsMultiSchedulingBlock;
 
+typedef struct {
+    Uint32 normal;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    Uint32 dirty_cpu;
+    Uint32 dirty_io;
+#endif
+} ErtsSchedTypeCounters;
+
 static struct {
     erts_smp_mtx_t mtx;
-    Uint32 online;
-    Uint32 curr_online;
-    Uint32 active;
+    ErtsSchedTypeCounters online;
+    ErtsSchedTypeCounters curr_online;
+    ErtsSchedTypeCounters active;
     erts_smp_atomic32_t changing;
     ErtsProcList *chngq;
     Eterm changer;
     ErtsMultiSchedulingBlock nmsb; /* Normal multi Scheduling Block */
     ErtsMultiSchedulingBlock msb; /* Multi Scheduling Block */
+#ifdef ERTS_DIRTY_SCHEDULERS
+    ErtsSchedType last_msb_dirty_type;
+#endif
 } schdlr_sspnd;
 
-#define ERTS_SCHDLR_SSPND_S_BITS	10
-#define ERTS_SCHDLR_SSPND_DCS_BITS	11
-#define ERTS_SCHDLR_SSPND_DIS_BITS	11
-
-#define ERTS_SCHDLR_SSPND_S_MASK	((1 << ERTS_SCHDLR_SSPND_S_BITS)-1)
-#define ERTS_SCHDLR_SSPND_DCS_MASK	((1 << ERTS_SCHDLR_SSPND_DCS_BITS)-1)
-#define ERTS_SCHDLR_SSPND_DIS_MASK	((1 << ERTS_SCHDLR_SSPND_DIS_BITS)-1)
-
-#define ERTS_SCHDLR_SSPND_S_SHIFT	0
-#define ERTS_SCHDLR_SSPND_DCS_SHIFT	(ERTS_SCHDLR_SSPND_S_SHIFT \
-					 + ERTS_SCHDLR_SSPND_S_BITS)
-#define ERTS_SCHDLR_SSPND_DIS_SHIFT	(ERTS_SCHDLR_SSPND_DCS_SHIFT \
-					 + ERTS_SCHDLR_SSPND_DCS_BITS)
-
-#if (ERTS_SCHDLR_SSPND_S_BITS		\
-     + ERTS_SCHDLR_SSPND_DCS_BITS	\
-     + ERTS_SCHDLR_SSPND_DIS_BITS) > 32
-#  error Wont fit in Uint32
-#endif
+static void init_scheduler_suspend(void);
 
-#if (ERTS_MAX_NO_OF_SCHEDULERS-1) > ERTS_SCHDLR_SSPND_S_MASK
-#  error Max no schedulers wont fit in its bit-field
-#endif
-#if ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS > ERTS_SCHDLR_SSPND_DCS_MASK
-#  error Max no dirty cpu schedulers wont fit in its bit-field
-#endif
-#if ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS > ERTS_SCHDLR_SSPND_DIS_MASK
-#  error Max no dirty io schedulers wont fit in its bit-field
+static ERTS_INLINE Uint32
+schdlr_sspnd_eq_nscheds(ErtsSchedTypeCounters *val1p, ErtsSchedTypeCounters *val2p)
+{
+    int res = val1p->normal == val2p->normal;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    res &= val1p->dirty_cpu == val2p->dirty_cpu;
+    res &= val1p->dirty_io == val2p->dirty_io;
 #endif
-
-#define ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(S, DCS, DIS)			\
-    ((((Uint32) (((S) & ERTS_SCHDLR_SSPND_S_MASK))-1)			\
-      << ERTS_SCHDLR_SSPND_S_SHIFT)					\
-     | ((((Uint32) ((DCS) & ERTS_SCHDLR_SSPND_DCS_MASK))		\
-	 << ERTS_SCHDLR_SSPND_DCS_SHIFT))				\
-     | ((((Uint32) ((DIS) & ERTS_SCHDLR_SSPND_DIS_MASK))		\
-	 << ERTS_SCHDLR_SSPND_DIS_SHIFT)))
-
-static void init_scheduler_suspend(void);
+    return res;
+}
 
 static ERTS_INLINE Uint32
-schdlr_sspnd_get_nscheds(Uint32 *valp, ErtsSchedType type)
+schdlr_sspnd_get_nscheds(ErtsSchedTypeCounters *valp,
+                         ErtsSchedType type)
 {
-    Uint32 res = (Uint32) (*valp);
     switch (type) {
     case ERTS_SCHED_NORMAL:
-	res >>= ERTS_SCHDLR_SSPND_S_SHIFT;
-	res &= (Uint32) ERTS_SCHDLR_SSPND_S_MASK;
-	res++;
-	break;
+        return valp->normal;
+#ifdef ERTS_DIRTY_SCHEDULERS
     case ERTS_SCHED_DIRTY_CPU:
-	res >>= ERTS_SCHDLR_SSPND_DCS_SHIFT;
-	res &= (Uint32) ERTS_SCHDLR_SSPND_DCS_MASK;
-	break;
+        return valp->dirty_cpu;
     case ERTS_SCHED_DIRTY_IO:
-	res >>= ERTS_SCHDLR_SSPND_DIS_SHIFT;
-	res &= (Uint32) ERTS_SCHDLR_SSPND_DIS_MASK;
-	break;
+        return valp->dirty_io;
+#else
+    case ERTS_SCHED_DIRTY_CPU:
+    case ERTS_SCHED_DIRTY_IO:
+        return 0;
+#endif
     default:
 	ERTS_INTERNAL_ERROR("Invalid scheduler type");
 	return 0;
     }
+}
 
+static ERTS_INLINE Uint32
+schdlr_sspnd_get_nscheds_tot(ErtsSchedTypeCounters *valp)
+{
+    Uint32 res = valp->normal;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    res += valp->dirty_cpu;
+    res += valp->dirty_io;
+#endif
     return res;
 }
 
 static ERTS_INLINE void
-schdlr_sspnd_dec_nscheds(Uint32 *valp, ErtsSchedType type)
+schdlr_sspnd_dec_nscheds(ErtsSchedTypeCounters *valp,
+                         ErtsSchedType type)
 {
     ASSERT(schdlr_sspnd_get_nscheds(valp, type) > 0);
 
     switch (type) {
     case ERTS_SCHED_NORMAL:
-	*valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT;
+        valp->normal--;
 	break;
+#ifdef ERTS_DIRTY_SCHEDULERS
     case ERTS_SCHED_DIRTY_CPU:
-	*valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT;
+        valp->dirty_cpu--;
 	break;
     case ERTS_SCHED_DIRTY_IO:
-	*valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT;
+        valp->dirty_io--;
 	break;
+#endif
     default:
 	ERTS_INTERNAL_ERROR("Invalid scheduler type");
     }
 }
 
 static ERTS_INLINE void
-schdlr_sspnd_inc_nscheds(Uint32 *valp, ErtsSchedType type)
+schdlr_sspnd_inc_nscheds(ErtsSchedTypeCounters *valp,
+                         ErtsSchedType type)
 {
     switch (type) {
     case ERTS_SCHED_NORMAL:
-	ASSERT(schdlr_sspnd_get_nscheds(valp, type)
-	       < ERTS_MAX_NO_OF_SCHEDULERS-1);
-	*valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT;
+        valp->normal++;
 	break;
+#ifdef ERTS_DIRTY_SCHEDULERS
     case ERTS_SCHED_DIRTY_CPU:
-	ASSERT(schdlr_sspnd_get_nscheds(valp, type)
-	       < ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS);
-	*valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT;
+        valp->dirty_cpu++;
 	break;
     case ERTS_SCHED_DIRTY_IO:
-	ASSERT(schdlr_sspnd_get_nscheds(valp, type)
-	       < ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS);
-	*valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT;
+        valp->dirty_io++;
 	break;
+#endif
     default:
 	ERTS_INTERNAL_ERROR("Invalid scheduler type");
     }
 }
 
 static ERTS_INLINE void
-schdlr_sspnd_set_nscheds(Uint32 *valp, ErtsSchedType type, Uint32 no)
+schdlr_sspnd_set_nscheds(ErtsSchedTypeCounters *valp,
+                         ErtsSchedType type, Uint32 no)
 {
-    Uint32 val = *valp;
-
     switch (type) {
     case ERTS_SCHED_NORMAL:
-	ASSERT(no > 0);
-	val &= ~(((Uint32) ERTS_SCHDLR_SSPND_S_MASK)
-		 << ERTS_SCHDLR_SSPND_S_SHIFT);
-	val |= (((no-1) & ((Uint32) ERTS_SCHDLR_SSPND_S_MASK))
-		<< ERTS_SCHDLR_SSPND_S_SHIFT);
+        valp->normal = no;
 	break;
+#ifdef ERTS_DIRTY_SCHEDULERS
     case ERTS_SCHED_DIRTY_CPU:
-	val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK)
-		 << ERTS_SCHDLR_SSPND_DCS_SHIFT);
-	val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK))
-		 << ERTS_SCHDLR_SSPND_DCS_SHIFT);
+        valp->dirty_cpu = no;
 	break;
     case ERTS_SCHED_DIRTY_IO:
-	val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK)
-		 << ERTS_SCHDLR_SSPND_DIS_SHIFT);
-	val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK))
-		<< ERTS_SCHDLR_SSPND_DIS_SHIFT);
+        valp->dirty_io = no;
 	break;
+#endif
     default:
 	ERTS_INTERNAL_ERROR("Invalid scheduler type");
     }
-
-    *valp = val;
 }
 
 static struct {
@@ -2293,7 +2293,8 @@ static ERTS_INLINE erts_aint32_t
 handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
 {
     unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_REAP_PORTS);
-    awdp->esdp->run_queue->halt_in_progress = 1;
+    ERTS_RUNQ_FLGS_SET(awdp->esdp->run_queue, ERTS_RUNQ_FLG_HALTING);
+
     if (erts_smp_atomic32_dec_read_acqb(&erts_halt_progress) == 0) {
 	int i, max = erts_ptab_max(&erts_port);
 	erts_smp_atomic32_set_nob(&erts_halt_progress, 1);
@@ -2833,11 +2834,12 @@ static erts_aint32_t
 sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi)
 {
     erts_aint32_t oflgs;
-    erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING
-			   | ERTS_SSI_FLG_WAITING);
+    erts_aint32_t nflgs;
     erts_aint32_t xflgs = 0;
 
     do {
+        nflgs = (xflgs & ERTS_SSI_FLG_MSB_EXEC);
+        nflgs |= ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING;
 	oflgs = erts_smp_atomic32_cmpxchg_acqb(&ssi->flags, nflgs, xflgs);
 	if (oflgs == xflgs)
 	    return nflgs;
@@ -2859,7 +2861,7 @@ sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi)
 	if (oflgs == xflgs)
 	    return nflgs;
 	xflgs = oflgs;
-	nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED;
+	nflgs |= oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC);
     } while (oflgs & ERTS_SSI_FLG_WAITING);
     return oflgs;
 }
@@ -2909,7 +2911,7 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type)
 	    return oflgs;
 	}
 	xflgs = oflgs;
-	nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED;
+	nflgs |= oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC);
     }
 }
 
@@ -3044,6 +3046,8 @@ aux_thread(void *unused)
     return NULL;
 }
 
+static void suspend_scheduler(ErtsSchedulerData *esdp);
+
 #endif /* ERTS_SMP */
 
 static void
@@ -3209,8 +3213,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
 	}
 
-	if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
-	    erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+	if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC))
+	    erts_smp_atomic32_read_band_nob(&ssi->flags,
+                                            (ERTS_SSI_FLG_SUSPENDED
+                                             | ERTS_SSI_FLG_MSB_EXEC));
 
 	if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
 	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
@@ -3409,8 +3415,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 	    erts_smp_runq_lock(rq);
 	}
 	clear_sys_scheduling();
-	if (flgs & ~ERTS_SSI_FLG_SUSPENDED)
-	    erts_smp_atomic32_read_band_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+	if (flgs & ~(ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC))
+	    erts_smp_atomic32_read_band_nob(&ssi->flags,
+                                            (ERTS_SSI_FLG_SUSPENDED
+                                             | ERTS_SSI_FLG_MSB_EXEC));
 #endif
 	if (!working)
 	    sched_wall_time_change(esdp, working = 1);
@@ -3433,17 +3441,54 @@ ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi)
 	oflgs = erts_smp_atomic32_cmpxchg_relb(&ssi->flags, nflgs, xflgs);
 	if (oflgs == xflgs)
 	    return oflgs;
-	nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED;
+	nflgs = oflgs & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC);
 	xflgs = oflgs;
     }
 }
 
+static ERTS_INLINE void
+ssi_wake(ErtsSchedulerSleepInfo *ssi)
+{
+    erts_sched_finish_poke(ssi, ssi_flags_set_wake(ssi));
+}
+
+#ifdef ERTS_DIRTY_SCHEDULERS
+
 static void
-wake_scheduler(ErtsRunQueue *rq)
+dcpu_sched_ix_suspend_wake(Uint ix)
 {
-    ErtsSchedulerSleepInfo *ssi;
-    erts_aint32_t flgs;
+    ErtsSchedulerSleepInfo* ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
+    erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+    ssi_wake(ssi);
+}
+
+static void
+dio_sched_ix_suspend_wake(Uint ix)
+{
+    ErtsSchedulerSleepInfo* ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
+    erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED);
+    ssi_wake(ssi);
+}
+
+static void
+dcpu_sched_ix_wake(Uint ix)
+{
+    ssi_wake(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix));
+}
+
+#if 0
+static void
+dio_sched_ix_wake(Uint ix)
+{
+    ssi_wake(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix));
+}
+#endif
 
+#endif
+
+static void
+wake_scheduler(ErtsRunQueue *rq)
+{
     /*
      * The unlocked run queue is not strictly necessary
      * from a thread safety or deadlock prevention
@@ -3455,10 +3500,7 @@ wake_scheduler(ErtsRunQueue *rq)
     ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)
 		       || ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
 
-    ssi = rq->scheduler->ssi;
-
-    flgs = ssi_flags_set_wake(ssi);
-    erts_sched_finish_poke(ssi, flgs);
+    ssi_wake(rq->scheduler->ssi);
 }
 
 #ifdef ERTS_DIRTY_SCHEDULERS
@@ -3505,6 +3547,13 @@ wake_dirty_schedulers(ErtsRunQueue *rq, int one)
 	} while (ssi);
     }
 }
+
+static void
+wake_dirty_scheduler(ErtsRunQueue *rq)
+{
+    wake_dirty_schedulers(rq, 1);
+}
+
 #endif
 
 #define ERTS_NO_USED_RUNQS_SHIFT 16
@@ -3645,7 +3694,7 @@ smp_notify_inc_runq(ErtsRunQueue *runq)
     if (runq) {
 #ifdef ERTS_DIRTY_SCHEDULERS
 	if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix))
-	    wake_dirty_schedulers(runq, 1);
+	    wake_dirty_scheduler(runq);
 	else
 #endif
 	    wake_scheduler(runq);
@@ -3949,8 +3998,7 @@ suspend_run_queue(ErtsRunQueue *rq)
     wake_scheduler(rq);
 }
 
-static void scheduler_ix_resume_wake(Uint ix);
-static void scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi);
+static void nrml_sched_ix_resume_wake(Uint ix);
 
 static ERTS_INLINE void
 resume_run_queue(ErtsRunQueue *rq)
@@ -3958,16 +4006,19 @@ resume_run_queue(ErtsRunQueue *rq)
     int pix;
     Uint32 oflgs;
 
+    ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
+
     erts_smp_runq_lock(rq);
 
     oflgs = ERTS_RUNQ_FLGS_READ_BSET(rq,
 				     (ERTS_RUNQ_FLG_OUT_OF_WORK
 				      | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK
-				      | ERTS_RUNQ_FLG_SUSPENDED),
+				      | ERTS_RUNQ_FLG_SUSPENDED
+                                      | ERTS_RUNQ_FLG_MSB_EXEC),
 				     (ERTS_RUNQ_FLG_OUT_OF_WORK
 				      | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK));
 
-    if (oflgs & ERTS_RUNQ_FLG_SUSPENDED) {
+    if (oflgs & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_MSB_EXEC)) {
 	erts_aint32_t len;
 
 	rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS;
@@ -3986,10 +4037,7 @@ resume_run_queue(ErtsRunQueue *rq)
 
     erts_smp_runq_unlock(rq);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-#endif
-	scheduler_ix_resume_wake(rq->ix);
+    nrml_sched_ix_resume_wake(rq->ix);
 }
 
 typedef struct {
@@ -4047,28 +4095,22 @@ evacuate_run_queue(ErtsRunQueue *rq,
     int prio_q;
     ErtsRunQueue *to_rq;
     ErtsMigrationPaths *mps;
-    ErtsMigrationPath *mp = NULL;
+    ErtsMigrationPath *mp;
 
     ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
 
     (void) ERTS_RUNQ_FLGS_UNSET(rq, ERTS_RUNQ_FLG_PROTECTED);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-#endif
-    {
-	mps = erts_get_migration_paths_managed();
-	mp = &mps->mpath[rq->ix];
-    }
+    ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
+
+    mps = erts_get_migration_paths_managed();
+    mp = &mps->mpath[rq->ix];
 
     /* Evacuate scheduled misc ops */
 
     if (rq->misc.start) {
 	ErtsMiscOpList *start, *end;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-	ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
-#endif
 	to_rq = mp->misc_evac_runq;
 	if (!to_rq)
 	    return;
@@ -4077,6 +4119,7 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	end = rq->misc.end;
 	rq->misc.start = NULL;
 	rq->misc.end = NULL;
+        ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP);
 	erts_smp_runq_unlock(rq);
 
 	erts_smp_runq_lock(to_rq);
@@ -4097,9 +4140,6 @@ evacuate_run_queue(ErtsRunQueue *rq,
     if (rq->ports.start) {
 	Port *prt;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-	ASSERT(!ERTS_RUNQ_IX_IS_DIRTY(rq->ix));
-#endif
 	to_rq = mp->prio[ERTS_PORT_PRIO_LEVEL].runq;
 	if (!to_rq)
 	    return;
@@ -4137,15 +4177,10 @@ evacuate_run_queue(ErtsRunQueue *rq,
 	int notify = 0;
 	to_rq = NULL;
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-	if (!ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-#endif
-	{
-	    if (!mp->prio[prio_q].runq)
-		return;
-	    if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq)
-		return;
-	}
+        if (!mp->prio[prio_q].runq)
+            return;
+        if (prio_q == PRIORITY_NORMAL && !mp->prio[PRIORITY_LOW].runq)
+            return;
 
 	proc = dequeue_process(rq, prio_q, &state);
 	while (proc) {
@@ -4201,11 +4236,6 @@ evacuate_run_queue(ErtsRunQueue *rq,
 		goto handle_next_proc;
 	    }
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-	    if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-		clear_proc_dirty_queue_bit(real_proc, rq, qbit);
-#endif
-
 	    if (ERTS_PSFLG_BOUND & real_state) {
 		/* Bound processes get stuck here... */
 		proc->next = NULL;
@@ -4219,16 +4249,7 @@ evacuate_run_queue(ErtsRunQueue *rq,
 		int prio = (int) ERTS_PSFLGS_GET_PRQ_PRIO(state);
 		erts_smp_runq_unlock(rq);
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-		if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix))
-		    /*
-		     * dirty run queues evacuate only to run
-		     * queue 0 during multi-scheduling blocking
-		     */
-		    to_rq = ERTS_RUNQ_IX(0);
-		else
-#endif
-		    to_rq = mp->prio[prio].runq;
+                to_rq = mp->prio[prio].runq;
 		RUNQ_SET_RQ(&proc->run_queue, to_rq);
 
 		erts_smp_runq_lock(to_rq);
@@ -4263,7 +4284,7 @@ try_steal_task_from_victim(ErtsRunQueue *rq, int *rq_lockedp, ErtsRunQueue *vrq,
 
     erts_smp_runq_lock(vrq);
 
-    if (rq->halt_in_progress)
+    if (ERTS_RUNQ_FLGS_GET_NOB(rq) & ERTS_RUNQ_FLG_HALTING)
 	goto no_procs;
 
     /*
@@ -4362,8 +4383,7 @@ check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix)
 {
     ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix);
     Uint32 flags = ERTS_RUNQ_FLGS_GET(vrq);
-    if ((flags & (ERTS_RUNQ_FLG_NONEMPTY
-		  | ERTS_RUNQ_FLG_PROTECTED)) == ERTS_RUNQ_FLG_NONEMPTY)
+    if (runq_got_work_to_execute_flags(flags) & (!(flags & ERTS_RUNQ_FLG_PROTECTED)))
 	return try_steal_task_from_victim(rq, rq_lockedp, vrq, flags);
     else
 	return 0;
@@ -4431,11 +4451,9 @@ try_steal_task(ErtsRunQueue *rq)
     if (!rq_locked)
 	erts_smp_runq_lock(rq);
 
-    if (!res)
-	res = rq->halt_in_progress ?
-	    !ERTS_EMPTY_RUNQ_PORTS(rq) : !ERTS_EMPTY_RUNQ(rq);
-
-    return res;
+    if (res)
+        return res;
+    return runq_got_work_to_execute(rq);
 }
 
 /* Run queue balancing */
@@ -5347,7 +5365,7 @@ wakeup_other_check(ErtsRunQueue *rq, Uint32 flags)
 #ifdef ERTS_DIRTY_SCHEDULERS
 		if (ERTS_RUNQ_IX_IS_DIRTY(rq->ix)) {
 		    if (rq->waiting) {
-			wake_dirty_schedulers(rq, 1);
+			wake_dirty_scheduler(rq);
 		    }
 		} else
 #endif
@@ -5710,13 +5728,29 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_BEAM_REGISTER,
 					   MAX_REG * sizeof(FloatDef));
 #ifdef ERTS_DIRTY_SCHEDULERS
+    esdp->run_queue = runq;
     if (ERTS_RUNQ_IX_IS_DIRTY(runq->ix)) {
 	esdp->no = 0;
+        if (runq == ERTS_DIRTY_CPU_RUNQ)
+            esdp->type = ERTS_SCHED_DIRTY_CPU;
+        else {
+            ASSERT(runq == ERTS_DIRTY_IO_RUNQ);
+            esdp->type = ERTS_SCHED_DIRTY_IO;
+        }
 	ERTS_DIRTY_SCHEDULER_NO(esdp) = (Uint) num;
+        if (num == 1) {
+            /*
+             * Multi-scheduling block functionality depends
+             * on finding dirty scheduler number 1 here...
+             */
+            runq->scheduler = esdp;
+        }
     }
     else {
+        esdp->type = ERTS_SCHED_NORMAL;
 	esdp->no = (Uint) num;
 	ERTS_DIRTY_SCHEDULER_NO(esdp) = 0;
+        runq->scheduler = esdp;
     }
     esdp->dirty_shadow_process = shadow_proc;
     if (shadow_proc) {
@@ -5728,6 +5762,8 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 	shadow_proc->static_flags = ERTS_STC_FLG_SHADOW_PROC;
     }
 #else
+    runq->scheduler = esdp;
+    esdp->run_queue = runq;
     esdp->no = (Uint) num;
 #endif
 
@@ -5740,9 +5776,6 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 
     erts_init_atom_cache_map(&esdp->atom_cache_map);
 
-    esdp->run_queue = runq;
-    esdp->run_queue->scheduler = esdp;
-
     esdp->last_monotonic_time = 0;
     esdp->check_time_reds = 0;
 
@@ -5858,7 +5891,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	erts_smp_atomic32_set_nob(&rq->len, 0);
 	rq->wakeup_other = 0;
 	rq->wakeup_other_reds = 0;
-	rq->halt_in_progress = 0;
 
 	rq->procs.pending_exiters = NULL;
 	rq->procs.context_switches = 0;
@@ -6974,15 +7006,8 @@ resume_process(Process *p, ErtsProcLocks locks)
 
 #ifdef ERTS_SMP
 
-static void
-scheduler_ix_resume_wake(Uint ix)
-{
-    ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix);
-    scheduler_ssi_resume_wake(ssi);
-}
-
-static void
-scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi)
+static ERTS_INLINE void
+sched_resume_wake__(ErtsSchedulerSleepInfo *ssi)
 {
     erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING
 			   | ERTS_SSI_FLG_TSE_SLEEPING
@@ -6996,9 +7021,31 @@ scheduler_ssi_resume_wake(ErtsSchedulerSleepInfo *ssi)
 	    break;
 	}
 	xflgs = oflgs;
-    } while (oflgs & ERTS_SSI_FLG_SUSPENDED);
+    } while (oflgs & (ERTS_SSI_FLG_MSB_EXEC|ERTS_SSI_FLG_SUSPENDED));
+}
+
+static void
+nrml_sched_ix_resume_wake(Uint ix)
+{
+    sched_resume_wake__(ERTS_SCHED_SLEEP_INFO_IX(ix));
 }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+static void
+dcpu_sched_ix_resume_wake(Uint ix)
+{
+    sched_resume_wake__(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix));
+}
+
+static void
+dio_sched_ix_resume_wake(Uint ix)
+{
+    sched_resume_wake__(ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix));
+}
+
+#endif
+
 static erts_aint32_t
 sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct)
 {
@@ -7078,9 +7125,18 @@ static void
 init_scheduler_suspend(void)
 {
     erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd");
-    schdlr_sspnd.online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0);
-    schdlr_sspnd.curr_online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0);
-    schdlr_sspnd.active = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0);
+    schdlr_sspnd.online.normal = 1;
+    schdlr_sspnd.curr_online.normal = 1;
+    schdlr_sspnd.active.normal = 1;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    schdlr_sspnd.online.dirty_cpu = 0;
+    schdlr_sspnd.curr_online.dirty_cpu = 0;
+    schdlr_sspnd.active.dirty_cpu = 0;
+    schdlr_sspnd.online.dirty_io = 0;
+    schdlr_sspnd.curr_online.dirty_io = 0;
+    schdlr_sspnd.active.dirty_io = 0;
+    schdlr_sspnd.last_msb_dirty_type = ERTS_SCHED_DIRTY_IO;
+#endif
     erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0);
     schdlr_sspnd.chngq = NULL;
     schdlr_sspnd.changer = am_false;
@@ -7141,6 +7197,252 @@ schdlr_sspnd_resume_procs(ErtsSchedType sched_type,
     }
 }
 
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+static ERTS_INLINE int
+have_dirty_work(void)
+{
+    return !(ERTS_EMPTY_RUNQ(ERTS_DIRTY_CPU_RUNQ)
+             | ERTS_EMPTY_RUNQ(ERTS_DIRTY_IO_RUNQ));
+}
+
+#define ERTS_MSB_NONE_PRIO_BIT PORT_BIT
+
+static ERTS_INLINE Uint32
+msb_runq_prio_bit(Uint32 flgs)
+{
+    int pbit;
+
+    pbit = (int) (flgs & ERTS_RUNQ_FLGS_PROCS_QMASK);
+    if (flgs & PORT_BIT) {
+        /* rate ports as proc prio high */
+        pbit |= HIGH_BIT;
+    }
+    if (flgs & ERTS_RUNQ_FLG_MISC_OP) {
+        /* rate misc ops as proc prio normal */
+        pbit |= NORMAL_BIT;
+    }
+    if (flgs & LOW_BIT) {
+        /* rate low prio as normal (avoid starvation) */
+        pbit |= NORMAL_BIT;
+    }
+    if (!pbit)
+        pbit = (int) ERTS_MSB_NONE_PRIO_BIT;
+    else
+        pbit &= -pbit; /* least significant bit set... */
+    ASSERT(pbit);
+
+    /* High prio low value; low prio high value... */
+    return (Uint32) pbit;
+}
+
+static ERTS_INLINE void
+msb_runq_prio_bits(Uint32 *nrmlp, Uint32 *dcpup, Uint32 *diop)
+{
+    Uint32 flgs = ERTS_RUNQ_FLGS_GET(ERTS_RUNQ_IX(0));
+    if (flgs & ERTS_RUNQ_FLG_HALTING) {
+        /*
+         * Emulator is halting; only execute port jobs
+         * on normal scheduler. Ensure that we switch
+         * to the normal scheduler.
+         */
+        *nrmlp = HIGH_BIT;
+        *dcpup = ERTS_MSB_NONE_PRIO_BIT;
+        *diop = ERTS_MSB_NONE_PRIO_BIT;
+    }
+    else {
+        *nrmlp = msb_runq_prio_bit(flgs);
+
+        flgs = ERTS_RUNQ_FLGS_GET(ERTS_DIRTY_CPU_RUNQ);
+        *dcpup = msb_runq_prio_bit(flgs);
+
+        flgs = ERTS_RUNQ_FLGS_GET(ERTS_DIRTY_IO_RUNQ);
+        *diop = msb_runq_prio_bit(flgs);
+    }
+}
+
+static int
+msb_scheduler_type_switch(ErtsSchedType sched_type,
+                          ErtsSchedulerData *esdp,
+                          long no)
+{
+    Uint32 nrml_prio, dcpu_prio, dio_prio;
+    ErtsSchedType exec_type;
+    ErtsRunQueue *exec_rq;
+#ifdef DEBUG
+    erts_aint32_t dbg_val;
+#endif
+
+    ASSERT(schdlr_sspnd.msb.ongoing);
+
+    /*
+     * This function determines how to switch
+     * between scheduler types when multi-scheduling
+     * is blocked.
+     *
+     * If no dirty work exist, we always select
+     * execution of normal scheduler. If nothing
+     * executes, normal scheduler 1 should be waiting
+     * in sys_schedule(), otherwise we cannot react
+     * on I/O events.
+     *
+     * We unconditionally switch back to normal
+     * scheduler after executing dirty in order to
+     * make sure we check for I/O...
+     */
+
+    msb_runq_prio_bits(&nrml_prio, &dcpu_prio, &dio_prio);
+
+    exec_type = ERTS_SCHED_NORMAL;
+    if (sched_type == ERTS_SCHED_NORMAL) {
+
+        /*
+         * Check priorities of work in the
+         * different run-queues and determine
+         * run-queue with highest prio job...
+         */
+
+        if ((dcpu_prio == ERTS_MSB_NONE_PRIO_BIT)
+            & (dio_prio == ERTS_MSB_NONE_PRIO_BIT)) {
+            /*
+             * No dirty work exist; continue on normal
+             * scheduler...
+             */
+            return 0;
+        }
+
+        if (dcpu_prio < nrml_prio) {
+            exec_type = ERTS_SCHED_DIRTY_CPU;
+            if (dio_prio < dcpu_prio)
+                exec_type = ERTS_SCHED_DIRTY_IO;
+        }
+        else {
+            if (dio_prio < nrml_prio)
+                exec_type = ERTS_SCHED_DIRTY_IO;
+        }
+
+        /*
+         * Make sure to alternate between dirty types
+         * inbetween normal execution if highest 
+         * priorities are equal.
+         */
+
+        if (exec_type == ERTS_SCHED_NORMAL) {
+            if (dcpu_prio == nrml_prio)
+                exec_type = ERTS_SCHED_DIRTY_CPU;
+            else if (dio_prio == nrml_prio)
+                exec_type = ERTS_SCHED_DIRTY_IO;
+            else {
+                /*
+                 * Normal work has higher prio than
+                 * dirty work; continue on normal
+                 * scheduler...
+                 */
+                return 0;
+            }
+        }
+
+        ASSERT(exec_type != ERTS_SCHED_NORMAL);
+        if (dio_prio == dcpu_prio) {
+            /* Alter between dirty types... */
+            if (schdlr_sspnd.last_msb_dirty_type == ERTS_SCHED_DIRTY_IO)
+                exec_type = ERTS_SCHED_DIRTY_CPU;
+            else
+                exec_type = ERTS_SCHED_DIRTY_IO;
+        }
+    }
+
+    ASSERT(sched_type != exec_type);
+
+    if (exec_type != ERTS_SCHED_NORMAL)
+        schdlr_sspnd.last_msb_dirty_type = exec_type;
+    else {
+        erts_aint32_t calls;
+        /*
+         * Going back to normal scheduler after
+         * dirty execution; make sure it will check
+         * for I/O...
+         */
+        if (ERTS_USE_MODIFIED_TIMING())
+            calls = ERTS_MODIFIED_TIMING_INPUT_REDS + 1;
+        else
+            calls = INPUT_REDUCTIONS + 1;
+        erts_smp_atomic32_set_nob(&function_calls, calls);
+
+        if ((nrml_prio == ERTS_MSB_NONE_PRIO_BIT)
+            & ((dcpu_prio != ERTS_MSB_NONE_PRIO_BIT)
+               | (dio_prio != ERTS_MSB_NONE_PRIO_BIT))) {
+            /*
+             * We have dirty work, but an empty
+             * normal run-queue.
+             *
+             * Since the normal run-queue is
+             * empty, the normal scheduler will
+             * go to sleep when selected for
+             * execution. We have dirty work to
+             * do, so we only want it to check
+             * I/O, and then come back here and
+             * switch to dirty execution.
+             *
+             * To prevent the scheduler from going
+             * to sleep we trick it into believing
+             * it has work to do...
+             */
+            ERTS_RUNQ_FLGS_SET_NOB(ERTS_RUNQ_IX(0),
+                                   ERTS_RUNQ_FLG_MISC_OP);
+        }
+    }
+
+    /*
+     * Suspend this scheduler and wake up scheduler
+     * number one of another type...
+     */
+#ifdef DEBUG
+    dbg_val =
+#else
+    (void)
+#endif
+        erts_smp_atomic32_read_bset_mb(&esdp->ssi->flags,
+                                       (ERTS_SSI_FLG_SUSPENDED
+                                        | ERTS_SSI_FLG_MSB_EXEC),
+                                       ERTS_SSI_FLG_SUSPENDED);
+    ASSERT(dbg_val & ERTS_SSI_FLG_MSB_EXEC);
+
+    switch (exec_type) {
+    case ERTS_SCHED_NORMAL:
+        exec_rq = ERTS_RUNQ_IX(0);
+        break;
+    case ERTS_SCHED_DIRTY_CPU:
+        exec_rq = ERTS_DIRTY_CPU_RUNQ;
+        break;
+    case ERTS_SCHED_DIRTY_IO:
+        exec_rq = ERTS_DIRTY_IO_RUNQ;
+        break;
+    default:
+        ERTS_INTERNAL_ERROR("Invalid scheduler type");
+        exec_rq = NULL;
+        break;
+    }
+
+#ifdef DEBUG
+    dbg_val =
+#else
+    (void)
+#endif
+        erts_smp_atomic32_read_bset_mb(&exec_rq->scheduler->ssi->flags,
+                                       (ERTS_SSI_FLG_SUSPENDED
+                                        | ERTS_SSI_FLG_MSB_EXEC),
+                                       ERTS_SSI_FLG_MSB_EXEC);
+    ASSERT(dbg_val & ERTS_SSI_FLG_SUSPENDED);
+
+    wake_scheduler(exec_rq);
+
+    return 1; /* suspend this scheduler... */
+
+}
+
+#endif
+
 static void
 suspend_scheduler(ErtsSchedulerData *esdp)
 {
@@ -7166,32 +7468,49 @@ suspend_scheduler(ErtsSchedulerData *esdp)
      * Regardless of why a scheduler is suspended, it ends up here.
      */
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-    if (ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+
+#if !defined(ERTS_DIRTY_SCHEDULERS)
+
+    sched_type = ERTS_SCHED_NORMAL;
+    online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN;
+    no = esdp->no;
+    ASSERT(no != 1);
+
+#else
+
+    sched_type = esdp->type;
+    switch (sched_type) {
+    case ERTS_SCHED_NORMAL:
+        online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN;
+        no = esdp->no;
+        break;
+    case ERTS_SCHED_DIRTY_CPU:
+        online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN;
+        no = ERTS_DIRTY_SCHEDULER_NO(esdp);
+        break;
+    case ERTS_SCHED_DIRTY_IO:
+        online_flag = 0;
 	no = ERTS_DIRTY_SCHEDULER_NO(esdp);
-	if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) {
-	    online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN;
-	    sched_type = ERTS_SCHED_DIRTY_CPU;
-	}
-	else {
-	    online_flag = 0;
-	    sched_type = ERTS_SCHED_DIRTY_IO;
-	}
+        break;
+    default:
+        ERTS_INTERNAL_ERROR("Invalid scheduler type");
+        return;
     }
-    else
-#endif
-    {
-	online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN;
-	no = esdp->no;
-	sched_type = ERTS_SCHED_NORMAL;
+
+    if (erts_smp_atomic32_read_nob(&ssi->flags) & ERTS_SSI_FLG_MSB_EXEC) {
+        ASSERT(no == 1);
+        if (!msb_scheduler_type_switch(sched_type, esdp, no))
+            return;
+        /* Suspend and let scheduler 1 of another type execute... */
     }
 
-    ASSERT(sched_type != ERTS_SCHED_NORMAL || no != 1);
+#endif
 
     if (sched_type != ERTS_SCHED_NORMAL)
 	erts_smp_runq_unlock(esdp->run_queue);
     else {
-	evacuate_run_queue(esdp->run_queue, &sbp);
+        if (no != 1)
+            evacuate_run_queue(esdp->run_queue, &sbp);
 
 	erts_smp_runq_unlock(esdp->run_queue);
 
@@ -7200,8 +7519,6 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 	if (erts_system_profile_flags.scheduler)
 	    profile_scheduler(make_small(esdp->no), am_inactive);
 
-	sched_wall_time_change(esdp, 0);
-
     }
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
@@ -7210,9 +7527,6 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 	schdlr_sspnd_dec_nscheds(&schdlr_sspnd.active, sched_type);
 
-	ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
-					ERTS_SCHED_NORMAL) >= 1);
-
 	changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing);
 
 	while (1) {
@@ -7234,9 +7548,13 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 						    ERTS_SCHED_NORMAL) == 1) {
 			clr_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB;
 		    }
-		    else if (schdlr_sspnd.active
-			     == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)) {
-			clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB;
+		    else if (schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+                                                      ERTS_SCHED_NORMAL) == 1
+                             && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+                                                         ERTS_SCHED_DIRTY_CPU) == 0
+                             && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+                                                         ERTS_SCHED_DIRTY_IO) == 0) {
+                        clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB;
 		    }
 
 		    if (clr_flg) {
@@ -7308,10 +7626,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 		}
 	    }
 
-	    if (curr_online
-		&& (sched_type == ERTS_SCHED_NORMAL
-		    ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)
-		    : !schdlr_sspnd.msb.ongoing)) {
+	    if (curr_online) {
 		flgs = erts_smp_atomic32_read_acqb(&ssi->flags);
 		if (!(flgs & ERTS_SSI_FLG_SUSPENDED))
 		    break;
@@ -7327,13 +7642,11 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 		if (sched_type != ERTS_SCHED_NORMAL)
 		    aux_work = 0;
 		else {
-		    erts_aint32_t qmask;
-		    qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue)
-			     & ERTS_RUNQ_FLGS_QMASK);
+                    int evacuate = no == 1 ? 0 : !ERTS_EMPTY_RUNQ(esdp->run_queue);
 
 		    aux_work = erts_atomic32_read_acqb(&ssi->aux_work);
 
-		    if (aux_work|qmask) {
+		    if (aux_work|evacuate) {
 			if (!thr_prgr_active) {
 			    erts_thr_progress_active(esdp, thr_prgr_active = 1);
 			    sched_wall_time_change(esdp, 1);
@@ -7345,7 +7658,7 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 			if (aux_work && erts_thr_progress_update(esdp))
 			    erts_thr_progress_leader_update(esdp);
-			if (qmask) {
+			if (evacuate) {
 			    erts_smp_runq_lock(esdp->run_queue);
 			    evacuate_run_queue(esdp->run_queue, &sbp);
 			    erts_smp_runq_unlock(esdp->run_queue);
@@ -7460,7 +7773,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
         if (changing) {
             if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB)
                 && !schdlr_sspnd.msb.ongoing
-                && schdlr_sspnd.online == schdlr_sspnd.active) {
+                && schdlr_sspnd_eq_nscheds(&schdlr_sspnd.online,
+                                           &schdlr_sspnd.active)) {
                 erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
                                                 ~ERTS_SCHDLR_SSPND_CHNG_MSB);
             }
@@ -7475,19 +7789,16 @@ suspend_scheduler(ErtsSchedulerData *esdp)
             }
         }
 	ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type));
-	ASSERT((sched_type == ERTS_SCHED_NORMAL
-		? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)
-		: !schdlr_sspnd.msb.ongoing));
     }
 
     erts_smp_mtx_unlock(&schdlr_sspnd.mtx);
 
-    ASSERT(!resume.msb.chngrs);
     schdlr_sspnd_resume_procs(sched_type, &resume);
 
     ASSERT(curr_online);
 
-    if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
+    if (sched_type == ERTS_SCHED_NORMAL) {
+	(void) erts_get_monotonic_time(esdp);
 	if (erts_system_profile_flags.scheduler)
 	    profile_scheduler(make_small(esdp->no), am_active);
 
@@ -7497,8 +7808,6 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 	}
     }
 
-    if (sched_type == ERTS_SCHED_NORMAL)
-	(void) erts_get_monotonic_time(esdp);
     erts_smp_runq_lock(esdp->run_queue);
     non_empty_runq(esdp->run_queue);
 
@@ -7768,10 +8077,8 @@ erts_set_schedulers_online(Process *p,
 		    erts_sched_poke(ssi);
 		}
 	    } else {
-		for (ix = dirty_online; ix < dirty_no; ix++) {
-		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-		    scheduler_ssi_resume_wake(ssi);
-		}
+		for (ix = dirty_online; ix < dirty_no; ix++)
+		    dcpu_sched_ix_resume_wake(ix);
 	    }
 	}
 	if (!dirty_only)
@@ -7799,19 +8106,19 @@ erts_set_schedulers_online(Process *p,
     else /* if decrease */ {
 #ifdef ERTS_DIRTY_SCHEDULERS
 	if (change_dirty) {
-	    ErtsSchedulerSleepInfo* ssi;
 	    if (schdlr_sspnd.msb.ongoing) {
-		for (ix = dirty_no; ix < dirty_online; ix++) {
-		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-		    erts_sched_poke(ssi);
-		}
-	    } else {
-		for (ix = dirty_no; ix < dirty_online; ix++) {
-		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-		    erts_smp_atomic32_read_bor_nob(&ssi->flags,
-						   ERTS_SSI_FLG_SUSPENDED);
-		}
-		wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
+		for (ix = dirty_no; ix < dirty_online; ix++)
+		    erts_sched_poke(ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix));
+	    }
+            else {
+		for (ix = dirty_no; ix < dirty_online; ix++)
+                    dcpu_sched_ix_suspend_wake(ix);
+                /*
+                 * Newly suspended scheduler may have just been
+                 * about to handle a task. Make sure someone takes
+                 * care of such a task...
+                 */
+                dcpu_sched_ix_wake(0);
 	    }
 	}
 	if (!dirty_only)
@@ -7874,9 +8181,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 {
     int resume_proc, ix, res, have_unlocked_plocks = 0;
     ErtsProcList *plp;
-#ifdef ERTS_DIRTY_SCHEDULERS
-    ErtsSchedulerSleepInfo* ssi;
-#endif
     ErtsMultiSchedulingBlock *msbp;
     erts_aint32_t chng_flg;
     int have_blckd_flg;
@@ -7923,8 +8227,9 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 	    erts_proclist_store_last(&msbp->blckrs, plp);
 	    p->flags |= have_blckd_flg;
 	    ASSERT(normal
-		   ? 1 == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, ERTS_SCHED_NORMAL)
-		   : schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0));
+                   ? 1 == schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
+                                                   ERTS_SCHED_NORMAL)
+                   : schdlr_sspnd_get_nscheds_tot(&schdlr_sspnd.active) == 1);
 	    ASSERT(erts_proc_sched_data(p)->no == 1);
 	    if (schdlr_sspnd.msb.ongoing)
 		res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
@@ -7942,59 +8247,41 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 	    }
 	    ASSERT(!msbp->ongoing);
 	    msbp->ongoing = 1;
-	    if (schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)
-		|| (normal && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
-						       ERTS_SCHED_NORMAL) == 1)) {
-		ASSERT(erts_proc_sched_data(p)->no == 1);
-		plp = proclist_create(p);
-		erts_proclist_store_last(&msbp->blckrs, plp);
-		if (schdlr_sspnd.msb.ongoing)
-		    res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED;
-		else
-		    res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED;
-	    }
-	    else {
-		erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing,
-					       chng_flg);
-		change_no_used_runqs(1);
-		for (ix = 1; ix < erts_no_run_queues; ix++)
-		    suspend_run_queue(ERTS_RUNQ_IX(ix));
 
-		for (ix = 1; ix < online; ix++) {
-		    ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
-		    wake_scheduler(rq);
-		}
+            erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing,
+                                           chng_flg);
+            change_no_used_runqs(1);
+            for (ix = 1; ix < erts_no_run_queues; ix++)
+                suspend_run_queue(ERTS_RUNQ_IX(ix));
 
-#ifdef ERTS_DIRTY_SCHEDULERS
-		if (!normal) {
-		    for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) {
-			ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-			erts_smp_atomic32_read_bor_nob(&ssi->flags,
-						       ERTS_SSI_FLG_SUSPENDED);
-		    }
-		    wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0);
+            for (ix = 1; ix < online; ix++) {
+                ErtsRunQueue *rq = ERTS_RUNQ_IX(ix);
+                wake_scheduler(rq);
+            }
 
-		    for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
-			ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
-			erts_smp_atomic32_read_bor_nob(&ssi->flags,
-						       ERTS_SSI_FLG_SUSPENDED);
-		    }
-		    wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0);
-		}
+#ifdef ERTS_DIRTY_SCHEDULERS
+            if (!normal) {
+                ERTS_RUNQ_FLGS_SET_NOB(ERTS_RUNQ_IX(0), ERTS_RUNQ_FLG_MSB_EXEC);
+                erts_smp_atomic32_read_bor_nob(&ERTS_RUNQ_IX(0)->scheduler->ssi->flags,
+                                               ERTS_SSI_FLG_MSB_EXEC);
+                for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++)
+                    dcpu_sched_ix_suspend_wake(ix);
+                for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++)
+                    dio_sched_ix_suspend_wake(ix);
+            }
 #endif
 
-	    wait_until_msb:
+        wait_until_msb:
 
-		ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing));
+            ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing));
 
-		plp = proclist_create(p);
-		erts_proclist_store_last(&msbp->chngq, plp);
-		resume_proc = 0;
-		if (schdlr_sspnd.msb.ongoing)
-		    res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED;
-		else
-		    res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED;
-	    }
+            plp = proclist_create(p);
+            erts_proclist_store_last(&msbp->chngq, plp);
+            resume_proc = 0;
+            if (schdlr_sspnd.msb.ongoing)
+                res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED;
+            else
+                res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED;
 	    ASSERT(erts_proc_sched_data(p));
 	}
     }
@@ -8026,27 +8313,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 	    }
 	}
 	if (!msbp->blckrs && !msbp->chngq) {
-	    int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
-							ERTS_SCHED_NORMAL);
+	    int online;
 	    erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing,
 					   chng_flg);
 	    p->flags &= ~have_blckd_flg;
 	    msbp->ongoing = 0;
-	    if (online == 1) {
-		/* No normal schedulers to resume */
-		ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active,
-						ERTS_SCHED_NORMAL) == 1);
-#ifndef ERTS_DIRTY_SCHEDULERS
-		erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing,
-						~chng_flg);
-#endif
-	    }
-	    else if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) {
-		if (plocks) {
+            if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) {
+                if (plocks) {
 		    have_unlocked_plocks = 1;
 		    erts_smp_proc_unlock(p, plocks);
 		}
 
+                online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
+							ERTS_SCHED_NORMAL);
 		change_no_used_runqs(online);
 
 		/* Resume all online run queues */
@@ -8057,19 +8336,15 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal
 		    suspend_run_queue(ERTS_RUNQ_IX(ix));
 	    }
 #ifdef ERTS_DIRTY_SCHEDULERS
-	    if (!normal) {
-		ASSERT(!schdlr_sspnd.msb.ongoing);
+            if (!schdlr_sspnd.msb.ongoing) {
+                /* Get rid of msb-exec flag in run-queue of scheduler 1 */
+                resume_run_queue(ERTS_RUNQ_IX(0));
 		online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online,
 							ERTS_SCHED_DIRTY_CPU);
-		for (ix = 0; ix < online; ix++) {
-		    ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix);
-		    scheduler_ssi_resume_wake(ssi);
-		}
-
-		for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) {
-		    ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix);
-		    scheduler_ssi_resume_wake(ssi);
-		}
+		for (ix = 0; ix < online; ix++)
+		    dcpu_sched_ix_resume_wake(ix);
+		for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++)
+		    dio_sched_ix_resume_wake(ix);
 	    }
 #endif
 	}
@@ -9734,7 +10009,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	if (!is_normal_sched) {
 	    if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
-		& ERTS_SSI_FLG_SUSPENDED) {
+		& (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) {
 		suspend_scheduler(esdp);
 	    }
 	}
@@ -9744,8 +10019,10 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	    ASSERT(is_normal_sched);
 
-	    if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND|ERTS_RUNQ_FLG_SUSPENDED)) {
-		if (flags & ERTS_RUNQ_FLG_SUSPENDED) {
+	    if (flags & (ERTS_RUNQ_FLG_CHK_CPU_BIND
+                         | ERTS_RUNQ_FLG_SUSPENDED
+                         | ERTS_RUNQ_FLG_MSB_EXEC)) {
+		if (flags & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_MSB_EXEC)) {
 		    (void) ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_EXEC);
 		    suspend_scheduler(esdp);
 		    flags = ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_EXEC);
@@ -9784,13 +10061,12 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
 
-	if (!is_normal_sched && rq->halt_in_progress) {
+	if (!is_normal_sched & !!(flags & ERTS_RUNQ_FLG_HALTING)) {
 	    /* Wait for emulator to terminate... */
 	    while (1)
 		erts_milli_sleep(1000*1000);
 	}
-	else if ((!(flags & ERTS_RUNQ_FLGS_QMASK) && !rq->misc.start)
-		 || (rq->halt_in_progress && ERTS_EMPTY_RUNQ_PORTS(rq))) {
+	else if (!runq_got_work_to_execute_flags(flags)) {
 	    /* Prepare for scheduler wait */
 #ifdef ERTS_SMP
 	    ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq));
@@ -9804,21 +10080,44 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	    if (flags & ERTS_RUNQ_FLG_INACTIVE)
 		empty_runq(rq);
 	    else {
-		if (is_normal_sched && try_steal_task(rq))
-		    goto continue_check_activities_to_run;
-
-		empty_runq(rq);
-
-		/*
-		 * Check for ERTS_RUNQ_FLG_SUSPENDED has to be done
-		 * after trying to steal a task.
-		 */
-		flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
-		if (flags & ERTS_RUNQ_FLG_SUSPENDED) {
-		    non_empty_runq(rq);
-		    flags |= ERTS_RUNQ_FLG_NONEMPTY;
-		    goto continue_check_activities_to_run_known_flags;
+                ASSERT(!runq_got_work_to_execute(rq));
+		if (!is_normal_sched) {
+                    /* Dirty scheduler */
+                    if (erts_smp_atomic32_read_acqb(&esdp->ssi->flags)
+                        & (ERTS_SSI_FLG_SUSPENDED|ERTS_SSI_FLG_MSB_EXEC)) {
+                        /* Go suspend... */
+                        goto continue_check_activities_to_run_known_flags;
+                    }
+                }
+                else {
+                    /* Normal scheduler */
+                    if (try_steal_task(rq))
+                        goto continue_check_activities_to_run;
+                    /*
+                     * Check for suspend has to be done after trying
+                     * to steal a task...
+                     */
+                    flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
+                    if ((flags & ERTS_RUNQ_FLG_SUSPENDED)
+#ifdef ERTS_DIRTY_SCHEDULERS
+                        /* If multi scheduling block and we have
+                         * dirty work, suspend and let dirty
+                         * scheduler handle work... */
+                        || ((((flags & (ERTS_RUNQ_FLG_HALTING
+                                       | ERTS_RUNQ_FLG_MSB_EXEC))
+                              == ERTS_RUNQ_FLG_MSB_EXEC))
+                            && have_dirty_work())
+#endif
+                        ) {
+                        non_empty_runq(rq);
+                        flags |= ERTS_RUNQ_FLG_NONEMPTY;
+                        /*
+                         * Go suspend...
+                         */
+                        goto continue_check_activities_to_run_known_flags;
+                    }
 		}
+		empty_runq(rq);
 	    }
 #endif
 
@@ -9870,7 +10169,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 #endif
 	}
 
-	if (rq->misc.start)
+        if (flags & ERTS_RUNQ_FLG_MISC_OP)
 	    exec_misc_ops(rq);
 
 #ifdef ERTS_SMP
@@ -9881,13 +10180,15 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 	 * Find a new port to run.
 	 */
 
-	if (RUNQ_READ_LEN(&rq->ports.info.len)) {
+        flags = ERTS_RUNQ_FLGS_GET_NOB(rq);
+
+	if (flags & PORT_BIT) {
 	    int have_outstanding_io;
 	    have_outstanding_io = erts_port_task_execute(rq, &esdp->current_port);
 	    if ((!erts_eager_check_io
 		 && have_outstanding_io
 		 && fcalls > 2*input_reductions)
-		|| rq->halt_in_progress) {
+		|| (flags & ERTS_RUNQ_FLG_HALTING)) {
 		/*
 		 * If we have performed more than 2*INPUT_REDUCTIONS since
 		 * last call to erl_sys_schedule() and we still haven't
@@ -11419,6 +11720,8 @@ erts_schedule_misc_op(void (*func)(void *), void *arg)
     non_empty_runq(rq);
 #endif
 
+    ERTS_RUNQ_FLGS_SET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP);
+
     erts_smp_runq_unlock(rq);
 
     smp_notify_inc_runq(rq);
@@ -11449,6 +11752,9 @@ exec_misc_ops(ErtsRunQueue *rq)
 	    rq->misc.end = NULL;
     }
 
+    if (!rq->misc.start)
+        ERTS_RUNQ_FLGS_UNSET_NOB(rq, ERTS_RUNQ_FLG_MISC_OP);
+
     erts_smp_runq_unlock(rq);
 
     while (molp) {
@@ -13576,6 +13882,8 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
                 erts_print(to, to_arg, "WAITING"); break;
             case ERTS_SSI_FLG_SUSPENDED:
                 erts_print(to, to_arg, "SUSPENDED"); break;
+            case ERTS_SSI_FLG_MSB_EXEC:
+                erts_print(to, to_arg, "MSB_EXEC"); break;
             default:
                 erts_print(to, to_arg, "UNKNOWN(%d)", flg); break;
             }
@@ -13685,6 +13993,12 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
                 erts_print(to, to_arg, "NONEMPTY"); break;
             case ERTS_RUNQ_FLG_PROTECTED:
                 erts_print(to, to_arg, "PROTECTED"); break;
+            case ERTS_RUNQ_FLG_EXEC:
+                erts_print(to, to_arg, "EXEC"); break;
+            case ERTS_RUNQ_FLG_MSB_EXEC:
+                erts_print(to, to_arg, "MSB_EXEC"); break;
+            case ERTS_RUNQ_FLG_MISC_OP:
+                erts_print(to, to_arg, "MISC_OP"); break;
             default:
                 erts_print(to, to_arg, "UNKNOWN(%d)", flg); break;
             }
@@ -13734,11 +14048,11 @@ erts_print_scheduler_info(fmtfn_t to, void *to_arg, ErtsSchedulerData *esdp) {
  * A nice system halt closing all open port goes as follows:
  * 1) This function schedules the aux work ERTS_SSI_AUX_WORK_REAP_PORTS
  *    on all schedulers, then schedules itself out.
- * 2) All shedulers detect this and set the flag halt_in_progress
+ * 2) All shedulers detect this and set the flag ERTS_RUNQ_FLG_HALTING
  *    on their run queue. The last scheduler sets all non-closed ports
  *    ERTS_PORT_SFLG_HALT. Global atomic erts_halt_progress is used
  *    as refcount to determine which is last.
- * 3) While the run ques has flag halt_in_progress no processes
+ * 3) While the run queues has flag ERTS_RUNQ_FLG_HALTING no processes
  *    will be scheduled, only ports.
  * 4) When the last port closes that scheduler calls erlang:halt/1.
  *    The same global atomic is used as refcount.
@@ -13753,8 +14067,8 @@ void erts_halt(int code)
 					     erts_no_schedulers,
 					     -1)) {
 #ifdef ERTS_DIRTY_SCHEDULERS
-	ERTS_DIRTY_CPU_RUNQ->halt_in_progress = 1;
-	ERTS_DIRTY_IO_RUNQ->halt_in_progress = 1;
+        ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_CPU_RUNQ, ERTS_RUNQ_FLG_HALTING);
+        ERTS_RUNQ_FLGS_SET(ERTS_DIRTY_IO_RUNQ, ERTS_RUNQ_FLG_HALTING);
 #endif
 	erts_halt_code = code;
 	notify_reap_ports_relb();
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 3938c4c913..fc2055332b 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -170,8 +170,14 @@ extern int erts_sched_thread_suggested_stack_size;
   (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 6))
 #define ERTS_RUNQ_FLG_EXEC \
   (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 7))
+#define ERTS_RUNQ_FLG_MSB_EXEC \
+  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 8))
+#define ERTS_RUNQ_FLG_MISC_OP \
+  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 9))
+#define ERTS_RUNQ_FLG_HALTING \
+  (((Uint32) 1) << (ERTS_RUNQ_FLG_BASE2 + 10))
 
-#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 8)
+#define ERTS_RUNQ_FLG_MAX (ERTS_RUNQ_FLG_BASE2 + 11)
 
 #define ERTS_RUNQ_FLGS_MIGRATION_QMASKS	\
   (ERTS_RUNQ_FLGS_EMIGRATE_QMASK	\
@@ -262,8 +268,9 @@ typedef enum {
 #define ERTS_SSI_FLG_TSE_SLEEPING 	(((erts_aint32_t) 1) << 2)
 #define ERTS_SSI_FLG_WAITING		(((erts_aint32_t) 1) << 3)
 #define ERTS_SSI_FLG_SUSPENDED	 	(((erts_aint32_t) 1) << 4)
+#define ERTS_SSI_FLG_MSB_EXEC	 	(((erts_aint32_t) 1) << 5)
 
-#define ERTS_SSI_FLGS_MAX                                       5
+#define ERTS_SSI_FLGS_MAX                                       6
 
 #define ERTS_SSI_FLGS_SLEEP_TYPE			\
  (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING)
@@ -274,7 +281,8 @@ typedef enum {
 #define ERTS_SSI_FLGS_ALL				\
  (ERTS_SSI_FLGS_SLEEP					\
   | ERTS_SSI_FLG_WAITING				\
-  | ERTS_SSI_FLG_SUSPENDED)
+  | ERTS_SSI_FLG_SUSPENDED                              \
+  | ERTS_SSI_FLG_MSB_EXEC)
 
 /*
  * Keep ERTS_SSI_AUX_WORK flags ordered in expected frequency
@@ -393,6 +401,12 @@ typedef struct {
     Process* last;
 } ErtsRunPrioQueue;
 
+typedef enum {
+    ERTS_SCHED_NORMAL,
+    ERTS_SCHED_DIRTY_CPU,
+    ERTS_SCHED_DIRTY_IO
+} ErtsSchedType;
+
 typedef struct ErtsSchedulerData_ ErtsSchedulerData;
 
 typedef struct ErtsRunQueue_ ErtsRunQueue;
@@ -478,7 +492,6 @@ struct ErtsRunQueue_ {
     erts_smp_atomic32_t len;
     int wakeup_other;
     int wakeup_other_reds;
-    int halt_in_progress;
 
     struct {
 	ErtsProcList *pending_exiters;
@@ -642,6 +655,7 @@ struct ErtsSchedulerData_ {
 #endif
     ErtsSchedulerSleepInfo *ssi;
     Process *current_process;
+    ErtsSchedType type;
     Uint no;			/* Scheduler number for normal schedulers */
 #ifdef ERTS_DIRTY_SCHEDULERS
     ErtsDirtySchedId dirty_no;  /* Scheduler number for dirty schedulers */
-- 
cgit v1.2.3


From fd1fb9070333fe5b91f4528841a4b0fe71bbf096 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 12 Jan 2017 14:25:30 +0100
Subject: Always return info from system_info(dirty_[cpu|io]_schedulers*)

---
 erts/emulator/beam/erl_bif_info.c | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 4659cb5a7b..5f2122b969 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -2680,20 +2680,30 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
 	erts_schedulers_state(NULL, NULL, &active, NULL, NULL, NULL, NULL, NULL);
 	BIF_RET(make_small(active));
 #endif
-#if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS)
     } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers", BIF_ARG_1)) {
 	Uint dirty_cpu;
+#ifdef ERTS_DIRTY_SCHEDULERS
 	erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, NULL, NULL);
+#else
+        dirty_cpu = 0;
+#endif
 	BIF_RET(make_small(dirty_cpu));
     } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers_online", BIF_ARG_1)) {
 	Uint dirty_cpu_onln;
+#ifdef ERTS_DIRTY_SCHEDULERS
 	erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, NULL, NULL);
+#else
+        dirty_cpu_onln = 0;
+#endif
 	BIF_RET(make_small(dirty_cpu_onln));
     } else if (ERTS_IS_ATOM_STR("dirty_io_schedulers", BIF_ARG_1)) {
 	Uint dirty_io;
+#ifdef ERTS_DIRTY_SCHEDULERS
 	erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, NULL, &dirty_io, NULL);
-	BIF_RET(make_small(dirty_io));
+#else
+        dirty_io = 0;
 #endif
+	BIF_RET(make_small(dirty_io));
     } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) {
 	res = make_small(erts_no_run_queues);
 	BIF_RET(res);
-- 
cgit v1.2.3


From f9459092940943876dff040ee997515b96fd5d50 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Wed, 4 Jan 2017 18:10:26 +0100
Subject: Scheduler wall time support for dirty schedulers

---
 erts/emulator/beam/atom.names     |   3 +-
 erts/emulator/beam/beam_debug.c   |  13 +-
 erts/emulator/beam/bif.c          |   2 +-
 erts/emulator/beam/erl_bif_info.c |   7 +-
 erts/emulator/beam/erl_process.c  | 366 +++++++++++++++++++++++++++++++-------
 erts/emulator/beam/erl_process.h  |   9 +-
 6 files changed, 327 insertions(+), 73 deletions(-)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names
index 7df350116a..dd0a42b5ba 100644
--- a/erts/emulator/beam/atom.names
+++ b/erts/emulator/beam/atom.names
@@ -574,6 +574,8 @@ atom safe
 atom save_calls
 atom scheduler 
 atom scheduler_id
+atom scheduler_wall_time
+atom scheduler_wall_time_all
 atom schedulers_online
 atom scheme
 atom scientific
@@ -678,7 +680,6 @@ atom waiting
 atom wall_clock
 atom warning
 atom warning_msg
-atom scheduler_wall_time
 atom wordsize
 atom write_concurrency
 atom xor
diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c
index 3ffbb0364c..8326d348af 100644
--- a/erts/emulator/beam/beam_debug.c
+++ b/erts/emulator/beam/beam_debug.c
@@ -994,13 +994,20 @@ dirty_test(Process *c_p, Eterm type, Eterm arg1, Eterm arg2, UWord *I)
 	Eterm *hp, *hp2;
 	Uint sz;
 	int i;
+	ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
+        int dirty_io = esdp->type == ERTS_SCHED_DIRTY_IO;
+
 	if (ERTS_PROC_IS_EXITING(real_c_p))
 	    goto badarg;
 	dirty_send_message(c_p, arg2, am_alive);
-	/* Wait until dead */
 
-	while (!ERTS_PROC_IS_EXITING(real_c_p))
-	    erts_thr_yield();
+	/* Wait until dead */
+	while (!ERTS_PROC_IS_EXITING(real_c_p)) {
+            if (dirty_io)
+                ms_wait(c_p, make_small(100), 0);
+            else
+                erts_thr_yield();
+        }
 
 	ms_wait(c_p, make_small(1000), 0);
 
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index a5b66db35c..65c370c55b 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -4601,7 +4601,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2)
 	    erts_aint32_t new = BIF_ARG_2 == am_true ? 1 : 0;
 	    erts_aint32_t old = erts_smp_atomic32_xchg_nob(&sched_wall_time,
 							   new);
-	    Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new);
+	    Eterm ref = erts_sched_wall_time_request(BIF_P, 1, new, 0, 0);
 	    ASSERT(is_value(ref));
 	    BIF_TRAP2(await_sched_wall_time_mod_trap,
 		      BIF_P,
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 5f2122b969..5ee19aead8 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -3385,7 +3385,12 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1)
     Eterm* hp;
 
     if (BIF_ARG_1 == am_scheduler_wall_time) {
-	res = erts_sched_wall_time_request(BIF_P, 0, 0);
+	res = erts_sched_wall_time_request(BIF_P, 0, 0, 1, 0);
+	if (is_non_value(res))
+	    BIF_RET(am_undefined);
+	BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res);
+    } else if (BIF_ARG_1 == am_scheduler_wall_time_all) {
+	res = erts_sched_wall_time_request(BIF_P, 0, 0, 1, 1);
 	if (is_non_value(res))
 	    BIF_RET(am_undefined);
 	BIF_TRAP1(gather_sched_wall_time_res_trap, BIF_P, res);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 084ecb14ec..8c9aa45a81 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -804,15 +804,28 @@ erts_late_init_process(void)
 
 }
 
+#define ERTS_SCHED_WTIME_IDLE ~((Uint64) 0)
+
 static void
-init_sched_wall_time(ErtsSchedWallTime *swtp)
+init_sched_wall_time(ErtsSchedulerData *esdp, Uint64 time_stamp)
 {
-    swtp->need = erts_sched_balance_util;
-    swtp->enabled = 0;
-    swtp->start = 0;
-    swtp->working.total = 0;
-    swtp->working.start = 0;
-    swtp->working.currently = 0;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (esdp->type != ERTS_SCHED_NORMAL) {
+        erts_atomic32_init_nob(&esdp->sched_wall_time.u.mod, 0);
+        esdp->sched_wall_time.enabled = 1;
+        esdp->sched_wall_time.start = time_stamp;
+        esdp->sched_wall_time.working.total = 0;
+        esdp->sched_wall_time.working.start = ERTS_SCHED_WTIME_IDLE;
+    }
+    else
+#endif
+    {
+        esdp->sched_wall_time.u.need = erts_sched_balance_util;
+        esdp->sched_wall_time.enabled = 0;
+        esdp->sched_wall_time.start = 0;
+        esdp->sched_wall_time.working.total = 0;
+        esdp->sched_wall_time.working.start = 0;
+    }
 }
 
 static ERTS_INLINE Uint64
@@ -1003,31 +1016,145 @@ init_runq_sched_util(ErtsRunQueueSchedUtil *rqsu, int enabled)
 
 #endif /* ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT */
 
-static ERTS_INLINE void
+#ifdef ERTS_DIRTY_SCHEDULERS
+
+typedef struct {
+    Uint64 working;
+    Uint64 total;
+} ErtsDirtySchedWallTime;
+
+static void
+read_dirty_sched_wall_time(ErtsSchedulerData *esdp, ErtsDirtySchedWallTime *info)
+{
+    erts_aint32_t mod1;
+    Uint64 working, start, ts;
+
+    mod1 = erts_atomic32_read_nob(&esdp->sched_wall_time.u.mod);
+
+    while (1) {
+        erts_aint32_t mod2;
+
+        /* Spin until values are not written... */
+        while (1) {
+            if ((mod1 & 1) == 0)
+                break;
+            ERTS_SPIN_BODY;
+            mod1 = erts_atomic32_read_nob(&esdp->sched_wall_time.u.mod);
+        }
+
+        ERTS_THR_READ_MEMORY_BARRIER;
+
+        working = esdp->sched_wall_time.working.total;
+        start = esdp->sched_wall_time.working.start;
+
+        ERTS_THR_READ_MEMORY_BARRIER;
+
+        mod2 = erts_atomic32_read_nob(&esdp->sched_wall_time.u.mod);
+        if (mod1 == mod2)
+            break;
+        mod1 = mod2;
+    }
+
+    ts = sched_wall_time_ts();
+    ts -= esdp->sched_wall_time.start;
+
+    info->total = ts;
+
+    if (start == ERTS_SCHED_WTIME_IDLE || ts < start)
+        info->working = working;
+    else
+        info->working = working + (ts - start);
+
+    if (info->working > info->total)
+        info->working = info->total;
+}
+
+#endif
+
+#ifdef ERTS_SMP
+
+static void
+dirty_sched_wall_time_change(ErtsSchedulerData *esdp, int working)
+{
+    erts_aint32_t mod;
+    Uint64 ts = sched_wall_time_ts();
+
+    ts -= esdp->sched_wall_time.start;
+
+    /*
+     * This thread is the only thread writing in
+     * this sched_wall_time struct. We set 'mod' to
+     * an odd value while writing...
+     */
+    mod = erts_atomic32_read_dirty(&esdp->sched_wall_time.u.mod);
+    ASSERT((mod & 1) == 0);
+    mod++;
+
+    erts_atomic32_set_nob(&esdp->sched_wall_time.u.mod, mod);
+    ERTS_THR_WRITE_MEMORY_BARRIER;
+
+    if (working) {
+        ASSERT(esdp->sched_wall_time.working.start
+               == ERTS_SCHED_WTIME_IDLE);
+
+        esdp->sched_wall_time.working.start = ts;
+
+    }
+    else {
+        Uint64 total;
+
+        ASSERT(esdp->sched_wall_time.working.start
+               != ERTS_SCHED_WTIME_IDLE);
+
+        total = esdp->sched_wall_time.working.total;
+        total += ts - esdp->sched_wall_time.working.start;
+
+        esdp->sched_wall_time.working.total = total;
+        esdp->sched_wall_time.working.start = ERTS_SCHED_WTIME_IDLE;
+
+
+    }
+
+    ERTS_THR_WRITE_MEMORY_BARRIER;
+    mod++;
+    erts_atomic32_set_nob(&esdp->sched_wall_time.u.mod, mod);
+
+#if 0
+    if (!working) {
+        ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_BUSY_WAIT);
+    } else {
+        ERTS_MSACC_SET_STATE_M_X(ERTS_MSACC_STATE_OTHER);
+    }
+#endif
+}
+
+#endif /* ERTS_SMP */
+
+static void
 sched_wall_time_change(ErtsSchedulerData *esdp, int working)
 {
-    if (esdp->sched_wall_time.need) {
+    if (esdp->sched_wall_time.u.need) {
 	Uint64 ts = sched_wall_time_ts();
 #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT
-	update_avg_sched_util(esdp, ts, working);
+        update_avg_sched_util(esdp, ts, working);
 #endif
 	if (esdp->sched_wall_time.enabled) {
 	    if (working) {
-#ifdef DEBUG
-		ASSERT(!esdp->sched_wall_time.working.currently);
-		esdp->sched_wall_time.working.currently = 1;
-#endif
+                ASSERT(esdp->sched_wall_time.working.start
+                       == ERTS_SCHED_WTIME_IDLE);
 		ts -= esdp->sched_wall_time.start;
 		esdp->sched_wall_time.working.start = ts;
 	    }
 	    else {
-#ifdef DEBUG
-		ASSERT(esdp->sched_wall_time.working.currently);
-		esdp->sched_wall_time.working.currently = 0;
-#endif
+                ASSERT(esdp->sched_wall_time.working.start
+                       != ERTS_SCHED_WTIME_IDLE);
 		ts -= esdp->sched_wall_time.start;
 		ts -= esdp->sched_wall_time.working.start;
 		esdp->sched_wall_time.working.total += ts;
+#ifdef DEBUG
+                esdp->sched_wall_time.working.start
+                    = ERTS_SCHED_WTIME_IDLE;
+#endif
 	    }
 	}
     }
@@ -1047,6 +1174,10 @@ typedef struct {
     Eterm ref_heap[REF_THING_SIZE];
     Uint req_sched;
     erts_smp_atomic32_t refc;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    int want_dirty_cpu;
+    int want_dirty_io;
+#endif
 } ErtsSchedWallTimeReq;
 
 typedef struct {
@@ -1068,6 +1199,7 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(screq,
 				 5,
 				 ERTS_ALC_T_SYS_CHECK_REQ)
 
+
 static void
 reply_sched_wall_time(void *vswtrp)
 {
@@ -1091,23 +1223,23 @@ reply_sched_wall_time(void *vswtrp)
 #endif
     if (swtrp->set) {
 	if (!swtrp->enable && esdp->sched_wall_time.enabled) {
-	    esdp->sched_wall_time.need = erts_sched_balance_util;
+	    esdp->sched_wall_time.u.need = erts_sched_balance_util;
 	    esdp->sched_wall_time.enabled = 0;
 	}
 	else if (swtrp->enable && !esdp->sched_wall_time.enabled) {
 	    Uint64 ts = sched_wall_time_ts();
-	    esdp->sched_wall_time.need = 1;
+	    esdp->sched_wall_time.u.need = 1;
 	    esdp->sched_wall_time.enabled = 1;
 	    esdp->sched_wall_time.start = ts;
 	    esdp->sched_wall_time.working.total = 0;
 	    esdp->sched_wall_time.working.start = 0;
-	    esdp->sched_wall_time.working.currently = 1;
 	}
     }
 
     if (esdp->sched_wall_time.enabled) {
 	Uint64 ts = sched_wall_time_ts();
-	ASSERT(esdp->sched_wall_time.working.currently);
+        ASSERT(esdp->sched_wall_time.working.start
+               != ERTS_SCHED_WTIME_IDLE);
 	ts -= esdp->sched_wall_time.start;
 	total = ts;
 	ts -= esdp->sched_wall_time.working.start;
@@ -1118,30 +1250,117 @@ reply_sched_wall_time(void *vswtrp)
     hpp = NULL;
     szp = &sz;
 
-    while (1) {
-	if (hpp)
-	    ref_copy = STORE_NC(hpp, ohp, swtrp->ref);
-	else
-	    *szp += REF_THING_SIZE;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    if (esdp->sched_wall_time.enabled
+        && swtrp->req_sched == esdp->no
+        && (swtrp->want_dirty_cpu || swtrp->want_dirty_io)) {
+        /* Reply with info about this scheduler and all dirty schedulers... */
+        ErtsDirtySchedWallTime *dswt;
+        int ix, no_dirty_scheds, want_dcpu, want_dio, soffset;
+
+        want_dcpu = swtrp->want_dirty_cpu;
+        want_dio = swtrp->want_dirty_io;
+
+        no_dirty_scheds = 0;
+        if (want_dcpu)
+            no_dirty_scheds += erts_no_dirty_cpu_schedulers;
+        if (want_dio)
+            no_dirty_scheds += erts_no_dirty_io_schedulers;
+
+        ASSERT(no_dirty_scheds);
+
+        dswt = erts_alloc(ERTS_ALC_T_TMP,
+                          sizeof(ErtsDirtySchedWallTime)
+                          * no_dirty_scheds);
+
+        for (ix = 0; ix < no_dirty_scheds; ix++) {
+            ErtsSchedulerData *esdp;
+            if (want_dcpu && ix < erts_no_dirty_cpu_schedulers)
+                esdp = &erts_aligned_dirty_cpu_scheduler_data[ix].esd;
+            else {
+                int dio_ix = ix - erts_no_dirty_cpu_schedulers;
+                esdp = &erts_aligned_dirty_io_scheduler_data[dio_ix].esd;
+            }
+            read_dirty_sched_wall_time(esdp, &dswt[ix]);
+        }
 
-	if (swtrp->set)
-	    msg = ref_copy;
-	else {
-	    msg = (!esdp->sched_wall_time.enabled
-		   ? am_notsup
-		   : erts_bld_tuple(hpp, szp, 3,
-				    make_small(esdp->no),
-				    erts_bld_uint64(hpp, szp, working),
-				    erts_bld_uint64(hpp, szp, total)));
+        soffset = erts_no_schedulers + 1;
 
-	    msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg);
-	}
-	if (hpp)
-	    break;
+        if (!want_dcpu) {
+            ASSERT(want_dio);
+            soffset += erts_no_dirty_cpu_schedulers;
+        }
+
+        while (1) {
+            if (hpp)
+                ref_copy = STORE_NC(hpp, ohp, swtrp->ref);
+            else
+                *szp += REF_THING_SIZE;
+
+            ASSERT(!swtrp->set);
+
+            /* info about dirty schedulers... */
+            msg = NIL;
+            for (ix = no_dirty_scheds-1; ix >= 0; ix--) {
+                msg = erts_bld_cons(hpp, szp,
+                                    erts_bld_tuple(hpp, szp, 3,
+                                                   make_small(ix+soffset),
+                                                   erts_bld_uint64(hpp, szp,
+                                                                   dswt[ix].working),
+                                                   erts_bld_uint64(hpp, szp,
+                                                                   dswt[ix].total)),
+                                    msg);
+            }
+            /* info about this scheduler... */
+            msg = erts_bld_cons(hpp, szp,
+                                erts_bld_tuple(hpp, szp, 3,
+                                               make_small(esdp->no),
+                                               erts_bld_uint64(hpp, szp, working),
+                                               erts_bld_uint64(hpp, szp, total)),
+                                msg);
+
+            msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg);
+
+            if (hpp)
+                break;
+
+            mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
+            szp = NULL;
+            hpp = &hp;
+        }
 
-	mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
-	szp = NULL;
-	hpp = &hp;
+        erts_free(ERTS_ALC_T_TMP, dswt);
+    }
+    else
+#endif
+    {
+        /* Reply with info about this scheduler only... */
+
+        while (1) {
+            if (hpp)
+                ref_copy = STORE_NC(hpp, ohp, swtrp->ref);
+            else
+                *szp += REF_THING_SIZE;
+
+            if (swtrp->set)
+                msg = ref_copy;
+            else {
+                msg = (!esdp->sched_wall_time.enabled
+                       ? am_undefined
+                       : erts_bld_tuple(hpp, szp, 3,
+                                        make_small(esdp->no),
+                                        erts_bld_uint64(hpp, szp, working),
+                                        erts_bld_uint64(hpp, szp, total)));
+
+                msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg);
+            }
+            if (hpp)
+                break;
+
+            mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
+            szp = NULL;
+            hpp = &hp;
+        }
     }
 
     erts_queue_message(rp, rp_locks, mp, msg, am_system);
@@ -1159,7 +1378,8 @@ reply_sched_wall_time(void *vswtrp)
 }
 
 Eterm
-erts_sched_wall_time_request(Process *c_p, int set, int enable)
+erts_sched_wall_time_request(Process *c_p, int set, int enable,
+                             int want_dirty_cpu, int want_dirty_io)
 {
     ErtsSchedulerData *esdp = erts_proc_sched_data(c_p);
     Eterm ref;
@@ -1181,6 +1401,10 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable)
     swtrp->proc = c_p;
     swtrp->ref = STORE_NC(&hp, NULL, ref);
     swtrp->req_sched = esdp->no;
+#ifdef ERTS_DIRTY_SCHEDULERS
+    swtrp->want_dirty_cpu = want_dirty_cpu;
+    swtrp->want_dirty_io = want_dirty_io;
+#endif
     erts_smp_atomic32_init_nob(&swtrp->refc,
 			       (erts_aint32_t) erts_no_schedulers);
 
@@ -3106,8 +3330,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
 
     tse_wait:
 
-	if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && thr_prgr_active != working)
-	    sched_wall_time_change(esdp, thr_prgr_active);
+	if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+	    dirty_sched_wall_time_change(esdp, working = 0);
+        else if (thr_prgr_active != working)
+	    sched_wall_time_change(esdp, working = thr_prgr_active);
 
 	while (1) {
 	    ErtsMonotonicTime current_time = 0;
@@ -3218,10 +3444,12 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq)
                                             (ERTS_SSI_FLG_SUSPENDED
                                              | ERTS_SSI_FLG_MSB_EXEC));
 
-	if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && !thr_prgr_active) {
+	if (ERTS_SCHEDULER_IS_DIRTY(esdp))
+	    dirty_sched_wall_time_change(esdp, working = 1);
+        else if (!thr_prgr_active) {
 	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
 	    sched_wall_time_change(esdp, 1);
-	}
+        }
 
 	erts_smp_runq_lock(rq);
 	sched_active(esdp->no, rq);
@@ -5712,7 +5940,8 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 		    ErtsSchedulerSleepInfo* ssi,
 		    ErtsRunQueue* runq,
 		    char** daww_ptr, size_t daww_sz,
-		    Process *shadow_proc)
+		    Process *shadow_proc,
+                    Uint64 time_stamp)
 {
     esdp->timer_wheel = NULL;
 #ifdef ERTS_SMP
@@ -5794,7 +6023,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
 
     esdp->reductions = 0;
 
-    init_sched_wall_time(&esdp->sched_wall_time);
+    init_sched_wall_time(esdp, time_stamp);
     erts_port_task_handle_init(&esdp->nosuspend_port_task_handle);
 }
 
@@ -6008,17 +6237,18 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 
     erts_aligned_scheduler_data = 
 	erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA,
-					   n*sizeof(ErtsAlignedSchedulerData));					   
+					   n*sizeof(ErtsAlignedSchedulerData));
 
     for (ix = 0; ix < n; ix++) {
 	ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix);
 	init_scheduler_data(esdp, ix+1, ERTS_SCHED_SLEEP_INFO_IX(ix),
 			    ERTS_RUNQ_IX(ix), &daww_ptr, daww_sz,
-			    NULL);
+			    NULL, 0);
     }
 
 #ifdef ERTS_DIRTY_SCHEDULERS
     {
+        Uint64 ts = sched_wall_time_ts();
 	int dirty_scheds = no_dirty_cpu_schedulers + no_dirty_io_schedulers;
 	int adspix = 0;
 	ErtsAlignedDirtyShadowProcess *adsp =
@@ -6038,13 +6268,13 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix);
 	    init_scheduler_data(esdp, ix+1, ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix),
 				ERTS_DIRTY_CPU_RUNQ, NULL, 0,
-				&adsp[adspix++].dsp);
+				&adsp[adspix++].dsp, ts);
 	}
 	for (ix = 0; ix < no_dirty_io_schedulers; ix++) {
 	    ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix);
 	    init_scheduler_data(esdp, ix+1, ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix),
 				ERTS_DIRTY_IO_RUNQ, NULL, 0,
-				&adsp[adspix++].dsp);
+				&adsp[adspix++].dsp, ts);
 	}
     }
 #endif
@@ -7506,8 +7736,10 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 #endif
 
-    if (sched_type != ERTS_SCHED_NORMAL)
+    if (sched_type != ERTS_SCHED_NORMAL) {
 	erts_smp_runq_unlock(esdp->run_queue);
+        dirty_sched_wall_time_change(esdp, 0);
+    }
     else {
         if (no != 1)
             evacuate_run_queue(esdp->run_queue, &sbp);
@@ -7518,8 +7750,8 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
 	if (erts_system_profile_flags.scheduler)
 	    profile_scheduler(make_small(esdp->no), am_inactive);
-
     }
+
     erts_smp_mtx_lock(&schdlr_sspnd.mtx);
 
     flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED);
@@ -7797,15 +8029,17 @@ suspend_scheduler(ErtsSchedulerData *esdp)
 
     ASSERT(curr_online);
 
-    if (sched_type == ERTS_SCHED_NORMAL) {
-	(void) erts_get_monotonic_time(esdp);
-	if (erts_system_profile_flags.scheduler)
-	    profile_scheduler(make_small(esdp->no), am_active);
+    if (sched_type != ERTS_SCHED_NORMAL)
+        dirty_sched_wall_time_change(esdp, 1);
+    else {
+        (void) erts_get_monotonic_time(esdp);
+        if (erts_system_profile_flags.scheduler)
+            profile_scheduler(make_small(esdp->no), am_active);
 
-	if (!thr_prgr_active) {
-	    erts_thr_progress_active(esdp, thr_prgr_active = 1);
-	    sched_wall_time_change(esdp, 1);
-	}
+        if (!thr_prgr_active) {
+            erts_thr_progress_active(esdp, thr_prgr_active = 1);
+            sched_wall_time_change(esdp, 1);
+        }
     }
 
     erts_smp_runq_lock(esdp->run_queue);
@@ -8518,6 +8752,8 @@ sched_dirty_cpu_thread_func(void *vesdp)
     callbacks.wait = NULL;
     callbacks.finalize_wait = NULL;
 
+    dirty_sched_wall_time_change(esdp, 1);
+
     esdp->thr_id += erts_no_schedulers;
 
     erts_msacc_init_thread("dirty_cpu_scheduler", no, 0);
@@ -8565,6 +8801,8 @@ sched_dirty_io_thread_func(void *vesdp)
     callbacks.wait = NULL;
     callbacks.finalize_wait = NULL;
 
+    dirty_sched_wall_time_change(esdp, 1);
+
     esdp->thr_id += erts_no_schedulers + erts_no_dirty_cpu_schedulers;
 
     erts_msacc_init_thread("dirty_io_scheduler", no, 0);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index fc2055332b..ce0989883c 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -550,13 +550,15 @@ do {								\
 } while (0)
 
 typedef struct {
-    int need; /* "+sbu true" or scheduler_wall_time enabled */
+    union {
+        erts_atomic32_t mod; /* on dirty schedulers */
+        int need; /* "+sbu true" or scheduler_wall_time enabled */
+    } u;
     int enabled;
     Uint64 start;
     struct {
 	Uint64 total;
 	Uint64 start;
-	int currently;
     } working;
 } ErtsSchedWallTime;
 
@@ -1598,7 +1600,8 @@ void erts_init_scheduling(int, int
 void erts_execute_dirty_system_task(Process *c_p);
 #endif
 int erts_set_gc_state(Process *c_p, int enable);
-Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable);
+Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable,
+                                   int dirty_cpu, int want_dirty_io);
 Eterm erts_system_check_request(Process *c_p);
 Eterm erts_gc_info_request(Process *c_p);
 Uint64 erts_get_proc_interval(void);
-- 
cgit v1.2.3


From 6bebb90f7c94e1aa95aff06fc5c40dd07696edc2 Mon Sep 17 00:00:00 2001
From: Rickard Green <rickard@erlang.org>
Date: Thu, 19 Jan 2017 16:28:35 +0100
Subject: Introduce erts_proc_lookup_inc_refc()

---
 erts/emulator/beam/erl_process_lock.c | 35 +++++++++++++++++++++++++++++++++++
 erts/emulator/beam/erl_process_lock.h |  2 ++
 2 files changed, 37 insertions(+)

(limited to 'erts/emulator/beam')

diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c
index 180df229eb..a93f1755c8 100644
--- a/erts/emulator/beam/erl_process_lock.c
+++ b/erts/emulator/beam/erl_process_lock.c
@@ -1006,6 +1006,41 @@ erts_pid2proc_opt(Process *c_p,
     return proc;
 }
 
+static ERTS_INLINE
+Process *proc_lookup_inc_refc(Eterm pid, int allow_exit)
+{
+    Process *proc;
+#ifdef ERTS_SMP
+    ErtsThrPrgrDelayHandle dhndl;
+
+    dhndl = erts_thr_progress_unmanaged_delay();
+#endif
+
+    proc = erts_proc_lookup_raw(pid);
+    if (proc) {
+        if (!allow_exit && ERTS_PROC_IS_EXITING(proc))
+            proc = NULL;
+        else
+            erts_proc_inc_refc(proc);
+    }
+
+#ifdef ERTS_SMP
+    erts_thr_progress_unmanaged_continue(dhndl);
+#endif
+
+    return proc;
+}
+
+Process *erts_proc_lookup_inc_refc(Eterm pid)
+{
+    return proc_lookup_inc_refc(pid, 0);
+}
+
+Process *erts_proc_lookup_raw_inc_refc(Eterm pid)
+{
+    return proc_lookup_inc_refc(pid, 1);
+}
+
 void
 erts_proc_lock_init(Process *p)
 {
diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h
index 2cccf0697a..773529384f 100644
--- a/erts/emulator/beam/erl_process_lock.h
+++ b/erts/emulator/beam/erl_process_lock.h
@@ -940,6 +940,8 @@ void erts_proc_safelock(Process *a_proc,
 #define erts_pid2proc(PROC, HL, PID, NL) \
   erts_pid2proc_opt((PROC), (HL), (PID), (NL), 0)
 
+Process *erts_proc_lookup_inc_refc(Eterm pid);
+Process *erts_proc_lookup_raw_inc_refc(Eterm pid);
 
 ERTS_GLB_INLINE Process *erts_pix2proc(int ix);
 ERTS_GLB_INLINE Process *erts_proc_lookup_raw(Eterm pid);
-- 
cgit v1.2.3