Merge branch 'maint'

17 files changed, 211 insertions, 180 deletions

diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml
index e402ce448f..2b7a3b85e5 100644
--- a/erts/doc/src/erlang.xml
+++ b/erts/doc/src/erlang.xml
@@ -6145,6 +6145,10 @@ lists:map(
         <taglist>
           <tag><c>scheduler</c></tag>
           <item>The main execution threads that do most of the work.</item>
+          <tag><c>dirty_cpu_scheduler</c></tag>
+          <item>The threads for long running cpu intensive work.</item>
+          <tag><c>dirty_io_scheduler</c></tag>
+          <item>The threads for long running I/O work.</item>
           <tag><c>async</c></tag>
           <item>Async threads are used by various linked-in drivers (mainly the
             file drivers) do offload non-CPU intensive work.</item>
diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h
index fbd0528009..ae60904785 100644
--- a/erts/emulator/beam/atom.h
+++ b/erts/emulator/beam/atom.h
@@ -21,10 +21,7 @@
 #ifndef __ATOM_H__
 #define __ATOM_H__
 
-#ifndef __INDEX_H__
 #include "index.h"
-#endif
-
 #include "erl_atom_table.h"
 
 #define MAX_ATOM_CHARACTERS 255
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index c9fb5a3022..a05ca5c602 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -2826,13 +2826,7 @@ do {						\
             goto context_switch3;
         }
 
-        if (ERTS_MSACC_IS_ENABLED_CACHED_X()) {
-            if (GET_BIF_MODULE(Arg(0)) == am_ets) {
-                ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS);
-            } else {
-                ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF);
-            }
-        }
+        ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(Arg(0)), GET_BIF_ADDRESS(Arg(0)));
 
 	bf = GET_BIF_ADDRESS(Arg(0));
 
@@ -3593,13 +3587,7 @@ do {						\
                 goto context_switch;
             }
 
-            if (ERTS_MSACC_IS_ENABLED_CACHED_X()) {
-                if ((Eterm)I[-3] == am_ets) {
-                    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS);
-                } else {
-                    ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF);
-                }
-            }
+            ERTS_MSACC_SET_BIF_STATE_CACHED_X((Eterm)I[-3], (BifFunction)Arg(0));
 
 	    c_p->current = I-3;	/* In case we apply process_info/1,2 or load_nif/1 */
 	    c_p->i = I;		/* In case we apply check_process_code/2. */
@@ -7000,7 +6988,11 @@ update_map_assoc(Process* p, Eterm* reg, Eterm map, BeamInstr* I)
 
     /* The expensive case, need to build a hashmap */
     if (n > MAP_SMALL_MAP_LIMIT) {
-	res = erts_hashmap_from_ks_and_vs(p,flatmap_get_keys(mp),flatmap_get_values(mp),n);
+        ErtsHeapFactory factory;
+        erts_factory_proc_init(&factory, p);
+        res = erts_hashmap_from_ks_and_vs(&factory,flatmap_get_keys(mp),
+                                          flatmap_get_values(mp),n);
+        erts_factory_close(&factory);
     }
     return res;
 }
diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h
index 464acd67f6..4a96d971c3 100644
--- a/erts/emulator/beam/big.h
+++ b/erts/emulator/beam/big.h
@@ -21,17 +21,8 @@
 #ifndef __BIG_H__
 #define __BIG_H__
 
-#ifndef __SYS_H__
 #include "sys.h"
-#endif
-
-#ifndef __CONFIG_H__
-#include "erl_vm.h"
-#endif
-
-#ifndef __GLOBAL_H__
 #include "global.h"
-#endif
 
 typedef Uint     ErtsDigit;
 
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index d0d74bbf44..a224383493 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -672,6 +672,7 @@ do_major_collection:
        killed before a GC could be done. */
     if (reds == -2) {
         ErtsProcLocks locks = ERTS_PROC_LOCKS_ALL;
+        int res;
 
         erts_smp_proc_lock(p, ERTS_PROC_LOCKS_ALL_MINOR);
         erts_send_exit_signal(p, p->common.id, p, &locks,
@@ -683,7 +684,9 @@ do_major_collection:
         erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
 
         /* We have to make sure that we have space for need on the heap */
-        return delay_garbage_collection(p, live_hf_end, need, fcalls);
+        res = delay_garbage_collection(p, live_hf_end, need, fcalls);
+        ERTS_MSACC_POP_STATE_M();
+        return res;
     }
 
     erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c
index 8efc983f04..979a0040b0 100644
--- a/erts/emulator/beam/erl_map.c
+++ b/erts/emulator/beam/erl_map.c
@@ -497,18 +497,50 @@ Eterm erts_hashmap_from_array(ErtsHeapFactory* factory, Eterm *leafs, Uint n,
     return res;
 }
 
+Eterm erts_map_from_ks_and_vs(ErtsHeapFactory *factory, Eterm *ks0, Eterm *vs0, Uint n)
+{
+    if (n < MAP_SMALL_MAP_LIMIT) {
+        Eterm *ks, *vs, *hp;
+	flatmap_t *mp;
+	Eterm keys;
 
-Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n,
+        hp    = erts_produce_heap(factory, 3 + 1 + (2 * n), 0);
+	keys  = make_tuple(hp);
+	*hp++ = make_arityval(n);
+	ks    = hp;
+	hp   += n;
+	mp    = (flatmap_t*)hp;
+	hp   += MAP_HEADER_FLATMAP_SZ;
+	vs    = hp;
+
+        mp->thing_word = MAP_HEADER_FLATMAP;
+	mp->size = n;
+	mp->keys = keys;
+
+        sys_memcpy(ks, ks0, n * sizeof(Eterm));
+        sys_memcpy(vs, vs0, n * sizeof(Eterm));
+
+        erts_validate_and_sort_flatmap(mp);
+
+        return make_flatmap(mp);
+    } else {
+        return erts_hashmap_from_ks_and_vs(factory, ks0, vs0, n);
+    }
+    return THE_NON_VALUE;
+}
+
+
+Eterm erts_hashmap_from_ks_and_vs_extra(ErtsHeapFactory *factory,
+                                        Eterm *ks, Eterm *vs, Uint n,
 					Eterm key, Eterm value) {
     Uint32 sw, hx;
     Uint i,sz;
     hxnode_t *hxns;
-    ErtsHeapFactory factory;
     Eterm *hp, res;
 
     sz = (key == THE_NON_VALUE) ? n : (n + 1);
     ASSERT(sz > MAP_SMALL_MAP_LIMIT);
-    hp = HAlloc(p, 2 * sz);
+    hp = erts_produce_heap(factory, 2 * sz, 0);
 
     /* create tmp hx values and leaf ptrs */
     hxns = (hxnode_t *)erts_alloc(ERTS_ALC_T_TMP, sz * sizeof(hxnode_t));
@@ -531,12 +563,9 @@ Eterm erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n
 	hxns[i].i    = i;
     }
 
-    erts_factory_proc_init(&factory, p);
-    res = hashmap_from_unsorted_array(&factory, hxns, sz, 0);
-    erts_factory_close(&factory);
+    res = hashmap_from_unsorted_array(factory, hxns, sz, 0);
 
     erts_free(ERTS_ALC_T_TMP, (void *) hxns);
-    ERTS_VERIFY_UNUSED_TEMP_ALLOC(p);
 
     return res;
 }
@@ -1780,11 +1809,14 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) {
 	/* the map will grow */
 
 	if (n >= MAP_SMALL_MAP_LIMIT) {
+            ErtsHeapFactory factory;
 	    HRelease(p, shp + MAP_HEADER_FLATMAP_SZ + n, shp);
 	    ks = flatmap_get_keys(mp);
 	    vs = flatmap_get_values(mp);
 
-	    res = erts_hashmap_from_ks_and_vs_extra(p,ks,vs,n,key,value);
+            erts_factory_proc_init(&factory, p);
+	    res = erts_hashmap_from_ks_and_vs_extra(&factory,ks,vs,n,key,value);
+            erts_factory_close(&factory);
 
 	    return res;
 	}
diff --git a/erts/emulator/beam/erl_map.h b/erts/emulator/beam/erl_map.h
index 8b5c9582ba..61a841f7f0 100644
--- a/erts/emulator/beam/erl_map.h
+++ b/erts/emulator/beam/erl_map.h
@@ -98,10 +98,12 @@ Eterm* hashmap_iterator_prev(struct ErtsWStack_* s);
 int    hashmap_key_hash_cmp(Eterm* ap, Eterm* bp);
 Eterm  erts_hashmap_from_array(ErtsHeapFactory*, Eterm *leafs, Uint n, int reject_dupkeys);
 
-#define erts_hashmap_from_ks_and_vs(P, KS, VS, N) \
-    erts_hashmap_from_ks_and_vs_extra((P), (KS), (VS), (N), THE_NON_VALUE, THE_NON_VALUE);
+#define erts_hashmap_from_ks_and_vs(F, KS, VS, N) \
+    erts_hashmap_from_ks_and_vs_extra((F), (KS), (VS), (N), THE_NON_VALUE, THE_NON_VALUE);
 
-Eterm  erts_hashmap_from_ks_and_vs_extra(Process *p, Eterm *ks, Eterm *vs, Uint n,
+Eterm erts_map_from_ks_and_vs(ErtsHeapFactory *factory, Eterm *ks, Eterm *vs, Uint n);
+Eterm  erts_hashmap_from_ks_and_vs_extra(ErtsHeapFactory *factory,
+                                         Eterm *ks, Eterm *vs, Uint n,
 					 Eterm k, Eterm v);
 
 const Eterm *erts_maps_get(Eterm key, Eterm map);
diff --git a/erts/emulator/beam/erl_msacc.c b/erts/emulator/beam/erl_msacc.c
index 544bc8b983..421445fbad 100644
--- a/erts/emulator/beam/erl_msacc.c
+++ b/erts/emulator/beam/erl_msacc.c
@@ -40,10 +40,11 @@
 #include "erl_bif_unique.h"
 #include "erl_map.h"
 #include "erl_msacc.h"
+#include "erl_bif_table.h"
 
 #if ERTS_ENABLE_MSACC
 
-static Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, Eterm **hpp, Uint *szp);
+static Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, ErtsHeapFactory *factory);
 static void erts_msacc_reset(ErtsMsAcc *msacc);
 static ErtsMsAcc* get_msacc(void);
 
@@ -52,7 +53,9 @@ erts_tsd_key_t ERTS_WRITE_UNLIKELY(erts_msacc_key);
 #else
 ErtsMsAcc *ERTS_WRITE_UNLIKELY(erts_msacc) = NULL;
 #endif
+#ifndef ERTS_MSACC_ALWAYS_ON
 int ERTS_WRITE_UNLIKELY(erts_msacc_enabled);
+#endif
 
 static Eterm *erts_msacc_state_atoms = NULL;
 static erts_rwmtx_t msacc_mutex;
@@ -62,6 +65,12 @@ static ErtsMsAcc *msacc_unmanaged = NULL;
 static Uint msacc_unmanaged_count = 0;
 #endif
 
+#if ERTS_MSACC_STATE_COUNT < MAP_SMALL_MAP_LIMIT
+#define DEFAULT_MSACC_MSG_SIZE (3 + 1 + ERTS_MSACC_STATE_COUNT * 2 + 3 + REF_THING_SIZE)
+#else
+#define DEFAULT_MSACC_MSG_SIZE (3 + ERTS_MSACC_STATE_COUNT * 3 + 3 + REF_THING_SIZE)
+#endif
+
 /* we have to split initiation as atoms are not inited in early init */
 void erts_msacc_early_init(void) {
 #ifndef ERTS_MSACC_ALWAYS_ON
@@ -88,7 +97,8 @@ void erts_msacc_init(void) {
 void erts_msacc_init_thread(char *type, int id, int managed) {
     ErtsMsAcc *msacc;
 
-    msacc = erts_alloc(ERTS_ALC_T_MSACC, sizeof(ErtsMsAcc));
+    msacc = erts_alloc(ERTS_ALC_T_MSACC, sizeof(ErtsMsAcc) +
+                       sizeof(ErtsMsAccPerfCntr) * ERTS_MSACC_STATE_COUNT);
 
     msacc->type = strdup(type);
     msacc->id = make_small(id);
@@ -122,79 +132,80 @@ void erts_msacc_init_thread(char *type, int id, int managed) {
 #endif
 }
 
+#ifdef ERTS_MSACC_EXTENDED_STATES
+
+void erts_msacc_set_bif_state(ErtsMsAcc *__erts_msacc_cache, Eterm mod, void *fn) {
+
+#ifdef ERTS_MSACC_EXTENDED_BIFS
+#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num)                           \
+    if (fn == &FuncAddr) {                                             \
+        ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATIC_STATE_COUNT + Num); \
+    } else
+#include "erl_bif_list.h"
+#undef BIF_LIST
+    { /* The last else in the macro expansion,
+         this happens for internal bifs, i.e. traps etc */
+        ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF);
+    }
+#else
+    if (mod == am_ets) {
+        ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_ETS);
+    } else {
+        ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_BIF);
+    }
+#endif
+}
+
+#endif
+
 /*
  * Creates a structure looking like this
  * #{ type => scheduler, id => 1, counters => #{ State1 => Counter1 ... StateN => CounterN}}
  */
 static
-Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, Eterm **hpp, Uint *szp) {
+Eterm erts_msacc_gather_stats(ErtsMsAcc *msacc, ErtsHeapFactory *factory) {
+    Uint sz = 0;
+    Eterm *hp, cvs[ERTS_MSACC_STATE_COUNT];
+    Eterm key, state_map;
     int i;
-    Eterm *hp;
-    Eterm key, state_key, state_map;
-    Eterm res = THE_NON_VALUE;
-    flatmap_t *map;
-
-    if (szp) {
-      *szp += MAP_HEADER_FLATMAP_SZ + 1 + 2*(3);
-      *szp += MAP_HEADER_FLATMAP_SZ + 1 + 2*(ERTS_MSACC_STATE_COUNT);
-      for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) {
-          (void)erts_bld_sint64(NULL,szp,(Sint64)msacc->perf_counters[i]);
+    flatmap_t *map; 
+
+    hp = erts_produce_heap(factory, 4, 0);
+    key = TUPLE3(hp,am_counters,am_id,am_type);
+
+    for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) {
+        cvs[i] = erts_bld_sint64(NULL, &sz,(Sint64)msacc->counters[i].pc);
 #ifdef ERTS_MSACC_STATE_COUNTERS
-          (void)erts_bld_uint64(NULL,szp,msacc->state_counters[i]);
-          *szp += 3; /* tuple to put state+perf counter in */
+        erts_bld_uint64(NULL,&sz,msacc->counters[i].sc);
+        sz += 3;
 #endif
-      }
     }
 
-    if (hpp) {
-        Eterm counters[ERTS_MSACC_STATE_COUNT];
-        hp = *hpp;
-        for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) {
-            Eterm counter = erts_bld_sint64(&hp,NULL,(Sint64)msacc->perf_counters[i]);
+    hp = erts_produce_heap(factory, sz, 0);
+
+    for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) {
+        cvs[i] = erts_bld_sint64(&hp,NULL,(Sint64)msacc->counters[i].pc);
 #ifdef ERTS_MSACC_STATE_COUNTERS
-            Eterm counter__ = erts_bld_uint64(&hp,NULL,msacc->state_counters[i]);
-            counters[i] = TUPLE2(hp,counter,counter__);
-            hp += 3;
-#else
-            counters[i] = counter;
+        Eterm counter__ = erts_bld_uint64(&hp,NULL,msacc->counters[i].sc);
+        cvs[i] = TUPLE2(hp,cvs[i],counter__);
+        hp += 3;
 #endif
-        }
-
-      key = TUPLE3(hp,am_counters,am_id,am_type);
-      hp += 4;
-
-      state_key = make_tuple(hp);
-      hp[0] = make_arityval(ERTS_MSACC_STATE_COUNT);
-      
-      for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++)
-        hp[1+i] = erts_msacc_state_atoms[i];
-      hp += 1 + ERTS_MSACC_STATE_COUNT;
-
-      map = (flatmap_t*)hp;
-      hp += MAP_HEADER_FLATMAP_SZ;
-      map->thing_word = MAP_HEADER_FLATMAP;
-      map->size = ERTS_MSACC_STATE_COUNT;
-      map->keys = state_key;
-      for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++)
-            hp[i] = counters[i];
-      hp += ERTS_MSACC_STATE_COUNT;
-      state_map = make_flatmap(map);
-
-      map = (flatmap_t*)hp;
-      hp += MAP_HEADER_FLATMAP_SZ;
-      map->thing_word = MAP_HEADER_FLATMAP;
-      map->size = 3;
-      map->keys = key;
-      hp[0] = state_map;
-      hp[1] = msacc->id;
-      hp[2] = am_atom_put(msacc->type,strlen(msacc->type));
-      hp += 3;
-
-      *hpp = hp;
-      res = make_flatmap(map);
     }
 
-    return res;
+    state_map = erts_map_from_ks_and_vs(factory, erts_msacc_state_atoms, cvs,
+                                        ERTS_MSACC_STATE_COUNT);
+
+    hp = erts_produce_heap(factory, MAP_HEADER_FLATMAP_SZ + 3, 0);
+    map = (flatmap_t*)hp;
+    hp += MAP_HEADER_FLATMAP_SZ;
+    map->thing_word = MAP_HEADER_FLATMAP;
+    map->size = 3;
+    map->keys = key;
+    hp[0] = state_map;
+    hp[1] = msacc->id;
+    hp[2] = am_atom_put(msacc->type,strlen(msacc->type));
+
+    return make_flatmap(map);
 }
 
 typedef struct {
@@ -222,40 +233,31 @@ static void send_reply(ErtsMsAcc *msacc, ErtsMSAccReq *msaccrp) {
     ErtsSchedulerData *esdp = erts_get_scheduler_data();
     Process *rp = msaccrp->proc;
     ErtsMessage *msgp = NULL;
-    Eterm **hpp, *hp;
+    Eterm *hp;
     Eterm ref_copy = NIL, msg;
-    Uint sz, *szp;
-    ErlOffHeap *ohp = NULL;
     ErtsProcLocks rp_locks = (esdp && msaccrp->req_sched == esdp->no
                               ? ERTS_PROC_LOCK_MAIN : 0);
+    ErtsHeapFactory factory;
 
-    sz = 0;
-    hpp = NULL;
-    szp = &sz;
-
-    if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx);
-
-    while (1) {
-	if (hpp)
-            ref_copy = STORE_NC(hpp, ohp, msaccrp->ref);
-	else
-            *szp += REF_THING_SIZE;
-
-	if (msaccrp->action != ERTS_MSACC_GATHER)
-            msg = ref_copy;
-	else {
-            msg = erts_msacc_gather_stats(msacc, hpp, szp);
-            msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg);
-	}
-	if (hpp)
-            break;
-
-	msgp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp);
-        hpp = &hp;
-	szp = NULL;
-    }
+    if (msaccrp->action == ERTS_MSACC_GATHER) {
+
+        msgp = erts_factory_message_create(&factory, rp, &rp_locks, DEFAULT_MSACC_MSG_SIZE);
+
+        if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx);
 
-    if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx);
+        hp = erts_produce_heap(&factory, REF_THING_SIZE + 3 /* tuple */, 0);
+        ref_copy = STORE_NC(&hp, &msgp->hfrag.off_heap, msaccrp->ref);
+        msg = erts_msacc_gather_stats(msacc, &factory);
+        msg = TUPLE2(hp, ref_copy, msg);
+
+        if (msacc->unmanaged) erts_mtx_unlock(&msacc->mtx);
+
+        erts_factory_close(&factory);
+    } else {
+        ErlOffHeap *ohp = NULL;
+        msgp = erts_alloc_message_heap(rp, &rp_locks, REF_THING_SIZE, &hp, &ohp);
+        msg = STORE_NC(&hp, &msgp->hfrag.off_heap, msaccrp->ref);
+    }
 
     erts_queue_message(rp, rp_locks, msgp, msg, am_system);
 
@@ -308,9 +310,9 @@ static void erts_msacc_reset(ErtsMsAcc *msacc) {
   if (msacc->unmanaged) erts_mtx_lock(&msacc->mtx);
 
   for (i = 0; i < ERTS_MSACC_STATE_COUNT; i++) {
-      msacc->perf_counters[i] = 0;
+      msacc->counters[i].pc = 0;
 #ifdef ERTS_MSACC_STATE_COUNTERS
-      msacc->state_counters[i] = 0;
+      msacc->counters[i].sc = 0;
 #endif
   }
 
@@ -415,7 +417,7 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads)
                 ErtsSysPerfCounter perf_counter;
                 /* if enabled update stats */
                 perf_counter = erts_sys_perf_counter();
-                unmanaged[i]->perf_counters[unmanaged[i]->state] +=
+                unmanaged[i]->counters[unmanaged[i]->state].pc +=
                     perf_counter - unmanaged[i]->perf_counter;
                 unmanaged[i]->perf_counter = perf_counter;
             }
@@ -454,7 +456,7 @@ erts_msacc_request(Process *c_p, int action, Eterm *threads)
         for (msacc = msacc_unmanaged; msacc != NULL; msacc = msacc->next) {
             erts_mtx_lock(&msacc->mtx);
             perf_counter = erts_sys_perf_counter();
-            msacc->perf_counters[msacc->state] += perf_counter - msacc->perf_counter;
+            msacc->counters[msacc->state].pc += perf_counter - msacc->perf_counter;
             msacc->perf_counter = 0;
             erts_mtx_unlock(&msacc->mtx);
         }
diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h
index 284388f7aa..ad7c8c5eee 100644
--- a/erts/emulator/beam/erl_msacc.h
+++ b/erts/emulator/beam/erl_msacc.h
@@ -35,6 +35,10 @@
    this reduces overhead a little bit when profiling */
 /* #define ERTS_MSACC_ALWAYS_ON 1 */
 
+/* Uncomment this to keep individual stats for all
+   of the bifs when extended states is enabled */
+/* #define ERTS_MSACC_EXTENDED_BIFS 1 */
+
 #define ERTS_MSACC_DISABLE 0
 #define ERTS_MSACC_ENABLE  1
 #define ERTS_MSACC_RESET   2
@@ -92,7 +96,13 @@ static char *erts_msacc_states[] = {
 #define ERTS_MSACC_STATE_SLEEP     13
 #define ERTS_MSACC_STATE_TIMERS    14
 
-#define ERTS_MSACC_STATE_COUNT 15
+#define ERTS_MSACC_STATIC_STATE_COUNT 15
+
+#ifdef ERTS_MSACC_EXTENDED_BIFS
+#define ERTS_MSACC_STATE_COUNT (ERTS_MSACC_STATIC_STATE_COUNT + BIF_SIZE)
+#else
+#define ERTS_MSACC_STATE_COUNT ERTS_MSACC_STATIC_STATE_COUNT
+#endif
 
 #if ERTS_MSACC_STATE_STRINGS
 static char *erts_msacc_states[] = {
@@ -111,22 +121,26 @@ static char *erts_msacc_states[] = {
     "send",
     "sleep",
     "timers"
+#ifdef ERTS_MSACC_EXTENDED_BIFS
+#define BIF_LIST(Mod,Func,Arity,FuncAddr,Num)   \
+        ,"bif_" #Mod "_" #Func "_" #Arity
+#include "erl_bif_list.h"
+#undef BIF_LIST
+#endif
 };
 #endif
 
 #endif
 
 typedef struct erl_msacc_t_ ErtsMsAcc;
-
-struct erl_msacc_t_ {
-
-    /* the the values below are protected by mtx iff unmanaged = 1 */
-    ErtsSysPerfCounter perf_counter;
-    ErtsSysPerfCounter perf_counters[ERTS_MSACC_STATE_COUNT];
+typedef struct erl_msacc_p_cnt_t_ {
+    ErtsSysPerfCounter pc;
 #ifdef ERTS_MSACC_STATE_COUNTERS
-    Uint64 state_counters[ERTS_MSACC_STATE_COUNT];
+    Uint64 sc;
 #endif
-    Uint state;
+} ErtsMsAccPerfCntr;
+
+struct erl_msacc_t_ {
 
     /* protected by msacc_mutex in erl_msacc.c, and should be constant */
     int unmanaged;
@@ -135,12 +149,16 @@ struct erl_msacc_t_ {
     erts_tid_t tid;
     Eterm id;
     char *type;
+
+    /* the the values below are protected by mtx iff unmanaged = 1 */
+    ErtsSysPerfCounter perf_counter;
+    Uint state;
+    ErtsMsAccPerfCntr counters[];
+
 };
 
 #if ERTS_ENABLE_MSACC
 
-#define ERTS_MSACC_INLINE ERTS_GLB_INLINE
-
 #ifdef USE_THREADS
 extern erts_tsd_key_t erts_msacc_key;
 #else
@@ -276,20 +294,20 @@ void erts_msacc_init_thread(char *type, int id, int liberty);
 #define ERTS_MSACC_PUSH_AND_SET_STATE_M(state)                    \
     ERTS_MSACC_PUSH_STATE_M(); ERTS_MSACC_SET_STATE_CACHED_M(state)
 
-ERTS_MSACC_INLINE
+ERTS_GLB_INLINE
 void erts_msacc_set_state_um__(ErtsMsAcc *msacc,Uint state,int increment);
-ERTS_MSACC_INLINE
+ERTS_GLB_INLINE
 void erts_msacc_set_state_m__(ErtsMsAcc *msacc,Uint state,int increment);
 
-ERTS_MSACC_INLINE
+ERTS_GLB_INLINE
 Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc);
-ERTS_MSACC_INLINE
+ERTS_GLB_INLINE
 Uint erts_msacc_get_state_m__(ErtsMsAcc *msacc);
 
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-ERTS_MSACC_INLINE
+ERTS_GLB_INLINE
 Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc) {
     Uint state;
     if (msacc->unmanaged)
@@ -300,12 +318,12 @@ Uint erts_msacc_get_state_um__(ErtsMsAcc *msacc) {
     return state;
 }
 
-ERTS_MSACC_INLINE
+ERTS_GLB_INLINE
 Uint erts_msacc_get_state_m__(ErtsMsAcc *msacc) {
     return msacc->state;
 }
 
-ERTS_MSACC_INLINE
+ERTS_GLB_INLINE
 void erts_msacc_set_state_um__(ErtsMsAcc *msacc, Uint new_state, int increment) {
     if (ERTS_UNLIKELY(msacc->unmanaged)) {
         erts_mtx_lock(&msacc->mtx);
@@ -322,7 +340,7 @@ void erts_msacc_set_state_um__(ErtsMsAcc *msacc, Uint new_state, int increment)
         erts_mtx_unlock(&msacc->mtx);
 }
 
-ERTS_MSACC_INLINE
+ERTS_GLB_INLINE
 void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) {
     ErtsSysPerfCounter prev_perf_counter;
     Sint64 diff;
@@ -334,9 +352,9 @@ void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) {
     msacc->perf_counter = erts_sys_perf_counter();
     diff = msacc->perf_counter - prev_perf_counter;
     ASSERT(diff >= 0);
-    msacc->perf_counters[msacc->state] += diff;
+    msacc->counters[msacc->state].pc += diff;
 #ifdef ERTS_MSACC_STATE_COUNTERS
-    msacc->state_counters[new_state] += increment;
+    msacc->counters[new_state].sc += increment;
 #endif
     msacc->state = new_state;
 }
@@ -364,7 +382,7 @@ void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) {
 #define ERTS_MSACC_SET_STATE_CACHED_M(state)
 #define ERTS_MSACC_POP_STATE_M()
 #define ERTS_MSACC_PUSH_AND_SET_STATE_M(state)
-
+#define ERTS_MSACC_SET_BIF_STATE_CACHED_X(Mod,Addr)
 
 #endif /* ERTS_ENABLE_MSACC */
 
@@ -385,9 +403,13 @@ void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) {
 #define ERTS_MSACC_SET_STATE_CACHED_M_X(state)
 #define ERTS_MSACC_POP_STATE_M_X()
 #define ERTS_MSACC_PUSH_AND_SET_STATE_M_X(state)
+#define ERTS_MSACC_PUSH_AND_SET_STATE_CACHED_M_X(state)
+#define ERTS_MSACC_SET_BIF_STATE_CACHED_X(Mod,Addr)
 
 #else
 
+void erts_msacc_set_bif_state(ErtsMsAcc *msacc, Eterm mod, void *addr);
+
 #define ERTS_MSACC_PUSH_STATE_X() ERTS_MSACC_PUSH_STATE()
 #define ERTS_MSACC_POP_STATE_X() ERTS_MSACC_POP_STATE()
 #define ERTS_MSACC_SET_STATE_X(state) ERTS_MSACC_SET_STATE(state)
@@ -403,6 +425,9 @@ void erts_msacc_set_state_m__(ErtsMsAcc *msacc, Uint new_state, int increment) {
 #define ERTS_MSACC_SET_STATE_CACHED_M_X(state) ERTS_MSACC_SET_STATE_CACHED_M(state)
 #define ERTS_MSACC_POP_STATE_M_X() ERTS_MSACC_POP_STATE_M()
 #define ERTS_MSACC_PUSH_AND_SET_STATE_M_X(state) ERTS_MSACC_PUSH_AND_SET_STATE_M(state)
+#define ERTS_MSACC_SET_BIF_STATE_CACHED_X(Mod,Addr)       \
+    if (ERTS_MSACC_IS_ENABLED_CACHED_X())               \
+        erts_msacc_set_bif_state(__erts_msacc_cache, Mod, Addr)
 
 #endif /* !ERTS_MSACC_EXTENDED_STATES */
 
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index f0f5b13f9d..b5d8c5bc75 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -8177,6 +8177,8 @@ sched_dirty_cpu_thread_func(void *vesdp)
 
     esdp->thr_id += erts_no_schedulers;
 
+    erts_msacc_init_thread("dirty_cpu_scheduler", no, 0);
+
     erts_thr_progress_register_unmanaged_thread(&callbacks);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
@@ -8222,6 +8224,8 @@ sched_dirty_io_thread_func(void *vesdp)
 
     esdp->thr_id += erts_no_schedulers + erts_no_dirty_cpu_schedulers;
 
+    erts_msacc_init_thread("dirty_io_scheduler", no, 0);
+
     erts_thr_progress_register_unmanaged_thread(&callbacks);
 #ifdef ERTS_ENABLE_LOCK_CHECK
     {
@@ -9823,8 +9827,6 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 		goto check_activities_to_run;
 	    }
 
-            ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR);
-
 	    /*
 	     * Take the chosen process out of the queue.
 	     */
@@ -9930,6 +9932,8 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls)
 
 	}
 
+        ERTS_MSACC_SET_STATE_CACHED_M(ERTS_MSACC_STATE_EMULATOR);
+
 #ifdef ERTS_SMP
 
 	if (flags & ERTS_RUNQ_FLG_PROTECTED)
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 7c98b60647..714e63e986 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -989,8 +989,7 @@ struct process {
     Uint32 rcount;		/* suspend count */
     int  schedule_count;	/* Times left to reschedule a low prio process */
     Uint reds;			/* No of reductions for this process  */
-    Eterm group_leader;		/* Pid in charge
-				   (can be boxed) */
+    Eterm group_leader;		/* Pid in charge (can be boxed) */
     Uint flags;			/* Trap exit, etc (no trace flags anymore) */
     Eterm fvalue;		/* Exit & Throw value (failure reason) */
     Uint freason;		/* Reason for detected failure */
diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h
index 8c81cbd410..1e7bb8514b 100644
--- a/erts/emulator/beam/export.h
+++ b/erts/emulator/beam/export.h
@@ -21,14 +21,8 @@
 #ifndef __EXPORT_H__
 #define __EXPORT_H__
 
-#ifndef __SYS_H__
 #include "sys.h"
-#endif
-
-#ifndef __INDEX_H__
 #include "index.h"
-#endif
-
 #include "code_ix.h"
 
 /*
diff --git a/erts/emulator/beam/hash.h b/erts/emulator/beam/hash.h
index 9f773d8faa..4e769c0119 100644
--- a/erts/emulator/beam/hash.h
+++ b/erts/emulator/beam/hash.h
@@ -25,9 +25,7 @@
 #ifndef __HASH_H__
 #define __HASH_H__
 
-#ifndef __SYS_H__
 #include "sys.h"
-#endif
 
 typedef unsigned long HashValue;
 typedef struct hash Hash;
diff --git a/erts/emulator/beam/index.h b/erts/emulator/beam/index.h
index 218779c33b..0a109d8699 100644
--- a/erts/emulator/beam/index.h
+++ b/erts/emulator/beam/index.h
@@ -26,13 +26,8 @@
 #ifndef __INDEX_H__
 #define __INDEX_H__
 
-#ifndef __HASH_H__
 #include "hash.h"
-#endif
-
-#ifndef ERL_ALLOC_H__
 #include "erl_alloc.h"
-#endif
 
 typedef struct index_slot 
 {
diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h
index 4e12731d85..5a60bc90d9 100644
--- a/erts/emulator/beam/module.h
+++ b/erts/emulator/beam/module.h
@@ -21,9 +21,7 @@
 #ifndef __MODULE_H__
 #define __MODULE_H__
 
-#ifndef __INDEX_H__
 #include "index.h"
-#endif
 
 struct erl_module_instance {
     BeamCodeHeader* code_hdr;
diff --git a/erts/emulator/beam/safe_hash.h b/erts/emulator/beam/safe_hash.h
index 6910b33004..285103cb17 100644
--- a/erts/emulator/beam/safe_hash.h
+++ b/erts/emulator/beam/safe_hash.h
@@ -26,14 +26,9 @@
 #ifndef __SAFE_HASH_H__
 #define __SAFE_HASH_H__
 
-
-#ifndef __SYS_H__
 #include "sys.h"
-#endif
-
 #include "erl_alloc.h"
 
-
 typedef unsigned long SafeHashValue;
 
 typedef int (*SHCMP_FUN)(void*, void*);
diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h
index 3a0d23cd36..b64b0d87f6 100644
--- a/erts/emulator/sys/unix/erl_unix_sys.h
+++ b/erts/emulator/sys/unix/erl_unix_sys.h
@@ -282,7 +282,7 @@ ERTS_GLB_INLINE ErtsSysPerfCounter erts_sys_perf_counter(void);
 
 #if ERTS_GLB_INLINE_INCL_FUNC_DEF
 
-ERTS_GLB_INLINE ErtsSysPerfCounter
+ERTS_GLB_FORCE_INLINE ErtsSysPerfCounter
 erts_sys_perf_counter()
 {
     return (*erts_sys_time_data__.r.o.perf_counter)();