Refactor GC

2 files changed, 334 insertions, 323 deletions

diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index e316ab95ab..2d7b7cafa4 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -42,11 +42,76 @@
 #include "dtrace-wrapper.h"
 #include "erl_bif_unique.h"
 
+#define ERTS_CONTINUOUS_NEW_HEAP
+
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
 #define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20
 #define ERTS_INACT_WR_PB_LEAVE_LIMIT 10
 #define ERTS_INACT_WR_PB_LEAVE_PERCENTAGE 10
 
+#if defined(DEBUG) || 0
+#define ERTS_GC_DEBUG
+#else
+#undef ERTS_GC_DEBUG
+#endif
+#ifdef ERTS_GC_DEBUG
+#  define ERTS_GC_ASSERT ASSERT
+#else
+#  define ERTS_GC_ASSERT(B) ((void) 1)
+#endif
+
+#ifdef ERTS_CONTINUOUS_NEW_HEAP
+#define ERTS_IS_NEW_HEAP_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    ErtsInArea((Ptr), (NhPtr), (NhSz))
+#define ERTS_IS_LITERAL_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    (!ErtsInArea((Ptr), (NhPtr), (NhSz))				\
+     && !ErtsInArea((Ptr), (OhPtr), (OhSz)))
+
+#ifdef ERTS_GC_DEBUG
+#define ERTS_IS_NEW_HEAP_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    (ERTS_IS_NEW_HEAP_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
+     ? (ERTS_GC_ASSERT(!erts_is_literal((TPtr), (Ptr))			\
+		       && !ErtsInArea((Ptr), (OhPtr), (OhSz))), 1)	\
+     : (ERTS_GC_ASSERT(erts_is_literal((TPtr), (Ptr))			\
+		       || ErtsInArea((Ptr), (OhPtr), (OhSz))), 0))
+#define ERTS_IS_LITERAL_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    (ERTS_IS_LITERAL_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
+     ? (ERTS_GC_ASSERT(erts_is_literal((TPtr), (Ptr))), 1)		\
+     : (ERTS_GC_ASSERT(!erts_is_literal((TPtr), (Ptr))), 0))
+#endif
+
+#else
+
+#define ERTS_IS_NEW_HEAP_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    (!erts_is_literal((TPtr), (Ptr)) && !ErtsInArea((Ptr), (OhPtr), (OhSz)))
+#define ERTS_IS_LITERAL_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    (erts_is_literal((TPtr), (Ptr)))
+
+#ifdef ERTS_GC_DEBUG
+#define ERTS_IS_NEW_HEAP_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    (ERTS_IS_NEW_HEAP_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
+     ? (ERTS_GC_ASSERT(ErtsInArea((Ptr), (NhPtr), (NhSz))), 1)		\
+     : (ERTS_GC_ASSERT(!ErtsInArea((Ptr), (NhPtr), (NhSz))), 0))
+#define ERTS_IS_LITERAL_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    (ERTS_IS_LITERAL_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
+     ? (ERTS_GC_ASSERT(!ErtsInArea((Ptr), (NhPtr), (NhSz))		\
+		       && !ErtsInArea((Ptr), (OhPtr), (OhSz))), 1)	\
+     : (ERTS_GC_ASSERT(ErtsInArea((Ptr), (NhPtr), (NhSz))		\
+		       || ErtsInArea((Ptr), (OhPtr), (OhSz))), 0))
+#endif
+
+#endif
+
+#ifndef ERTS_IS_NEW_HEAP_PTR
+#define ERTS_IS_NEW_HEAP_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    ERTS_IS_NEW_HEAP_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz))
+#endif
+
+#ifndef ERTS_IS_LITERAL_PTR
+#define ERTS_IS_LITERAL_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz)	\
+    ERTS_IS_LITERAL_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz))
+#endif
+
 /*
  * Returns number of elements in an array.
  */
@@ -100,13 +165,27 @@ static Uint setup_rootset(Process*, Eterm*, int, Rootset*);
 static void cleanup_rootset(Rootset *rootset);
 static Uint combined_message_size(Process* p);
 static void remove_message_buffers(Process* p);
+static Eterm *full_sweep_heaps(Process *p,
+			       int hibernate,
+			       Eterm *n_heap, Eterm* n_htop,
+			       char *h, Uint h_size,
+			       char *oh, Uint oh_size,
+			       Eterm *objv, int nobj);
 static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
 static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
 static void do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj);
-static Eterm* sweep_rootset(Rootset *rootset, Eterm* htop, char* src, Uint src_size);
-static Eterm* sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size);
-static Eterm* sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
-			     char* src, Uint src_size);
+static Eterm *sweep_new_heap(Eterm *n_hp, Eterm *n_htop,
+			     char* new_heap, Uint new_heap_size,
+			     char* old_heap, Uint old_heap_size);
+static Eterm *sweep_heaps(Eterm *n_hp, Eterm *n_htop,
+			  char* new_heap, Uint new_heap_size,
+			  char* old_heap, Uint old_heap_size);
+static Eterm* sweep_literal_area(Eterm* n_hp, Eterm* n_htop,
+				    char* new_heap, Uint new_heap_size,
+				    char* old_heap, Uint old_heap_size,
+				    char* src, Uint src_size);
+static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
+					 char* src, Uint src_size);
 static Eterm* collect_heap_frags(Process* p, Eterm* heap,
 				 Eterm* htop, Eterm* objv, int nobj);
 static void adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj);
@@ -514,9 +593,6 @@ erts_garbage_collect_hibernate(Process* p)
     Uint heap_size;
     Eterm* heap;
     Eterm* htop;
-    Rootset rootset;
-    char* src;
-    Uint src_size;
     Uint actual_size;
     char* area;
     Uint area_size;
@@ -544,41 +620,16 @@ erts_garbage_collect_hibernate(Process* p)
 				    sizeof(Eterm)*heap_size);
     htop = heap;
 
-    (void) setup_rootset(p, p->arg_reg, p->arity, &rootset);
-#if HIPE
-    hipe_empty_nstack(p);
-#endif
-
-    src = (char *) p->heap;
-    src_size = (char *) p->htop - src;
-    htop = sweep_rootset(&rootset, htop, src, src_size);
-    htop = sweep_one_area(heap, htop, src, src_size);
-
-    if (p->old_heap) {
-	src = (char *) p->old_heap;
-	src_size = (char *) p->old_htop - src;
-	htop = sweep_rootset(&rootset, htop, src, src_size);
-	htop = sweep_one_area(heap, htop, src, src_size);
-    }
-
-    cleanup_rootset(&rootset);
-
-    if (MSO(p).first) {
-        sweep_off_heap(p, 1);
-    }
-
-    /*
-     *  Update all pointers.
-     */
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
-		   (void*)HEAP_START(p),
-		   HEAP_SIZE(p) * sizeof(Eterm));
-    if (p->old_heap) {
-	ERTS_HEAP_FREE(ERTS_ALC_T_OLD_HEAP,
-		       (void*)p->old_heap,
-		       (p->old_hend - p->old_heap) * sizeof(Eterm));
-	p->old_heap = p->old_htop = p->old_hend = 0;
-    }
+    htop = full_sweep_heaps(p,
+			    1,
+			    heap,
+			    htop,
+			    (char *) p->heap,
+			    (char *) p->htop - (char *) p->heap,
+			    (char *) p->old_heap,
+			    (char *) p->old_htop - (char *) p->old_heap,
+			    p->arg_reg,
+			    p->arity);
 
     p->heap = heap;
     p->high_water = htop;
@@ -724,7 +775,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
                 if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
-		} else if (in_area(ptr, area, area_size)) {
+		} else if (ErtsInArea(ptr, area, area_size)) {
                     MOVE_BOXED(ptr,val,old_htop,g_ptr++);
 		} else {
 		    g_ptr++;
@@ -735,7 +786,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
                 val = *ptr;
                 if (IS_MOVED_CONS(val)) { /* Moved */
                     *g_ptr++ = ptr[1];
-		} else if (in_area(ptr, area, area_size)) {
+		} else if (ErtsInArea(ptr, area, area_size)) {
                     MOVE_CONS(ptr,val,old_htop,g_ptr++);
                 } else {
 		    g_ptr++;
@@ -755,8 +806,11 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
      * Now we'll have to go through all heaps updating all other references.
      */
 
-    old_htop = sweep_one_heap(p->heap, p->htop, old_htop, area, area_size);
-    old_htop = sweep_one_area(p->old_heap, old_htop, area, area_size);
+    old_htop = sweep_literals_to_old_heap(p->heap, p->htop, old_htop, area, area_size);
+    old_htop = sweep_literal_area(p->old_heap, old_htop,
+				  (char *) p->heap, sizeof(Eterm)*p->heap_sz,
+				  (char *) p->old_heap, sizeof(Eterm)*old_heap_size,
+				  area, area_size);
     ASSERT(p->old_htop <= old_htop && old_htop <= p->old_hend);
     p->old_htop = old_htop;
 
@@ -1034,15 +1088,13 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
                 if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
                     *g_ptr++ = val;
-                } else if (in_area(ptr, heap, mature_size)) {
+                } else if (ErtsInArea(ptr, heap, mature_size)) {
                     MOVE_BOXED(ptr,val,old_htop,g_ptr++);
-                } else if (in_area(ptr, heap, heap_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr)
-			   && !in_area(ptr, oh, oh_size));
+                } else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
+						heap, heap_size,
+						oh, oh_size)) {
                     MOVE_BOXED(ptr,val,n_htop,g_ptr++);
                 } else {
-		    ASSERT(erts_is_literal(gval, ptr)
-			   || in_area(ptr, oh, oh_size));
 		    g_ptr++;
 		}
                 break;
@@ -1053,15 +1105,13 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
                 val = *ptr;
                 if (IS_MOVED_CONS(val)) { /* Moved */
                     *g_ptr++ = ptr[1];
-                } else if (in_area(ptr, heap, mature_size)) {
+                } else if (ErtsInArea(ptr, heap, mature_size)) {
                     MOVE_CONS(ptr,val,old_htop,g_ptr++);
-                } else if (in_area(ptr, heap, heap_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr)
-			   && !in_area(ptr, oh, oh_size));
+                } else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
+						heap, heap_size,
+						oh, oh_size)) {
                     MOVE_CONS(ptr,val,n_htop,g_ptr++);
                 } else {
-		    ASSERT(erts_is_literal(gval, ptr)
-			   || in_area(ptr, oh, oh_size));
 		    g_ptr++;
 		}
 		break;
@@ -1084,7 +1134,8 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
      */
 
     if (mature_size == 0) {
-	n_htop = sweep_one_area(n_heap, n_htop, heap, heap_size);
+	n_htop = sweep_new_heap(n_heap, n_htop, heap, heap_size,
+				oh, oh_size);
     } else {
 	Eterm* n_hp = n_heap;
 	Eterm* ptr;
@@ -1101,15 +1152,13 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 		if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
 		    *n_hp++ = val;
-		} else if (in_area(ptr, heap, mature_size)) {
+		} else if (ErtsInArea(ptr, heap, mature_size)) {
 		    MOVE_BOXED(ptr,val,old_htop,n_hp++);
-		} else if (in_area(ptr, heap, heap_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr)
-			   && !in_area(ptr, oh, oh_size));
+		} else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
+						heap, heap_size,
+						oh, oh_size)) {
 		    MOVE_BOXED(ptr,val,n_htop,n_hp++);
 		} else {
-		    ASSERT(erts_is_literal(gval, ptr)
-			   || in_area(ptr, oh, oh_size));
 		    n_hp++;
 		}
 		break;
@@ -1119,15 +1168,13 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 		val = *ptr;
 		if (IS_MOVED_CONS(val)) {
 		    *n_hp++ = ptr[1];
-		} else if (in_area(ptr, heap, mature_size)) {
+		} else if (ErtsInArea(ptr, heap, mature_size)) {
 		    MOVE_CONS(ptr,val,old_htop,n_hp++);
-		} else if (in_area(ptr, heap, heap_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr)
-			   && !in_area(ptr, oh, oh_size));
+		} else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
+						heap, heap_size,
+						oh, oh_size)) {
 		    MOVE_CONS(ptr,val,n_htop,n_hp++);
 		} else {
-		    ASSERT(erts_is_literal(gval, ptr)
-			   || in_area(ptr, oh, oh_size));
 		    n_hp++;
 		}
 		break;
@@ -1145,19 +1192,15 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 			if (IS_MOVED_BOXED(val)) {
 			    *origptr = val;
 			    mb->base = binary_bytes(val);
-			} else if (in_area(ptr, heap, mature_size)) {
+			} else if (ErtsInArea(ptr, heap, mature_size)) {
 			    MOVE_BOXED(ptr,val,old_htop,origptr);
 			    mb->base = binary_bytes(mb->orig);
-			} else if (in_area(ptr, heap, heap_size)) {
-			    ASSERT(!erts_is_literal(*origptr, origptr)
-				   && !in_area(ptr, oh, oh_size));
+			} else if (ERTS_IS_NEW_HEAP_PTR(*origptr, ptr,
+							heap, heap_size,
+							oh, oh_size)) {
 			    MOVE_BOXED(ptr,val,n_htop,origptr);
 			    mb->base = binary_bytes(mb->orig);
 			}
-			else {
-			    ASSERT(erts_is_literal(*origptr, origptr)
-				   || in_area(ptr, oh, oh_size));
-			}
 		    }
 		    n_hp += (thing_arityval(gval)+1);
 		}
@@ -1176,7 +1219,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
      */
 
     if (OLD_HTOP(p) < old_htop) {
-	old_htop = sweep_one_area(OLD_HTOP(p), old_htop, heap, heap_size);
+	old_htop = sweep_new_heap(OLD_HTOP(p), old_htop,
+				  heap, heap_size,
+				  oh, oh_size);
     }
     OLD_HTOP(p) = old_htop;
     HIGH_WATER(p) = n_htop;
@@ -1229,8 +1274,6 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
 static int
 major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 {
-    Rootset rootset;
-    Roots* roots;
     const Uint size_before = ((HEAP_TOP(p) - HEAP_START(p))
                               + (OLD_HTOP(p) - OLD_HEAP(p))
                               + MBUF_SIZE(p));
@@ -1240,8 +1283,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
     Uint src_size = (char *) HEAP_TOP(p) - src;
     char* oh = (char *) OLD_HEAP(p);
     Uint oh_size = (char *) OLD_HTOP(p) - oh;
-    Uint n;
-    Uint new_sz;
+    Uint new_sz, stk_sz;
 
     /*
      * Do a fullsweep GC. First figure out the size of the heap
@@ -1272,13 +1314,93 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	n_htop = collect_heap_frags(p, n_heap, n_htop, objv, nobj);
     }
 
+    n_htop = full_sweep_heaps(p, 0, n_heap, n_htop, src, src_size,
+			      oh, oh_size, objv, nobj);
+
+    /* Move the stack to the end of the heap */
+    stk_sz = HEAP_END(p) - p->stop;
+    sys_memcpy(n_heap + new_sz - stk_sz, p->stop, stk_sz * sizeof(Eterm));
+    p->stop = n_heap + new_sz - stk_sz;
+
+#ifdef USE_VM_PROBES
+    if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) {
+        DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
+
+        dtrace_proc_str(p, pidbuf);
+        DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz);
+    }
+#endif
+
+    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
+		   (void *) HEAP_START(p),
+		   (HEAP_END(p) - HEAP_START(p)) * sizeof(Eterm));
+    HEAP_START(p) = n_heap;
+    HEAP_TOP(p) = n_htop;
+    HEAP_SIZE(p) = new_sz;
+    HEAP_END(p) = n_heap + new_sz;
+    GEN_GCS(p) = 0;
+
+    HIGH_WATER(p) = HEAP_TOP(p);
+
+    ErtsGcQuickSanityCheck(p);
+
+    *recl += size_before - (HEAP_TOP(p) - HEAP_START(p));
+
+    {
+	ErlMessage *msgp;
+
+	/*
+	 * Copy newly received message onto the end of the new heap.
+	 */
+        for (msgp = p->msg.first; msgp; msgp = msgp->next) {
+	    if (msgp->data.attached) {
+                ErtsHeapFactory factory;
+                erts_factory_proc_prealloc_init(&factory, p,
+                                                erts_msg_attached_data_size(msgp));
+		erts_move_msg_attached_data_to_heap(&factory, msgp);
+                erts_factory_close(&factory);
+		ErtsGcQuickSanityCheck(p);
+	    }
+	}
+    }
+
+    adjust_after_fullsweep(p, need, objv, nobj);
+
+#ifdef HARDDEBUG
+    disallow_heap_frag_ref_in_heap(p);
+#endif
+    remove_message_buffers(p);
+
+    ErtsGcQuickSanityCheck(p);
+    return 1;			/* We are done. */
+}
+
+static Eterm *
+full_sweep_heaps(Process *p,
+		 int hibernate,
+		 Eterm *n_heap, Eterm* n_htop,
+		 char *h, Uint h_size,
+		 char *oh, Uint oh_size,
+		 Eterm *objv, int nobj)
+{
+    Rootset rootset;
+    Roots *roots;
+    Uint n;
+
     /*
      * Copy all top-level terms directly referenced by the rootset to
      * the new new_heap.
      */
 
     n = setup_rootset(p, objv, nobj, &rootset);
-    n_htop = fullsweep_nstack(p, n_htop);
+
+#ifdef HIPE
+    if (hibernate)
+	hipe_empty_nstack(p);
+    else
+	n_htop = fullsweep_nstack(p, n_htop);
+#endif
+
     roots = rootset.roots;
     while (n--) {
 	Eterm* g_ptr = roots->v;
@@ -1298,11 +1420,11 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 		if (IS_MOVED_BOXED(val)) {
 		    ASSERT(is_boxed(val));
 		    *g_ptr++ = val;
-		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr));
+		} else if (!ERTS_IS_LITERAL_PTR(gval, ptr,
+						h, h_size,
+						oh, oh_size)) {
 		    MOVE_BOXED(ptr,val,n_htop,g_ptr++);
 		} else {
-		    ASSERT(erts_is_literal(gval, ptr));
 		    g_ptr++;
 		}
 		continue;
@@ -1313,11 +1435,11 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 		val = *ptr;
 		if (IS_MOVED_CONS(val)) {
 		    *g_ptr++ = ptr[1];
-		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr));
+		} else if (!ERTS_IS_LITERAL_PTR(gval, ptr,
+						h, h_size,
+						oh, oh_size)) {
 		    MOVE_CONS(ptr,val,n_htop,g_ptr++);
 		} else {
-		    ASSERT(erts_is_literal(gval, ptr));
 		    g_ptr++;
 		}
 		continue;
@@ -1340,82 +1462,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
      * until all is copied.
      */
 
-    if (oh_size == 0) {
-	n_htop = sweep_one_area(n_heap, n_htop, src, src_size);
-    } else {
-	Eterm* n_hp = n_heap;
-
-	while (n_hp != n_htop) {
-	    Eterm* ptr;
-	    Eterm val;
-	    Eterm gval = *n_hp;
-
-	    switch (primary_tag(gval)) {
-	    case TAG_PRIMARY_BOXED: {
-		ptr = boxed_val(gval);
-		val = *ptr;
-		if (IS_MOVED_BOXED(val)) {
-		    ASSERT(is_boxed(val));
-		    *n_hp++ = val;
-		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr));
-		    MOVE_BOXED(ptr,val,n_htop,n_hp++);
-		} else {
-		    ASSERT(erts_is_literal(gval, ptr));
-		    n_hp++;
-		}
-		break;
-	    }
-	    case TAG_PRIMARY_LIST: {
-		ptr = list_val(gval);
-		val = *ptr;
-		if (IS_MOVED_CONS(val)) {
-		    *n_hp++ = ptr[1];
-		} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
-		    ASSERT(!erts_is_literal(gval, ptr));
-		    MOVE_CONS(ptr,val,n_htop,n_hp++);
-		} else {
-		    ASSERT(erts_is_literal(gval, ptr));
-		    n_hp++;
-		}
-		break;
-	    }
-	    case TAG_PRIMARY_HEADER: {
-		if (!header_is_thing(gval))
-		    n_hp++;
-		else {
-		    if (header_is_bin_matchstate(gval)) {
-			ErlBinMatchState *ms = (ErlBinMatchState*) n_hp;
-			ErlBinMatchBuffer *mb = &(ms->mb);
-			Eterm* origptr;	
-			origptr = &(mb->orig);
-			ptr = boxed_val(*origptr);
-			val = *ptr;
-			if (IS_MOVED_BOXED(val)) {
-			    *origptr = val;
-			    mb->base = binary_bytes(*origptr);
-			} else if (in_area(ptr, src, src_size) ||
--				   in_area(ptr, oh, oh_size)) {
-			    ASSERT(!erts_is_literal(*origptr, origptr));
-			    MOVE_BOXED(ptr,val,n_htop,origptr); 
-			    mb->base = binary_bytes(*origptr);
-			    ptr = boxed_val(*origptr);
-			    val = *ptr;
-			}
-			else {
-			    ASSERT(erts_is_literal(*origptr, origptr));
-			}
-		    }
-		    n_hp += (thing_arityval(gval)+1);
-		}
-		break;
-	    }
-	    default:
-		n_hp++;
-		break;
-	    }
-	}
-    }
+    n_htop = sweep_heaps(n_heap, n_htop, h, h_size, oh, oh_size);
 
     if (MSO(p).first) {
 	sweep_off_heap(p, 1);
@@ -1428,62 +1475,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
 	OLD_HEAP(p) = OLD_HTOP(p) = OLD_HEND(p) = NULL;
     }
 
-    /* Move the stack to the end of the heap */
-    n = HEAP_END(p) - p->stop;
-    sys_memcpy(n_heap + new_sz - n, p->stop, n * sizeof(Eterm));
-    p->stop = n_heap + new_sz - n;
-
-#ifdef USE_VM_PROBES
-    if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) {
-        DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
-
-        dtrace_proc_str(p, pidbuf);
-        DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz);
-    }
-#endif
-
-    ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
-		   (void *) HEAP_START(p),
-		   (HEAP_END(p) - HEAP_START(p)) * sizeof(Eterm));
-    HEAP_START(p) = n_heap;
-    HEAP_TOP(p) = n_htop;
-    HEAP_SIZE(p) = new_sz;
-    HEAP_END(p) = n_heap + new_sz;
-    GEN_GCS(p) = 0;
-
-    HIGH_WATER(p) = HEAP_TOP(p);
-
-    ErtsGcQuickSanityCheck(p);
-
-    *recl += size_before - (HEAP_TOP(p) - HEAP_START(p));
-
-    {
-	ErlMessage *msgp;
-
-	/*
-	 * Copy newly received message onto the end of the new heap.
-	 */
-        for (msgp = p->msg.first; msgp; msgp = msgp->next) {
-	    if (msgp->data.attached) {
-                ErtsHeapFactory factory;
-                erts_factory_proc_prealloc_init(&factory, p,
-                                                erts_msg_attached_data_size(msgp));
-		erts_move_msg_attached_data_to_heap(&factory, msgp);
-                erts_factory_close(&factory);
-		ErtsGcQuickSanityCheck(p);
-	    }
-	}
-    }
-
-    adjust_after_fullsweep(p, need, objv, nobj);
-
-#ifdef HARDDEBUG
-    disallow_heap_frag_ref_in_heap(p);
-#endif
-    remove_message_buffers(p);
-
-    ErtsGcQuickSanityCheck(p);
-    return 1;			/* We are done. */
+    return n_htop;
 }
 
 static void
@@ -1585,7 +1577,7 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj)
 		objv++;
 	    } else {
  		for (qb = mbuf; qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+		    if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1601,7 +1593,7 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj)
 		objv++;
 	    } else {
 		for (qb = mbuf; qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+		    if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1644,9 +1636,9 @@ disallow_heap_frag_ref_in_heap(Process* p)
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
 	    ptr = _unchecked_boxed_val(val);
-	    if (!in_area(ptr, heap, heap_size)) {
+	    if (!ErtsInArea(ptr, heap, heap_size)) {
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+		    if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1654,9 +1646,9 @@ disallow_heap_frag_ref_in_heap(Process* p)
 	    break;
 	case TAG_PRIMARY_LIST:
 	    ptr = _unchecked_list_val(val);
-	    if (!in_area(ptr, heap, heap_size)) {
+	    if (!ErtsInArea(ptr, heap, heap_size)) {
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+		    if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1699,12 +1691,12 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_BOXED:
 	    ptr = (Eterm *) val;
-	    if (!in_area(ptr, old_heap, old_heap_size)) {
-		if (in_area(ptr, new_heap, new_heap_size)) {
+	    if (!ErtsInArea(ptr, old_heap, old_heap_size)) {
+		if (ErtsInArea(ptr, new_heap, new_heap_size)) {
 		    abort();
 		}
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+		    if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1712,12 +1704,12 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
 	    break;
 	case TAG_PRIMARY_LIST:
 	    ptr = (Eterm *) val;
-	    if (!in_area(ptr, old_heap, old_heap_size)) {
-		if (in_area(ptr, new_heap, new_heap_size)) {
+	    if (!ErtsInArea(ptr, old_heap, old_heap_size)) {
+		if (ErtsInArea(ptr, new_heap, new_heap_size)) {
 		    abort();
 		}
 		for (qb = MBUF(p); qb != NULL; qb = qb->next) {
-		    if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+		    if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
 			abort();
 		    }
 		}
@@ -1726,7 +1718,7 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
 	case TAG_PRIMARY_HEADER:
 	    if (header_is_thing(val)) {
 		hp += _unchecked_thing_arityval(val);
-		if (!in_area(hp, old_heap, old_heap_size+1)) {
+		if (!ErtsInArea(hp, old_heap, old_heap_size+1)) {
 		    abort();
 		}
 	    }
@@ -1736,66 +1728,35 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
 }
 #endif
 
-static Eterm*
-sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size)
+typedef enum {
+    ErtsSweepNewHeap,
+    ErtsSweepHeaps,
+    ErtsSweepLiteralArea
+} ErtsSweepType;
+
+static ERTS_FORCE_INLINE Eterm *
+sweep(Eterm *n_hp, Eterm *n_htop,
+      ErtsSweepType type,
+      char *h, Uint hsz,
+      char *oh, Uint ohsz,
+      char *src, Uint src_size)
 {
-    Roots* roots = rootset->roots;
-    Uint n = rootset->num_roots;
     Eterm* ptr;
-    Eterm gval;
     Eterm val;
+    Eterm gval;
 
-    while (n--) {
-        Eterm* g_ptr = roots->v;
-        Uint g_sz = roots->sz;
-
-	roots++;
-        while (g_sz--) {
-            gval = *g_ptr;
-
-            switch (primary_tag(gval)) {
-	    case TAG_PRIMARY_BOXED: {
-		ptr = boxed_val(gval);
-                val = *ptr;
-                if (IS_MOVED_BOXED(val)) {
-		    ASSERT(is_boxed(val));
-                    *g_ptr++ = val;
-                } else if (in_area(ptr, src, src_size)) {
-                    MOVE_BOXED(ptr,val,htop,g_ptr++);
-                } else {
-		    g_ptr++;
-		}
-                break;
-	    }
-	    case TAG_PRIMARY_LIST: {
-                ptr = list_val(gval);
-                val = *ptr;
-                if (IS_MOVED_CONS(val)) {
-                    *g_ptr++ = ptr[1];
-                } else if (in_area(ptr, src, src_size)) {
-                    MOVE_CONS(ptr,val,htop,g_ptr++);
-                } else {
-		    g_ptr++;
-		}
-		break;
-	    }
-
-	    default:
-                g_ptr++;
-		break;
-            }
-        }
-    }
-    return htop;
-}
-
+#undef ERTS_IS_IN_SWEEP_AREA
 
-static Eterm*
-sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
-{
-    Eterm* ptr;
-    Eterm val;
-    Eterm gval;
+#define ERTS_IS_IN_SWEEP_AREA(TPtr, Ptr)				\
+    (type == ErtsSweepHeaps						\
+     ? !ERTS_IS_LITERAL_PTR((TPtr), (Ptr), h, hsz, oh, ohsz)		\
+     : (type == ErtsSweepNewHeap					\
+	? ERTS_IS_NEW_HEAP_PTR((TPtr), (Ptr), h, hsz, oh, ohsz)		\
+	: (ErtsInArea((Ptr), src, src_size)				\
+	   ? (ERTS_GC_ASSERT(erts_is_literal((TPtr), (Ptr))		\
+			     && !ErtsInArea((Ptr), h, hsz)		\
+			     && !ErtsInArea((Ptr), oh, ohsz)), 1)	\
+	   : 0)))
 
     while (n_hp != n_htop) {
 	ASSERT(n_hp < n_htop);
@@ -1807,7 +1768,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
 	    if (IS_MOVED_BOXED(val)) {
 		ASSERT(is_boxed(val));
 		*n_hp++ = val;
-	    } else if (in_area(ptr, src, src_size)) {
+	    } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) {
 		MOVE_BOXED(ptr,val,n_htop,n_hp++);
 	    } else {
 		n_hp++;
@@ -1819,7 +1780,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
 	    val = *ptr;
 	    if (IS_MOVED_CONS(val)) {
 		*n_hp++ = ptr[1];
-	    } else if (in_area(ptr, src, src_size)) {
+	    } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) {
 		MOVE_CONS(ptr,val,n_htop,n_hp++);
 	    } else {
 		n_hp++;
@@ -1840,7 +1801,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
 		    if (IS_MOVED_BOXED(val)) {
 			*origptr = val;
 			mb->base = binary_bytes(*origptr);
-		    } else if (in_area(ptr, src, src_size)) {
+		    } else if (ERTS_IS_IN_SWEEP_AREA(*origptr, ptr)) {
 			MOVE_BOXED(ptr,val,n_htop,origptr); 
 			mb->base = binary_bytes(*origptr);
 		    }
@@ -1855,10 +1816,49 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
 	}
     }
     return n_htop;
+#undef ERTS_IS_IN_SWEEP_AREA
+}
+
+static Eterm *
+sweep_new_heap(Eterm *n_hp, Eterm *n_htop,
+	       char* new_heap, Uint new_heap_size,
+	       char* old_heap, Uint old_heap_size)
+{
+    return sweep(n_hp, n_htop,
+		 ErtsSweepNewHeap,
+		 new_heap, new_heap_size,
+		 old_heap, old_heap_size,
+		 NULL, 0);
+}
+
+static Eterm *
+sweep_heaps(Eterm *n_hp, Eterm *n_htop,
+	    char* new_heap, Uint new_heap_size,
+	    char* old_heap, Uint old_heap_size)
+{
+    return sweep(n_hp, n_htop,
+		 ErtsSweepHeaps,
+		 new_heap, new_heap_size,
+		 old_heap, old_heap_size,
+		 NULL, 0);
+}
+
+static Eterm *
+sweep_literal_area(Eterm *n_hp, Eterm *n_htop,
+		   char* new_heap, Uint new_heap_size,
+		   char* old_heap, Uint old_heap_size,
+		   char* src, Uint src_size)
+{
+    return sweep(n_hp, n_htop,
+		 ErtsSweepLiteralArea,
+		 new_heap, new_heap_size,
+		 old_heap, old_heap_size,
+		 src, src_size);
 }
 
 static Eterm*
-sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint src_size)
+sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
+			   char* src, Uint src_size)
 {
     while (heap_ptr < heap_end) {
 	Eterm* ptr;
@@ -1872,7 +1872,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
 	    if (IS_MOVED_BOXED(val)) {
 		ASSERT(is_boxed(val));
 		*heap_ptr++ = val;
-	    } else if (in_area(ptr, src, src_size)) {
+	    } else if (ErtsInArea(ptr, src, src_size)) {
 		MOVE_BOXED(ptr,val,htop,heap_ptr++);
 	    } else {
 		heap_ptr++;
@@ -1884,7 +1884,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
 	    val = *ptr;
 	    if (IS_MOVED_CONS(val)) {
 		*heap_ptr++ = ptr[1];
-	    } else if (in_area(ptr, src, src_size)) {
+	    } else if (ErtsInArea(ptr, src, src_size)) {
 		MOVE_CONS(ptr,val,htop,heap_ptr++);
 	    } else {
 		heap_ptr++;
@@ -1905,7 +1905,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
 		    if (IS_MOVED_BOXED(val)) {
 			*origptr = val;
 			mb->base = binary_bytes(*origptr);
-		    } else if (in_area(ptr, src, src_size)) {
+		    } else if (ErtsInArea(ptr, src, src_size)) {
 			MOVE_BOXED(ptr,val,htop,origptr);
 			mb->base = binary_bytes(*origptr);
 		    }
@@ -2340,11 +2340,11 @@ sweep_off_heap(Process *p, int fullsweep)
      */
     while (ptr) {
 	if (IS_MOVED_BOXED(ptr->thing_word)) {
-	    ASSERT(!in_area(ptr, oheap, oheap_sz));
+	    ASSERT(!ErtsInArea(ptr, oheap, oheap_sz));
 	    *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word);
 	    ASSERT(!IS_MOVED_BOXED(ptr->thing_word));
 	    if (ptr->thing_word == HEADER_PROC_BIN) {
-		int to_new_heap = !in_area(ptr, oheap, oheap_sz);
+		int to_new_heap = !ErtsInArea(ptr, oheap, oheap_sz);
 		ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1)));
 		if (to_new_heap) {
 		    bin_vheap += ptr->size / sizeof(Eterm);
@@ -2358,7 +2358,7 @@ sweep_off_heap(Process *p, int fullsweep)
 		ptr = ptr->next;
 	    }
 	}
-	else if (!in_area(ptr, oheap, oheap_sz)) {
+	else if (!ErtsInArea(ptr, oheap, oheap_sz)) {
 	    /* garbage */
 	    switch (thing_subtag(ptr->thing_word)) {
 	    case REFC_BINARY_SUBTAG:
@@ -2390,7 +2390,7 @@ sweep_off_heap(Process *p, int fullsweep)
      * generational collection - keep objects in list.
      */
     while (ptr) {
-	ASSERT(in_area(ptr, oheap, oheap_sz));
+	ASSERT(ErtsInArea(ptr, oheap, oheap_sz));
 	ASSERT(!IS_MOVED_BOXED(ptr->thing_word));       
 	if (ptr->thing_word == HEADER_PROC_BIN) {
 	    BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
@@ -2480,7 +2480,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 	switch (primary_tag(val)) {
 	  case TAG_PRIMARY_LIST:
 	  case TAG_PRIMARY_BOXED:
-	      if (in_area(ptr_val(val), area, area_size)) {
+	      if (ErtsInArea(ptr_val(val), area, area_size)) {
 		  *hp = offset_ptr(val, offs);
 	      }
 	      hp++;
@@ -2502,7 +2502,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 		  {
 		      struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp;
 
-		      if (in_area(oh->next, area, area_size)) {
+		      if (ErtsInArea(oh->next, area, area_size)) {
 			  Eterm** uptr = (Eterm **) (void *) &oh->next;
 			  *uptr += offs; /* Patch the mso chain */
 		      }
@@ -2512,7 +2512,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 		{	
 		  ErlBinMatchState *ms = (ErlBinMatchState*) hp;
 		  ErlBinMatchBuffer *mb = &(ms->mb);
-		  if (in_area(ptr_val(mb->orig), area, area_size)) {
+		  if (ErtsInArea(ptr_val(mb->orig), area, area_size)) {
 		      mb->orig = offset_ptr(mb->orig, offs);
 		      mb->base = binary_bytes(mb->orig);
 		  }
@@ -2542,7 +2542,7 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 	switch (primary_tag(val)) {
 	case TAG_PRIMARY_LIST:
 	case TAG_PRIMARY_BOXED:
-	    if (in_area(ptr_val(val), area, area_size)) {
+	    if (ErtsInArea(ptr_val(val), area, area_size)) {
 		*hp = offset_ptr(val, offs);
 	    }
 	    hp++;
@@ -2557,7 +2557,7 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
 static void
 offset_off_heap(Process* p, Sint offs, char* area, Uint area_size)
 {
-    if (MSO(p).first && in_area((Eterm *)MSO(p).first, area, area_size)) {
+    if (MSO(p).first && ErtsInArea((Eterm *)MSO(p).first, area, area_size)) {
         Eterm** uptr = (Eterm**) (void *) &MSO(p).first;
         *uptr += offs;
     }
@@ -2577,19 +2577,19 @@ offset_mqueue(Process *p, Sint offs, char* area, Uint area_size)
 	    switch (primary_tag(mesg)) {
 	    case TAG_PRIMARY_LIST:
 	    case TAG_PRIMARY_BOXED:
-		if (in_area(ptr_val(mesg), area, area_size)) {
+		if (ErtsInArea(ptr_val(mesg), area, area_size)) {
 		    ERL_MESSAGE_TERM(mp) = offset_ptr(mesg, offs);
 		}
 		break;
 	    }
 	}
 	mesg = ERL_MESSAGE_TOKEN(mp);
-	if (is_boxed(mesg) && in_area(ptr_val(mesg), area, area_size)) {
+	if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) {
 	    ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs);
         }
 #ifdef USE_VM_PROBES
 	mesg = ERL_MESSAGE_DT_UTAG(mp);
-	if (is_boxed(mesg) && in_area(ptr_val(mesg), area, area_size)) {
+	if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) {
 	    ERL_MESSAGE_DT_UTAG(mp) = offset_ptr(mesg, offs);
         }
 #endif	
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index d63e81cb5e..5db7ee6d7c 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -21,6 +21,19 @@
 #ifndef __SYS_H__
 #define __SYS_H__
 
+#if !defined(__GNUC__)
+#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
+#elif !defined(__GNUC_MINOR__)
+#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+  ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#elif !defined(__GNUC_PATCHLEVEL__)
+#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#else
+#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#endif
+
 #ifdef ERTS_INLINE
 #  ifndef ERTS_CAN_INLINE
 #    define ERTS_CAN_INLINE 1
@@ -38,6 +51,17 @@
 #  endif
 #endif
 
+#ifndef ERTS_FORCE_INLINE
+#  if ERTS_AT_LEAST_GCC_VSN__(3,1,1)
+#    define ERTS_FORCE_INLINE __inline__ __attribute__((__always_inline__))
+#  elif defined(__WIN32__)
+#    define ERTS_FORCE_INLINE __forceinline
+#  endif
+#  ifndef ERTS_FORCE_INLINE
+#    define ERTS_FORCE_INLINE ERTS_INLINE
+#  endif
+#endif
+
 #if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
 #  undef ERTS_CAN_INLINE
 #  define ERTS_CAN_INLINE 0
@@ -112,19 +136,6 @@ typedef int ErtsSysFdType;
 typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
 #endif
 
-#if !defined(__GNUC__)
-#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
-#elif !defined(__GNUC_MINOR__)
-#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
-  ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
-#elif !defined(__GNUC_PATCHLEVEL__)
-#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
-  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
-#else
-#  define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
-  (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
-#endif
-
 #if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0)
 #  define ERTS_LIKELY(BOOL)   __builtin_expect((BOOL), !0)
 #  define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0)