aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRickard Green <rickard@erlang.org>2015-09-16 15:02:19 +0200
committerRickard Green <rickard@erlang.org>2015-11-12 15:25:47 +0100
commitec4a7e7150c47523a553cac806c86ec08ab2dae7 (patch)
tree75263372c8351e10c771534dd13cb40d106e757a
parentb21b604137c5cb5f5039a40994e429871e5b707b (diff)
downloadotp-ec4a7e7150c47523a553cac806c86ec08ab2dae7.tar.gz
otp-ec4a7e7150c47523a553cac806c86ec08ab2dae7.tar.bz2
otp-ec4a7e7150c47523a553cac806c86ec08ab2dae7.zip
Refactor GC
-rw-r--r--erts/emulator/beam/erl_gc.c620
-rw-r--r--erts/emulator/beam/sys.h37
2 files changed, 334 insertions, 323 deletions
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index e316ab95ab..2d7b7cafa4 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -42,11 +42,76 @@
#include "dtrace-wrapper.h"
#include "erl_bif_unique.h"
+#define ERTS_CONTINUOUS_NEW_HEAP
+
#define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
#define ERTS_INACT_WR_PB_LEAVE_MUCH_PERCENTAGE 20
#define ERTS_INACT_WR_PB_LEAVE_LIMIT 10
#define ERTS_INACT_WR_PB_LEAVE_PERCENTAGE 10
+#if defined(DEBUG) || 0
+#define ERTS_GC_DEBUG
+#else
+#undef ERTS_GC_DEBUG
+#endif
+#ifdef ERTS_GC_DEBUG
+# define ERTS_GC_ASSERT ASSERT
+#else
+# define ERTS_GC_ASSERT(B) ((void) 1)
+#endif
+
+#ifdef ERTS_CONTINUOUS_NEW_HEAP
+#define ERTS_IS_NEW_HEAP_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz) \
+ ErtsInArea((Ptr), (NhPtr), (NhSz))
+#define ERTS_IS_LITERAL_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz) \
+ (!ErtsInArea((Ptr), (NhPtr), (NhSz)) \
+ && !ErtsInArea((Ptr), (OhPtr), (OhSz)))
+
+#ifdef ERTS_GC_DEBUG
+#define ERTS_IS_NEW_HEAP_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz) \
+ (ERTS_IS_NEW_HEAP_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
+ ? (ERTS_GC_ASSERT(!erts_is_literal((TPtr), (Ptr)) \
+ && !ErtsInArea((Ptr), (OhPtr), (OhSz))), 1) \
+ : (ERTS_GC_ASSERT(erts_is_literal((TPtr), (Ptr)) \
+ || ErtsInArea((Ptr), (OhPtr), (OhSz))), 0))
+#define ERTS_IS_LITERAL_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz) \
+ (ERTS_IS_LITERAL_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
+ ? (ERTS_GC_ASSERT(erts_is_literal((TPtr), (Ptr))), 1) \
+ : (ERTS_GC_ASSERT(!erts_is_literal((TPtr), (Ptr))), 0))
+#endif
+
+#else
+
+#define ERTS_IS_NEW_HEAP_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz) \
+ (!erts_is_literal((TPtr), (Ptr)) && !ErtsInArea((Ptr), (OhPtr), (OhSz)))
+#define ERTS_IS_LITERAL_PTR__(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz) \
+ (erts_is_literal((TPtr), (Ptr)))
+
+#ifdef ERTS_GC_DEBUG
+#define ERTS_IS_NEW_HEAP_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz) \
+ (ERTS_IS_NEW_HEAP_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
+ ? (ERTS_GC_ASSERT(ErtsInArea((Ptr), (NhPtr), (NhSz))), 1) \
+ : (ERTS_GC_ASSERT(!ErtsInArea((Ptr), (NhPtr), (NhSz))), 0))
+#define ERTS_IS_LITERAL_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz) \
+ (ERTS_IS_LITERAL_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz)) \
+ ? (ERTS_GC_ASSERT(!ErtsInArea((Ptr), (NhPtr), (NhSz)) \
+ && !ErtsInArea((Ptr), (OhPtr), (OhSz))), 1) \
+ : (ERTS_GC_ASSERT(ErtsInArea((Ptr), (NhPtr), (NhSz)) \
+ || ErtsInArea((Ptr), (OhPtr), (OhSz))), 0))
+#endif
+
+#endif
+
+#ifndef ERTS_IS_NEW_HEAP_PTR
+#define ERTS_IS_NEW_HEAP_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz) \
+ ERTS_IS_NEW_HEAP_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz))
+#endif
+
+#ifndef ERTS_IS_LITERAL_PTR
+#define ERTS_IS_LITERAL_PTR(TPtr, Ptr, NhPtr, NhSz, OhPtr, OhSz) \
+ ERTS_IS_LITERAL_PTR__((TPtr), (Ptr), (NhPtr), (NhSz), (OhPtr), (OhSz))
+#endif
+
/*
* Returns number of elements in an array.
*/
@@ -100,13 +165,27 @@ static Uint setup_rootset(Process*, Eterm*, int, Rootset*);
static void cleanup_rootset(Rootset *rootset);
static Uint combined_message_size(Process* p);
static void remove_message_buffers(Process* p);
+static Eterm *full_sweep_heaps(Process *p,
+ int hibernate,
+ Eterm *n_heap, Eterm* n_htop,
+ char *h, Uint h_size,
+ char *oh, Uint oh_size,
+ Eterm *objv, int nobj);
static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
static void do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj);
-static Eterm* sweep_rootset(Rootset *rootset, Eterm* htop, char* src, Uint src_size);
-static Eterm* sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size);
-static Eterm* sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
- char* src, Uint src_size);
+static Eterm *sweep_new_heap(Eterm *n_hp, Eterm *n_htop,
+ char* new_heap, Uint new_heap_size,
+ char* old_heap, Uint old_heap_size);
+static Eterm *sweep_heaps(Eterm *n_hp, Eterm *n_htop,
+ char* new_heap, Uint new_heap_size,
+ char* old_heap, Uint old_heap_size);
+static Eterm* sweep_literal_area(Eterm* n_hp, Eterm* n_htop,
+ char* new_heap, Uint new_heap_size,
+ char* old_heap, Uint old_heap_size,
+ char* src, Uint src_size);
+static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
+ char* src, Uint src_size);
static Eterm* collect_heap_frags(Process* p, Eterm* heap,
Eterm* htop, Eterm* objv, int nobj);
static void adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj);
@@ -514,9 +593,6 @@ erts_garbage_collect_hibernate(Process* p)
Uint heap_size;
Eterm* heap;
Eterm* htop;
- Rootset rootset;
- char* src;
- Uint src_size;
Uint actual_size;
char* area;
Uint area_size;
@@ -544,41 +620,16 @@ erts_garbage_collect_hibernate(Process* p)
sizeof(Eterm)*heap_size);
htop = heap;
- (void) setup_rootset(p, p->arg_reg, p->arity, &rootset);
-#if HIPE
- hipe_empty_nstack(p);
-#endif
-
- src = (char *) p->heap;
- src_size = (char *) p->htop - src;
- htop = sweep_rootset(&rootset, htop, src, src_size);
- htop = sweep_one_area(heap, htop, src, src_size);
-
- if (p->old_heap) {
- src = (char *) p->old_heap;
- src_size = (char *) p->old_htop - src;
- htop = sweep_rootset(&rootset, htop, src, src_size);
- htop = sweep_one_area(heap, htop, src, src_size);
- }
-
- cleanup_rootset(&rootset);
-
- if (MSO(p).first) {
- sweep_off_heap(p, 1);
- }
-
- /*
- * Update all pointers.
- */
- ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
- (void*)HEAP_START(p),
- HEAP_SIZE(p) * sizeof(Eterm));
- if (p->old_heap) {
- ERTS_HEAP_FREE(ERTS_ALC_T_OLD_HEAP,
- (void*)p->old_heap,
- (p->old_hend - p->old_heap) * sizeof(Eterm));
- p->old_heap = p->old_htop = p->old_hend = 0;
- }
+ htop = full_sweep_heaps(p,
+ 1,
+ heap,
+ htop,
+ (char *) p->heap,
+ (char *) p->htop - (char *) p->heap,
+ (char *) p->old_heap,
+ (char *) p->old_htop - (char *) p->old_heap,
+ p->arg_reg,
+ p->arity);
p->heap = heap;
p->high_water = htop;
@@ -724,7 +775,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
*g_ptr++ = val;
- } else if (in_area(ptr, area, area_size)) {
+ } else if (ErtsInArea(ptr, area, area_size)) {
MOVE_BOXED(ptr,val,old_htop,g_ptr++);
} else {
g_ptr++;
@@ -735,7 +786,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
val = *ptr;
if (IS_MOVED_CONS(val)) { /* Moved */
*g_ptr++ = ptr[1];
- } else if (in_area(ptr, area, area_size)) {
+ } else if (ErtsInArea(ptr, area, area_size)) {
MOVE_CONS(ptr,val,old_htop,g_ptr++);
} else {
g_ptr++;
@@ -755,8 +806,11 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
* Now we'll have to go through all heaps updating all other references.
*/
- old_htop = sweep_one_heap(p->heap, p->htop, old_htop, area, area_size);
- old_htop = sweep_one_area(p->old_heap, old_htop, area, area_size);
+ old_htop = sweep_literals_to_old_heap(p->heap, p->htop, old_htop, area, area_size);
+ old_htop = sweep_literal_area(p->old_heap, old_htop,
+ (char *) p->heap, sizeof(Eterm)*p->heap_sz,
+ (char *) p->old_heap, sizeof(Eterm)*old_heap_size,
+ area, area_size);
ASSERT(p->old_htop <= old_htop && old_htop <= p->old_hend);
p->old_htop = old_htop;
@@ -1034,15 +1088,13 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
*g_ptr++ = val;
- } else if (in_area(ptr, heap, mature_size)) {
+ } else if (ErtsInArea(ptr, heap, mature_size)) {
MOVE_BOXED(ptr,val,old_htop,g_ptr++);
- } else if (in_area(ptr, heap, heap_size)) {
- ASSERT(!erts_is_literal(gval, ptr)
- && !in_area(ptr, oh, oh_size));
+ } else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
+ heap, heap_size,
+ oh, oh_size)) {
MOVE_BOXED(ptr,val,n_htop,g_ptr++);
} else {
- ASSERT(erts_is_literal(gval, ptr)
- || in_area(ptr, oh, oh_size));
g_ptr++;
}
break;
@@ -1053,15 +1105,13 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
val = *ptr;
if (IS_MOVED_CONS(val)) { /* Moved */
*g_ptr++ = ptr[1];
- } else if (in_area(ptr, heap, mature_size)) {
+ } else if (ErtsInArea(ptr, heap, mature_size)) {
MOVE_CONS(ptr,val,old_htop,g_ptr++);
- } else if (in_area(ptr, heap, heap_size)) {
- ASSERT(!erts_is_literal(gval, ptr)
- && !in_area(ptr, oh, oh_size));
+ } else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
+ heap, heap_size,
+ oh, oh_size)) {
MOVE_CONS(ptr,val,n_htop,g_ptr++);
} else {
- ASSERT(erts_is_literal(gval, ptr)
- || in_area(ptr, oh, oh_size));
g_ptr++;
}
break;
@@ -1084,7 +1134,8 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
*/
if (mature_size == 0) {
- n_htop = sweep_one_area(n_heap, n_htop, heap, heap_size);
+ n_htop = sweep_new_heap(n_heap, n_htop, heap, heap_size,
+ oh, oh_size);
} else {
Eterm* n_hp = n_heap;
Eterm* ptr;
@@ -1101,15 +1152,13 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
*n_hp++ = val;
- } else if (in_area(ptr, heap, mature_size)) {
+ } else if (ErtsInArea(ptr, heap, mature_size)) {
MOVE_BOXED(ptr,val,old_htop,n_hp++);
- } else if (in_area(ptr, heap, heap_size)) {
- ASSERT(!erts_is_literal(gval, ptr)
- && !in_area(ptr, oh, oh_size));
+ } else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
+ heap, heap_size,
+ oh, oh_size)) {
MOVE_BOXED(ptr,val,n_htop,n_hp++);
} else {
- ASSERT(erts_is_literal(gval, ptr)
- || in_area(ptr, oh, oh_size));
n_hp++;
}
break;
@@ -1119,15 +1168,13 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
val = *ptr;
if (IS_MOVED_CONS(val)) {
*n_hp++ = ptr[1];
- } else if (in_area(ptr, heap, mature_size)) {
+ } else if (ErtsInArea(ptr, heap, mature_size)) {
MOVE_CONS(ptr,val,old_htop,n_hp++);
- } else if (in_area(ptr, heap, heap_size)) {
- ASSERT(!erts_is_literal(gval, ptr)
- && !in_area(ptr, oh, oh_size));
+ } else if (ERTS_IS_NEW_HEAP_PTR(gval, ptr,
+ heap, heap_size,
+ oh, oh_size)) {
MOVE_CONS(ptr,val,n_htop,n_hp++);
} else {
- ASSERT(erts_is_literal(gval, ptr)
- || in_area(ptr, oh, oh_size));
n_hp++;
}
break;
@@ -1145,19 +1192,15 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
if (IS_MOVED_BOXED(val)) {
*origptr = val;
mb->base = binary_bytes(val);
- } else if (in_area(ptr, heap, mature_size)) {
+ } else if (ErtsInArea(ptr, heap, mature_size)) {
MOVE_BOXED(ptr,val,old_htop,origptr);
mb->base = binary_bytes(mb->orig);
- } else if (in_area(ptr, heap, heap_size)) {
- ASSERT(!erts_is_literal(*origptr, origptr)
- && !in_area(ptr, oh, oh_size));
+ } else if (ERTS_IS_NEW_HEAP_PTR(*origptr, ptr,
+ heap, heap_size,
+ oh, oh_size)) {
MOVE_BOXED(ptr,val,n_htop,origptr);
mb->base = binary_bytes(mb->orig);
}
- else {
- ASSERT(erts_is_literal(*origptr, origptr)
- || in_area(ptr, oh, oh_size));
- }
}
n_hp += (thing_arityval(gval)+1);
}
@@ -1176,7 +1219,9 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
*/
if (OLD_HTOP(p) < old_htop) {
- old_htop = sweep_one_area(OLD_HTOP(p), old_htop, heap, heap_size);
+ old_htop = sweep_new_heap(OLD_HTOP(p), old_htop,
+ heap, heap_size,
+ oh, oh_size);
}
OLD_HTOP(p) = old_htop;
HIGH_WATER(p) = n_htop;
@@ -1229,8 +1274,6 @@ do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
static int
major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
{
- Rootset rootset;
- Roots* roots;
const Uint size_before = ((HEAP_TOP(p) - HEAP_START(p))
+ (OLD_HTOP(p) - OLD_HEAP(p))
+ MBUF_SIZE(p));
@@ -1240,8 +1283,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
Uint src_size = (char *) HEAP_TOP(p) - src;
char* oh = (char *) OLD_HEAP(p);
Uint oh_size = (char *) OLD_HTOP(p) - oh;
- Uint n;
- Uint new_sz;
+ Uint new_sz, stk_sz;
/*
* Do a fullsweep GC. First figure out the size of the heap
@@ -1272,13 +1314,93 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
n_htop = collect_heap_frags(p, n_heap, n_htop, objv, nobj);
}
+ n_htop = full_sweep_heaps(p, 0, n_heap, n_htop, src, src_size,
+ oh, oh_size, objv, nobj);
+
+ /* Move the stack to the end of the heap */
+ stk_sz = HEAP_END(p) - p->stop;
+ sys_memcpy(n_heap + new_sz - stk_sz, p->stop, stk_sz * sizeof(Eterm));
+ p->stop = n_heap + new_sz - stk_sz;
+
+#ifdef USE_VM_PROBES
+ if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) {
+ DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
+
+ dtrace_proc_str(p, pidbuf);
+ DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz);
+ }
+#endif
+
+ ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
+ (void *) HEAP_START(p),
+ (HEAP_END(p) - HEAP_START(p)) * sizeof(Eterm));
+ HEAP_START(p) = n_heap;
+ HEAP_TOP(p) = n_htop;
+ HEAP_SIZE(p) = new_sz;
+ HEAP_END(p) = n_heap + new_sz;
+ GEN_GCS(p) = 0;
+
+ HIGH_WATER(p) = HEAP_TOP(p);
+
+ ErtsGcQuickSanityCheck(p);
+
+ *recl += size_before - (HEAP_TOP(p) - HEAP_START(p));
+
+ {
+ ErlMessage *msgp;
+
+ /*
+ * Copy newly received message onto the end of the new heap.
+ */
+ for (msgp = p->msg.first; msgp; msgp = msgp->next) {
+ if (msgp->data.attached) {
+ ErtsHeapFactory factory;
+ erts_factory_proc_prealloc_init(&factory, p,
+ erts_msg_attached_data_size(msgp));
+ erts_move_msg_attached_data_to_heap(&factory, msgp);
+ erts_factory_close(&factory);
+ ErtsGcQuickSanityCheck(p);
+ }
+ }
+ }
+
+ adjust_after_fullsweep(p, need, objv, nobj);
+
+#ifdef HARDDEBUG
+ disallow_heap_frag_ref_in_heap(p);
+#endif
+ remove_message_buffers(p);
+
+ ErtsGcQuickSanityCheck(p);
+ return 1; /* We are done. */
+}
+
+static Eterm *
+full_sweep_heaps(Process *p,
+ int hibernate,
+ Eterm *n_heap, Eterm* n_htop,
+ char *h, Uint h_size,
+ char *oh, Uint oh_size,
+ Eterm *objv, int nobj)
+{
+ Rootset rootset;
+ Roots *roots;
+ Uint n;
+
/*
* Copy all top-level terms directly referenced by the rootset to
* the new new_heap.
*/
n = setup_rootset(p, objv, nobj, &rootset);
- n_htop = fullsweep_nstack(p, n_htop);
+
+#ifdef HIPE
+ if (hibernate)
+ hipe_empty_nstack(p);
+ else
+ n_htop = fullsweep_nstack(p, n_htop);
+#endif
+
roots = rootset.roots;
while (n--) {
Eterm* g_ptr = roots->v;
@@ -1298,11 +1420,11 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
*g_ptr++ = val;
- } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
- ASSERT(!erts_is_literal(gval, ptr));
+ } else if (!ERTS_IS_LITERAL_PTR(gval, ptr,
+ h, h_size,
+ oh, oh_size)) {
MOVE_BOXED(ptr,val,n_htop,g_ptr++);
} else {
- ASSERT(erts_is_literal(gval, ptr));
g_ptr++;
}
continue;
@@ -1313,11 +1435,11 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
val = *ptr;
if (IS_MOVED_CONS(val)) {
*g_ptr++ = ptr[1];
- } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
- ASSERT(!erts_is_literal(gval, ptr));
+ } else if (!ERTS_IS_LITERAL_PTR(gval, ptr,
+ h, h_size,
+ oh, oh_size)) {
MOVE_CONS(ptr,val,n_htop,g_ptr++);
} else {
- ASSERT(erts_is_literal(gval, ptr));
g_ptr++;
}
continue;
@@ -1340,82 +1462,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
* until all is copied.
*/
- if (oh_size == 0) {
- n_htop = sweep_one_area(n_heap, n_htop, src, src_size);
- } else {
- Eterm* n_hp = n_heap;
-
- while (n_hp != n_htop) {
- Eterm* ptr;
- Eterm val;
- Eterm gval = *n_hp;
-
- switch (primary_tag(gval)) {
- case TAG_PRIMARY_BOXED: {
- ptr = boxed_val(gval);
- val = *ptr;
- if (IS_MOVED_BOXED(val)) {
- ASSERT(is_boxed(val));
- *n_hp++ = val;
- } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
- ASSERT(!erts_is_literal(gval, ptr));
- MOVE_BOXED(ptr,val,n_htop,n_hp++);
- } else {
- ASSERT(erts_is_literal(gval, ptr));
- n_hp++;
- }
- break;
- }
- case TAG_PRIMARY_LIST: {
- ptr = list_val(gval);
- val = *ptr;
- if (IS_MOVED_CONS(val)) {
- *n_hp++ = ptr[1];
- } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
- ASSERT(!erts_is_literal(gval, ptr));
- MOVE_CONS(ptr,val,n_htop,n_hp++);
- } else {
- ASSERT(erts_is_literal(gval, ptr));
- n_hp++;
- }
- break;
- }
- case TAG_PRIMARY_HEADER: {
- if (!header_is_thing(gval))
- n_hp++;
- else {
- if (header_is_bin_matchstate(gval)) {
- ErlBinMatchState *ms = (ErlBinMatchState*) n_hp;
- ErlBinMatchBuffer *mb = &(ms->mb);
- Eterm* origptr;
- origptr = &(mb->orig);
- ptr = boxed_val(*origptr);
- val = *ptr;
- if (IS_MOVED_BOXED(val)) {
- *origptr = val;
- mb->base = binary_bytes(*origptr);
- } else if (in_area(ptr, src, src_size) ||
-- in_area(ptr, oh, oh_size)) {
- ASSERT(!erts_is_literal(*origptr, origptr));
- MOVE_BOXED(ptr,val,n_htop,origptr);
- mb->base = binary_bytes(*origptr);
- ptr = boxed_val(*origptr);
- val = *ptr;
- }
- else {
- ASSERT(erts_is_literal(*origptr, origptr));
- }
- }
- n_hp += (thing_arityval(gval)+1);
- }
- break;
- }
- default:
- n_hp++;
- break;
- }
- }
- }
+ n_htop = sweep_heaps(n_heap, n_htop, h, h_size, oh, oh_size);
if (MSO(p).first) {
sweep_off_heap(p, 1);
@@ -1428,62 +1475,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
OLD_HEAP(p) = OLD_HTOP(p) = OLD_HEND(p) = NULL;
}
- /* Move the stack to the end of the heap */
- n = HEAP_END(p) - p->stop;
- sys_memcpy(n_heap + new_sz - n, p->stop, n * sizeof(Eterm));
- p->stop = n_heap + new_sz - n;
-
-#ifdef USE_VM_PROBES
- if (HEAP_SIZE(p) != new_sz && DTRACE_ENABLED(process_heap_grow)) {
- DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE);
-
- dtrace_proc_str(p, pidbuf);
- DTRACE3(process_heap_grow, pidbuf, HEAP_SIZE(p), new_sz);
- }
-#endif
-
- ERTS_HEAP_FREE(ERTS_ALC_T_HEAP,
- (void *) HEAP_START(p),
- (HEAP_END(p) - HEAP_START(p)) * sizeof(Eterm));
- HEAP_START(p) = n_heap;
- HEAP_TOP(p) = n_htop;
- HEAP_SIZE(p) = new_sz;
- HEAP_END(p) = n_heap + new_sz;
- GEN_GCS(p) = 0;
-
- HIGH_WATER(p) = HEAP_TOP(p);
-
- ErtsGcQuickSanityCheck(p);
-
- *recl += size_before - (HEAP_TOP(p) - HEAP_START(p));
-
- {
- ErlMessage *msgp;
-
- /*
- * Copy newly received message onto the end of the new heap.
- */
- for (msgp = p->msg.first; msgp; msgp = msgp->next) {
- if (msgp->data.attached) {
- ErtsHeapFactory factory;
- erts_factory_proc_prealloc_init(&factory, p,
- erts_msg_attached_data_size(msgp));
- erts_move_msg_attached_data_to_heap(&factory, msgp);
- erts_factory_close(&factory);
- ErtsGcQuickSanityCheck(p);
- }
- }
- }
-
- adjust_after_fullsweep(p, need, objv, nobj);
-
-#ifdef HARDDEBUG
- disallow_heap_frag_ref_in_heap(p);
-#endif
- remove_message_buffers(p);
-
- ErtsGcQuickSanityCheck(p);
- return 1; /* We are done. */
+ return n_htop;
}
static void
@@ -1585,7 +1577,7 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj)
objv++;
} else {
for (qb = mbuf; qb != NULL; qb = qb->next) {
- if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+ if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
abort();
}
}
@@ -1601,7 +1593,7 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj)
objv++;
} else {
for (qb = mbuf; qb != NULL; qb = qb->next) {
- if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+ if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
abort();
}
}
@@ -1644,9 +1636,9 @@ disallow_heap_frag_ref_in_heap(Process* p)
switch (primary_tag(val)) {
case TAG_PRIMARY_BOXED:
ptr = _unchecked_boxed_val(val);
- if (!in_area(ptr, heap, heap_size)) {
+ if (!ErtsInArea(ptr, heap, heap_size)) {
for (qb = MBUF(p); qb != NULL; qb = qb->next) {
- if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+ if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
abort();
}
}
@@ -1654,9 +1646,9 @@ disallow_heap_frag_ref_in_heap(Process* p)
break;
case TAG_PRIMARY_LIST:
ptr = _unchecked_list_val(val);
- if (!in_area(ptr, heap, heap_size)) {
+ if (!ErtsInArea(ptr, heap, heap_size)) {
for (qb = MBUF(p); qb != NULL; qb = qb->next) {
- if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+ if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
abort();
}
}
@@ -1699,12 +1691,12 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
switch (primary_tag(val)) {
case TAG_PRIMARY_BOXED:
ptr = (Eterm *) val;
- if (!in_area(ptr, old_heap, old_heap_size)) {
- if (in_area(ptr, new_heap, new_heap_size)) {
+ if (!ErtsInArea(ptr, old_heap, old_heap_size)) {
+ if (ErtsInArea(ptr, new_heap, new_heap_size)) {
abort();
}
for (qb = MBUF(p); qb != NULL; qb = qb->next) {
- if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+ if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
abort();
}
}
@@ -1712,12 +1704,12 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
break;
case TAG_PRIMARY_LIST:
ptr = (Eterm *) val;
- if (!in_area(ptr, old_heap, old_heap_size)) {
- if (in_area(ptr, new_heap, new_heap_size)) {
+ if (!ErtsInArea(ptr, old_heap, old_heap_size)) {
+ if (ErtsInArea(ptr, new_heap, new_heap_size)) {
abort();
}
for (qb = MBUF(p); qb != NULL; qb = qb->next) {
- if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
+ if (ErtsInArea(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
abort();
}
}
@@ -1726,7 +1718,7 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
case TAG_PRIMARY_HEADER:
if (header_is_thing(val)) {
hp += _unchecked_thing_arityval(val);
- if (!in_area(hp, old_heap, old_heap_size+1)) {
+ if (!ErtsInArea(hp, old_heap, old_heap_size+1)) {
abort();
}
}
@@ -1736,66 +1728,35 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
}
#endif
-static Eterm*
-sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size)
+typedef enum {
+ ErtsSweepNewHeap,
+ ErtsSweepHeaps,
+ ErtsSweepLiteralArea
+} ErtsSweepType;
+
+static ERTS_FORCE_INLINE Eterm *
+sweep(Eterm *n_hp, Eterm *n_htop,
+ ErtsSweepType type,
+ char *h, Uint hsz,
+ char *oh, Uint ohsz,
+ char *src, Uint src_size)
{
- Roots* roots = rootset->roots;
- Uint n = rootset->num_roots;
Eterm* ptr;
- Eterm gval;
Eterm val;
+ Eterm gval;
- while (n--) {
- Eterm* g_ptr = roots->v;
- Uint g_sz = roots->sz;
-
- roots++;
- while (g_sz--) {
- gval = *g_ptr;
-
- switch (primary_tag(gval)) {
- case TAG_PRIMARY_BOXED: {
- ptr = boxed_val(gval);
- val = *ptr;
- if (IS_MOVED_BOXED(val)) {
- ASSERT(is_boxed(val));
- *g_ptr++ = val;
- } else if (in_area(ptr, src, src_size)) {
- MOVE_BOXED(ptr,val,htop,g_ptr++);
- } else {
- g_ptr++;
- }
- break;
- }
- case TAG_PRIMARY_LIST: {
- ptr = list_val(gval);
- val = *ptr;
- if (IS_MOVED_CONS(val)) {
- *g_ptr++ = ptr[1];
- } else if (in_area(ptr, src, src_size)) {
- MOVE_CONS(ptr,val,htop,g_ptr++);
- } else {
- g_ptr++;
- }
- break;
- }
-
- default:
- g_ptr++;
- break;
- }
- }
- }
- return htop;
-}
-
+#undef ERTS_IS_IN_SWEEP_AREA
-static Eterm*
-sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
-{
- Eterm* ptr;
- Eterm val;
- Eterm gval;
+#define ERTS_IS_IN_SWEEP_AREA(TPtr, Ptr) \
+ (type == ErtsSweepHeaps \
+ ? !ERTS_IS_LITERAL_PTR((TPtr), (Ptr), h, hsz, oh, ohsz) \
+ : (type == ErtsSweepNewHeap \
+ ? ERTS_IS_NEW_HEAP_PTR((TPtr), (Ptr), h, hsz, oh, ohsz) \
+ : (ErtsInArea((Ptr), src, src_size) \
+ ? (ERTS_GC_ASSERT(erts_is_literal((TPtr), (Ptr)) \
+ && !ErtsInArea((Ptr), h, hsz) \
+ && !ErtsInArea((Ptr), oh, ohsz)), 1) \
+ : 0)))
while (n_hp != n_htop) {
ASSERT(n_hp < n_htop);
@@ -1807,7 +1768,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
*n_hp++ = val;
- } else if (in_area(ptr, src, src_size)) {
+ } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) {
MOVE_BOXED(ptr,val,n_htop,n_hp++);
} else {
n_hp++;
@@ -1819,7 +1780,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
val = *ptr;
if (IS_MOVED_CONS(val)) {
*n_hp++ = ptr[1];
- } else if (in_area(ptr, src, src_size)) {
+ } else if (ERTS_IS_IN_SWEEP_AREA(gval, ptr)) {
MOVE_CONS(ptr,val,n_htop,n_hp++);
} else {
n_hp++;
@@ -1840,7 +1801,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
if (IS_MOVED_BOXED(val)) {
*origptr = val;
mb->base = binary_bytes(*origptr);
- } else if (in_area(ptr, src, src_size)) {
+ } else if (ERTS_IS_IN_SWEEP_AREA(*origptr, ptr)) {
MOVE_BOXED(ptr,val,n_htop,origptr);
mb->base = binary_bytes(*origptr);
}
@@ -1855,10 +1816,49 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
}
}
return n_htop;
+#undef ERTS_IS_IN_SWEEP_AREA
+}
+
+static Eterm *
+sweep_new_heap(Eterm *n_hp, Eterm *n_htop,
+ char* new_heap, Uint new_heap_size,
+ char* old_heap, Uint old_heap_size)
+{
+ return sweep(n_hp, n_htop,
+ ErtsSweepNewHeap,
+ new_heap, new_heap_size,
+ old_heap, old_heap_size,
+ NULL, 0);
+}
+
+static Eterm *
+sweep_heaps(Eterm *n_hp, Eterm *n_htop,
+ char* new_heap, Uint new_heap_size,
+ char* old_heap, Uint old_heap_size)
+{
+ return sweep(n_hp, n_htop,
+ ErtsSweepHeaps,
+ new_heap, new_heap_size,
+ old_heap, old_heap_size,
+ NULL, 0);
+}
+
+static Eterm *
+sweep_literal_area(Eterm *n_hp, Eterm *n_htop,
+ char* new_heap, Uint new_heap_size,
+ char* old_heap, Uint old_heap_size,
+ char* src, Uint src_size)
+{
+ return sweep(n_hp, n_htop,
+ ErtsSweepLiteralArea,
+ new_heap, new_heap_size,
+ old_heap, old_heap_size,
+ src, src_size);
}
static Eterm*
-sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint src_size)
+sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
+ char* src, Uint src_size)
{
while (heap_ptr < heap_end) {
Eterm* ptr;
@@ -1872,7 +1872,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
*heap_ptr++ = val;
- } else if (in_area(ptr, src, src_size)) {
+ } else if (ErtsInArea(ptr, src, src_size)) {
MOVE_BOXED(ptr,val,htop,heap_ptr++);
} else {
heap_ptr++;
@@ -1884,7 +1884,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
val = *ptr;
if (IS_MOVED_CONS(val)) {
*heap_ptr++ = ptr[1];
- } else if (in_area(ptr, src, src_size)) {
+ } else if (ErtsInArea(ptr, src, src_size)) {
MOVE_CONS(ptr,val,htop,heap_ptr++);
} else {
heap_ptr++;
@@ -1905,7 +1905,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
if (IS_MOVED_BOXED(val)) {
*origptr = val;
mb->base = binary_bytes(*origptr);
- } else if (in_area(ptr, src, src_size)) {
+ } else if (ErtsInArea(ptr, src, src_size)) {
MOVE_BOXED(ptr,val,htop,origptr);
mb->base = binary_bytes(*origptr);
}
@@ -2340,11 +2340,11 @@ sweep_off_heap(Process *p, int fullsweep)
*/
while (ptr) {
if (IS_MOVED_BOXED(ptr->thing_word)) {
- ASSERT(!in_area(ptr, oheap, oheap_sz));
+ ASSERT(!ErtsInArea(ptr, oheap, oheap_sz));
*prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word);
ASSERT(!IS_MOVED_BOXED(ptr->thing_word));
if (ptr->thing_word == HEADER_PROC_BIN) {
- int to_new_heap = !in_area(ptr, oheap, oheap_sz);
+ int to_new_heap = !ErtsInArea(ptr, oheap, oheap_sz);
ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1)));
if (to_new_heap) {
bin_vheap += ptr->size / sizeof(Eterm);
@@ -2358,7 +2358,7 @@ sweep_off_heap(Process *p, int fullsweep)
ptr = ptr->next;
}
}
- else if (!in_area(ptr, oheap, oheap_sz)) {
+ else if (!ErtsInArea(ptr, oheap, oheap_sz)) {
/* garbage */
switch (thing_subtag(ptr->thing_word)) {
case REFC_BINARY_SUBTAG:
@@ -2390,7 +2390,7 @@ sweep_off_heap(Process *p, int fullsweep)
* generational collection - keep objects in list.
*/
while (ptr) {
- ASSERT(in_area(ptr, oheap, oheap_sz));
+ ASSERT(ErtsInArea(ptr, oheap, oheap_sz));
ASSERT(!IS_MOVED_BOXED(ptr->thing_word));
if (ptr->thing_word == HEADER_PROC_BIN) {
BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
@@ -2480,7 +2480,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
switch (primary_tag(val)) {
case TAG_PRIMARY_LIST:
case TAG_PRIMARY_BOXED:
- if (in_area(ptr_val(val), area, area_size)) {
+ if (ErtsInArea(ptr_val(val), area, area_size)) {
*hp = offset_ptr(val, offs);
}
hp++;
@@ -2502,7 +2502,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
{
struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp;
- if (in_area(oh->next, area, area_size)) {
+ if (ErtsInArea(oh->next, area, area_size)) {
Eterm** uptr = (Eterm **) (void *) &oh->next;
*uptr += offs; /* Patch the mso chain */
}
@@ -2512,7 +2512,7 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
{
ErlBinMatchState *ms = (ErlBinMatchState*) hp;
ErlBinMatchBuffer *mb = &(ms->mb);
- if (in_area(ptr_val(mb->orig), area, area_size)) {
+ if (ErtsInArea(ptr_val(mb->orig), area, area_size)) {
mb->orig = offset_ptr(mb->orig, offs);
mb->base = binary_bytes(mb->orig);
}
@@ -2542,7 +2542,7 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
switch (primary_tag(val)) {
case TAG_PRIMARY_LIST:
case TAG_PRIMARY_BOXED:
- if (in_area(ptr_val(val), area, area_size)) {
+ if (ErtsInArea(ptr_val(val), area, area_size)) {
*hp = offset_ptr(val, offs);
}
hp++;
@@ -2557,7 +2557,7 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
static void
offset_off_heap(Process* p, Sint offs, char* area, Uint area_size)
{
- if (MSO(p).first && in_area((Eterm *)MSO(p).first, area, area_size)) {
+ if (MSO(p).first && ErtsInArea((Eterm *)MSO(p).first, area, area_size)) {
Eterm** uptr = (Eterm**) (void *) &MSO(p).first;
*uptr += offs;
}
@@ -2577,19 +2577,19 @@ offset_mqueue(Process *p, Sint offs, char* area, Uint area_size)
switch (primary_tag(mesg)) {
case TAG_PRIMARY_LIST:
case TAG_PRIMARY_BOXED:
- if (in_area(ptr_val(mesg), area, area_size)) {
+ if (ErtsInArea(ptr_val(mesg), area, area_size)) {
ERL_MESSAGE_TERM(mp) = offset_ptr(mesg, offs);
}
break;
}
}
mesg = ERL_MESSAGE_TOKEN(mp);
- if (is_boxed(mesg) && in_area(ptr_val(mesg), area, area_size)) {
+ if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) {
ERL_MESSAGE_TOKEN(mp) = offset_ptr(mesg, offs);
}
#ifdef USE_VM_PROBES
mesg = ERL_MESSAGE_DT_UTAG(mp);
- if (is_boxed(mesg) && in_area(ptr_val(mesg), area, area_size)) {
+ if (is_boxed(mesg) && ErtsInArea(ptr_val(mesg), area, area_size)) {
ERL_MESSAGE_DT_UTAG(mp) = offset_ptr(mesg, offs);
}
#endif
diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h
index d63e81cb5e..5db7ee6d7c 100644
--- a/erts/emulator/beam/sys.h
+++ b/erts/emulator/beam/sys.h
@@ -21,6 +21,19 @@
#ifndef __SYS_H__
#define __SYS_H__
+#if !defined(__GNUC__)
+# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
+#elif !defined(__GNUC_MINOR__)
+# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+ ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#elif !defined(__GNUC_PATCHLEVEL__)
+# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+ (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#else
+# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
+ (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
+#endif
+
#ifdef ERTS_INLINE
# ifndef ERTS_CAN_INLINE
# define ERTS_CAN_INLINE 1
@@ -38,6 +51,17 @@
# endif
#endif
+#ifndef ERTS_FORCE_INLINE
+# if ERTS_AT_LEAST_GCC_VSN__(3,1,1)
+# define ERTS_FORCE_INLINE __inline__ __attribute__((__always_inline__))
+# elif defined(__WIN32__)
+# define ERTS_FORCE_INLINE __forceinline
+# endif
+# ifndef ERTS_FORCE_INLINE
+# define ERTS_FORCE_INLINE ERTS_INLINE
+# endif
+#endif
+
#if defined(DEBUG) || defined(ERTS_ENABLE_LOCK_CHECK)
# undef ERTS_CAN_INLINE
# define ERTS_CAN_INLINE 0
@@ -112,19 +136,6 @@ typedef int ErtsSysFdType;
typedef ERTS_SYS_FD_TYPE ErtsSysFdType;
#endif
-#if !defined(__GNUC__)
-# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) 0
-#elif !defined(__GNUC_MINOR__)
-# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
- ((__GNUC__ << 24) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
-#elif !defined(__GNUC_PATCHLEVEL__)
-# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
- (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12)) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
-#else
-# define ERTS_AT_LEAST_GCC_VSN__(MAJ, MIN, PL) \
- (((__GNUC__ << 24) | (__GNUC_MINOR__ << 12) | __GNUC_PATCHLEVEL__) >= (((MAJ) << 24) | ((MIN) << 12) | (PL)))
-#endif
-
#if ERTS_AT_LEAST_GCC_VSN__(2, 96, 0)
# define ERTS_LIKELY(BOOL) __builtin_expect((BOOL), !0)
# define ERTS_UNLIKELY(BOOL) __builtin_expect((BOOL), 0)