aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/erl_gc.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/erl_gc.c')
-rw-r--r--erts/emulator/beam/erl_gc.c654
1 files changed, 261 insertions, 393 deletions
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index e9bf37a173..e3445bcdc5 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2002-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -33,6 +33,7 @@
#include "erl_gc.h"
#if HIPE
#include "hipe_stack.h"
+#include "hipe_mode_switch.h"
#endif
#define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1
@@ -99,20 +100,18 @@ static Uint combined_message_size(Process* p);
static void remove_message_buffers(Process* p);
static int major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
static int minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl);
-static void do_minor(Process *p, int new_sz, Eterm* objv, int nobj);
+static void do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj);
static Eterm* sweep_rootset(Rootset *rootset, Eterm* htop, char* src, Uint src_size);
static Eterm* sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size);
static Eterm* sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop,
char* src, Uint src_size);
static Eterm* collect_heap_frags(Process* p, Eterm* heap,
Eterm* htop, Eterm* objv, int nobj);
-static Uint adjust_after_fullsweep(Process *p, int size_before,
+static Uint adjust_after_fullsweep(Process *p, Uint size_before,
int need, Eterm *objv, int nobj);
static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj);
static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj);
-static void sweep_proc_bins(Process *p, int fullsweep);
-static void sweep_proc_funs(Process *p, int fullsweep);
-static void sweep_proc_externals(Process *p, int fullsweep);
+static void sweep_off_heap(Process *p, int fullsweep);
static void offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size);
static void offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size);
static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size,
@@ -126,7 +125,7 @@ static void disallow_heap_frag_ref_in_old_heap(Process* p);
static void disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj);
#endif
-#ifdef ARCH_64
+#if defined(ARCH_64) && !HALFWORD_HEAP
# define MAX_HEAP_SIZES 154
#else
# define MAX_HEAP_SIZES 55
@@ -145,6 +144,16 @@ erts_init_gc(void)
{
int i = 0;
+ ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word));
+ ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word));
+ ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header));
+ ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size));
+ ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size));
+ ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size));
+ ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next));
+ ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next));
+ ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next));
+
erts_smp_spinlock_init(&info_lck, "gc_info");
garbage_cols = 0;
reclaimed = 0;
@@ -206,7 +215,7 @@ erts_next_heap_size(Uint size, Uint offset)
low = mid + 1;
}
}
- erl_exit(1, "no next heap size found: %d, offset %d\n", size, offset);
+ erl_exit(1, "no next heap size found: %lu, offset %lu\n", (unsigned long)size, (unsigned long)offset);
}
return 0;
}
@@ -286,25 +295,14 @@ erts_offset_heap_ptr(Eterm* hp, Uint sz, Sint offs,
offset_heap_ptr(hp, sz, offs, (char *) low, ((char *)high)-((char *)low));
}
+
#define ptr_within(ptr, low, high) ((ptr) < (high) && (ptr) >= (low))
void
erts_offset_off_heap(ErlOffHeap *ohp, Sint offs, Eterm* low, Eterm* high)
{
- if (ohp->mso && ptr_within((Eterm *)ohp->mso, low, high)) {
- Eterm** uptr = (Eterm**) (void *) &ohp->mso;
- *uptr += offs;
- }
-
-#ifndef HYBRID /* FIND ME! */
- if (ohp->funs && ptr_within((Eterm *)ohp->funs, low, high)) {
- Eterm** uptr = (Eterm**) (void *) &ohp->funs;
- *uptr += offs;
- }
-#endif
-
- if (ohp->externals && ptr_within((Eterm *)ohp->externals, low, high)) {
- Eterm** uptr = (Eterm**) (void *) &ohp->externals;
+ if (ohp->first && ptr_within((Eterm *)ohp->first, low, high)) {
+ Eterm** uptr = (Eterm**) (void *) &ohp->first;
*uptr += offs;
}
}
@@ -443,7 +441,15 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
p->last_old_htop = p->old_htop;
#endif
- return ((int) (HEAP_TOP(p) - HEAP_START(p)) / 10);
+ /* FIXME: This function should really return an Sint, i.e., a possibly
+ 64 bit wide signed integer, but that requires updating all the code
+ that calls it. For now, we just return INT_MAX if the result is too
+ large for an int. */
+ {
+ Sint result = (HEAP_TOP(p) - HEAP_START(p)) / 10;
+ if (result >= INT_MAX) return INT_MAX;
+ else return (int) result;
+ }
}
/*
@@ -457,7 +463,6 @@ erts_garbage_collect_hibernate(Process* p)
Eterm* heap;
Eterm* htop;
Rootset rootset;
- int n;
char* src;
Uint src_size;
Uint actual_size;
@@ -488,7 +493,10 @@ erts_garbage_collect_hibernate(Process* p)
sizeof(Eterm)*heap_size);
htop = heap;
- n = setup_rootset(p, p->arg_reg, p->arity, &rootset);
+ (void) setup_rootset(p, p->arg_reg, p->arity, &rootset);
+#if HIPE
+ hipe_empty_nstack(p);
+#endif
src = (char *) p->heap;
src_size = (char *) p->htop - src;
@@ -504,14 +512,8 @@ erts_garbage_collect_hibernate(Process* p)
cleanup_rootset(&rootset);
- if (MSO(p).mso) {
- sweep_proc_bins(p, 1);
- }
- if (MSO(p).funs) {
- sweep_proc_funs(p, 1);
- }
- if (MSO(p).externals) {
- sweep_proc_externals(p, 1);
+ if (MSO(p).first) {
+ sweep_off_heap(p, 1);
}
/*
@@ -605,7 +607,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
char* area;
Uint area_size;
Eterm* old_htop;
- int n;
+ Uint n;
/*
* Set GC state.
@@ -667,7 +669,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
case TAG_PRIMARY_BOXED:
ptr = boxed_val(gval);
val = *ptr;
- if (IS_MOVED(val)) {
+ if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
*g_ptr++ = val;
} else if (in_area(ptr, area, area_size)) {
@@ -679,7 +681,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size)
case TAG_PRIMARY_LIST:
ptr = list_val(gval);
val = *ptr;
- if (is_non_value(val)) { /* Moved */
+ if (IS_MOVED_CONS(val)) { /* Moved */
*g_ptr++ = ptr[1];
} else if (in_area(ptr, area, area_size)) {
MOVE_CONS(ptr,val,old_htop,g_ptr++);
@@ -737,7 +739,7 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
* This improved Estone by more than 1200 estones on my computer
* (Ultra Sparc 10).
*/
- size_t new_sz = erts_next_heap_size(HEAP_TOP(p) - HEAP_START(p), 1);
+ Uint new_sz = erts_next_heap_size(HEAP_TOP(p) - HEAP_START(p), 1);
/* Create new, empty old_heap */
n_old = (Eterm *) ERTS_HEAP_ALLOC(ERTS_ALC_T_OLD_HEAP,
@@ -752,7 +754,10 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
* is large enough.
*/
- if (OLD_HEAP(p) && mature <= OLD_HEND(p) - OLD_HTOP(p)) {
+ if (OLD_HEAP(p) &&
+ ((mature <= OLD_HEND(p) - OLD_HTOP(p)) &&
+ ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) &&
+ ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) {
ErlMessage *msgp;
Uint size_after;
Uint need_after;
@@ -874,12 +879,12 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
#endif /* HIPE */
static void
-do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
+do_minor(Process *p, Uint new_sz, Eterm* objv, int nobj)
{
Rootset rootset; /* Rootset for GC (stack, dictionary, etc). */
Roots* roots;
Eterm* n_htop;
- int n;
+ Uint n;
Eterm* ptr;
Eterm val;
Eterm gval;
@@ -913,7 +918,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
case TAG_PRIMARY_BOXED: {
ptr = boxed_val(gval);
val = *ptr;
- if (IS_MOVED(val)) {
+ if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
*g_ptr++ = val;
} else if (in_area(ptr, heap, mature_size)) {
@@ -929,7 +934,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
case TAG_PRIMARY_LIST: {
ptr = list_val(gval);
val = *ptr;
- if (is_non_value(val)) { /* Moved */
+ if (IS_MOVED_CONS(val)) { /* Moved */
*g_ptr++ = ptr[1];
} else if (in_area(ptr, heap, mature_size)) {
MOVE_CONS(ptr,val,old_htop,g_ptr++);
@@ -972,7 +977,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
case TAG_PRIMARY_BOXED: {
ptr = boxed_val(gval);
val = *ptr;
- if (IS_MOVED(val)) {
+ if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
*n_hp++ = val;
} else if (in_area(ptr, heap, mature_size)) {
@@ -987,7 +992,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
case TAG_PRIMARY_LIST: {
ptr = list_val(gval);
val = *ptr;
- if (is_non_value(val)) {
+ if (IS_MOVED_CONS(val)) {
*n_hp++ = ptr[1];
} else if (in_area(ptr, heap, mature_size)) {
MOVE_CONS(ptr,val,old_htop,n_hp++);
@@ -1008,7 +1013,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
Eterm* origptr = &(mb->orig);
ptr = boxed_val(*origptr);
val = *ptr;
- if (IS_MOVED(val)) {
+ if (IS_MOVED_BOXED(val)) {
*origptr = val;
mb->base = binary_bytes(val);
} else if (in_area(ptr, heap, mature_size)) {
@@ -1041,15 +1046,8 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj)
OLD_HTOP(p) = old_htop;
HIGH_WATER(p) = (HEAP_START(p) != HIGH_WATER(p)) ? n_heap : n_htop;
- if (MSO(p).mso) {
- sweep_proc_bins(p, 0);
- }
-
- if (MSO(p).funs) {
- sweep_proc_funs(p, 0);
- }
- if (MSO(p).externals) {
- sweep_proc_externals(p, 0);
+ if (MSO(p).first) {
+ sweep_off_heap(p, 0);
}
#ifdef HARDDEBUG
@@ -1089,14 +1087,14 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
{
Rootset rootset;
Roots* roots;
- int size_before;
+ Uint size_before;
Eterm* n_heap;
Eterm* n_htop;
char* src = (char *) HEAP_START(p);
Uint src_size = (char *) HEAP_TOP(p) - src;
char* oh = (char *) OLD_HEAP(p);
Uint oh_size = (char *) OLD_HTOP(p) - oh;
- int n;
+ Uint n;
Uint new_sz;
Uint fragments = MBUF_SIZE(p) + combined_message_size(p);
ErlMessage *msgp;
@@ -1161,7 +1159,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
case TAG_PRIMARY_BOXED: {
ptr = boxed_val(gval);
val = *ptr;
- if (IS_MOVED(val)) {
+ if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
*g_ptr++ = val;
} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
@@ -1175,7 +1173,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
case TAG_PRIMARY_LIST: {
ptr = list_val(gval);
val = *ptr;
- if (is_non_value(val)) {
+ if (IS_MOVED_CONS(val)) {
*g_ptr++ = ptr[1];
} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
MOVE_CONS(ptr,val,n_htop,g_ptr++);
@@ -1216,7 +1214,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
case TAG_PRIMARY_BOXED: {
ptr = boxed_val(gval);
val = *ptr;
- if (IS_MOVED(val)) {
+ if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
*n_hp++ = val;
} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
@@ -1229,7 +1227,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
case TAG_PRIMARY_LIST: {
ptr = list_val(gval);
val = *ptr;
- if (is_non_value(val)) {
+ if (IS_MOVED_CONS(val)) {
*n_hp++ = ptr[1];
} else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) {
MOVE_CONS(ptr,val,n_htop,n_hp++);
@@ -1249,7 +1247,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
origptr = &(mb->orig);
ptr = boxed_val(*origptr);
val = *ptr;
- if (IS_MOVED(val)) {
+ if (IS_MOVED_BOXED(val)) {
*origptr = val;
mb->base = binary_bytes(*origptr);
} else if (in_area(ptr, src, src_size) ||
@@ -1271,17 +1269,11 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
}
}
- if (MSO(p).mso) {
- sweep_proc_bins(p, 1);
- }
- if (MSO(p).funs) {
- sweep_proc_funs(p, 1);
- }
- if (MSO(p).externals) {
- sweep_proc_externals(p, 1);
+ if (MSO(p).first) {
+ sweep_off_heap(p, 1);
}
- if (OLD_HEAP(p) != NULL) {
+ if (OLD_HEAP(p) != NULL) {
ERTS_HEAP_FREE(ERTS_ALC_T_OLD_HEAP,
OLD_HEAP(p),
(OLD_HEND(p) - OLD_HEAP(p)) * sizeof(Eterm));
@@ -1305,6 +1297,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
HIGH_WATER(p) = HEAP_TOP(p);
ErtsGcQuickSanityCheck(p);
+
/*
* Copy newly received message onto the end of the new heap.
*/
@@ -1327,10 +1320,10 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl)
}
static Uint
-adjust_after_fullsweep(Process *p, int size_before, int need, Eterm *objv, int nobj)
+adjust_after_fullsweep(Process *p, Uint size_before, int need, Eterm *objv, int nobj)
{
- int wanted, sz, size_after, need_after;
- int stack_size = STACK_SZ_ON_HEAP(p);
+ Uint wanted, sz, size_after, need_after;
+ Uint stack_size = STACK_SZ_ON_HEAP(p);
Uint reclaimed_now;
size_after = (HEAP_TOP(p) - HEAP_START(p));
@@ -1392,17 +1385,12 @@ combined_message_size(Process* p)
static void
remove_message_buffers(Process* p)
{
- ErlHeapFragment* bp = MBUF(p);
-
- MBUF(p) = NULL;
- MBUF_SIZE(p) = 0;
- while (bp != NULL) {
- ErlHeapFragment* next_bp = bp->next;
- free_message_buffer(bp);
- bp = next_bp;
- }
+ if (MBUF(p) != NULL) {
+ free_message_buffer(MBUF(p));
+ MBUF(p) = NULL;
+ }
+ MBUF_SIZE(p) = 0;
}
-
#ifdef HARDDEBUG
/*
@@ -1433,12 +1421,12 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj)
case TAG_PRIMARY_BOXED: {
ptr = _unchecked_boxed_val(gval);
val = *ptr;
- if (IS_MOVED(val)) {
+ if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
objv++;
} else {
for (qb = mbuf; qb != NULL; qb = qb->next) {
- if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+ if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
abort();
}
}
@@ -1450,11 +1438,11 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj)
case TAG_PRIMARY_LIST: {
ptr = _unchecked_list_val(gval);
val = *ptr;
- if (is_non_value(val)) {
+ if (IS_MOVED_CONS(val)) {
objv++;
} else {
for (qb = mbuf; qb != NULL; qb = qb->next) {
- if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+ if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
abort();
}
}
@@ -1499,7 +1487,7 @@ disallow_heap_frag_ref_in_heap(Process* p)
ptr = _unchecked_boxed_val(val);
if (!in_area(ptr, heap, heap_size)) {
for (qb = MBUF(p); qb != NULL; qb = qb->next) {
- if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+ if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
abort();
}
}
@@ -1509,7 +1497,7 @@ disallow_heap_frag_ref_in_heap(Process* p)
ptr = _unchecked_list_val(val);
if (!in_area(ptr, heap, heap_size)) {
for (qb = MBUF(p); qb != NULL; qb = qb->next) {
- if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+ if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
abort();
}
}
@@ -1551,26 +1539,26 @@ disallow_heap_frag_ref_in_old_heap(Process* p)
val = *hp++;
switch (primary_tag(val)) {
case TAG_PRIMARY_BOXED:
- ptr = (Eterm *) val;
+ ptr = (Eterm *) EXPAND_POINTER(val);
if (!in_area(ptr, old_heap, old_heap_size)) {
if (in_area(ptr, new_heap, new_heap_size)) {
abort();
}
for (qb = MBUF(p); qb != NULL; qb = qb->next) {
- if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+ if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
abort();
}
}
}
break;
case TAG_PRIMARY_LIST:
- ptr = (Eterm *) val;
+ ptr = (Eterm *) EXPAND_POINTER(val);
if (!in_area(ptr, old_heap, old_heap_size)) {
if (in_area(ptr, new_heap, new_heap_size)) {
abort();
}
for (qb = MBUF(p); qb != NULL; qb = qb->next) {
- if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) {
+ if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) {
abort();
}
}
@@ -1610,7 +1598,7 @@ sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size)
case TAG_PRIMARY_BOXED: {
ptr = boxed_val(gval);
val = *ptr;
- if (IS_MOVED(val)) {
+ if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
*g_ptr++ = val;
} else if (in_area(ptr, src, src_size)) {
@@ -1623,7 +1611,7 @@ sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size)
case TAG_PRIMARY_LIST: {
ptr = list_val(gval);
val = *ptr;
- if (is_non_value(val)) { /* Moved */
+ if (IS_MOVED_CONS(val)) {
*g_ptr++ = ptr[1];
} else if (in_area(ptr, src, src_size)) {
MOVE_CONS(ptr,val,htop,g_ptr++);
@@ -1657,7 +1645,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
case TAG_PRIMARY_BOXED: {
ptr = boxed_val(gval);
val = *ptr;
- if (IS_MOVED(val)) {
+ if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
*n_hp++ = val;
} else if (in_area(ptr, src, src_size)) {
@@ -1670,7 +1658,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
case TAG_PRIMARY_LIST: {
ptr = list_val(gval);
val = *ptr;
- if (is_non_value(val)) {
+ if (IS_MOVED_CONS(val)) {
*n_hp++ = ptr[1];
} else if (in_area(ptr, src, src_size)) {
MOVE_CONS(ptr,val,n_htop,n_hp++);
@@ -1690,7 +1678,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size)
origptr = &(mb->orig);
ptr = boxed_val(*origptr);
val = *ptr;
- if (IS_MOVED(val)) {
+ if (IS_MOVED_BOXED(val)) {
*origptr = val;
mb->base = binary_bytes(*origptr);
} else if (in_area(ptr, src, src_size)) {
@@ -1722,7 +1710,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
case TAG_PRIMARY_BOXED: {
ptr = boxed_val(gval);
val = *ptr;
- if (IS_MOVED(val)) {
+ if (IS_MOVED_BOXED(val)) {
ASSERT(is_boxed(val));
*heap_ptr++ = val;
} else if (in_area(ptr, src, src_size)) {
@@ -1735,7 +1723,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr
case TAG_PRIMARY_LIST: {
ptr = list_val(gval);
val = *ptr;
- if (is_non_value(val)) {
+ if (IS_MOVED_CONS(val)) {
*heap_ptr++ = ptr[1];
} else if (in_area(ptr, src, src_size)) {
MOVE_CONS(ptr,val,htop,heap_ptr++);
@@ -1830,28 +1818,6 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop,
return n_htop;
}
-#ifdef DEBUG
-static Eterm follow_moved(Eterm term)
-{
- Eterm* ptr;
- switch (primary_tag(term)) {
- case TAG_PRIMARY_IMMED1:
- break;
- case TAG_PRIMARY_BOXED:
- ptr = boxed_val(term);
- if (IS_MOVED(*ptr)) term = *ptr;
- break;
- case TAG_PRIMARY_LIST:
- ptr = list_val(term);
- if (is_non_value(ptr[0])) term = ptr[1];
- break;
- default:
- abort();
- }
- return term;
-}
-#endif
-
static Uint
setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset)
{
@@ -1957,8 +1923,8 @@ static void
grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj)
{
Eterm* new_heap;
- int heap_size = HEAP_TOP(p) - HEAP_START(p);
- int stack_size = p->hend - p->stop;
+ Uint heap_size = HEAP_TOP(p) - HEAP_START(p);
+ Uint stack_size = p->hend - p->stop;
Sint offs;
ASSERT(HEAP_SIZE(p) < new_sz);
@@ -1996,10 +1962,10 @@ static void
shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj)
{
Eterm* new_heap;
- int heap_size = HEAP_TOP(p) - HEAP_START(p);
+ Uint heap_size = HEAP_TOP(p) - HEAP_START(p);
Sint offs;
- int stack_size = p->hend - p->stop;
+ Uint stack_size = p->hend - p->stop;
ASSERT(new_sz < p->heap_sz);
sys_memmove(p->heap + new_sz - stack_size, p->stop, stack_size *
@@ -2030,8 +1996,8 @@ shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj)
HEAP_SIZE(p) = new_sz;
}
-static Uint
-do_next_vheap_size(Uint vheap, Uint vheap_sz) {
+static Uint64
+do_next_vheap_size(Uint64 vheap, Uint64 vheap_sz) {
/* grow
*
@@ -2048,131 +2014,53 @@ do_next_vheap_size(Uint vheap, Uint vheap_sz) {
* ----------------------
*/
- if (vheap > (Uint) (vheap_sz*3/4)) {
+ if ((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) {
+ Uint64 new_vheap_sz = vheap_sz;
- while(vheap > (Uint) (vheap_sz*3/4)) {
- vheap_sz = vheap_sz*2;
+ while((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) {
+ /* the golden ratio = 1.618 */
+ new_vheap_sz = (Uint64) vheap_sz * 1.618;
+ if (new_vheap_sz < vheap_sz ) {
+ return vheap_sz;
+ }
+ vheap_sz = new_vheap_sz;
}
- return erts_next_heap_size(vheap_sz, 0);
+ return vheap_sz;
}
- if (vheap < (Uint) (vheap_sz/4)) {
- return erts_next_heap_size((Uint) (vheap_sz / 2), 0);
+ if (vheap < (Uint64) (vheap_sz/4)) {
+ return (vheap_sz >> 1);
}
return vheap_sz;
}
-static Uint
-next_vheap_size(Process* p, Uint vheap, Uint vheap_sz) {
- vheap_sz = do_next_vheap_size(vheap, vheap_sz);
- return vheap_sz < p->min_vheap_size ? p->min_vheap_size : vheap_sz;
-}
-
-static void
-sweep_proc_externals(Process *p, int fullsweep)
-{
- ExternalThing** prev;
- ExternalThing* ptr;
- char* oh = 0;
- Uint oh_size = 0;
-
- if (fullsweep == 0) {
- oh = (char *) OLD_HEAP(p);
- oh_size = (char *) OLD_HEND(p) - oh;
- }
-
- prev = &MSO(p).externals;
- ptr = MSO(p).externals;
-
- while (ptr) {
- Eterm* ppt = (Eterm *) ptr;
-
- if (IS_MOVED(*ppt)) { /* Object is alive */
- ExternalThing* ro = external_thing_ptr(*ppt);
-
- *prev = ro; /* Patch to moved pos */
- prev = &ro->next;
- ptr = ro->next;
- } else if (in_area(ppt, oh, oh_size)) {
- /*
- * Object resides on old heap, and we just did a
- * generational collection - keep object in list.
- */
- prev = &ptr->next;
- ptr = ptr->next;
- } else { /* Object has not been moved - deref it */
- erts_deref_node_entry(ptr->node);
- *prev = ptr = ptr->next;
- }
- }
- ASSERT(*prev == NULL);
-}
-
-static void
-sweep_proc_funs(Process *p, int fullsweep)
-{
- ErlFunThing** prev;
- ErlFunThing* ptr;
- char* oh = 0;
- Uint oh_size = 0;
-
- if (fullsweep == 0) {
- oh = (char *) OLD_HEAP(p);
- oh_size = (char *) OLD_HEND(p) - oh;
- }
-
- prev = &MSO(p).funs;
- ptr = MSO(p).funs;
-
- while (ptr) {
- Eterm* ppt = (Eterm *) ptr;
-
- if (IS_MOVED(*ppt)) { /* Object is alive */
- ErlFunThing* ro = (ErlFunThing *) fun_val(*ppt);
-
- *prev = ro; /* Patch to moved pos */
- prev = &ro->next;
- ptr = ro->next;
- } else if (in_area(ppt, oh, oh_size)) {
- /*
- * Object resides on old heap, and we just did a
- * generational collection - keep object in list.
- */
- prev = &ptr->next;
- ptr = ptr->next;
- } else { /* Object has not been moved - deref it */
- ErlFunEntry* fe = ptr->fe;
-
- *prev = ptr = ptr->next;
- if (erts_refc_dectest(&fe->refc, 0) == 0) {
- erts_erase_fun_entry(fe);
- }
- }
- }
- ASSERT(*prev == NULL);
+static Uint64
+next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz) {
+ Uint64 new_vheap_sz = do_next_vheap_size(vheap, vheap_sz);
+ return new_vheap_sz < p->min_vheap_size ? p->min_vheap_size : new_vheap_sz;
}
struct shrink_cand_data {
- ProcBin* new_candidates;
- ProcBin* new_candidates_end;
- ProcBin* old_candidates;
+ struct erl_off_heap_header* new_candidates;
+ struct erl_off_heap_header* new_candidates_end;
+ struct erl_off_heap_header* old_candidates;
Uint no_of_candidates;
Uint no_of_active;
};
static ERTS_INLINE void
link_live_proc_bin(struct shrink_cand_data *shrink,
- ProcBin ***prevppp,
- ProcBin **pbpp,
+ struct erl_off_heap_header*** prevppp,
+ struct erl_off_heap_header** currpp,
int new_heap)
{
- ProcBin *pbp = *pbpp;
-
- *pbpp = pbp->next;
+ ProcBin *pbp = (ProcBin*) *currpp;
+ ASSERT(**prevppp == *currpp);
+ *currpp = pbp->next;
if (pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) {
ASSERT(((pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE))
== (PB_ACTIVE_WRITER|PB_IS_WRITABLE))
@@ -2189,15 +2077,16 @@ link_live_proc_bin(struct shrink_cand_data *shrink,
/* Our allocators are 8 byte aligned, i.e., shrinking with
less than 8 bytes will have no real effect */
if (unused >= 8) { /* A shrink candidate; save in candidate list */
+ **prevppp = pbp->next;
if (new_heap) {
if (!shrink->new_candidates)
- shrink->new_candidates_end = pbp;
+ shrink->new_candidates_end = (struct erl_off_heap_header*)pbp;
pbp->next = shrink->new_candidates;
- shrink->new_candidates = pbp;
+ shrink->new_candidates = (struct erl_off_heap_header*)pbp;
}
else {
pbp->next = shrink->old_candidates;
- shrink->old_candidates = pbp;
+ shrink->old_candidates = (struct erl_off_heap_header*)pbp;
}
shrink->no_of_candidates++;
return;
@@ -2205,83 +2094,117 @@ link_live_proc_bin(struct shrink_cand_data *shrink,
}
}
- /* Not a shrink candidate; keep in original mso list */
- **prevppp = pbp;
+ /* Not a shrink candidate; keep in original mso list */
*prevppp = &pbp->next;
-
}
-static void
-sweep_proc_bins(Process *p, int fullsweep)
+static void
+sweep_off_heap(Process *p, int fullsweep)
{
struct shrink_cand_data shrink = {0};
- ProcBin** prev;
- ProcBin* ptr;
- Binary* bptr;
- char* oh = NULL;
- Uint oh_size = 0;
- Uint bin_vheap = 0;
+ struct erl_off_heap_header* ptr;
+ struct erl_off_heap_header** prev;
+ char* oheap = NULL;
+ Uint oheap_sz = 0;
+ Uint64 bin_vheap = 0;
+#ifdef DEBUG
+ int seen_mature = 0;
+#endif
if (fullsweep == 0) {
- oh = (char *) OLD_HEAP(p);
- oh_size = (char *) OLD_HEND(p) - oh;
+ oheap = (char *) OLD_HEAP(p);
+ oheap_sz = (char *) OLD_HEND(p) - oheap;
}
BIN_OLD_VHEAP(p) = 0;
- prev = &MSO(p).mso;
- ptr = MSO(p).mso;
+ prev = &MSO(p).first;
+ ptr = MSO(p).first;
- /*
- * Note: In R7 we no longer force a fullsweep when we find binaries
- * on the old heap. The reason is that with the introduction of the
- * bit syntax we can expect binaries to be used a lot more. Note that
- * in earlier releases a brand new binary (or any other term) could
- * be put on the old heap during a gen-gc fullsweep, but this is
- * no longer the case in R7.
+ /* Firts part of the list will reside on the (old) new-heap.
+ * Keep if moved, otherwise deref.
*/
while (ptr) {
- Eterm* ppt = (Eterm *) ptr;
-
- if (IS_MOVED(*ppt)) { /* Object is alive */
- bin_vheap += ptr->size / sizeof(Eterm);
- ptr = (ProcBin*) binary_val(*ppt);
- link_live_proc_bin(&shrink,
- &prev,
- &ptr,
- !in_area(ptr, oh, oh_size));
- } else if (in_area(ppt, oh, oh_size)) {
- /*
- * Object resides on old heap, and we just did a
- * generational collection - keep object in list.
- */
- BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
- link_live_proc_bin(&shrink, &prev, &ptr, 0);
- } else { /* Object has not been moved - deref it */
-
- *prev = ptr->next;
- bptr = ptr->val;
- if (erts_refc_dectest(&bptr->refc, 0) == 0)
- erts_bin_free(bptr);
- ptr = *prev;
- }
+ if (IS_MOVED_BOXED(ptr->thing_word)) {
+ ASSERT(!in_area(ptr, oheap, oheap_sz));
+ *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word);
+ ASSERT(!IS_MOVED_BOXED(ptr->thing_word));
+ if (ptr->thing_word == HEADER_PROC_BIN) {
+ int to_new_heap = !in_area(ptr, oheap, oheap_sz);
+ ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1)));
+ if (to_new_heap) {
+ bin_vheap += ptr->size / sizeof(Eterm);
+ } else {
+ BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
+ }
+ link_live_proc_bin(&shrink, &prev, &ptr, to_new_heap);
+ }
+ else {
+ prev = &ptr->next;
+ ptr = ptr->next;
+ }
+ }
+ else if (!in_area(ptr, oheap, oheap_sz)) {
+ /* garbage */
+ switch (thing_subtag(ptr->thing_word)) {
+ case REFC_BINARY_SUBTAG:
+ {
+ Binary* bptr = ((ProcBin*)ptr)->val;
+ if (erts_refc_dectest(&bptr->refc, 0) == 0) {
+ erts_bin_free(bptr);
+ }
+ break;
+ }
+ case FUN_SUBTAG:
+ {
+ ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe;
+ if (erts_refc_dectest(&fe->refc, 0) == 0) {
+ erts_erase_fun_entry(fe);
+ }
+ break;
+ }
+ default:
+ ASSERT(is_external_header(ptr->thing_word));
+ erts_deref_node_entry(((ExternalThing*)ptr)->node);
+ }
+ *prev = ptr = ptr->next;
+ }
+ else break; /* and let old-heap loop continue */
}
- if (BIN_OLD_VHEAP(p) >= BIN_OLD_VHEAP_SZ(p)) {
- FLAGS(p) |= F_NEED_FULLSWEEP;
+ /* The rest of the list resides on old-heap, and we just did a
+ * generational collection - keep objects in list.
+ */
+ while (ptr) {
+ ASSERT(in_area(ptr, oheap, oheap_sz));
+ ASSERT(!IS_MOVED_BOXED(ptr->thing_word));
+ if (ptr->thing_word == HEADER_PROC_BIN) {
+ BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/
+ link_live_proc_bin(&shrink, &prev, &ptr, 0);
+ }
+ else {
+ ASSERT(is_fun_header(ptr->thing_word) ||
+ is_external_header(ptr->thing_word));
+ prev = &ptr->next;
+ ptr = ptr->next;
+ }
}
- BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p));
- BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p), BIN_OLD_VHEAP_SZ(p));
- MSO(p).overhead = bin_vheap;
+ if (fullsweep) {
+ BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p) + MSO(p).overhead, BIN_OLD_VHEAP_SZ(p));
+ }
+ BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p));
+ MSO(p).overhead = bin_vheap;
+ BIN_VHEAP_MATURE(p) = bin_vheap;
/*
* If we got any shrink candidates, check them out.
*/
if (shrink.no_of_candidates) {
- ProcBin *candlist[] = {shrink.new_candidates, shrink.old_candidates};
+ ProcBin *candlist[] = { (ProcBin*)shrink.new_candidates,
+ (ProcBin*)shrink.old_candidates };
Uint leave_unused = 0;
int i;
@@ -2293,21 +2216,21 @@ sweep_proc_bins(Process *p, int fullsweep)
}
for (i = 0; i < sizeof(candlist)/sizeof(candlist[0]); i++) {
-
- for (ptr = candlist[i]; ptr; ptr = ptr->next) {
- Uint new_size = ptr->size;
+ ProcBin* pb;
+ for (pb = candlist[i]; pb; pb = (ProcBin*)pb->next) {
+ Uint new_size = pb->size;
if (leave_unused) {
new_size += (new_size * 100) / leave_unused;
/* Our allocators are 8 byte aligned, i.e., shrinking with
less than 8 bytes will have no real effect */
- if (new_size + 8 >= ptr->val->orig_size)
+ if (new_size + 8 >= pb->val->orig_size)
continue;
}
- ptr->val = erts_bin_realloc(ptr->val, new_size);
- ptr->val->orig_size = new_size;
- ptr->bytes = (byte *) ptr->val->orig_bytes;
+ pb->val = erts_bin_realloc(pb->val, new_size);
+ pb->val->orig_size = new_size;
+ pb->bytes = (byte *) pb->val->orig_bytes;
}
}
@@ -2316,21 +2239,20 @@ sweep_proc_bins(Process *p, int fullsweep)
* We now potentially have the mso list divided into three lists:
* - shrink candidates on new heap (inactive writable with unused data)
* - shrink candidates on old heap (inactive writable with unused data)
- * - other binaries (read only + active writable ...)
+ * - other binaries (read only + active writable ...) + funs and externals
*
* Put them back together: new candidates -> other -> old candidates
* This order will ensure that the list only refers from new
* generation to old and never from old to new *which is important*.
*/
if (shrink.new_candidates) {
- if (prev == &MSO(p).mso) /* empty other binaries list */
+ if (prev == &MSO(p).first) /* empty other binaries list */
prev = &shrink.new_candidates_end->next;
else
- shrink.new_candidates_end->next = MSO(p).mso;
- MSO(p).mso = shrink.new_candidates;
+ shrink.new_candidates_end->next = MSO(p).first;
+ MSO(p).first = shrink.new_candidates;
}
}
-
*prev = shrink.old_candidates;
}
@@ -2361,15 +2283,17 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
tari = thing_arityval(val);
switch (thing_subtag(val)) {
case REFC_BINARY_SUBTAG:
+ case FUN_SUBTAG:
+ case EXTERNAL_PID_SUBTAG:
+ case EXTERNAL_PORT_SUBTAG:
+ case EXTERNAL_REF_SUBTAG:
{
- ProcBin* pb = (ProcBin*) hp;
- Eterm** uptr = (Eterm **) (void *) &pb->next;
+ struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp;
- if (*uptr && in_area((Eterm *)pb->next, area, area_size)) {
+ if (in_area(oh->next, area, area_size)) {
+ Eterm** uptr = (Eterm **) (void *) &oh->next;
*uptr += offs; /* Patch the mso chain */
}
- sz -= tari;
- hp += tari + 1;
}
break;
case BIN_MATCHSTATE_SUBTAG:
@@ -2380,40 +2304,11 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
mb->orig = offset_ptr(mb->orig, offs);
mb->base = binary_bytes(mb->orig);
}
- sz -= tari;
- hp += tari + 1;
}
break;
- case FUN_SUBTAG:
- {
- ErlFunThing* funp = (ErlFunThing *) hp;
- Eterm** uptr = (Eterm **) (void *) &funp->next;
-
- if (*uptr && in_area((Eterm *)funp->next, area, area_size)) {
- *uptr += offs;
- }
- sz -= tari;
- hp += tari + 1;
- }
- break;
- case EXTERNAL_PID_SUBTAG:
- case EXTERNAL_PORT_SUBTAG:
- case EXTERNAL_REF_SUBTAG:
- {
- ExternalThing* etp = (ExternalThing *) hp;
- Eterm** uptr = (Eterm **) (void *) &etp->next;
-
- if (*uptr && in_area((Eterm *)etp->next, area, area_size)) {
- *uptr += offs;
- }
- sz -= tari;
- hp += tari + 1;
- }
- break;
- default:
- sz -= tari;
- hp += tari + 1;
}
+ sz -= tari;
+ hp += tari + 1;
break;
}
default:
@@ -2450,18 +2345,8 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
static void
offset_off_heap(Process* p, Sint offs, char* area, Uint area_size)
{
- if (MSO(p).mso && in_area((Eterm *)MSO(p).mso, area, area_size)) {
- Eterm** uptr = (Eterm**) (void *) &MSO(p).mso;
- *uptr += offs;
- }
-
- if (MSO(p).funs && in_area((Eterm *)MSO(p).funs, area, area_size)) {
- Eterm** uptr = (Eterm**) (void *) &MSO(p).funs;
- *uptr += offs;
- }
-
- if (MSO(p).externals && in_area((Eterm *)MSO(p).externals, area, area_size)) {
- Eterm** uptr = (Eterm**) (void *) &MSO(p).externals;
+ if (MSO(p).first && in_area((Eterm *)MSO(p).first, area, area_size)) {
+ Eterm** uptr = (Eterm**) (void *) &MSO(p).first;
*uptr += offs;
}
}
@@ -2542,7 +2427,7 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop)
return 1;
}
while (bp != NULL) {
- if (bp->mem <= ptr && ptr < bp->mem + bp->size) {
+ if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) {
return 1;
}
bp = bp->next;
@@ -2556,7 +2441,7 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop)
hfp = erts_dist_ext_trailer(mp->data.dist_ext);
else
hfp = NULL;
- if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->size)
+ if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->used_size)
return 1;
}
mp = mp->next;
@@ -2582,8 +2467,8 @@ do { \
__FILE__, __LINE__, #EXP); \
} while (0)
-#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST
-# define ERTS_EXTERNAL_VISITED_BIT ((Eterm) 1 << 31)
+#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST
+# define ERTS_OFFHEAP_VISITED_BIT ((Eterm) 1 << 31)
#endif
@@ -2593,62 +2478,45 @@ erts_check_off_heap2(Process *p, Eterm *htop)
Eterm *oheap = (Eterm *) OLD_HEAP(p);
Eterm *ohtop = (Eterm *) OLD_HTOP(p);
int old;
- ProcBin *pb;
- ErlFunThing *eft;
- ExternalThing *et;
+ union erl_off_heap_ptr u;
old = 0;
- for (pb = MSO(p).mso; pb; pb = pb->next) {
- Eterm *ptr = (Eterm *) pb;
- long refc = erts_refc_read(&pb->val->refc, 1);
+ for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) {
+ erts_aint_t refc;
+ switch (thing_subtag(u.hdr->thing_word)) {
+ case REFC_BINARY_SUBTAG:
+ refc = erts_refc_read(&u.pb->val->refc, 1);
+ break;
+ case FUN_SUBTAG:
+ refc = erts_refc_read(&u.fun->fe->refc, 1);
+ break;
+ case EXTERNAL_PID_SUBTAG:
+ case EXTERNAL_PORT_SUBTAG:
+ case EXTERNAL_REF_SUBTAG:
+ refc = erts_refc_read(&u.ext->node->refc, 1);
+ break;
+ default:
+ ASSERT(!!"erts_check_off_heap2: Invalid thing_word");
+ }
ERTS_CHK_OFFHEAP_ASSERT(refc >= 1);
+#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST
+ ERTS_CHK_OFFHEAP_ASSERT(!(u.hdr->thing_word & ERTS_EXTERNAL_VISITED_BIT));
+ u.hdr->thing_word |= ERTS_OFFHEAP_VISITED_BIT;
+#endif
if (old) {
- ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop);
+ ERTS_CHK_OFFHEAP_ASSERT(oheap <= u.ep && u.ep < ohtop);
}
- else if (oheap <= ptr && ptr < ohtop)
+ else if (oheap <= u.ep && u.ep < ohtop)
old = 1;
else {
- ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop));
+ ERTS_CHK_OFFHEAP_ASSERT(within2(u.ep, p, htop));
}
}
- old = 0;
- for (eft = MSO(p).funs; eft; eft = eft->next) {
- Eterm *ptr = (Eterm *) eft;
- long refc = erts_refc_read(&eft->fe->refc, 1);
- ERTS_CHK_OFFHEAP_ASSERT(refc >= 1);
- if (old)
- ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop);
- else if (oheap <= ptr && ptr < ohtop)
- old = 1;
- else
- ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop));
- }
-
- old = 0;
- for (et = MSO(p).externals; et; et = et->next) {
- Eterm *ptr = (Eterm *) et;
- long refc = erts_refc_read(&et->node->refc, 1);
- ERTS_CHK_OFFHEAP_ASSERT(refc >= 1);
#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST
- ERTS_CHK_OFFHEAP_ASSERT(!(et->header & ERTS_EXTERNAL_VISITED_BIT));
+ for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next)
+ u.hdr->thing_word &= ~ERTS_OFFHEAP_VISITED_BIT;
#endif
- if (old)
- ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop);
- else if (oheap <= ptr && ptr < ohtop)
- old = 1;
- else
- ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop));
-#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST
- et->header |= ERTS_EXTERNAL_VISITED_BIT;
-#endif
- }
-
-#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST
- for (et = MSO(p).externals; et; et = et->next)
- et->header &= ~ERTS_EXTERNAL_VISITED_BIT;
-#endif
-
}
void
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-09 12:30:45 +0000