aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator/beam/erl_gc.c
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator/beam/erl_gc.c')
-rw-r--r--erts/emulator/beam/erl_gc.c95
1 files changed, 81 insertions, 14 deletions
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 3a50b294d1..13b1f8ab4d 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -151,6 +151,7 @@ static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj);
static void sweep_off_heap(Process *p, int fullsweep);
static void offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size);
static void offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size);
+static void offset_heap_ptr_nstack(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size);
static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size,
Eterm* objv, int nobj);
static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size);
@@ -577,7 +578,7 @@ force_reschedule:
}
static ERTS_FORCE_INLINE Uint
-young_gen_usage(Process *p)
+young_gen_usage(Process *p, Uint *ext_msg_usage)
{
Uint hsz;
Eterm *aheap;
@@ -604,7 +605,10 @@ young_gen_usage(Process *p)
if (ERTS_SIG_IS_MSG(mp)
&& mp->data.attached
&& mp->data.attached != ERTS_MSG_COMBINED_HFRAG) {
- hsz += erts_msg_attached_data_size(mp);
+ Uint sz = erts_msg_attached_data_size(mp);
+ if (ERTS_SIG_IS_EXTERNAL_MSG(mp))
+ *ext_msg_usage += sz;
+ hsz += sz;
}
});
}
@@ -676,6 +680,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
{
Uint reclaimed_now = 0;
Uint ygen_usage;
+ Uint ext_msg_usage = 0;
Eterm gc_trace_end_tag;
int reds;
ErtsMonotonicTime start_time;
@@ -698,7 +703,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
return delay_garbage_collection(p, live_hf_end, need, fcalls);
}
- ygen_usage = max_young_gen_usage ? max_young_gen_usage : young_gen_usage(p);
+ ygen_usage = max_young_gen_usage ? max_young_gen_usage : young_gen_usage(p, &ext_msg_usage);
if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) {
check_for_possibly_long_gc(p, ygen_usage);
@@ -739,7 +744,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
trace_gc(p, am_gc_minor_start, need, THE_NON_VALUE);
}
DTRACE2(gc_minor_start, pidbuf, need);
- reds = minor_collection(p, live_hf_end, need, objv, nobj,
+ reds = minor_collection(p, live_hf_end, need + ext_msg_usage, objv, nobj,
ygen_usage, &reclaimed_now);
DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
if (reds == -1) {
@@ -764,7 +769,7 @@ do_major_collection:
trace_gc(p, am_gc_major_start, need, THE_NON_VALUE);
}
DTRACE2(gc_major_start, pidbuf, need);
- reds = major_collection(p, live_hf_end, need, objv, nobj,
+ reds = major_collection(p, live_hf_end, need + ext_msg_usage, objv, nobj,
ygen_usage, &reclaimed_now);
if (ERTS_SCHEDULER_IS_DIRTY(esdp))
p->flags &= ~(F_DIRTY_MAJOR_GC|F_DIRTY_MINOR_GC);
@@ -1050,9 +1055,10 @@ erts_garbage_collect_hibernate(Process* p)
n_htop = tmp_n_htop; \
} while(0)
+
/*
* offset_nstack() can ignore the descriptor-based traversal the other
- * nstack procedures use and simply call offset_heap_ptr() instead.
+ * nstack procedures use and do a simpler word by word traversal instead.
* This relies on two facts:
* 1. The only live non-Erlang terms on an nstack are return addresses,
* and they will be skipped thanks to the low/high range check.
@@ -1067,14 +1073,51 @@ static ERTS_INLINE void offset_nstack(Process* p, Sint offs,
{
if (p->hipe.nstack) {
ASSERT(p->hipe.nsp && p->hipe.nstend);
- offset_heap_ptr(hipe_nstack_start(p), hipe_nstack_used(p),
- offs, area, area_size);
+ offset_heap_ptr_nstack(hipe_nstack_start(p), hipe_nstack_used(p),
+ offs, area, area_size);
}
else {
ASSERT(!p->hipe.nsp && !p->hipe.nstend);
}
}
+/*
+ * This is the same as offset_heap_ptr()
+ *
+ * Except for VALGRIND. It allows benign offsetting of undefined (dead) words
+ * on the nstack while also retaining them as undefined. This suppresses
+ * valgrinds "Conditional jump or move depends on uninitialised value(s)".
+ */
+static void
+offset_heap_ptr_nstack(Eterm* hp, Uint sz, Sint offs,
+ char* area, Uint area_size)
+{
+ while (sz--) {
+ Eterm val = *hp;
+#ifdef VALGRIND
+ Eterm val_vbits;
+ VALGRIND_GET_VBITS(&val, &val_vbits, sizeof(val));
+ VALGRIND_MAKE_MEM_DEFINED(&val, sizeof(val));
+#endif
+ switch (primary_tag(val)) {
+ case TAG_PRIMARY_LIST:
+ case TAG_PRIMARY_BOXED:
+ if (ErtsInArea(ptr_val(val), area, area_size)) {
+#ifdef VALGRIND
+ VALGRIND_SET_VBITS(&val, val_vbits, sizeof(val));
+#endif
+ *hp = offset_ptr(val, offs);
+ }
+ hp++;
+ break;
+ default:
+ hp++;
+ break;
+ }
+ }
+}
+
+
#else /* !HIPE */
#define fullsweep_nstack(p,n_htop) (n_htop)
@@ -1101,6 +1144,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
Eterm* old_htop;
Uint n;
Uint ygen_usage = 0;
+ Uint ext_msg_usage = 0;
struct erl_off_heap_header** prev = NULL;
Sint64 reds;
int hibernated = !!(p->flags & F_HIBERNATED);
@@ -1118,7 +1162,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
p->flags &= ~F_DIRTY_CLA;
else {
Uint size = byte_lit_size/sizeof(Uint);
- ygen_usage = young_gen_usage(p);
+ ygen_usage = young_gen_usage(p, &ext_msg_usage);
if (hibernated)
size = size*2 + 3*ygen_usage;
else
@@ -1130,7 +1174,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
}
}
- reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, 0,
+ reds = (Sint64) garbage_collect(p, ERTS_INVALID_HFRAG_PTR, ext_msg_usage,
p->arg_reg, p->arity, fcalls,
ygen_usage);
if (ERTS_PROC_IS_EXITING(p)) {
@@ -1303,7 +1347,8 @@ erts_garbage_collect_literals(Process* p, Eterm* literals,
ExternalThing *etp;
ASSERT(is_external_header(ptr->thing_word));
etp = (ExternalThing *) ptr;
- erts_refc_inc(&etp->node->refc, 1);
+ erts_ref_node_entry(etp->node, 1,
+ make_boxed(&oh->thing_word));
break;
}
}
@@ -1347,6 +1392,9 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
Eterm *mature = p->abandoned_heap ? p->abandoned_heap : p->heap;
Uint mature_size = p->high_water - mature;
Uint size_before = ygen_usage;
+#ifdef DEBUG
+ Uint debug_tmp = 0;
+#endif
/*
* Check if we have gone past the max heap size limit
@@ -1483,7 +1531,7 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
process from there */
ASSERT(!MAX_HEAP_SIZE_GET(p) ||
!(MAX_HEAP_SIZE_FLAGS_GET(p) & MAX_HEAP_SIZE_KILL) ||
- MAX_HEAP_SIZE_GET(p) > (young_gen_usage(p) +
+ MAX_HEAP_SIZE_GET(p) > (young_gen_usage(p, &debug_tmp) +
(OLD_HEND(p) - OLD_HEAP(p)) +
(HEAP_END(p) - HEAP_TOP(p))));
@@ -2836,7 +2884,11 @@ sweep_off_heap(Process *p, int fullsweep)
while (ptr) {
if (IS_MOVED_BOXED(ptr->thing_word)) {
ASSERT(!ErtsInArea(ptr, oheap, oheap_sz));
- *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word);
+ if (is_external_header(((struct erl_off_heap_header*) boxed_val(ptr->thing_word))->thing_word))
+ erts_node_bookkeep(((ExternalThing*)ptr)->node,
+ make_boxed(&ptr->thing_word),
+ ERL_NODE_DEC, __FILE__, __LINE__);
+ *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word);
ASSERT(!IS_MOVED_BOXED(ptr->thing_word));
switch (ptr->thing_word) {
case HEADER_PROC_BIN: {
@@ -2863,6 +2915,11 @@ sweep_off_heap(Process *p, int fullsweep)
/* fall through... */
}
default:
+ if (is_external_header(ptr->thing_word)) {
+ erts_node_bookkeep(((ExternalThing*)ptr)->node,
+ make_boxed(&ptr->thing_word),
+ ERL_NODE_INC, __FILE__, __LINE__);
+ }
prev = &ptr->next;
ptr = ptr->next;
}
@@ -2896,7 +2953,8 @@ sweep_off_heap(Process *p, int fullsweep)
}
default:
ASSERT(is_external_header(ptr->thing_word));
- erts_deref_node_entry(((ExternalThing*)ptr)->node);
+ erts_deref_node_entry(((ExternalThing*)ptr)->node,
+ make_boxed(&ptr->thing_word));
}
*prev = ptr = ptr->next;
}
@@ -3029,6 +3087,15 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size)
{
struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp;
+ if (is_external_header(oh->thing_word)) {
+ erts_node_bookkeep(((ExternalThing*)oh)->node,
+ make_boxed(((Eterm*)oh)-offs),
+ ERL_NODE_DEC, __FILE__, __LINE__);
+ erts_node_bookkeep(((ExternalThing*)oh)->node,
+ make_boxed((Eterm*)oh), ERL_NODE_INC,
+ __FILE__, __LINE__);
+ }
+
if (ErtsInArea(oh->next, area, area_size)) {
Eterm** uptr = (Eterm **) (void *) &oh->next;
*uptr += offs; /* Patch the mso chain */
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-31 15:26:39 +0000