aboutsummaryrefslogtreecommitdiffstats
path: root/erts
diff options
context:
space:
mode:
Diffstat (limited to 'erts')
-rw-r--r--erts/emulator/beam/beam_bif_load.c2
-rw-r--r--erts/emulator/beam/beam_emu.c108
-rw-r--r--erts/emulator/beam/bif.c6
-rw-r--r--erts/emulator/beam/erl_gc.c126
-rw-r--r--erts/emulator/beam/erl_gc.h3
-rw-r--r--erts/emulator/beam/erl_process.c10
-rw-r--r--erts/emulator/beam/erl_process_dict.c6
-rw-r--r--erts/emulator/hipe/hipe_mode_switch.c2
-rw-r--r--erts/emulator/hipe/hipe_mode_switch.h2
-rw-r--r--erts/emulator/hipe/hipe_native_bif.c2
10 files changed, 159 insertions, 108 deletions
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c
index 22b4e26c77..c3ebf71a01 100644
--- a/erts/emulator/beam/beam_bif_load.c
+++ b/erts/emulator/beam/beam_bif_load.c
@@ -935,7 +935,7 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp)
if (need_gc & ERTS_ORDINARY_GC__) {
FLAGS(rp) |= F_NEED_FULLSWEEP;
- *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity);
+ *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity);
done_gc |= ERTS_ORDINARY_GC__;
}
if (need_gc & ERTS_LITERAL_GC__) {
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 9521997987..f74f182863 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -237,6 +237,14 @@ void** beam_ops;
HEAP_TOP(c_p) = HTOP; \
c_p->stop = E
+#define HEAVY_SWAPIN \
+ SWAPIN; \
+ FCALLS = c_p->fcalls
+
+#define HEAVY_SWAPOUT \
+ SWAPOUT; \
+ c_p->fcalls = FCALLS
+
/*
* Use LIGHT_SWAPOUT when the called function
* will call HeapOnlyAlloc() (and never HAlloc()).
@@ -297,7 +305,7 @@ void** beam_ops;
if (E - HTOP < (needed + (HeapNeed))) { \
SWAPOUT; \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \
+ FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), reg, (M)); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
SWAPIN; \
@@ -349,7 +357,7 @@ void** beam_ops;
if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\
SWAPOUT; \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \
+ FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
SWAPIN; \
@@ -370,7 +378,7 @@ void** beam_ops;
if (E - HTOP < need) { \
SWAPOUT; \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \
+ FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));\
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
SWAPIN; \
@@ -391,7 +399,7 @@ void** beam_ops;
SWAPOUT; \
reg[Live] = Extra; \
PROCESS_MAIN_CHK_LOCKS(c_p); \
- FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1); \
+ FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1); \
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \
PROCESS_MAIN_CHK_LOCKS(c_p); \
Extra = reg[Live]; \
@@ -415,9 +423,9 @@ void** beam_ops;
#define MakeFun(FunP, NumFree) \
do { \
- SWAPOUT; \
+ HEAVY_SWAPOUT; \
r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree); \
- SWAPIN; \
+ HEAVY_SWAPIN; \
} while (0)
#define PutTuple(Dst, Arity) \
@@ -1402,11 +1410,11 @@ void process_main(void)
}
live = Arg(2);
- SWAPOUT;
+ HEAVY_SWAPOUT;
reg[live] = increment_reg_val;
reg[live+1] = make_small(increment_val);
result = erts_gc_mixed_plus(c_p, reg, live);
- SWAPIN;
+ HEAVY_SWAPIN;
ERTS_HOLE_CHECK(c_p);
if (is_value(result)) {
StoreBifResult(3, result);
@@ -1420,11 +1428,11 @@ void process_main(void)
Eterm result; \
Uint live = Arg(1); \
\
- SWAPOUT; \
+ HEAVY_SWAPOUT; \
reg[live] = Op1; \
reg[live+1] = Op2; \
result = erts_gc_##name(c_p, reg, live); \
- SWAPIN; \
+ HEAVY_SWAPIN; \
ERTS_HOLE_CHECK(c_p); \
if (is_value(result)) { \
StoreBifResult(4, result); \
@@ -1744,7 +1752,7 @@ void process_main(void)
if (E - HTOP < 3) {
SWAPOUT;
PROCESS_MAIN_CHK_LOCKS(c_p);
- FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1);
+ FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1);
ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
PROCESS_MAIN_CHK_LOCKS(c_p);
SWAPIN;
@@ -2382,9 +2390,9 @@ void process_main(void)
OpCase(new_map_dII): {
Eterm res;
- SWAPOUT;
+ HEAVY_SWAPOUT;
res = new_map(c_p, reg, I-1);
- SWAPIN;
+ HEAVY_SWAPIN;
StoreResult(res, Arg(0));
Next(3+Arg(2));
}
@@ -2472,9 +2480,9 @@ do { \
Eterm map;
GetArg1(1, map);
- SWAPOUT;
+ HEAVY_SWAPOUT;
res = update_map_assoc(c_p, reg, map, I);
- SWAPIN;
+ HEAVY_SWAPIN;
if (is_value(res)) {
StoreResult(res, Arg(2));
Next(5+Arg(4));
@@ -2494,9 +2502,9 @@ do { \
Eterm map;
GetArg1(1, map);
- SWAPOUT;
+ HEAVY_SWAPOUT;
res = update_map_exact(c_p, reg, map, I);
- SWAPIN;
+ HEAVY_SWAPIN;
if (is_value(res)) {
StoreResult(res, Arg(2));
Next(5+Arg(4));
@@ -3072,10 +3080,10 @@ do { \
bnot_val = make_small(~signed_val(bnot_val));
} else {
Uint live = Arg(2);
- SWAPOUT;
+ HEAVY_SWAPOUT;
reg[live] = bnot_val;
bnot_val = erts_gc_bnot(c_p, reg, live);
- SWAPIN;
+ HEAVY_SWAPIN;
ERTS_HOLE_CHECK(c_p);
if (is_nil(bnot_val)) {
goto lb_Cl_error;
@@ -3090,9 +3098,9 @@ do { \
OpCase(i_apply): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = apply(c_p, r(0), x(1), x(2), reg);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
SET_CP(c_p, I+1);
SET_I(next);
@@ -3104,9 +3112,9 @@ do { \
OpCase(i_apply_last_P): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = apply(c_p, r(0), x(1), x(2), reg);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
SET_CP(c_p, (BeamInstr *) E[0]);
E = ADD_BYTE_OFFSET(E, Arg(0));
@@ -3119,9 +3127,9 @@ do { \
OpCase(i_apply_only): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = apply(c_p, r(0), x(1), x(2), reg);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
SET_I(next);
Dispatch();
@@ -3133,9 +3141,9 @@ do { \
OpCase(apply_I): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = fixed_apply(c_p, reg, Arg(0));
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
SET_CP(c_p, I+2);
SET_I(next);
@@ -3148,9 +3156,9 @@ do { \
OpCase(apply_last_IP): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = fixed_apply(c_p, reg, Arg(0));
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
SET_CP(c_p, (BeamInstr *) E[0]);
E = ADD_BYTE_OFFSET(E, Arg(1));
@@ -3164,9 +3172,9 @@ do { \
OpCase(i_apply_fun): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = apply_fun(c_p, r(0), x(1), reg);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
SET_CP(c_p, I+1);
SET_I(next);
@@ -3178,9 +3186,9 @@ do { \
OpCase(i_apply_fun_last_P): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = apply_fun(c_p, r(0), x(1), reg);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
SET_CP(c_p, (BeamInstr *) E[0]);
E = ADD_BYTE_OFFSET(E, Arg(0));
@@ -3193,9 +3201,9 @@ do { \
OpCase(i_apply_fun_only): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = apply_fun(c_p, r(0), x(1), reg);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
SET_I(next);
Dispatchfun();
@@ -3206,9 +3214,9 @@ do { \
OpCase(i_call_fun_I): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
SET_CP(c_p, I+2);
SET_I(next);
@@ -3220,9 +3228,9 @@ do { \
OpCase(i_call_fun_last_IP): {
BeamInstr *next;
- SWAPOUT;
+ HEAVY_SWAPOUT;
next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
SET_CP(c_p, (BeamInstr *) E[0]);
E = ADD_BYTE_OFFSET(E, Arg(1));
@@ -3421,9 +3429,9 @@ do { \
* code[3]: &&call_error_handler
* code[4]: Not used
*/
- SWAPOUT;
+ HEAVY_SWAPOUT;
I = call_error_handler(c_p, I-3, reg, am_undefined_function);
- SWAPIN;
+ HEAVY_SWAPIN;
if (I) {
Goto(*I);
}
@@ -3977,10 +3985,10 @@ do { \
Eterm Size;
GetArg1(4, Size);
- SWAPOUT;
+ HEAVY_SWAPOUT;
reg[live] = x(SCRATCH_X_REG);
res = erts_bs_append(c_p, reg, live, Size, Arg(1), Arg(3));
- SWAPIN;
+ HEAVY_SWAPIN;
if (is_non_value(res)) {
/* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */
goto lb_Cl_error;
@@ -4005,9 +4013,9 @@ do { \
}
OpCase(bs_init_writable): {
- SWAPOUT;
+ HEAVY_SWAPOUT;
r(0) = erts_bs_init_writable(c_p, r(0));
- SWAPIN;
+ HEAVY_SWAPIN;
Next(0);
}
@@ -4835,7 +4843,7 @@ do { \
BeamInstr *next;
next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE);
- SWAPIN;
+ HEAVY_SWAPIN;
if (next != NULL) {
SET_I(next);
Dispatchfun();
@@ -4884,20 +4892,22 @@ do { \
}
OpCase(i_hibernate): {
- SWAPOUT;
+ HEAVY_SWAPOUT;
if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) {
+ FCALLS = c_p->fcalls;
c_p->flags &= ~F_HIBERNATE_SCHED;
goto do_schedule;
} else {
+ HEAVY_SWAPIN;
I = handle_error(c_p, I, reg, hibernate_3);
goto post_error_handling;
}
}
OpCase(i_debug_breakpoint): {
- SWAPOUT;
+ HEAVY_SWAPOUT;
I = call_error_handler(c_p, I-3, reg, am_breakpoint);
- SWAPIN;
+ HEAVY_SWAPIN;
if (I) {
Goto(*I);
}
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index e4283ac945..57dd045193 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -3839,11 +3839,9 @@ BIF_RETTYPE now_0(BIF_ALIST_0)
BIF_RETTYPE garbage_collect_0(BIF_ALIST_0)
{
- int reds;
-
FLAGS(BIF_P) |= F_NEED_FULLSWEEP;
- reds = erts_garbage_collect(BIF_P, 0, NULL, 0);
- BIF_RET2(am_true, reds);
+ erts_garbage_collect(BIF_P, 0, NULL, 0);
+ BIF_RET(am_true);
}
/**********************************************************************/
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 6a52e1a890..7163b4839d 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -135,7 +135,7 @@ static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm
char* src, Uint src_size);
static Eterm* collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end,
Eterm* heap, Eterm* htop, Eterm* objv, int nobj);
-static void adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj);
+static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj);
static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj);
static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj);
static void sweep_off_heap(Process *p, int fullsweep);
@@ -172,6 +172,20 @@ typedef struct {
erts_smp_atomic32_t refc;
} ErtsGCInfoReq;
+static ERTS_INLINE int
+gc_cost(Uint gc_moved_live_words, Uint resize_moved_words)
+{
+ Sint reds;
+
+ reds = gc_moved_live_words/10;
+ reds += resize_moved_words/100;
+ if (reds < 1)
+ return 1;
+ if (reds > INT_MAX)
+ return INT_MAX;
+ return (int) reds;
+}
+
ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq,
ErtsGCInfoReq,
5,
@@ -413,6 +427,7 @@ static ERTS_INLINE void reset_active_writer(Process *p)
}
#define ERTS_DELAY_GC_EXTRA_FREE 40
+#define ERTS_ABANDON_HEAP_COST 10
static int
delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
@@ -421,6 +436,7 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
Eterm *orig_heap, *orig_hend, *orig_htop, *orig_stop;
Eterm *stop, *hend;
Uint hsz, ssz;
+ int reds_left;
ERTS_HOLE_CHECK(p);
@@ -482,7 +498,12 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need)
/* Make sure that we do a proper GC as soon as possible... */
p->flags |= F_FORCE_GC;
- return CONTEXT_REDS;
+ reds_left = ERTS_BIF_REDS_LEFT(p);
+ if (reds_left > ERTS_ABANDON_HEAP_COST) {
+ int vreds = reds_left - ERTS_ABANDON_HEAP_COST;
+ ERTS_VBUMP_REDS(p, vreds);
+ }
+ return ERTS_ABANDON_HEAP_COST;
}
static ERTS_FORCE_INLINE Uint
@@ -536,7 +557,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
int need, Eterm* objv, int nobj)
{
Uint reclaimed_now = 0;
- int done = 0;
+ int reds;
ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */
ErtsSchedulerData *esdp;
#ifdef USE_VM_PROBES
@@ -562,9 +583,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
ERTS_CHK_OFFHEAP(p);
ErtsGcQuickSanityCheck(p);
- if (GEN_GCS(p) >= MAX_GEN_GCS(p)) {
- FLAGS(p) |= F_NEED_FULLSWEEP;
- }
+
#ifdef USE_VM_PROBES
*pidbuf = '\0';
if (DTRACE_ENABLED(gc_major_start)
@@ -577,17 +596,21 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
/*
* Test which type of GC to do.
*/
- while (!done) {
- if ((FLAGS(p) & F_NEED_FULLSWEEP) != 0) {
- DTRACE2(gc_major_start, pidbuf, need);
- done = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
- DTRACE2(gc_major_end, pidbuf, reclaimed_now);
- } else {
- DTRACE2(gc_minor_start, pidbuf, need);
- done = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
- DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
- }
+
+ if (GEN_GCS(p) < MAX_GEN_GCS(p) && !(FLAGS(p) & F_NEED_FULLSWEEP)) {
+ DTRACE2(gc_minor_start, pidbuf, need);
+ reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
+ DTRACE2(gc_minor_end, pidbuf, reclaimed_now);
+ if (reds < 0)
+ goto do_major_collection;
+ }
+ else {
+ do_major_collection:
+ DTRACE2(gc_major_start, pidbuf, need);
+ reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now);
+ DTRACE2(gc_major_end, pidbuf, reclaimed_now);
}
+
reset_active_writer(p);
/*
@@ -648,23 +671,22 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end,
p->last_old_htop = p->old_htop;
#endif
- /* FIXME: This function should really return an Sint, i.e., a possibly
- 64 bit wide signed integer, but that requires updating all the code
- that calls it. For now, we just return INT_MAX if the result is too
- large for an int. */
- {
- Sint result = (HEAP_TOP(p) - HEAP_START(p)) / 10;
- if (result >= INT_MAX) return INT_MAX;
- else return (int) result;
- }
+ return reds;
}
int
-erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
+erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj)
{
return garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj);
}
+void
+erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj)
+{
+ int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj);
+ BUMP_REDS(p, reds);
+}
+
/*
* Place all living data on a the new heap; deallocate any old heap.
* Meant to be used by hibernate/3.
@@ -679,6 +701,7 @@ erts_garbage_collect_hibernate(Process* p)
char* area;
Uint area_size;
Sint offs;
+ int reds;
if (p->flags & F_DISABLE_GC)
ERTS_INTERNAL_ERROR("GC disabled");
@@ -774,6 +797,9 @@ erts_garbage_collect_hibernate(Process* p)
ErtsGcQuickSanityCheck(p);
erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC);
+
+ reds = gc_cost(actual_size, actual_size);
+ BUMP_REDS(p, reds);
}
@@ -989,16 +1015,21 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
((mature_size <= OLD_HEND(p) - OLD_HTOP(p)) &&
((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) &&
((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) {
- Uint stack_size, size_after, need_after, new_sz;
+ Eterm *prev_old_htop;
+ Uint stack_size, size_after, adjust_size, need_after, new_sz, new_mature;
stack_size = p->hend - p->stop;
new_sz = stack_size + size_before;
new_sz = next_heap_size(p, new_sz, 0);
+ prev_old_htop = p->old_htop;
do_minor(p, live_hf_end, (char *) mature, mature_size*sizeof(Eterm),
new_sz, objv, nobj);
- size_after = HEAP_TOP(p) - HEAP_START(p);
+ new_mature = p->old_htop - prev_old_htop;
+
+ size_after = new_mature;
+ size_after += HEAP_TOP(p) - HEAP_START(p);
*recl += (size_before - size_after);
ErtsGcQuickSanityCheck(p);
@@ -1017,6 +1048,8 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
* the heap size is substantial, we don't want to shrink.
*/
+ adjust_size = 0;
+
if ((HEAP_SIZE(p) > 3000) && (4 * need_after < HEAP_SIZE(p)) &&
((HEAP_SIZE(p) > 8000) ||
(HEAP_SIZE(p) > (OLD_HEND(p) - OLD_HEAP(p))))) {
@@ -1038,31 +1071,33 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end,
: next_heap_size(p, wanted, 0);
if (wanted < HEAP_SIZE(p)) {
shrink_new_heap(p, wanted, objv, nobj);
+ adjust_size = p->htop - p->heap;
}
- ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
- ASSERT(MBUF(p) == NULL);
- return 1; /* We are done. */
+ goto done;
}
if (HEAP_SIZE(p) >= need_after) {
/*
* The heap size turned out to be just right. We are done.
*/
- ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
- ASSERT(MBUF(p) == NULL);
- return 1;
+ goto done;
}
grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj);
- return 1;
+ adjust_size = p->htop - p->heap;
+
+ done:
+ ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0));
+ ASSERT(MBUF(p) == NULL);
+
+ return gc_cost(size_after, adjust_size);
}
/*
* Not enough room for a minor collection. Must force a major collection.
*/
- FLAGS(p) |= F_NEED_FULLSWEEP;
- return 0;
+ return -1;
}
/*
@@ -1341,12 +1376,13 @@ static int
major_collection(Process* p, ErlHeapFragment *live_hf_end,
int need, Eterm* objv, int nobj, Uint *recl)
{
- Uint size_before, stack_size;
+ Uint size_before, size_after, stack_size;
Eterm* n_heap;
Eterm* n_htop;
char* oh = (char *) OLD_HEAP(p);
Uint oh_size = (char *) OLD_HTOP(p) - oh;
Uint new_sz, stk_sz;
+ int adjusted;
/*
* Do a fullsweep GC. First figure out the size of the heap
@@ -1413,9 +1449,10 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
ErtsGcQuickSanityCheck(p);
- *recl += size_before - (HEAP_TOP(p) - HEAP_START(p));
+ size_after = HEAP_TOP(p) - HEAP_START(p);
+ *recl += size_before - size_after;
- adjust_after_fullsweep(p, need, objv, nobj);
+ adjusted = adjust_after_fullsweep(p, need, objv, nobj);
#ifdef HARDDEBUG
disallow_heap_frag_ref_in_heap(p);
@@ -1423,7 +1460,8 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end,
remove_message_buffers(p);
ErtsGcQuickSanityCheck(p);
- return 1; /* We are done. */
+
+ return gc_cost(size_after, adjusted ? size_after : 0);
}
static Eterm *
@@ -1524,9 +1562,10 @@ full_sweep_heaps(Process *p,
return n_htop;
}
-static void
+static int
adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
{
+ int adjusted = 0;
Uint wanted, sz, need_after;
Uint stack_size = STACK_SZ_ON_HEAP(p);
@@ -1539,6 +1578,7 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
/* Too small - grow to match requested need */
sz = next_heap_size(p, need_after, 0);
grow_new_heap(p, sz, objv, nobj);
+ adjusted = 1;
} else if (3 * HEAP_SIZE(p) < 4 * need_after){
/* Need more than 75% of current, postpone to next GC.*/
FLAGS(p) |= F_HEAP_GROW;
@@ -1555,8 +1595,10 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj)
if (sz < HEAP_SIZE(p)) {
shrink_new_heap(p, sz, objv, nobj);
+ adjusted = 1;
}
}
+ return adjusted;
}
/*
diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h
index a496c5f008..d603866cbf 100644
--- a/erts/emulator/beam/erl_gc.h
+++ b/erts/emulator/beam/erl_gc.h
@@ -134,7 +134,8 @@ typedef struct {
void erts_gc_info(ErtsGCInfo *gcip);
void erts_init_gc(void);
-int erts_garbage_collect(struct process*, int, Eterm*, int);
+int erts_garbage_collect_nobump(struct process*, int, Eterm*, int);
+void erts_garbage_collect(struct process*, int, Eterm*, int);
void erts_garbage_collect_hibernate(struct process* p);
Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end,
Eterm result, Eterm* regs, Uint arity);
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index e490ffea5a..5e7ed0d151 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -9774,7 +9774,7 @@ Process *schedule(Process *p, int calls)
if (ERTS_IS_GC_DESIRED(p)) {
if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & F_DISABLE_GC)) {
- reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity);
+ reds -= erts_garbage_collect_nobump(p, 0, p->arg_reg, p->arity);
if (reds <= 0) {
p->fcalls = reds;
goto sched_out_proc;
@@ -10057,10 +10057,10 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds)
else {
if (!garbage_collected) {
FLAGS(c_p) |= F_NEED_FULLSWEEP;
- reds += erts_garbage_collect(c_p,
- 0,
- c_p->arg_reg,
- c_p->arity);
+ reds += erts_garbage_collect_nobump(c_p,
+ 0,
+ c_p->arg_reg,
+ c_p->arity);
garbage_collected = 1;
}
st_res = am_true;
diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c
index 8606371bdf..f82cad745a 100644
--- a/erts/emulator/beam/erl_process_dict.c
+++ b/erts/emulator/beam/erl_process_dict.c
@@ -583,7 +583,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value)
root[0] = id;
root[1] = value;
root[2] = old;
- BUMP_REDS(p, erts_garbage_collect(p, needed, root, 3));
+ erts_garbage_collect(p, needed, root, 3);
id = root[0];
value = root[1];
old = root[2];
@@ -715,7 +715,7 @@ static void shrink(Process *p, Eterm* ret)
needed = 2*erts_list_length(hi);
}
if (HeapWordsLeft(p) < needed) {
- BUMP_REDS(p, erts_garbage_collect(p, needed, ret, 1));
+ erts_garbage_collect(p, needed, ret, 1);
hi = pd->data[(pd->splitPosition + pd->homeSize)];
lo = pd->data[pd->splitPosition];
}
@@ -811,7 +811,7 @@ static void grow(Process *p)
}
}
if (HeapWordsLeft(p) < needed) {
- BUMP_REDS(p, erts_garbage_collect(p, needed, 0, 0));
+ erts_garbage_collect(p, needed, 0, 0);
}
#ifdef DEBUG
hp_limit = p->htop + needed;
diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c
index 968452a641..976180cc30 100644
--- a/erts/emulator/hipe/hipe_mode_switch.c
+++ b/erts/emulator/hipe/hipe_mode_switch.c
@@ -196,7 +196,7 @@ hipe_push_beam_trap_frame(Process *p, Eterm reg[], unsigned arity)
ASSERT(!(p->flags & F_DISABLE_GC));
if ((p->stop - 2) < p->htop) {
DPRINTF("calling gc to increase BEAM stack size");
- p->fcalls -= erts_garbage_collect(p, 2, reg, arity);
+ erts_garbage_collect(p, 2, reg, arity);
ASSERT(!((p->stop - 2) < p->htop));
}
p->stop -= 2;
diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h
index bc863a4f36..620cc6356b 100644
--- a/erts/emulator/hipe/hipe_mode_switch.h
+++ b/erts/emulator/hipe/hipe_mode_switch.h
@@ -95,7 +95,7 @@ ERTS_GLB_INLINE void hipe_reserve_beam_trap_frame(Process *p, Eterm reg[], unsig
/* ensure that at least 2 words are available on the BEAM stack */
if ((p->stop - 2) < p->htop) {
- p->fcalls -= erts_garbage_collect(p, 2, reg, arity);
+ erts_garbage_collect(p, 2, reg, arity);
ASSERT(!((p->stop - 2) < p->htop));
}
p->stop -= 2;
diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c
index ceae3497c5..aa81008b12 100644
--- a/erts/emulator/hipe/hipe_native_bif.c
+++ b/erts/emulator/hipe/hipe_native_bif.c
@@ -80,7 +80,7 @@ Eterm hipe_show_nstack_1(BIF_ALIST_1)
void hipe_gc(Process *p, Eterm need)
{
hipe_set_narity(p, 1);
- p->fcalls -= erts_garbage_collect(p, unsigned_val(need), NULL, 0);
+ erts_garbage_collect(p, unsigned_val(need), NULL, 0);
hipe_set_narity(p, 0);
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 20:05:29 +0000