From 9c6f45b901ee701553afe34c0b33b7d931d73fd9 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Wed, 11 Nov 2015 11:39:32 +0100 Subject: Bump reductions on GC --- erts/emulator/beam/beam_bif_load.c | 2 +- erts/emulator/beam/beam_emu.c | 108 ++++++++++++++++------------- erts/emulator/beam/bif.c | 6 +- erts/emulator/beam/erl_gc.c | 126 ++++++++++++++++++++++------------ erts/emulator/beam/erl_gc.h | 3 +- erts/emulator/beam/erl_process.c | 10 +-- erts/emulator/beam/erl_process_dict.c | 6 +- 7 files changed, 156 insertions(+), 105 deletions(-) (limited to 'erts/emulator/beam') diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 22b4e26c77..c3ebf71a01 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -935,7 +935,7 @@ check_process_code(Process* rp, Module* modp, int allow_gc, int *redsp) if (need_gc & ERTS_ORDINARY_GC__) { FLAGS(rp) |= F_NEED_FULLSWEEP; - *redsp += erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); + *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity); done_gc |= ERTS_ORDINARY_GC__; } if (need_gc & ERTS_LITERAL_GC__) { diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 9521997987..f74f182863 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -237,6 +237,14 @@ void** beam_ops; HEAP_TOP(c_p) = HTOP; \ c_p->stop = E +#define HEAVY_SWAPIN \ + SWAPIN; \ + FCALLS = c_p->fcalls + +#define HEAVY_SWAPOUT \ + SWAPOUT; \ + c_p->fcalls = FCALLS + /* * Use LIGHT_SWAPOUT when the called function * will call HeapOnlyAlloc() (and never HAlloc()). @@ -297,7 +305,7 @@ void** beam_ops; if (E - HTOP < (needed + (HeapNeed))) { \ SWAPOUT; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, needed + (HeapNeed), reg, (M)); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ SWAPIN; \ @@ -349,7 +357,7 @@ void** beam_ops; if ((E - HTOP < need) || (MSO(c_p).overhead + (VNh) >= BIN_VHEAP_SZ(c_p))) {\ SWAPOUT; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ SWAPIN; \ @@ -370,7 +378,7 @@ void** beam_ops; if (E - HTOP < need) { \ SWAPOUT; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live));\ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ SWAPIN; \ @@ -391,7 +399,7 @@ void** beam_ops; SWAPOUT; \ reg[Live] = Extra; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ - FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1); \ + FCALLS -= erts_garbage_collect_nobump(c_p, need, reg, (Live)+1); \ ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ Extra = reg[Live]; \ @@ -415,9 +423,9 @@ void** beam_ops; #define MakeFun(FunP, NumFree) \ do { \ - SWAPOUT; \ + HEAVY_SWAPOUT; \ r(0) = new_fun(c_p, reg, (ErlFunEntry *) FunP, NumFree); \ - SWAPIN; \ + HEAVY_SWAPIN; \ } while (0) #define PutTuple(Dst, Arity) \ @@ -1402,11 +1410,11 @@ void process_main(void) } live = Arg(2); - SWAPOUT; + HEAVY_SWAPOUT; reg[live] = increment_reg_val; reg[live+1] = make_small(increment_val); result = erts_gc_mixed_plus(c_p, reg, live); - SWAPIN; + HEAVY_SWAPIN; ERTS_HOLE_CHECK(c_p); if (is_value(result)) { StoreBifResult(3, result); @@ -1420,11 +1428,11 @@ void process_main(void) Eterm result; \ Uint live = Arg(1); \ \ - SWAPOUT; \ + HEAVY_SWAPOUT; \ reg[live] = Op1; \ reg[live+1] = Op2; \ result = erts_gc_##name(c_p, reg, live); \ - SWAPIN; \ + HEAVY_SWAPIN; \ ERTS_HOLE_CHECK(c_p); \ if (is_value(result)) { \ StoreBifResult(4, result); \ @@ -1744,7 +1752,7 @@ void process_main(void) if (E - HTOP < 3) { SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); - FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1); + FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+2, 1); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; @@ -2382,9 +2390,9 @@ void process_main(void) OpCase(new_map_dII): { Eterm res; - SWAPOUT; + HEAVY_SWAPOUT; res = new_map(c_p, reg, I-1); - SWAPIN; + HEAVY_SWAPIN; StoreResult(res, Arg(0)); Next(3+Arg(2)); } @@ -2472,9 +2480,9 @@ do { \ Eterm map; GetArg1(1, map); - SWAPOUT; + HEAVY_SWAPOUT; res = update_map_assoc(c_p, reg, map, I); - SWAPIN; + HEAVY_SWAPIN; if (is_value(res)) { StoreResult(res, Arg(2)); Next(5+Arg(4)); @@ -2494,9 +2502,9 @@ do { \ Eterm map; GetArg1(1, map); - SWAPOUT; + HEAVY_SWAPOUT; res = update_map_exact(c_p, reg, map, I); - SWAPIN; + HEAVY_SWAPIN; if (is_value(res)) { StoreResult(res, Arg(2)); Next(5+Arg(4)); @@ -3072,10 +3080,10 @@ do { \ bnot_val = make_small(~signed_val(bnot_val)); } else { Uint live = Arg(2); - SWAPOUT; + HEAVY_SWAPOUT; reg[live] = bnot_val; bnot_val = erts_gc_bnot(c_p, reg, live); - SWAPIN; + HEAVY_SWAPIN; ERTS_HOLE_CHECK(c_p); if (is_nil(bnot_val)) { goto lb_Cl_error; @@ -3090,9 +3098,9 @@ do { \ OpCase(i_apply): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, I+1); SET_I(next); @@ -3104,9 +3112,9 @@ do { \ OpCase(i_apply_last_P): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(0)); @@ -3119,9 +3127,9 @@ do { \ OpCase(i_apply_only): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply(c_p, r(0), x(1), x(2), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { SET_I(next); Dispatch(); @@ -3133,9 +3141,9 @@ do { \ OpCase(apply_I): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = fixed_apply(c_p, reg, Arg(0)); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, I+2); SET_I(next); @@ -3148,9 +3156,9 @@ do { \ OpCase(apply_last_IP): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = fixed_apply(c_p, reg, Arg(0)); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(1)); @@ -3164,9 +3172,9 @@ do { \ OpCase(i_apply_fun): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, I+1); SET_I(next); @@ -3178,9 +3186,9 @@ do { \ OpCase(i_apply_fun_last_P): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(0)); @@ -3193,9 +3201,9 @@ do { \ OpCase(i_apply_fun_only): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = apply_fun(c_p, r(0), x(1), reg); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { SET_I(next); Dispatchfun(); @@ -3206,9 +3214,9 @@ do { \ OpCase(i_call_fun_I): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, I+2); SET_I(next); @@ -3220,9 +3228,9 @@ do { \ OpCase(i_call_fun_last_IP): { BeamInstr *next; - SWAPOUT; + HEAVY_SWAPOUT; next = call_fun(c_p, Arg(0), reg, THE_NON_VALUE); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { SET_CP(c_p, (BeamInstr *) E[0]); E = ADD_BYTE_OFFSET(E, Arg(1)); @@ -3421,9 +3429,9 @@ do { \ * code[3]: &&call_error_handler * code[4]: Not used */ - SWAPOUT; + HEAVY_SWAPOUT; I = call_error_handler(c_p, I-3, reg, am_undefined_function); - SWAPIN; + HEAVY_SWAPIN; if (I) { Goto(*I); } @@ -3977,10 +3985,10 @@ do { \ Eterm Size; GetArg1(4, Size); - SWAPOUT; + HEAVY_SWAPOUT; reg[live] = x(SCRATCH_X_REG); res = erts_bs_append(c_p, reg, live, Size, Arg(1), Arg(3)); - SWAPIN; + HEAVY_SWAPIN; if (is_non_value(res)) { /* c_p->freason is already set (may be either BADARG or SYSTEM_LIMIT). */ goto lb_Cl_error; @@ -4005,9 +4013,9 @@ do { \ } OpCase(bs_init_writable): { - SWAPOUT; + HEAVY_SWAPOUT; r(0) = erts_bs_init_writable(c_p, r(0)); - SWAPIN; + HEAVY_SWAPIN; Next(0); } @@ -4835,7 +4843,7 @@ do { \ BeamInstr *next; next = call_fun(c_p, c_p->arity - 1, reg, THE_NON_VALUE); - SWAPIN; + HEAVY_SWAPIN; if (next != NULL) { SET_I(next); Dispatchfun(); @@ -4884,20 +4892,22 @@ do { \ } OpCase(i_hibernate): { - SWAPOUT; + HEAVY_SWAPOUT; if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) { + FCALLS = c_p->fcalls; c_p->flags &= ~F_HIBERNATE_SCHED; goto do_schedule; } else { + HEAVY_SWAPIN; I = handle_error(c_p, I, reg, hibernate_3); goto post_error_handling; } } OpCase(i_debug_breakpoint): { - SWAPOUT; + HEAVY_SWAPOUT; I = call_error_handler(c_p, I-3, reg, am_breakpoint); - SWAPIN; + HEAVY_SWAPIN; if (I) { Goto(*I); } diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index e4283ac945..57dd045193 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -3839,11 +3839,9 @@ BIF_RETTYPE now_0(BIF_ALIST_0) BIF_RETTYPE garbage_collect_0(BIF_ALIST_0) { - int reds; - FLAGS(BIF_P) |= F_NEED_FULLSWEEP; - reds = erts_garbage_collect(BIF_P, 0, NULL, 0); - BIF_RET2(am_true, reds); + erts_garbage_collect(BIF_P, 0, NULL, 0); + BIF_RET(am_true); } /**********************************************************************/ diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 6a52e1a890..7163b4839d 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -135,7 +135,7 @@ static Eterm* sweep_literals_to_old_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm char* src, Uint src_size); static Eterm* collect_live_heap_frags(Process* p, ErlHeapFragment *live_hf_end, Eterm* heap, Eterm* htop, Eterm* objv, int nobj); -static void adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj); +static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); static void sweep_off_heap(Process *p, int fullsweep); @@ -172,6 +172,20 @@ typedef struct { erts_smp_atomic32_t refc; } ErtsGCInfoReq; +static ERTS_INLINE int +gc_cost(Uint gc_moved_live_words, Uint resize_moved_words) +{ + Sint reds; + + reds = gc_moved_live_words/10; + reds += resize_moved_words/100; + if (reds < 1) + return 1; + if (reds > INT_MAX) + return INT_MAX; + return (int) reds; +} + ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq, ErtsGCInfoReq, 5, @@ -413,6 +427,7 @@ static ERTS_INLINE void reset_active_writer(Process *p) } #define ERTS_DELAY_GC_EXTRA_FREE 40 +#define ERTS_ABANDON_HEAP_COST 10 static int delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need) @@ -421,6 +436,7 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need) Eterm *orig_heap, *orig_hend, *orig_htop, *orig_stop; Eterm *stop, *hend; Uint hsz, ssz; + int reds_left; ERTS_HOLE_CHECK(p); @@ -482,7 +498,12 @@ delay_garbage_collection(Process *p, ErlHeapFragment *live_hf_end, int need) /* Make sure that we do a proper GC as soon as possible... */ p->flags |= F_FORCE_GC; - return CONTEXT_REDS; + reds_left = ERTS_BIF_REDS_LEFT(p); + if (reds_left > ERTS_ABANDON_HEAP_COST) { + int vreds = reds_left - ERTS_ABANDON_HEAP_COST; + ERTS_VBUMP_REDS(p, vreds); + } + return ERTS_ABANDON_HEAP_COST; } static ERTS_FORCE_INLINE Uint @@ -536,7 +557,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, int need, Eterm* objv, int nobj) { Uint reclaimed_now = 0; - int done = 0; + int reds; ErtsMonotonicTime start_time = 0; /* Shut up faulty warning... */ ErtsSchedulerData *esdp; #ifdef USE_VM_PROBES @@ -562,9 +583,7 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, ERTS_CHK_OFFHEAP(p); ErtsGcQuickSanityCheck(p); - if (GEN_GCS(p) >= MAX_GEN_GCS(p)) { - FLAGS(p) |= F_NEED_FULLSWEEP; - } + #ifdef USE_VM_PROBES *pidbuf = '\0'; if (DTRACE_ENABLED(gc_major_start) @@ -577,17 +596,21 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, /* * Test which type of GC to do. */ - while (!done) { - if ((FLAGS(p) & F_NEED_FULLSWEEP) != 0) { - DTRACE2(gc_major_start, pidbuf, need); - done = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); - DTRACE2(gc_major_end, pidbuf, reclaimed_now); - } else { - DTRACE2(gc_minor_start, pidbuf, need); - done = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); - DTRACE2(gc_minor_end, pidbuf, reclaimed_now); - } + + if (GEN_GCS(p) < MAX_GEN_GCS(p) && !(FLAGS(p) & F_NEED_FULLSWEEP)) { + DTRACE2(gc_minor_start, pidbuf, need); + reds = minor_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); + DTRACE2(gc_minor_end, pidbuf, reclaimed_now); + if (reds < 0) + goto do_major_collection; + } + else { + do_major_collection: + DTRACE2(gc_major_start, pidbuf, need); + reds = major_collection(p, live_hf_end, need, objv, nobj, &reclaimed_now); + DTRACE2(gc_major_end, pidbuf, reclaimed_now); } + reset_active_writer(p); /* @@ -648,23 +671,22 @@ garbage_collect(Process* p, ErlHeapFragment *live_hf_end, p->last_old_htop = p->old_htop; #endif - /* FIXME: This function should really return an Sint, i.e., a possibly - 64 bit wide signed integer, but that requires updating all the code - that calls it. For now, we just return INT_MAX if the result is too - large for an int. */ - { - Sint result = (HEAP_TOP(p) - HEAP_START(p)) / 10; - if (result >= INT_MAX) return INT_MAX; - else return (int) result; - } + return reds; } int -erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) +erts_garbage_collect_nobump(Process* p, int need, Eterm* objv, int nobj) { return garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj); } +void +erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) +{ + int reds = garbage_collect(p, ERTS_INVALID_HFRAG_PTR, need, objv, nobj); + BUMP_REDS(p, reds); +} + /* * Place all living data on a the new heap; deallocate any old heap. * Meant to be used by hibernate/3. @@ -679,6 +701,7 @@ erts_garbage_collect_hibernate(Process* p) char* area; Uint area_size; Sint offs; + int reds; if (p->flags & F_DISABLE_GC) ERTS_INTERNAL_ERROR("GC disabled"); @@ -774,6 +797,9 @@ erts_garbage_collect_hibernate(Process* p) ErtsGcQuickSanityCheck(p); erts_smp_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_GC); + + reds = gc_cost(actual_size, actual_size); + BUMP_REDS(p, reds); } @@ -989,16 +1015,21 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, ((mature_size <= OLD_HEND(p) - OLD_HTOP(p)) && ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) && ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) { - Uint stack_size, size_after, need_after, new_sz; + Eterm *prev_old_htop; + Uint stack_size, size_after, adjust_size, need_after, new_sz, new_mature; stack_size = p->hend - p->stop; new_sz = stack_size + size_before; new_sz = next_heap_size(p, new_sz, 0); + prev_old_htop = p->old_htop; do_minor(p, live_hf_end, (char *) mature, mature_size*sizeof(Eterm), new_sz, objv, nobj); - size_after = HEAP_TOP(p) - HEAP_START(p); + new_mature = p->old_htop - prev_old_htop; + + size_after = new_mature; + size_after += HEAP_TOP(p) - HEAP_START(p); *recl += (size_before - size_after); ErtsGcQuickSanityCheck(p); @@ -1017,6 +1048,8 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, * the heap size is substantial, we don't want to shrink. */ + adjust_size = 0; + if ((HEAP_SIZE(p) > 3000) && (4 * need_after < HEAP_SIZE(p)) && ((HEAP_SIZE(p) > 8000) || (HEAP_SIZE(p) > (OLD_HEND(p) - OLD_HEAP(p))))) { @@ -1038,31 +1071,33 @@ minor_collection(Process* p, ErlHeapFragment *live_hf_end, : next_heap_size(p, wanted, 0); if (wanted < HEAP_SIZE(p)) { shrink_new_heap(p, wanted, objv, nobj); + adjust_size = p->htop - p->heap; } - ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); - ASSERT(MBUF(p) == NULL); - return 1; /* We are done. */ + goto done; } if (HEAP_SIZE(p) >= need_after) { /* * The heap size turned out to be just right. We are done. */ - ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); - ASSERT(MBUF(p) == NULL); - return 1; + goto done; } grow_new_heap(p, next_heap_size(p, need_after, 0), objv, nobj); - return 1; + adjust_size = p->htop - p->heap; + + done: + ASSERT(HEAP_SIZE(p) == next_heap_size(p, HEAP_SIZE(p), 0)); + ASSERT(MBUF(p) == NULL); + + return gc_cost(size_after, adjust_size); } /* * Not enough room for a minor collection. Must force a major collection. */ - FLAGS(p) |= F_NEED_FULLSWEEP; - return 0; + return -1; } /* @@ -1341,12 +1376,13 @@ static int major_collection(Process* p, ErlHeapFragment *live_hf_end, int need, Eterm* objv, int nobj, Uint *recl) { - Uint size_before, stack_size; + Uint size_before, size_after, stack_size; Eterm* n_heap; Eterm* n_htop; char* oh = (char *) OLD_HEAP(p); Uint oh_size = (char *) OLD_HTOP(p) - oh; Uint new_sz, stk_sz; + int adjusted; /* * Do a fullsweep GC. First figure out the size of the heap @@ -1413,9 +1449,10 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, ErtsGcQuickSanityCheck(p); - *recl += size_before - (HEAP_TOP(p) - HEAP_START(p)); + size_after = HEAP_TOP(p) - HEAP_START(p); + *recl += size_before - size_after; - adjust_after_fullsweep(p, need, objv, nobj); + adjusted = adjust_after_fullsweep(p, need, objv, nobj); #ifdef HARDDEBUG disallow_heap_frag_ref_in_heap(p); @@ -1423,7 +1460,8 @@ major_collection(Process* p, ErlHeapFragment *live_hf_end, remove_message_buffers(p); ErtsGcQuickSanityCheck(p); - return 1; /* We are done. */ + + return gc_cost(size_after, adjusted ? size_after : 0); } static Eterm * @@ -1524,9 +1562,10 @@ full_sweep_heaps(Process *p, return n_htop; } -static void +static int adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) { + int adjusted = 0; Uint wanted, sz, need_after; Uint stack_size = STACK_SZ_ON_HEAP(p); @@ -1539,6 +1578,7 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) /* Too small - grow to match requested need */ sz = next_heap_size(p, need_after, 0); grow_new_heap(p, sz, objv, nobj); + adjusted = 1; } else if (3 * HEAP_SIZE(p) < 4 * need_after){ /* Need more than 75% of current, postpone to next GC.*/ FLAGS(p) |= F_HEAP_GROW; @@ -1555,8 +1595,10 @@ adjust_after_fullsweep(Process *p, int need, Eterm *objv, int nobj) if (sz < HEAP_SIZE(p)) { shrink_new_heap(p, sz, objv, nobj); + adjusted = 1; } } + return adjusted; } /* diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index a496c5f008..d603866cbf 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -134,7 +134,8 @@ typedef struct { void erts_gc_info(ErtsGCInfo *gcip); void erts_init_gc(void); -int erts_garbage_collect(struct process*, int, Eterm*, int); +int erts_garbage_collect_nobump(struct process*, int, Eterm*, int); +void erts_garbage_collect(struct process*, int, Eterm*, int); void erts_garbage_collect_hibernate(struct process* p); Eterm erts_gc_after_bif_call_lhf(struct process* p, ErlHeapFragment *live_hf_end, Eterm result, Eterm* regs, Uint arity); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index e490ffea5a..5e7ed0d151 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -9774,7 +9774,7 @@ Process *schedule(Process *p, int calls) if (ERTS_IS_GC_DESIRED(p)) { if (!(state & ERTS_PSFLG_EXITING) && !(p->flags & F_DISABLE_GC)) { - reds -= erts_garbage_collect(p, 0, p->arg_reg, p->arity); + reds -= erts_garbage_collect_nobump(p, 0, p->arg_reg, p->arity); if (reds <= 0) { p->fcalls = reds; goto sched_out_proc; @@ -10057,10 +10057,10 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) else { if (!garbage_collected) { FLAGS(c_p) |= F_NEED_FULLSWEEP; - reds += erts_garbage_collect(c_p, - 0, - c_p->arg_reg, - c_p->arity); + reds += erts_garbage_collect_nobump(c_p, + 0, + c_p->arg_reg, + c_p->arity); garbage_collected = 1; } st_res = am_true; diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 8606371bdf..f82cad745a 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -583,7 +583,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) root[0] = id; root[1] = value; root[2] = old; - BUMP_REDS(p, erts_garbage_collect(p, needed, root, 3)); + erts_garbage_collect(p, needed, root, 3); id = root[0]; value = root[1]; old = root[2]; @@ -715,7 +715,7 @@ static void shrink(Process *p, Eterm* ret) needed = 2*erts_list_length(hi); } if (HeapWordsLeft(p) < needed) { - BUMP_REDS(p, erts_garbage_collect(p, needed, ret, 1)); + erts_garbage_collect(p, needed, ret, 1); hi = pd->data[(pd->splitPosition + pd->homeSize)]; lo = pd->data[pd->splitPosition]; } @@ -811,7 +811,7 @@ static void grow(Process *p) } } if (HeapWordsLeft(p) < needed) { - BUMP_REDS(p, erts_garbage_collect(p, needed, 0, 0)); + erts_garbage_collect(p, needed, 0, 0); } #ifdef DEBUG hp_limit = p->htop + needed; -- cgit v1.2.3